diff --git "a/checkpoint-50000/trainer_state.json" "b/checkpoint-50000/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-50000/trainer_state.json" @@ -0,0 +1,355551 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 1.37363466656075, + "global_step": 50000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 1.5873015873015873e-06, + "loss": 10.5355, + "theoretical_loss": 20.841284642178216, + "tokens_seen": 65536 + }, + { + "epoch": 0.0, + "learning_rate": 3.1746031746031746e-06, + "loss": 10.5405, + "theoretical_loss": 17.59466794495971, + "tokens_seen": 131072 + }, + { + "epoch": 0.0, + "learning_rate": 4.7619047619047615e-06, + "loss": 10.4096, + "theoretical_loss": 15.967943933467634, + "tokens_seen": 196608 + }, + { + "epoch": 0.0, + "learning_rate": 6.349206349206349e-06, + "loss": 10.1794, + "theoretical_loss": 14.920783596619636, + "tokens_seen": 262144 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507936e-06, + "loss": 9.9009, + "theoretical_loss": 14.164683778394041, + "tokens_seen": 327680 + }, + { + "epoch": 0.0, + "learning_rate": 9.523809523809523e-06, + "loss": 9.6355, + "theoretical_loss": 13.581028313181289, + "tokens_seen": 393216 + }, + { + "epoch": 0.0, + "learning_rate": 1.1111111111111112e-05, + "loss": 9.3831, + "theoretical_loss": 13.110275842032255, + "tokens_seen": 458752 + }, + { + "epoch": 0.0, + "learning_rate": 1.2698412698412699e-05, + "loss": 9.1719, + "theoretical_loss": 12.71859646611439, + "tokens_seen": 524288 + }, + { + "epoch": 0.0, + "learning_rate": 1.4285714285714285e-05, + "loss": 9.0143, + "theoretical_loss": 12.38505930450469, + "tokens_seen": 589824 + }, + { + "epoch": 0.0, + "learning_rate": 1.5873015873015872e-05, + "loss": 8.8536, + "theoretical_loss": 12.095879447666144, + "tokens_seen": 655360 + }, + { + "epoch": 0.0, + "learning_rate": 1.746031746031746e-05, + "loss": 8.8086, + "theoretical_loss": 11.841533071732742, + "tokens_seen": 720896 + }, + { + "epoch": 0.0, + "learning_rate": 1.9047619047619046e-05, + "loss": 8.74, + "theoretical_loss": 11.615186049337796, + "tokens_seen": 786432 + }, + { + "epoch": 0.0, + "learning_rate": 2.0634920634920636e-05, + "loss": 8.5264, + "theoretical_loss": 11.411780980817854, + "tokens_seen": 851968 + }, + { + "epoch": 0.0, + "learning_rate": 2.2222222222222223e-05, + "loss": 8.5191, + "theoretical_loss": 11.227478542742938, + "tokens_seen": 917504 + }, + { + "epoch": 0.0, + "learning_rate": 2.380952380952381e-05, + "loss": 8.4946, + "theoretical_loss": 11.059300758608979, + "tokens_seen": 983040 + }, + { + "epoch": 0.0, + "learning_rate": 2.5396825396825397e-05, + "loss": 8.3993, + "theoretical_loss": 10.904894927088016, + "tokens_seen": 1048576 + }, + { + "epoch": 0.0, + "learning_rate": 2.6984126984126984e-05, + "loss": 8.339, + "theoretical_loss": 10.76237258069379, + "tokens_seen": 1114112 + }, + { + "epoch": 0.0, + "learning_rate": 2.857142857142857e-05, + "loss": 8.3136, + "theoretical_loss": 10.630196716861345, + "tokens_seen": 1179648 + }, + { + "epoch": 0.0, + "learning_rate": 3.0158730158730158e-05, + "loss": 8.2016, + "theoretical_loss": 10.507101012987707, + "tokens_seen": 1245184 + }, + { + "epoch": 0.0, + "learning_rate": 3.1746031746031745e-05, + "loss": 8.2822, + "theoretical_loss": 10.392030784394397, + "tokens_seen": 1310720 + }, + { + "epoch": 0.0, + "learning_rate": 3.3333333333333335e-05, + "loss": 8.1899, + "theoretical_loss": 10.284099061527812, + "tokens_seen": 1376256 + }, + { + "epoch": 0.0, + "learning_rate": 3.492063492063492e-05, + "loss": 8.0618, + "theoretical_loss": 10.182553393901085, + "tokens_seen": 1441792 + }, + { + "epoch": 0.0, + "learning_rate": 3.650793650793651e-05, + "loss": 8.1167, + "theoretical_loss": 10.086750402363664, + "tokens_seen": 1507328 + }, + { + "epoch": 0.0, + "learning_rate": 3.809523809523809e-05, + "loss": 7.9888, + "theoretical_loss": 9.996136019471344, + "tokens_seen": 1572864 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-05, + "loss": 7.9389, + "theoretical_loss": 9.910229967024176, + "tokens_seen": 1638400 + }, + { + "epoch": 0.0, + "learning_rate": 4.126984126984127e-05, + "loss": 7.9611, + "theoretical_loss": 9.828613432171625, + "tokens_seen": 1703936 + }, + { + "epoch": 0.0, + "learning_rate": 4.2857142857142856e-05, + "loss": 7.8176, + "theoretical_loss": 9.75091918753851, + "tokens_seen": 1769472 + }, + { + "epoch": 0.0, + "learning_rate": 4.4444444444444447e-05, + "loss": 7.782, + "theoretical_loss": 9.676823599712613, + "tokens_seen": 1835008 + }, + { + "epoch": 0.0, + "learning_rate": 4.603174603174603e-05, + "loss": 7.8388, + "theoretical_loss": 9.606040111776705, + "tokens_seen": 1900544 + }, + { + "epoch": 0.0, + "learning_rate": 4.761904761904762e-05, + "loss": 7.7238, + "theoretical_loss": 9.538313887395919, + "tokens_seen": 1966080 + }, + { + "epoch": 0.0, + "learning_rate": 4.9206349206349204e-05, + "loss": 7.6459, + "theoretical_loss": 9.473417378263001, + "tokens_seen": 2031616 + }, + { + "epoch": 0.0, + "learning_rate": 5.0793650793650794e-05, + "loss": 7.4806, + "theoretical_loss": 9.411146631541524, + "tokens_seen": 2097152 + }, + { + "epoch": 0.0, + "learning_rate": 5.2380952380952384e-05, + "loss": 7.462, + "theoretical_loss": 9.351318194865048, + "tokens_seen": 2162688 + }, + { + "epoch": 0.0, + "learning_rate": 5.396825396825397e-05, + "loss": 7.4201, + "theoretical_loss": 9.293766507291341, + "tokens_seen": 2228224 + }, + { + "epoch": 0.0, + "learning_rate": 5.555555555555556e-05, + "loss": 7.38, + "theoretical_loss": 9.238341688075822, + "tokens_seen": 2293760 + }, + { + "epoch": 0.0, + "learning_rate": 5.714285714285714e-05, + "loss": 7.2879, + "theoretical_loss": 9.184907653139359, + "tokens_seen": 2359296 + }, + { + "epoch": 0.0, + "learning_rate": 5.873015873015873e-05, + "loss": 7.3045, + "theoretical_loss": 9.133340503044415, + "tokens_seen": 2424832 + }, + { + "epoch": 0.0, + "learning_rate": 6.0317460317460316e-05, + "loss": 7.1794, + "theoretical_loss": 9.0835271371648, + "tokens_seen": 2490368 + }, + { + "epoch": 0.0, + "learning_rate": 6.19047619047619e-05, + "loss": 7.1184, + "theoretical_loss": 9.035364057274894, + "tokens_seen": 2555904 + }, + { + "epoch": 0.0, + "learning_rate": 6.349206349206349e-05, + "loss": 7.0666, + "theoretical_loss": 8.988756330540422, + "tokens_seen": 2621440 + }, + { + "epoch": 0.0, + "learning_rate": 6.507936507936509e-05, + "loss": 6.982, + "theoretical_loss": 8.943616687271678, + "tokens_seen": 2686976 + }, + { + "epoch": 0.0, + "learning_rate": 6.666666666666667e-05, + "loss": 7.1173, + "theoretical_loss": 8.89986473310929, + "tokens_seen": 2752512 + }, + { + "epoch": 0.0, + "learning_rate": 6.825396825396825e-05, + "loss": 7.0194, + "theoretical_loss": 8.857426258784564, + "tokens_seen": 2818048 + }, + { + "epoch": 0.0, + "learning_rate": 6.984126984126984e-05, + "loss": 6.9244, + "theoretical_loss": 8.816232633409479, + "tokens_seen": 2883584 + }, + { + "epoch": 0.0, + "learning_rate": 7.142857142857143e-05, + "loss": 6.9294, + "theoretical_loss": 8.776220269542522, + "tokens_seen": 2949120 + }, + { + "epoch": 0.0, + "learning_rate": 7.301587301587302e-05, + "loss": 6.7326, + "theoretical_loss": 8.737330150151898, + "tokens_seen": 3014656 + }, + { + "epoch": 0.0, + "learning_rate": 7.460317460317461e-05, + "loss": 6.8233, + "theoretical_loss": 8.699507409139912, + "tokens_seen": 3080192 + }, + { + "epoch": 0.0, + "learning_rate": 7.619047619047618e-05, + "loss": 6.7542, + "theoretical_loss": 8.662700958366539, + "tokens_seen": 3145728 + }, + { + "epoch": 0.0, + "learning_rate": 7.777777777777778e-05, + "loss": 6.6128, + "theoretical_loss": 8.62686315516739, + "tokens_seen": 3211264 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507937e-05, + "loss": 6.6336, + "theoretical_loss": 8.591949505242134, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 8.095238095238096e-05, + "loss": 6.5844, + "theoretical_loss": 8.557918396526182, + "tokens_seen": 3342336 + }, + { + "epoch": 0.0, + "learning_rate": 8.253968253968255e-05, + "loss": 6.3935, + "theoretical_loss": 8.524730860277067, + "tokens_seen": 3407872 + }, + { + "epoch": 0.0, + "learning_rate": 8.412698412698413e-05, + "loss": 6.4484, + "theoretical_loss": 8.492350356128252, + "tokens_seen": 3473408 + }, + { + "epoch": 0.0, + "learning_rate": 8.571428571428571e-05, + "loss": 6.3623, + "theoretical_loss": 8.460742578303845, + "tokens_seen": 3538944 + }, + { + "epoch": 0.0, + "learning_rate": 8.730158730158731e-05, + "loss": 6.4215, + "theoretical_loss": 8.429875280561783, + "tokens_seen": 3604480 + }, + { + "epoch": 0.0, + "learning_rate": 8.888888888888889e-05, + "loss": 6.222, + "theoretical_loss": 8.399718117751275, + "tokens_seen": 3670016 + }, + { + "epoch": 0.0, + "learning_rate": 9.047619047619048e-05, + "loss": 6.3531, + "theoretical_loss": 8.370242502142142, + "tokens_seen": 3735552 + }, + { + "epoch": 0.0, + "learning_rate": 9.206349206349206e-05, + "loss": 6.2966, + "theoretical_loss": 8.341421472916394, + "tokens_seen": 3801088 + }, + { + "epoch": 0.0, + "learning_rate": 9.365079365079366e-05, + "loss": 6.2777, + "theoretical_loss": 8.313229577412248, + "tokens_seen": 3866624 + }, + { + "epoch": 0.0, + "learning_rate": 9.523809523809524e-05, + "loss": 6.1319, + "theoretical_loss": 8.28564276288293, + "tokens_seen": 3932160 + }, + { + "epoch": 0.0, + "learning_rate": 9.682539682539682e-05, + "loss": 6.1204, + "theoretical_loss": 8.258638277681179, + "tokens_seen": 3997696 + }, + { + "epoch": 0.0, + "learning_rate": 9.841269841269841e-05, + "loss": 6.2434, + "theoretical_loss": 8.232194580909036, + "tokens_seen": 4063232 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001, + "loss": 6.1409, + "theoretical_loss": 8.20629125968411, + "tokens_seen": 4128768 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010158730158730159, + "loss": 5.9151, + "theoretical_loss": 8.180908953270682, + "tokens_seen": 4194304 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010317460317460319, + "loss": 5.9821, + "theoretical_loss": 8.156029283408564, + "tokens_seen": 4259840 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010476190476190477, + "loss": 5.8359, + "theoretical_loss": 8.131634790246775, + "tokens_seen": 4325376 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010634920634920635, + "loss": 5.9882, + "theoretical_loss": 8.107708873353706, + "tokens_seen": 4390912 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010793650793650794, + "loss": 5.9366, + "theoretical_loss": 8.084235737332481, + "tokens_seen": 4456448 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010952380952380953, + "loss": 5.7864, + "theoretical_loss": 8.061200341620074, + "tokens_seen": 4521984 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011111111111111112, + "loss": 5.7847, + "theoretical_loss": 8.038588354092902, + "tokens_seen": 4587520 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001126984126984127, + "loss": 5.7529, + "theoretical_loss": 8.016386108140445, + "tokens_seen": 4653056 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011428571428571428, + "loss": 5.8326, + "theoretical_loss": 7.994580562902867, + "tokens_seen": 4718592 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001158730158730159, + "loss": 5.7328, + "theoretical_loss": 7.973159266399054, + "tokens_seen": 4784128 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011746031746031746, + "loss": 5.6849, + "theoretical_loss": 7.952110321298584, + "tokens_seen": 4849664 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011904761904761905, + "loss": 5.641, + "theoretical_loss": 7.931422353115133, + "tokens_seen": 4915200 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012063492063492063, + "loss": 5.7055, + "theoretical_loss": 7.911084480620269, + "tokens_seen": 4980736 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012222222222222224, + "loss": 5.7157, + "theoretical_loss": 7.891086288295694, + "tokens_seen": 5046272 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001238095238095238, + "loss": 5.7148, + "theoretical_loss": 7.871417800659003, + "tokens_seen": 5111808 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001253968253968254, + "loss": 5.5534, + "theoretical_loss": 7.8520694583133785, + "tokens_seen": 5177344 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012698412698412698, + "loss": 5.7642, + "theoretical_loss": 7.833032095585231, + "tokens_seen": 5242880 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012857142857142858, + "loss": 5.6881, + "theoretical_loss": 7.8142969196261385, + "tokens_seen": 5308416 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013015873015873017, + "loss": 5.5251, + "theoretical_loss": 7.79585549086643, + "tokens_seen": 5373952 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013174603174603174, + "loss": 5.5142, + "theoretical_loss": 7.777699704717685, + "tokens_seen": 5439488 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013333333333333334, + "loss": 5.5252, + "theoretical_loss": 7.759821774430384, + "tokens_seen": 5505024 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013492063492063494, + "loss": 5.6046, + "theoretical_loss": 7.742214215020986, + "tokens_seen": 5570560 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001365079365079365, + "loss": 5.4653, + "theoretical_loss": 7.72486982819, + "tokens_seen": 5636096 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001380952380952381, + "loss": 5.6301, + "theoretical_loss": 7.707781688159247, + "tokens_seen": 5701632 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013968253968253967, + "loss": 5.5605, + "theoretical_loss": 7.690943128362394, + "tokens_seen": 5767168 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001412698412698413, + "loss": 5.54, + "theoretical_loss": 7.674347728928345, + "tokens_seen": 5832704 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014285714285714287, + "loss": 5.574, + "theoretical_loss": 7.6579893049019265, + "tokens_seen": 5898240 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014444444444444444, + "loss": 5.5315, + "theoretical_loss": 7.641861895150762, + "tokens_seen": 5963776 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014603174603174603, + "loss": 5.4421, + "theoretical_loss": 7.625959751911346, + "tokens_seen": 6029312 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014761904761904763, + "loss": 5.4408, + "theoretical_loss": 7.610277330930954, + "tokens_seen": 6094848 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014920634920634923, + "loss": 5.4034, + "theoretical_loss": 7.5948092821654365, + "tokens_seen": 6160384 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001507936507936508, + "loss": 5.2766, + "theoretical_loss": 7.579550440996015, + "tokens_seen": 6225920 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015238095238095237, + "loss": 5.3709, + "theoretical_loss": 7.564495819930985, + "tokens_seen": 6291456 + }, + { + "epoch": 0.0, + "learning_rate": 0.000153968253968254, + "loss": 5.3441, + "theoretical_loss": 7.549640600760828, + "tokens_seen": 6356992 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015555555555555556, + "loss": 5.3515, + "theoretical_loss": 7.534980127137563, + "tokens_seen": 6422528 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015714285714285716, + "loss": 5.394, + "theoretical_loss": 7.520509897551315, + "tokens_seen": 6488064 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 10124, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 5.372183799743652, + "objective/train/theoretical_loss": 7.5062255586790965, + "objective/train/tokens_used": 27013600, + "theoretical_loss": 7.5062255586790965, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015873015873015873, + "loss": 5.3722, + "theoretical_loss": 7.5062255586790965, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016031746031746033, + "loss": 5.3875, + "theoretical_loss": 7.492122899082528, + "tokens_seen": 6619136 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016190476190476192, + "loss": 5.4057, + "theoretical_loss": 7.478197843232982, + "tokens_seen": 6684672 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001634920634920635, + "loss": 5.3362, + "theoretical_loss": 7.464446445844085, + "tokens_seen": 6750208 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001650793650793651, + "loss": 5.3139, + "theoretical_loss": 7.450864886492971, + "tokens_seen": 6815744 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001666666666666667, + "loss": 5.2791, + "theoretical_loss": 7.437449464512941, + "tokens_seen": 6881280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016825396825396826, + "loss": 5.2369, + "theoretical_loss": 7.424196594141411, + "tokens_seen": 6946816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016984126984126986, + "loss": 5.2822, + "theoretical_loss": 7.4111027999080985, + "tokens_seen": 7012352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017142857142857143, + "loss": 5.3693, + "theoretical_loss": 7.398164712249439, + "tokens_seen": 7077888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017301587301587302, + "loss": 5.4055, + "theoretical_loss": 7.3853790633361545, + "tokens_seen": 7143424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017460317460317462, + "loss": 5.1771, + "theoretical_loss": 7.3727426831017535, + "tokens_seen": 7208960 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001761904761904762, + "loss": 5.3257, + "theoretical_loss": 7.360252495460585, + "tokens_seen": 7274496 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017777777777777779, + "loss": 5.1899, + "theoretical_loss": 7.347905514704743, + "tokens_seen": 7340032 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017936507936507938, + "loss": 5.2891, + "theoretical_loss": 7.33569884206989, + "tokens_seen": 7405568 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018095238095238095, + "loss": 5.2128, + "theoretical_loss": 7.32362966246063, + "tokens_seen": 7471104 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018253968253968255, + "loss": 5.1534, + "theoretical_loss": 7.311695241326718, + "tokens_seen": 7536640 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018412698412698412, + "loss": 5.302, + "theoretical_loss": 7.299892921681898, + "tokens_seen": 7602176 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018571428571428572, + "loss": 5.2996, + "theoretical_loss": 7.288220121257689, + "tokens_seen": 7667712 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018730158730158731, + "loss": 5.2726, + "theoretical_loss": 7.276674329784936, + "tokens_seen": 7733248 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018888888888888888, + "loss": 5.1882, + "theoretical_loss": 7.265253106396328, + "tokens_seen": 7798784 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019047619047619048, + "loss": 5.0924, + "theoretical_loss": 7.253954077143563, + "tokens_seen": 7864320 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019206349206349208, + "loss": 5.282, + "theoretical_loss": 7.242774932623186, + "tokens_seen": 7929856 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019365079365079365, + "loss": 5.0143, + "theoretical_loss": 7.2317134257054665, + "tokens_seen": 7995392 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019523809523809525, + "loss": 5.1009, + "theoretical_loss": 7.220767369361081, + "tokens_seen": 8060928 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019682539682539682, + "loss": 4.9736, + "theoretical_loss": 7.2099346345805815, + "tokens_seen": 8126464 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019841269841269844, + "loss": 5.2631, + "theoretical_loss": 7.1992131483820305, + "tokens_seen": 8192000 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002, + "loss": 5.1442, + "theoretical_loss": 7.188600891902338, + "tokens_seen": 8257536 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020158730158730158, + "loss": 5.146, + "theoretical_loss": 7.178095898568204, + "tokens_seen": 8323072 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020317460317460318, + "loss": 5.2826, + "theoretical_loss": 7.167696252342704, + "tokens_seen": 8388608 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020476190476190475, + "loss": 5.1134, + "theoretical_loss": 7.15740008604386, + "tokens_seen": 8454144 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020634920634920637, + "loss": 5.1957, + "theoretical_loss": 7.147205579731682, + "tokens_seen": 8519680 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020793650793650797, + "loss": 5.0839, + "theoretical_loss": 7.1371109591604105, + "tokens_seen": 8585216 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020952380952380954, + "loss": 5.2144, + "theoretical_loss": 7.127114494292836, + "tokens_seen": 8650752 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021111111111111113, + "loss": 5.0342, + "theoretical_loss": 7.117214497873759, + "tokens_seen": 8716288 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002126984126984127, + "loss": 4.9848, + "theoretical_loss": 7.107409324059815, + "tokens_seen": 8781824 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002142857142857143, + "loss": 5.1246, + "theoretical_loss": 7.097697367103032, + "tokens_seen": 8847360 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021587301587301587, + "loss": 5.1589, + "theoretical_loss": 7.0880770600856335, + "tokens_seen": 8912896 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021746031746031744, + "loss": 5.1326, + "theoretical_loss": 7.0785468737037345, + "tokens_seen": 8978432 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021904761904761907, + "loss": 4.9242, + "theoretical_loss": 7.069105315097691, + "tokens_seen": 9043968 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022063492063492066, + "loss": 4.9897, + "theoretical_loss": 7.059750926726999, + "tokens_seen": 9109504 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022222222222222223, + "loss": 4.936, + "theoretical_loss": 7.050482285287744, + "tokens_seen": 9175040 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022380952380952383, + "loss": 4.9534, + "theoretical_loss": 7.041298000670675, + "tokens_seen": 9240576 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002253968253968254, + "loss": 4.972, + "theoretical_loss": 7.032196714958137, + "tokens_seen": 9306112 + }, + { + "epoch": 0.0, + "learning_rate": 0.000226984126984127, + "loss": 5.0425, + "theoretical_loss": 7.023177101458123, + "tokens_seen": 9371648 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022857142857142857, + "loss": 5.1426, + "theoretical_loss": 7.014237863773846, + "tokens_seen": 9437184 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023015873015873016, + "loss": 4.9935, + "theoretical_loss": 7.005377734907255, + "tokens_seen": 9502720 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002317460317460318, + "loss": 4.9806, + "theoretical_loss": 6.996595476395081, + "tokens_seen": 9568256 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023333333333333336, + "loss": 4.9519, + "theoretical_loss": 6.98788987747596, + "tokens_seen": 9633792 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023492063492063493, + "loss": 4.9623, + "theoretical_loss": 6.979259754287375, + "tokens_seen": 9699328 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023650793650793653, + "loss": 4.9584, + "theoretical_loss": 6.970703949091095, + "tokens_seen": 9764864 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002380952380952381, + "loss": 5.0187, + "theoretical_loss": 6.962221329525966, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002396825396825397, + "loss": 4.9565, + "theoretical_loss": 6.953810787886883, + "tokens_seen": 9895936 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024126984126984126, + "loss": 4.8708, + "theoretical_loss": 6.945471240428864, + "tokens_seen": 9961472 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024285714285714286, + "loss": 4.8598, + "theoretical_loss": 6.937201626695213, + "tokens_seen": 10027008 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002444444444444445, + "loss": 4.978, + "theoretical_loss": 6.929000908868754, + "tokens_seen": 10092544 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024603174603174605, + "loss": 5.0203, + "theoretical_loss": 6.920868071145226, + "tokens_seen": 10158080 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002476190476190476, + "loss": 5.0491, + "theoretical_loss": 6.912802119127937, + "tokens_seen": 10223616 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024920634920634925, + "loss": 4.8428, + "theoretical_loss": 6.904802079242816, + "tokens_seen": 10289152 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002507936507936508, + "loss": 4.8803, + "theoretical_loss": 6.896866998173063, + "tokens_seen": 10354688 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002523809523809524, + "loss": 4.9369, + "theoretical_loss": 6.888995942312617, + "tokens_seen": 10420224 + }, + { + "epoch": 0.0, + "learning_rate": 0.00025396825396825396, + "loss": 4.8271, + "theoretical_loss": 6.881187997237696, + "tokens_seen": 10485760 + }, + { + "epoch": 0.0, + "learning_rate": 0.00025555555555555553, + "loss": 4.872, + "theoretical_loss": 6.873442267195708, + "tokens_seen": 10551296 + }, + { + "epoch": 0.0, + "learning_rate": 0.00025714285714285715, + "loss": 5.0061, + "theoretical_loss": 6.8657578746108605, + "tokens_seen": 10616832 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002587301587301588, + "loss": 4.9092, + "theoretical_loss": 6.858133959605815, + "tokens_seen": 10682368 + }, + { + "epoch": 0.0, + "learning_rate": 0.00026031746031746035, + "loss": 5.0769, + "theoretical_loss": 6.8505696795387845, + "tokens_seen": 10747904 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002619047619047619, + "loss": 4.8411, + "theoretical_loss": 6.843064208555461, + "tokens_seen": 10813440 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002634920634920635, + "loss": 4.9565, + "theoretical_loss": 6.835616737155247, + "tokens_seen": 10878976 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002650793650793651, + "loss": 4.7779, + "theoretical_loss": 6.828226471771211, + "tokens_seen": 10944512 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002666666666666667, + "loss": 4.9304, + "theoretical_loss": 6.8208926343632905, + "tokens_seen": 11010048 + }, + { + "epoch": 0.0, + "learning_rate": 0.00026825396825396825, + "loss": 4.9079, + "theoretical_loss": 6.813614462024214, + "tokens_seen": 11075584 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002698412698412699, + "loss": 4.8657, + "theoretical_loss": 6.806391206597704, + "tokens_seen": 11141120 + }, + { + "epoch": 0.0, + "learning_rate": 0.00027142857142857144, + "loss": 4.8902, + "theoretical_loss": 6.799222134308489, + "tokens_seen": 11206656 + }, + { + "epoch": 0.0, + "learning_rate": 0.000273015873015873, + "loss": 4.8231, + "theoretical_loss": 6.792106525403691, + "tokens_seen": 11272192 + }, + { + "epoch": 0.0, + "learning_rate": 0.00027460317460317464, + "loss": 4.9382, + "theoretical_loss": 6.785043673805189, + "tokens_seen": 11337728 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002761904761904762, + "loss": 4.8235, + "theoretical_loss": 6.778032886772552, + "tokens_seen": 11403264 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002777777777777778, + "loss": 4.7207, + "theoretical_loss": 6.771073484576156, + "tokens_seen": 11468800 + }, + { + "epoch": 0.0, + "learning_rate": 0.00027936507936507935, + "loss": 4.9425, + "theoretical_loss": 6.764164800180141, + "tokens_seen": 11534336 + }, + { + "epoch": 0.0, + "learning_rate": 0.00028095238095238097, + "loss": 4.7528, + "theoretical_loss": 6.757306178934827, + "tokens_seen": 11599872 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002825396825396826, + "loss": 4.935, + "theoretical_loss": 6.750496978278292, + "tokens_seen": 11665408 + }, + { + "epoch": 0.0, + "learning_rate": 0.00028412698412698417, + "loss": 4.7528, + "theoretical_loss": 6.743736567446755, + "tokens_seen": 11730944 + }, + { + "epoch": 0.0, + "learning_rate": 0.00028571428571428574, + "loss": 4.8485, + "theoretical_loss": 6.737024327193478, + "tokens_seen": 11796480 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002873015873015873, + "loss": 4.8715, + "theoretical_loss": 6.73035964951589, + "tokens_seen": 11862016 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002888888888888889, + "loss": 4.7815, + "theoretical_loss": 6.723741937390625, + "tokens_seen": 11927552 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002904761904761905, + "loss": 4.7251, + "theoretical_loss": 6.717170604516246, + "tokens_seen": 11993088 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029206349206349207, + "loss": 4.7504, + "theoretical_loss": 6.71064507506334, + "tokens_seen": 12058624 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029365079365079364, + "loss": 4.7729, + "theoretical_loss": 6.704164783431781, + "tokens_seen": 12124160 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029523809523809526, + "loss": 4.6926, + "theoretical_loss": 6.6977291740148805, + "tokens_seen": 12189696 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029682539682539683, + "loss": 4.7032, + "theoretical_loss": 6.691337700970224, + "tokens_seen": 12255232 + }, + { + "epoch": 0.0, + "learning_rate": 0.00029841269841269846, + "loss": 4.6978, + "theoretical_loss": 6.684989827996945, + "tokens_seen": 12320768 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030000000000000003, + "loss": 4.913, + "theoretical_loss": 6.6786850281192365, + "tokens_seen": 12386304 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003015873015873016, + "loss": 4.6724, + "theoretical_loss": 6.672422783475897, + "tokens_seen": 12451840 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030317460317460317, + "loss": 4.7759, + "theoretical_loss": 6.666202585115692, + "tokens_seen": 12517376 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030476190476190474, + "loss": 4.7548, + "theoretical_loss": 6.66002393279837, + "tokens_seen": 12582912 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030634920634920636, + "loss": 4.6869, + "theoretical_loss": 6.653886334801115, + "tokens_seen": 12648448 + }, + { + "epoch": 0.0, + "learning_rate": 0.000307936507936508, + "loss": 4.7677, + "theoretical_loss": 6.647789307730289, + "tokens_seen": 12713984 + }, + { + "epoch": 0.0, + "learning_rate": 0.00030952380952380956, + "loss": 4.7758, + "theoretical_loss": 6.641732376338277, + "tokens_seen": 12779520 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003111111111111111, + "loss": 4.8743, + "theoretical_loss": 6.6357150733452785, + "tokens_seen": 12845056 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003126984126984127, + "loss": 4.7179, + "theoretical_loss": 6.629736939265876, + "tokens_seen": 12910592 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003142857142857143, + "loss": 4.9144, + "theoretical_loss": 6.623797522240246, + "tokens_seen": 12976128 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003158730158730159, + "loss": 4.8827, + "theoretical_loss": 6.617896377869851, + "tokens_seen": 13041664 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 12399, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 4.735827445983887, + "objective/train/theoretical_loss": 6.61203306905748, + "objective/train/tokens_used": 33567200, + "theoretical_loss": 6.61203306905748, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.00031746031746031746, + "loss": 4.7358, + "theoretical_loss": 6.61203306905748, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003190476190476191, + "loss": 4.7847, + "theoretical_loss": 6.6062071658514965, + "tokens_seen": 13172736 + }, + { + "epoch": 0.0, + "learning_rate": 0.00032063492063492065, + "loss": 4.7583, + "theoretical_loss": 6.600418245294164, + "tokens_seen": 13238272 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003222222222222222, + "loss": 4.6434, + "theoretical_loss": 6.594665891273929, + "tokens_seen": 13303808 + }, + { + "epoch": 0.0, + "learning_rate": 0.00032380952380952385, + "loss": 4.8477, + "theoretical_loss": 6.588949694381529, + "tokens_seen": 13369344 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003253968253968254, + "loss": 4.6934, + "theoretical_loss": 6.583269251769811, + "tokens_seen": 13434880 + }, + { + "epoch": 0.0, + "learning_rate": 0.000326984126984127, + "loss": 4.6676, + "theoretical_loss": 6.577624167017157, + "tokens_seen": 13500416 + }, + { + "epoch": 0.0, + "learning_rate": 0.00032857142857142856, + "loss": 4.6634, + "theoretical_loss": 6.5720140499943955, + "tokens_seen": 13565952 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003301587301587302, + "loss": 4.6706, + "theoretical_loss": 6.566438516735092, + "tokens_seen": 13631488 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003317460317460318, + "loss": 4.6321, + "theoretical_loss": 6.560897189309134, + "tokens_seen": 13697024 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003333333333333334, + "loss": 4.659, + "theoretical_loss": 6.5553896956994855, + "tokens_seen": 13762560 + }, + { + "epoch": 0.0, + "learning_rate": 0.00033492063492063495, + "loss": 4.7745, + "theoretical_loss": 6.5499156696820435, + "tokens_seen": 13828096 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003365079365079365, + "loss": 4.6713, + "theoretical_loss": 6.544474750708482, + "tokens_seen": 13893632 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003380952380952381, + "loss": 4.6694, + "theoretical_loss": 6.539066583792016, + "tokens_seen": 13959168 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003396825396825397, + "loss": 4.6684, + "theoretical_loss": 6.533690819395976, + "tokens_seen": 14024704 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003412698412698413, + "loss": 4.6629, + "theoretical_loss": 6.528347113325134, + "tokens_seen": 14090240 + }, + { + "epoch": 0.0, + "learning_rate": 0.00034285714285714285, + "loss": 4.8538, + "theoretical_loss": 6.523035126619684, + "tokens_seen": 14155776 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003444444444444445, + "loss": 4.63, + "theoretical_loss": 6.517754525451811, + "tokens_seen": 14221312 + }, + { + "epoch": 0.0, + "learning_rate": 0.00034603174603174604, + "loss": 4.8216, + "theoretical_loss": 6.512504981024765, + "tokens_seen": 14286848 + }, + { + "epoch": 0.0, + "learning_rate": 0.00034761904761904767, + "loss": 4.7503, + "theoretical_loss": 6.507286169474376, + "tokens_seen": 14352384 + }, + { + "epoch": 0.0, + "learning_rate": 0.00034920634920634924, + "loss": 4.6495, + "theoretical_loss": 6.502097771772935, + "tokens_seen": 14417920 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003507936507936508, + "loss": 4.6212, + "theoretical_loss": 6.496939473635374, + "tokens_seen": 14483456 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003523809523809524, + "loss": 4.6616, + "theoretical_loss": 6.491810965427682, + "tokens_seen": 14548992 + }, + { + "epoch": 0.0, + "learning_rate": 0.00035396825396825395, + "loss": 4.5403, + "theoretical_loss": 6.486711942077496, + "tokens_seen": 14614528 + }, + { + "epoch": 0.0, + "learning_rate": 0.00035555555555555557, + "loss": 4.5692, + "theoretical_loss": 6.481642102986796, + "tokens_seen": 14680064 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003571428571428572, + "loss": 4.6113, + "theoretical_loss": 6.476601151946659, + "tokens_seen": 14745600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00035873015873015877, + "loss": 4.6895, + "theoretical_loss": 6.471588797054005, + "tokens_seen": 14811136 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036031746031746034, + "loss": 4.6877, + "theoretical_loss": 6.46660475063028, + "tokens_seen": 14876672 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003619047619047619, + "loss": 4.5206, + "theoretical_loss": 6.46164872914203, + "tokens_seen": 14942208 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036349206349206353, + "loss": 4.725, + "theoretical_loss": 6.456720453123305, + "tokens_seen": 15007744 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003650793650793651, + "loss": 4.5871, + "theoretical_loss": 6.4518196470998515, + "tokens_seen": 15073280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036666666666666667, + "loss": 4.6798, + "theoretical_loss": 6.446946039515035, + "tokens_seen": 15138816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036825396825396824, + "loss": 4.5425, + "theoretical_loss": 6.442099362657457, + "tokens_seen": 15204352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00036984126984126986, + "loss": 4.5059, + "theoretical_loss": 6.437279352590206, + "tokens_seen": 15269888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00037142857142857143, + "loss": 4.6036, + "theoretical_loss": 6.432485749081721, + "tokens_seen": 15335424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00037301587301587306, + "loss": 4.6586, + "theoretical_loss": 6.427718295538199, + "tokens_seen": 15400960 + }, + { + "epoch": 0.0, + "learning_rate": 0.00037460317460317463, + "loss": 4.6508, + "theoretical_loss": 6.422976738937518, + "tokens_seen": 15466496 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003761904761904762, + "loss": 4.592, + "theoretical_loss": 6.418260829764641, + "tokens_seen": 15532032 + }, + { + "epoch": 0.0, + "learning_rate": 0.00037777777777777777, + "loss": 4.5609, + "theoretical_loss": 6.413570321948453, + "tokens_seen": 15597568 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003793650793650794, + "loss": 4.5308, + "theoretical_loss": 6.40890497279999, + "tokens_seen": 15663104 + }, + { + "epoch": 0.0, + "learning_rate": 0.00038095238095238096, + "loss": 4.4842, + "theoretical_loss": 6.404264542952033, + "tokens_seen": 15728640 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003825396825396826, + "loss": 4.5147, + "theoretical_loss": 6.399648796300032, + "tokens_seen": 15794176 + }, + { + "epoch": 0.0, + "learning_rate": 0.00038412698412698416, + "loss": 4.579, + "theoretical_loss": 6.395057499944313, + "tokens_seen": 15859712 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003857142857142857, + "loss": 4.5359, + "theoretical_loss": 6.390490424133546, + "tokens_seen": 15925248 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003873015873015873, + "loss": 4.6826, + "theoretical_loss": 6.3859473422094375, + "tokens_seen": 15990784 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003888888888888889, + "loss": 4.6003, + "theoretical_loss": 6.381428030552619, + "tokens_seen": 16056320 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003904761904761905, + "loss": 4.4455, + "theoretical_loss": 6.376932268529695, + "tokens_seen": 16121856 + }, + { + "epoch": 0.0, + "learning_rate": 0.00039206349206349206, + "loss": 4.6933, + "theoretical_loss": 6.372459838441432, + "tokens_seen": 16187392 + }, + { + "epoch": 0.0, + "learning_rate": 0.00039365079365079363, + "loss": 4.6425, + "theoretical_loss": 6.368010525472036, + "tokens_seen": 16252928 + }, + { + "epoch": 0.0, + "learning_rate": 0.00039523809523809526, + "loss": 4.4457, + "theoretical_loss": 6.363584117639526, + "tokens_seen": 16318464 + }, + { + "epoch": 0.0, + "learning_rate": 0.0003968253968253969, + "loss": 4.4832, + "theoretical_loss": 6.35918040574715, + "tokens_seen": 16384000 + }, + { + "epoch": 0.0, + "learning_rate": 0.00039841269841269845, + "loss": 4.6614, + "theoretical_loss": 6.354799183335816, + "tokens_seen": 16449536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004, + "loss": 4.5142, + "theoretical_loss": 6.350440246637536, + "tokens_seen": 16515072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004015873015873016, + "loss": 4.4264, + "theoretical_loss": 6.3461033945298295, + "tokens_seen": 16580608 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040317460317460316, + "loss": 4.3929, + "theoretical_loss": 6.341788428491087, + "tokens_seen": 16646144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004047619047619048, + "loss": 4.5313, + "theoretical_loss": 6.337495152556851, + "tokens_seen": 16711680 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040634920634920635, + "loss": 4.5278, + "theoretical_loss": 6.333223373277006, + "tokens_seen": 16777216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004079365079365079, + "loss": 4.5302, + "theoretical_loss": 6.32897289967384, + "tokens_seen": 16842752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004095238095238095, + "loss": 4.5178, + "theoretical_loss": 6.324743543200984, + "tokens_seen": 16908288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004111111111111111, + "loss": 4.4393, + "theoretical_loss": 6.320535117703164, + "tokens_seen": 16973824 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041269841269841274, + "loss": 4.4453, + "theoretical_loss": 6.316347439376797, + "tokens_seen": 17039360 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041428571428571437, + "loss": 4.4908, + "theoretical_loss": 6.312180326731362, + "tokens_seen": 17104896 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041587301587301594, + "loss": 4.4296, + "theoretical_loss": 6.308033600551573, + "tokens_seen": 17170432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004174603174603175, + "loss": 4.598, + "theoretical_loss": 6.303907083860294, + "tokens_seen": 17235968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004190476190476191, + "loss": 4.616, + "theoretical_loss": 6.2998006018822075, + "tokens_seen": 17301504 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042063492063492065, + "loss": 4.5066, + "theoretical_loss": 6.295713982008203, + "tokens_seen": 17367040 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042222222222222227, + "loss": 4.3981, + "theoretical_loss": 6.2916470537604745, + "tokens_seen": 17432576 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042380952380952384, + "loss": 4.4611, + "theoretical_loss": 6.287599648758312, + "tokens_seen": 17498112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004253968253968254, + "loss": 4.3655, + "theoretical_loss": 6.283571600684564, + "tokens_seen": 17563648 + }, + { + "epoch": 0.01, + "learning_rate": 0.000426984126984127, + "loss": 4.4687, + "theoretical_loss": 6.279562745252756, + "tokens_seen": 17629184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004285714285714286, + "loss": 4.5314, + "theoretical_loss": 6.2755729201748665, + "tokens_seen": 17694720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004301587301587302, + "loss": 4.3542, + "theoretical_loss": 6.271601965129717, + "tokens_seen": 17760256 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043174603174603174, + "loss": 4.4021, + "theoretical_loss": 6.267649721731981, + "tokens_seen": 17825792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004333333333333333, + "loss": 4.5365, + "theoretical_loss": 6.263716033501797, + "tokens_seen": 17891328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004349206349206349, + "loss": 4.3897, + "theoretical_loss": 6.259800745834964, + "tokens_seen": 17956864 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004365079365079365, + "loss": 4.3846, + "theoretical_loss": 6.255903705973709, + "tokens_seen": 18022400 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043809523809523813, + "loss": 4.4128, + "theoretical_loss": 6.252024762978021, + "tokens_seen": 18087936 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043968253968253976, + "loss": 4.2714, + "theoretical_loss": 6.248163767697523, + "tokens_seen": 18153472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004412698412698413, + "loss": 4.4651, + "theoretical_loss": 6.244320572743888, + "tokens_seen": 18219008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004428571428571429, + "loss": 4.2786, + "theoretical_loss": 6.240495032463768, + "tokens_seen": 18284544 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044444444444444447, + "loss": 4.3311, + "theoretical_loss": 6.236687002912243, + "tokens_seen": 18350080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004460317460317461, + "loss": 4.4895, + "theoretical_loss": 6.232896341826766, + "tokens_seen": 18415616 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044761904761904766, + "loss": 4.3794, + "theoretical_loss": 6.229122908601596, + "tokens_seen": 18481152 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044920634920634923, + "loss": 4.3418, + "theoretical_loss": 6.225366564262705, + "tokens_seen": 18546688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004507936507936508, + "loss": 4.4627, + "theoretical_loss": 6.2216271714431635, + "tokens_seen": 18612224 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045238095238095237, + "loss": 4.3152, + "theoretical_loss": 6.217904594358968, + "tokens_seen": 18677760 + }, + { + "epoch": 0.01, + "learning_rate": 0.000453968253968254, + "loss": 4.4017, + "theoretical_loss": 6.214198698785328, + "tokens_seen": 18743296 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045555555555555556, + "loss": 4.2326, + "theoretical_loss": 6.210509352033379, + "tokens_seen": 18808832 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045714285714285713, + "loss": 4.3124, + "theoretical_loss": 6.206836422927337, + "tokens_seen": 18874368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004587301587301587, + "loss": 4.4355, + "theoretical_loss": 6.203179781782049, + "tokens_seen": 18939904 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046031746031746033, + "loss": 4.2566, + "theoretical_loss": 6.199539300380979, + "tokens_seen": 19005440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004619047619047619, + "loss": 4.3537, + "theoretical_loss": 6.195914851954575, + "tokens_seen": 19070976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004634920634920636, + "loss": 4.3399, + "theoretical_loss": 6.19230631115903, + "tokens_seen": 19136512 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046507936507936515, + "loss": 4.3265, + "theoretical_loss": 6.188713554055438, + "tokens_seen": 19202048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004666666666666667, + "loss": 4.3022, + "theoretical_loss": 6.185136458089309, + "tokens_seen": 19267584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004682539682539683, + "loss": 4.3129, + "theoretical_loss": 6.181574902070463, + "tokens_seen": 19333120 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046984126984126986, + "loss": 4.2721, + "theoretical_loss": 6.178028766153277, + "tokens_seen": 19398656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004714285714285715, + "loss": 4.2608, + "theoretical_loss": 6.174497931817287, + "tokens_seen": 19464192 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047301587301587305, + "loss": 4.2924, + "theoretical_loss": 6.17098228184813, + "tokens_seen": 19529728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004746031746031746, + "loss": 4.3237, + "theoretical_loss": 6.167481700318827, + "tokens_seen": 19595264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 14832, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 4.324337482452393, + "objective/train/theoretical_loss": 6.163996072571392, + "objective/train/tokens_used": 40120800, + "theoretical_loss": 6.163996072571392, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004761904761904762, + "loss": 4.3243, + "theoretical_loss": 6.163996072571392, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004777777777777778, + "loss": 4.262, + "theoretical_loss": 6.160525285198771, + "tokens_seen": 19726336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004793650793650794, + "loss": 4.413, + "theoretical_loss": 6.157069226027087, + "tokens_seen": 19791872 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048095238095238095, + "loss": 4.4588, + "theoretical_loss": 6.153627784098204, + "tokens_seen": 19857408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004825396825396825, + "loss": 4.1598, + "theoretical_loss": 6.150200849652586, + "tokens_seen": 19922944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004841269841269841, + "loss": 4.2043, + "theoretical_loss": 6.146788314112464, + "tokens_seen": 19988480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004857142857142857, + "loss": 4.2098, + "theoretical_loss": 6.143390070065279, + "tokens_seen": 20054016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004873015873015873, + "loss": 4.2947, + "theoretical_loss": 6.140006011247426, + "tokens_seen": 20119552 + }, + { + "epoch": 0.01, + "learning_rate": 0.000488888888888889, + "loss": 4.3071, + "theoretical_loss": 6.136636032528262, + "tokens_seen": 20185088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004904761904761905, + "loss": 4.2235, + "theoretical_loss": 6.133280029894401, + "tokens_seen": 20250624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004920634920634921, + "loss": 4.2626, + "theoretical_loss": 6.129937900434269, + "tokens_seen": 20316160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004936507936507937, + "loss": 4.1477, + "theoretical_loss": 6.126609542322921, + "tokens_seen": 20381696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004952380952380952, + "loss": 4.1866, + "theoretical_loss": 6.123294854807119, + "tokens_seen": 20447232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004968253968253969, + "loss": 4.2563, + "theoretical_loss": 6.11999373819066, + "tokens_seen": 20512768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984126984126985, + "loss": 4.3367, + "theoretical_loss": 6.116706093819951, + "tokens_seen": 20578304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005, + "loss": 4.0307, + "theoretical_loss": 6.113431824069829, + "tokens_seen": 20643840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005015873015873016, + "loss": 4.2187, + "theoretical_loss": 6.110170832329612, + "tokens_seen": 20709376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005031746031746032, + "loss": 4.2966, + "theoretical_loss": 6.10692302298939, + "tokens_seen": 20774912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005047619047619048, + "loss": 4.4374, + "theoretical_loss": 6.103688301426538, + "tokens_seen": 20840448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005063492063492064, + "loss": 4.2097, + "theoretical_loss": 6.100466573992453, + "tokens_seen": 20905984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005079365079365079, + "loss": 4.2785, + "theoretical_loss": 6.097257747999514, + "tokens_seen": 20971520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005095238095238095, + "loss": 4.2787, + "theoretical_loss": 6.094061731708257, + "tokens_seen": 21037056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005111111111111111, + "loss": 4.2814, + "theoretical_loss": 6.090878434314753, + "tokens_seen": 21102592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005126984126984128, + "loss": 4.0601, + "theoretical_loss": 6.0877077659382, + "tokens_seen": 21168128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005142857142857143, + "loss": 4.3214, + "theoretical_loss": 6.084549637608715, + "tokens_seen": 21233664 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005158730158730159, + "loss": 4.0764, + "theoretical_loss": 6.081403961255324, + "tokens_seen": 21299200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005174603174603176, + "loss": 4.0942, + "theoretical_loss": 6.078270649694149, + "tokens_seen": 21364736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005190476190476191, + "loss": 4.1596, + "theoretical_loss": 6.07514961661678, + "tokens_seen": 21430272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005206349206349207, + "loss": 4.1015, + "theoretical_loss": 6.072040776578845, + "tokens_seen": 21495808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005222222222222222, + "loss": 4.2186, + "theoretical_loss": 6.068944044988752, + "tokens_seen": 21561344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005238095238095238, + "loss": 4.122, + "theoretical_loss": 6.065859338096619, + "tokens_seen": 21626880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005253968253968255, + "loss": 4.0439, + "theoretical_loss": 6.0627865729833745, + "tokens_seen": 21692416 + }, + { + "epoch": 0.01, + "learning_rate": 0.000526984126984127, + "loss": 4.2066, + "theoretical_loss": 6.059725667550044, + "tokens_seen": 21757952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005285714285714286, + "loss": 4.1576, + "theoretical_loss": 6.0566765405071905, + "tokens_seen": 21823488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005301587301587302, + "loss": 4.0447, + "theoretical_loss": 6.0536391113645305, + "tokens_seen": 21889024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005317460317460317, + "loss": 4.2702, + "theoretical_loss": 6.050613300420713, + "tokens_seen": 21954560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005333333333333334, + "loss": 4.1029, + "theoretical_loss": 6.047599028753265, + "tokens_seen": 22020096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005349206349206349, + "loss": 4.1342, + "theoretical_loss": 6.04459621820868, + "tokens_seen": 22085632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005365079365079365, + "loss": 4.2107, + "theoretical_loss": 6.041604791392675, + "tokens_seen": 22151168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005380952380952381, + "loss": 4.1167, + "theoretical_loss": 6.038624671660598, + "tokens_seen": 22216704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005396825396825397, + "loss": 4.2148, + "theoretical_loss": 6.035655783107973, + "tokens_seen": 22282240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005412698412698414, + "loss": 4.1704, + "theoretical_loss": 6.0326980505612084, + "tokens_seen": 22347776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005428571428571429, + "loss": 4.1784, + "theoretical_loss": 6.029751399568434, + "tokens_seen": 22413312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005444444444444445, + "loss": 4.065, + "theoretical_loss": 6.026815756390491, + "tokens_seen": 22478848 + }, + { + "epoch": 0.01, + "learning_rate": 0.000546031746031746, + "loss": 3.9732, + "theoretical_loss": 6.023891047992053, + "tokens_seen": 22544384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005476190476190477, + "loss": 4.2287, + "theoretical_loss": 6.020977202032884, + "tokens_seen": 22609920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005492063492063493, + "loss": 4.1212, + "theoretical_loss": 6.018074146859234, + "tokens_seen": 22675456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005507936507936508, + "loss": 4.0999, + "theoretical_loss": 6.015181811495358, + "tokens_seen": 22740992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005523809523809524, + "loss": 4.1779, + "theoretical_loss": 6.012300125635177, + "tokens_seen": 22806528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005539682539682539, + "loss": 4.0847, + "theoretical_loss": 6.009429019634049, + "tokens_seen": 22872064 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005555555555555556, + "loss": 4.1275, + "theoretical_loss": 6.006568424500674, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005571428571428572, + "loss": 4.042, + "theoretical_loss": 6.003718271889123, + "tokens_seen": 23003136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005587301587301587, + "loss": 4.0857, + "theoretical_loss": 6.000878494090979, + "tokens_seen": 23068672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005603174603174603, + "loss": 4.1695, + "theoretical_loss": 5.998049024027599, + "tokens_seen": 23134208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005619047619047619, + "loss": 4.0201, + "theoretical_loss": 5.995229795242498, + "tokens_seen": 23199744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005634920634920636, + "loss": 4.0275, + "theoretical_loss": 5.99242074189383, + "tokens_seen": 23265280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005650793650793652, + "loss": 4.0145, + "theoretical_loss": 5.989621798747002, + "tokens_seen": 23330816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005666666666666667, + "loss": 4.0395, + "theoretical_loss": 5.98683290116738, + "tokens_seen": 23396352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005682539682539683, + "loss": 4.0452, + "theoretical_loss": 5.9840539851131105, + "tokens_seen": 23461888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005698412698412698, + "loss": 4.003, + "theoretical_loss": 5.981284987128044, + "tokens_seen": 23527424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005714285714285715, + "loss": 3.9503, + "theoretical_loss": 5.97852584433477, + "tokens_seen": 23592960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005730158730158731, + "loss": 4.1692, + "theoretical_loss": 5.975776494427746, + "tokens_seen": 23658496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005746031746031746, + "loss": 4.0227, + "theoretical_loss": 5.973036875666525, + "tokens_seen": 23724032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005761904761904762, + "loss": 3.9862, + "theoretical_loss": 5.970306926869093, + "tokens_seen": 23789568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005777777777777778, + "loss": 4.0578, + "theoretical_loss": 5.967586587405295, + "tokens_seen": 23855104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005793650793650794, + "loss": 4.0431, + "theoretical_loss": 5.96487579719035, + "tokens_seen": 23920640 + }, + { + "epoch": 0.01, + "learning_rate": 0.000580952380952381, + "loss": 3.9127, + "theoretical_loss": 5.962174496678481, + "tokens_seen": 23986176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005825396825396825, + "loss": 3.8789, + "theoretical_loss": 5.959482626856606, + "tokens_seen": 24051712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005841269841269841, + "loss": 3.9908, + "theoretical_loss": 5.956800129238152, + "tokens_seen": 24117248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005857142857142857, + "loss": 3.9941, + "theoretical_loss": 5.954126945856933, + "tokens_seen": 24182784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005873015873015873, + "loss": 3.9969, + "theoretical_loss": 5.951463019261125, + "tokens_seen": 24248320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005888888888888889, + "loss": 4.1916, + "theoretical_loss": 5.948808292507333, + "tokens_seen": 24313856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005904761904761905, + "loss": 4.1251, + "theoretical_loss": 5.946162709154724, + "tokens_seen": 24379392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005920634920634922, + "loss": 3.9343, + "theoretical_loss": 5.943526213259268, + "tokens_seen": 24444928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005936507936507937, + "loss": 3.9681, + "theoretical_loss": 5.940898749368037, + "tokens_seen": 24510464 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005952380952380953, + "loss": 3.9707, + "theoretical_loss": 5.9382802625136, + "tokens_seen": 24576000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005968253968253969, + "loss": 4.0216, + "theoretical_loss": 5.935670698208488, + "tokens_seen": 24641536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005984126984126984, + "loss": 3.9762, + "theoretical_loss": 5.933070002439743, + "tokens_seen": 24707072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006000000000000001, + "loss": 3.9851, + "theoretical_loss": 5.930478121663538, + "tokens_seen": 24772608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006015873015873016, + "loss": 4.0456, + "theoretical_loss": 5.927895002799874, + "tokens_seen": 24838144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006031746031746032, + "loss": 4.0994, + "theoretical_loss": 5.925320593227351, + "tokens_seen": 24903680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006047619047619048, + "loss": 3.9298, + "theoretical_loss": 5.922754840778015, + "tokens_seen": 24969216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006063492063492063, + "loss": 3.9744, + "theoretical_loss": 5.9201976937322645, + "tokens_seen": 25034752 + }, + { + "epoch": 0.01, + "learning_rate": 0.000607936507936508, + "loss": 3.9431, + "theoretical_loss": 5.917649100813846, + "tokens_seen": 25100288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006095238095238095, + "loss": 4.1063, + "theoretical_loss": 5.915109011184899, + "tokens_seen": 25165824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006111111111111111, + "loss": 3.9948, + "theoretical_loss": 5.9125773744410814, + "tokens_seen": 25231360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006126984126984127, + "loss": 3.989, + "theoretical_loss": 5.91005414060676, + "tokens_seen": 25296896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006142857142857143, + "loss": 3.8734, + "theoretical_loss": 5.90753926013026, + "tokens_seen": 25362432 + }, + { + "epoch": 0.01, + "learning_rate": 0.000615873015873016, + "loss": 4.0352, + "theoretical_loss": 5.90503268387919, + "tokens_seen": 25427968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006174603174603175, + "loss": 3.871, + "theoretical_loss": 5.9025343631358185, + "tokens_seen": 25493504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006190476190476191, + "loss": 3.9888, + "theoretical_loss": 5.900044249592523, + "tokens_seen": 25559040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006206349206349206, + "loss": 3.9204, + "theoretical_loss": 5.897562295347297, + "tokens_seen": 25624576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006222222222222223, + "loss": 3.9253, + "theoretical_loss": 5.895088452899312, + "tokens_seen": 25690112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006238095238095239, + "loss": 3.8434, + "theoretical_loss": 5.892622675144553, + "tokens_seen": 25755648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006253968253968254, + "loss": 3.7913, + "theoretical_loss": 5.890164915371495, + "tokens_seen": 25821184 + }, + { + "epoch": 0.01, + "learning_rate": 0.000626984126984127, + "loss": 4.0049, + "theoretical_loss": 5.887715127256854, + "tokens_seen": 25886720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006285714285714286, + "loss": 3.8893, + "theoretical_loss": 5.885273264861381, + "tokens_seen": 25952256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006301587301587302, + "loss": 3.9636, + "theoretical_loss": 5.882839282625721, + "tokens_seen": 26017792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006317460317460318, + "loss": 3.9577, + "theoretical_loss": 5.880413135366323, + "tokens_seen": 26083328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006333333333333333, + "loss": 4.0493, + "theoretical_loss": 5.877994778271411, + "tokens_seen": 26148864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 17681, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 3.8979604244232178, + "objective/train/theoretical_loss": 5.8755841668969895, + "objective/train/tokens_used": 46674400, + "theoretical_loss": 5.8755841668969895, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006349206349206349, + "loss": 3.898, + "theoretical_loss": 5.8755841668969895, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006365079365079365, + "loss": 3.9651, + "theoretical_loss": 5.873181257162926, + "tokens_seen": 26279936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006380952380952382, + "loss": 3.8692, + "theoretical_loss": 5.870786005349071, + "tokens_seen": 26345472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006396825396825398, + "loss": 3.8808, + "theoretical_loss": 5.868398368091427, + "tokens_seen": 26411008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006412698412698413, + "loss": 3.9366, + "theoretical_loss": 5.866018302378377, + "tokens_seen": 26476544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006428571428571429, + "loss": 3.846, + "theoretical_loss": 5.863645765546956, + "tokens_seen": 26542080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006444444444444444, + "loss": 3.7786, + "theoretical_loss": 5.861280715279168, + "tokens_seen": 26607616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006460317460317461, + "loss": 3.9345, + "theoretical_loss": 5.858923109598366, + "tokens_seen": 26673152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006476190476190477, + "loss": 3.7997, + "theoretical_loss": 5.8565729068656545, + "tokens_seen": 26738688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006492063492063492, + "loss": 3.7713, + "theoretical_loss": 5.85423006577636, + "tokens_seen": 26804224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006507936507936508, + "loss": 3.8107, + "theoretical_loss": 5.851894545356539, + "tokens_seen": 26869760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006523809523809524, + "loss": 3.8922, + "theoretical_loss": 5.849566304959527, + "tokens_seen": 26935296 + }, + { + "epoch": 0.01, + "learning_rate": 0.000653968253968254, + "loss": 3.7606, + "theoretical_loss": 5.84724530426254, + "tokens_seen": 27000832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006555555555555556, + "loss": 3.7933, + "theoretical_loss": 5.844931503263309, + "tokens_seen": 27066368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006571428571428571, + "loss": 3.9081, + "theoretical_loss": 5.842624862276773, + "tokens_seen": 27131904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006587301587301587, + "loss": 3.832, + "theoretical_loss": 5.840325341931795, + "tokens_seen": 27197440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006603174603174604, + "loss": 3.8188, + "theoretical_loss": 5.838032903167934, + "tokens_seen": 27262976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006619047619047619, + "loss": 3.8649, + "theoretical_loss": 5.835747507232257, + "tokens_seen": 27328512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006634920634920636, + "loss": 3.8314, + "theoretical_loss": 5.833469115676177, + "tokens_seen": 27394048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006650793650793651, + "loss": 3.9019, + "theoretical_loss": 5.8311976903523535, + "tokens_seen": 27459584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006666666666666668, + "loss": 3.8903, + "theoretical_loss": 5.828933193411611, + "tokens_seen": 27525120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006682539682539683, + "loss": 3.9442, + "theoretical_loss": 5.826675587299913, + "tokens_seen": 27590656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006698412698412699, + "loss": 3.7999, + "theoretical_loss": 5.824424834755357, + "tokens_seen": 27656192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006714285714285715, + "loss": 3.9527, + "theoretical_loss": 5.8221808988052235, + "tokens_seen": 27721728 + }, + { + "epoch": 0.01, + "learning_rate": 0.000673015873015873, + "loss": 3.7933, + "theoretical_loss": 5.819943742763051, + "tokens_seen": 27787264 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006746031746031747, + "loss": 3.741, + "theoretical_loss": 5.817713330225754, + "tokens_seen": 27852800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006761904761904762, + "loss": 3.8241, + "theoretical_loss": 5.815489625070766, + "tokens_seen": 27918336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006777777777777778, + "loss": 3.7877, + "theoretical_loss": 5.813272591453236, + "tokens_seen": 27983872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006793650793650794, + "loss": 3.6882, + "theoretical_loss": 5.811062193803242, + "tokens_seen": 28049408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006809523809523809, + "loss": 3.7131, + "theoretical_loss": 5.808858396823043, + "tokens_seen": 28114944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006825396825396826, + "loss": 3.755, + "theoretical_loss": 5.806661165484377, + "tokens_seen": 28180480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006841269841269841, + "loss": 3.8132, + "theoretical_loss": 5.804470465025777, + "tokens_seen": 28246016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006857142857142857, + "loss": 3.7366, + "theoretical_loss": 5.802286260949924, + "tokens_seen": 28311552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006873015873015873, + "loss": 3.7987, + "theoretical_loss": 5.8001085190210375, + "tokens_seen": 28377088 + }, + { + "epoch": 0.01, + "learning_rate": 0.000688888888888889, + "loss": 3.6966, + "theoretical_loss": 5.797937205262292, + "tokens_seen": 28442624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006904761904761906, + "loss": 3.8535, + "theoretical_loss": 5.795772285953266, + "tokens_seen": 28508160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006920634920634921, + "loss": 3.7654, + "theoretical_loss": 5.793613727627429, + "tokens_seen": 28573696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006936507936507937, + "loss": 3.8714, + "theoretical_loss": 5.791461497069644, + "tokens_seen": 28639232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006952380952380953, + "loss": 3.7254, + "theoretical_loss": 5.789315561313717, + "tokens_seen": 28704768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006968253968253969, + "loss": 3.7796, + "theoretical_loss": 5.787175887639959, + "tokens_seen": 28770304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0006984126984126985, + "loss": 3.7973, + "theoretical_loss": 5.785042443572798, + "tokens_seen": 28835840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007, + "loss": 3.809, + "theoretical_loss": 5.782915196878396, + "tokens_seen": 28901376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007015873015873016, + "loss": 3.7745, + "theoretical_loss": 5.780794115562314, + "tokens_seen": 28966912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007031746031746032, + "loss": 3.8756, + "theoretical_loss": 5.778679167867196, + "tokens_seen": 29032448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007047619047619048, + "loss": 3.6832, + "theoretical_loss": 5.7765703222704765, + "tokens_seen": 29097984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007063492063492064, + "loss": 3.6919, + "theoretical_loss": 5.7744675474821285, + "tokens_seen": 29163520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007079365079365079, + "loss": 3.6528, + "theoretical_loss": 5.772370812442426, + "tokens_seen": 29229056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007095238095238095, + "loss": 3.7711, + "theoretical_loss": 5.770280086319733, + "tokens_seen": 29294592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007111111111111111, + "loss": 3.8099, + "theoretical_loss": 5.768195338508333, + "tokens_seen": 29360128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007126984126984128, + "loss": 3.8639, + "theoretical_loss": 5.766116538626263, + "tokens_seen": 29425664 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007142857142857144, + "loss": 3.8192, + "theoretical_loss": 5.7640436565131905, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007158730158730159, + "loss": 3.6903, + "theoretical_loss": 5.761976662228307, + "tokens_seen": 29556736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007174603174603175, + "loss": 3.8698, + "theoretical_loss": 5.759915526048243, + "tokens_seen": 29622272 + }, + { + "epoch": 0.01, + "learning_rate": 0.000719047619047619, + "loss": 3.7103, + "theoretical_loss": 5.757860218465023, + "tokens_seen": 29687808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007206349206349207, + "loss": 3.7779, + "theoretical_loss": 5.755810710184019, + "tokens_seen": 29753344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007222222222222223, + "loss": 3.7096, + "theoretical_loss": 5.753766972121954, + "tokens_seen": 29818880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007238095238095238, + "loss": 3.8155, + "theoretical_loss": 5.751728975404911, + "tokens_seen": 29884416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007253968253968254, + "loss": 3.6583, + "theoretical_loss": 5.7496966913663705, + "tokens_seen": 29949952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007269841269841271, + "loss": 3.7968, + "theoretical_loss": 5.747670091545274, + "tokens_seen": 30015488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007285714285714286, + "loss": 3.6869, + "theoretical_loss": 5.745649147684103, + "tokens_seen": 30081024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007301587301587302, + "loss": 3.6626, + "theoretical_loss": 5.743633831726987, + "tokens_seen": 30146560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007317460317460317, + "loss": 3.6203, + "theoretical_loss": 5.741624115817827, + "tokens_seen": 30212096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007333333333333333, + "loss": 3.6655, + "theoretical_loss": 5.739619972298446, + "tokens_seen": 30277632 + }, + { + "epoch": 0.01, + "learning_rate": 0.000734920634920635, + "loss": 3.5931, + "theoretical_loss": 5.737621373706753, + "tokens_seen": 30343168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007365079365079365, + "loss": 3.8355, + "theoretical_loss": 5.735628292774945, + "tokens_seen": 30408704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007380952380952382, + "loss": 3.6235, + "theoretical_loss": 5.7336407024276985, + "tokens_seen": 30474240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007396825396825397, + "loss": 3.7167, + "theoretical_loss": 5.731658575780418, + "tokens_seen": 30539776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007412698412698414, + "loss": 3.688, + "theoretical_loss": 5.729681886137476, + "tokens_seen": 30605312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007428571428571429, + "loss": 3.7001, + "theoretical_loss": 5.727710606990488, + "tokens_seen": 30670848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007444444444444445, + "loss": 3.5595, + "theoretical_loss": 5.725744712016601, + "tokens_seen": 30736384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007460317460317461, + "loss": 3.689, + "theoretical_loss": 5.723784175076802, + "tokens_seen": 30801920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007476190476190476, + "loss": 3.6387, + "theoretical_loss": 5.72182897021425, + "tokens_seen": 30867456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007492063492063493, + "loss": 3.7047, + "theoretical_loss": 5.719879071652615, + "tokens_seen": 30932992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007507936507936508, + "loss": 3.6256, + "theoretical_loss": 5.717934453794454, + "tokens_seen": 30998528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007523809523809524, + "loss": 3.6762, + "theoretical_loss": 5.715995091219584, + "tokens_seen": 31064064 + }, + { + "epoch": 0.01, + "learning_rate": 0.000753968253968254, + "loss": 3.7207, + "theoretical_loss": 5.7140609586834925, + "tokens_seen": 31129600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007555555555555555, + "loss": 3.5748, + "theoretical_loss": 5.712132031115749, + "tokens_seen": 31195136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007571428571428572, + "loss": 3.6296, + "theoretical_loss": 5.710208283618447, + "tokens_seen": 31260672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007587301587301588, + "loss": 3.6736, + "theoretical_loss": 5.708289691464657, + "tokens_seen": 31326208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007603174603174603, + "loss": 3.6819, + "theoretical_loss": 5.706376230096897, + "tokens_seen": 31391744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007619047619047619, + "loss": 3.6571, + "theoretical_loss": 5.704467875125619, + "tokens_seen": 31457280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007634920634920635, + "loss": 3.6361, + "theoretical_loss": 5.702564602327719, + "tokens_seen": 31522816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007650793650793652, + "loss": 3.5594, + "theoretical_loss": 5.700666387645049, + "tokens_seen": 31588352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007666666666666667, + "loss": 3.6851, + "theoretical_loss": 5.698773207182963, + "tokens_seen": 31653888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007682539682539683, + "loss": 3.6468, + "theoretical_loss": 5.696885037208865, + "tokens_seen": 31719424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007698412698412699, + "loss": 3.7059, + "theoretical_loss": 5.695001854150782, + "tokens_seen": 31784960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007714285714285715, + "loss": 3.6026, + "theoretical_loss": 5.693123634595938, + "tokens_seen": 31850496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007730158730158731, + "loss": 3.6567, + "theoretical_loss": 5.69125035528937, + "tokens_seen": 31916032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007746031746031746, + "loss": 3.6296, + "theoretical_loss": 5.689381993132532, + "tokens_seen": 31981568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007761904761904762, + "loss": 3.5629, + "theoretical_loss": 5.687518525181924, + "tokens_seen": 32047104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007777777777777778, + "loss": 3.6558, + "theoretical_loss": 5.685659928647743, + "tokens_seen": 32112640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007793650793650794, + "loss": 3.5843, + "theoretical_loss": 5.68380618089254, + "tokens_seen": 32178176 + }, + { + "epoch": 0.01, + "learning_rate": 0.000780952380952381, + "loss": 3.7462, + "theoretical_loss": 5.6819572594298915, + "tokens_seen": 32243712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007825396825396825, + "loss": 3.6714, + "theoretical_loss": 5.680113141923088, + "tokens_seen": 32309248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007841269841269841, + "loss": 3.6394, + "theoretical_loss": 5.67827380618384, + "tokens_seen": 32374784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007857142857142857, + "loss": 3.687, + "theoretical_loss": 5.6764392301709865, + "tokens_seen": 32440320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007873015873015873, + "loss": 3.6331, + "theoretical_loss": 5.674609391989232, + "tokens_seen": 32505856 + }, + { + "epoch": 0.01, + "learning_rate": 0.000788888888888889, + "loss": 3.6678, + "theoretical_loss": 5.672784269887885, + "tokens_seen": 32571392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007904761904761905, + "loss": 3.6251, + "theoretical_loss": 5.670963842259615, + "tokens_seen": 32636928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007920634920634921, + "loss": 3.5818, + "theoretical_loss": 5.669148087639222, + "tokens_seen": 32702464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 20023, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 3.6404943466186523, + "objective/train/theoretical_loss": 5.66733698470242, + "objective/train/tokens_used": 53228000, + "theoretical_loss": 5.66733698470242, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007936507936507938, + "loss": 3.6405, + "theoretical_loss": 5.66733698470242, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007952380952380953, + "loss": 3.5114, + "theoretical_loss": 5.665530512264631, + "tokens_seen": 32833536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968253968253969, + "loss": 3.7098, + "theoretical_loss": 5.663728649279793, + "tokens_seen": 32899072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984126984126984, + "loss": 3.4674, + "theoretical_loss": 5.6619313748391855, + "tokens_seen": 32964608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0008, + "loss": 3.5626, + "theoretical_loss": 5.6601386681702515, + "tokens_seen": 33030144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999839518555668, + "loss": 3.5969, + "theoretical_loss": 5.658350508635454, + "tokens_seen": 33095680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999679037111334, + "loss": 3.605, + "theoretical_loss": 5.656566875731127, + "tokens_seen": 33161216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999518555667002, + "loss": 3.5132, + "theoretical_loss": 5.654787749086345, + "tokens_seen": 33226752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999358074222668, + "loss": 3.5917, + "theoretical_loss": 5.6530131084618045, + "tokens_seen": 33292288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999197592778336, + "loss": 3.6407, + "theoretical_loss": 5.651242933748718, + "tokens_seen": 33357824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007999037111334002, + "loss": 3.6881, + "theoretical_loss": 5.649477204967717, + "tokens_seen": 33423360 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799887662988967, + "loss": 3.6215, + "theoretical_loss": 5.647715902267764, + "tokens_seen": 33488896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007998716148445336, + "loss": 3.4909, + "theoretical_loss": 5.645959005925086, + "tokens_seen": 33554432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007998555667001004, + "loss": 3.5438, + "theoretical_loss": 5.644206496342103, + "tokens_seen": 33619968 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799839518555667, + "loss": 3.4454, + "theoretical_loss": 5.642458354046386, + "tokens_seen": 33685504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007998234704112338, + "loss": 3.4821, + "theoretical_loss": 5.6407145596896076, + "tokens_seen": 33751040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007998074222668004, + "loss": 3.4895, + "theoretical_loss": 5.638975094046518, + "tokens_seen": 33816576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997913741223672, + "loss": 3.5686, + "theoretical_loss": 5.637239938013924, + "tokens_seen": 33882112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997753259779338, + "loss": 3.4646, + "theoretical_loss": 5.6355090726096755, + "tokens_seen": 33947648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997592778335006, + "loss": 3.7202, + "theoretical_loss": 5.633782478971672, + "tokens_seen": 34013184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997432296890672, + "loss": 3.5494, + "theoretical_loss": 5.632060138356873, + "tokens_seen": 34078720 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799727181544634, + "loss": 3.5303, + "theoretical_loss": 5.630342032140315, + "tokens_seen": 34144256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007997111334002006, + "loss": 3.6338, + "theoretical_loss": 5.628628141814151, + "tokens_seen": 34209792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996950852557674, + "loss": 3.6308, + "theoretical_loss": 5.62691844898668, + "tokens_seen": 34275328 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799679037111334, + "loss": 3.3829, + "theoretical_loss": 5.625212935381407, + "tokens_seen": 34340864 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996629889669008, + "loss": 3.4401, + "theoretical_loss": 5.6235115828361035, + "tokens_seen": 34406400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996469408224675, + "loss": 3.6042, + "theoretical_loss": 5.621814373301866, + "tokens_seen": 34471936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996308926780342, + "loss": 3.567, + "theoretical_loss": 5.620121288842209, + "tokens_seen": 34537472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007996148445336009, + "loss": 3.5176, + "theoretical_loss": 5.618432311632143, + "tokens_seen": 34603008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995987963891676, + "loss": 3.4325, + "theoretical_loss": 5.616747423957278, + "tokens_seen": 34668544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995827482447343, + "loss": 3.5763, + "theoretical_loss": 5.615066608212926, + "tokens_seen": 34734080 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799566700100301, + "loss": 3.4953, + "theoretical_loss": 5.613389846903216, + "tokens_seen": 34799616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995506519558677, + "loss": 3.5708, + "theoretical_loss": 5.611717122640225, + "tokens_seen": 34865152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995346038114344, + "loss": 3.7478, + "theoretical_loss": 5.610048418143098, + "tokens_seen": 34930688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995185556670011, + "loss": 3.5119, + "theoretical_loss": 5.6083837162372, + "tokens_seen": 34996224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007995025075225678, + "loss": 3.5977, + "theoretical_loss": 5.606722999853261, + "tokens_seen": 35061760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994864593781345, + "loss": 3.4637, + "theoretical_loss": 5.605066252026537, + "tokens_seen": 35127296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994704112337012, + "loss": 3.5509, + "theoretical_loss": 5.603413455895973, + "tokens_seen": 35192832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994543630892679, + "loss": 3.5226, + "theoretical_loss": 5.601764594703377, + "tokens_seen": 35258368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994383149448346, + "loss": 3.5264, + "theoretical_loss": 5.600119651792607, + "tokens_seen": 35323904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007994222668004013, + "loss": 3.5676, + "theoretical_loss": 5.598478610608758, + "tokens_seen": 35389440 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799406218655968, + "loss": 3.3829, + "theoretical_loss": 5.596841454697365, + "tokens_seen": 35454976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993901705115347, + "loss": 3.5647, + "theoretical_loss": 5.595208167703601, + "tokens_seen": 35520512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993741223671014, + "loss": 3.553, + "theoretical_loss": 5.5935787333714995, + "tokens_seen": 35586048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993580742226681, + "loss": 3.4864, + "theoretical_loss": 5.59195313554317, + "tokens_seen": 35651584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993420260782347, + "loss": 3.6239, + "theoretical_loss": 5.590331358158036, + "tokens_seen": 35717120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993259779338015, + "loss": 3.4815, + "theoretical_loss": 5.58871338525206, + "tokens_seen": 35782656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007993099297893681, + "loss": 3.5306, + "theoretical_loss": 5.587099200956995, + "tokens_seen": 35848192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992938816449349, + "loss": 3.467, + "theoretical_loss": 5.585488789499637, + "tokens_seen": 35913728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992778335005015, + "loss": 3.5261, + "theoretical_loss": 5.583882135201078, + "tokens_seen": 35979264 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992617853560683, + "loss": 3.4974, + "theoretical_loss": 5.582279222475973, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992457372116349, + "loss": 3.2454, + "theoretical_loss": 5.580680035831821, + "tokens_seen": 36110336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992296890672017, + "loss": 3.4453, + "theoretical_loss": 5.579084559868232, + "tokens_seen": 36175872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007992136409227683, + "loss": 3.5094, + "theoretical_loss": 5.577492779276222, + "tokens_seen": 36241408 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799197592778335, + "loss": 3.4054, + "theoretical_loss": 5.575904678837501, + "tokens_seen": 36306944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991815446339017, + "loss": 3.5655, + "theoretical_loss": 5.574320243423783, + "tokens_seen": 36372480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991654964894684, + "loss": 3.3753, + "theoretical_loss": 5.572739457996079, + "tokens_seen": 36438016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991494483450351, + "loss": 3.3295, + "theoretical_loss": 5.5711623076040215, + "tokens_seen": 36503552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991334002006018, + "loss": 3.3126, + "theoretical_loss": 5.569588777385177, + "tokens_seen": 36569088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991173520561685, + "loss": 3.4123, + "theoretical_loss": 5.568018852564379, + "tokens_seen": 36634624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007991013039117352, + "loss": 3.5944, + "theoretical_loss": 5.566452518453051, + "tokens_seen": 36700160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990852557673019, + "loss": 3.3454, + "theoretical_loss": 5.564889760448561, + "tokens_seen": 36765696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990692076228686, + "loss": 3.3713, + "theoretical_loss": 5.5633305640335475, + "tokens_seen": 36831232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990531594784353, + "loss": 3.4241, + "theoretical_loss": 5.561774914775286, + "tokens_seen": 36896768 + }, + { + "epoch": 0.01, + "learning_rate": 0.000799037111334002, + "loss": 3.4546, + "theoretical_loss": 5.5602227983250385, + "tokens_seen": 36962304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990210631895687, + "loss": 3.5998, + "theoretical_loss": 5.558674200417419, + "tokens_seen": 37027840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007990050150451354, + "loss": 3.5237, + "theoretical_loss": 5.557129106869764, + "tokens_seen": 37093376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989889669007021, + "loss": 3.3499, + "theoretical_loss": 5.555587503581506, + "tokens_seen": 37158912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989729187562688, + "loss": 3.3352, + "theoretical_loss": 5.554049376533554, + "tokens_seen": 37224448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989568706118355, + "loss": 3.4595, + "theoretical_loss": 5.552514711787683, + "tokens_seen": 37289984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989408224674022, + "loss": 3.3808, + "theoretical_loss": 5.550983495485925, + "tokens_seen": 37355520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989247743229689, + "loss": 3.4302, + "theoretical_loss": 5.549455713849963, + "tokens_seen": 37421056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007989087261785356, + "loss": 3.3324, + "theoretical_loss": 5.547931353180543, + "tokens_seen": 37486592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988926780341023, + "loss": 3.3144, + "theoretical_loss": 5.546410399856875, + "tokens_seen": 37552128 + }, + { + "epoch": 0.01, + "learning_rate": 0.000798876629889669, + "loss": 3.39, + "theoretical_loss": 5.544892840336053, + "tokens_seen": 37617664 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988605817452357, + "loss": 3.4015, + "theoretical_loss": 5.543378661152472, + "tokens_seen": 37683200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988445336008024, + "loss": 3.4097, + "theoretical_loss": 5.5418678489172555, + "tokens_seen": 37748736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988284854563691, + "loss": 3.3702, + "theoretical_loss": 5.540360390317682, + "tokens_seen": 37814272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007988124373119358, + "loss": 3.4479, + "theoretical_loss": 5.538856272116626, + "tokens_seen": 37879808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987963891675025, + "loss": 3.3019, + "theoretical_loss": 5.5373554811519945, + "tokens_seen": 37945344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987803410230693, + "loss": 3.4943, + "theoretical_loss": 5.535858004336175, + "tokens_seen": 38010880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987642928786359, + "loss": 3.2602, + "theoretical_loss": 5.534363828655488, + "tokens_seen": 38076416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987482447342027, + "loss": 3.2579, + "theoretical_loss": 5.532872941169639, + "tokens_seen": 38141952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987321965897693, + "loss": 3.4578, + "theoretical_loss": 5.531385329011181, + "tokens_seen": 38207488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987161484453361, + "loss": 3.3188, + "theoretical_loss": 5.529900979384985, + "tokens_seen": 38273024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007987001003009027, + "loss": 3.3775, + "theoretical_loss": 5.5284198795677, + "tokens_seen": 38338560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986840521564695, + "loss": 3.3057, + "theoretical_loss": 5.526942016907242, + "tokens_seen": 38404096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986680040120361, + "loss": 3.5022, + "theoretical_loss": 5.5254673788222615, + "tokens_seen": 38469632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986519558676029, + "loss": 3.3356, + "theoretical_loss": 5.523995952801634, + "tokens_seen": 38535168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986359077231695, + "loss": 3.293, + "theoretical_loss": 5.522527726403953, + "tokens_seen": 38600704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986198595787363, + "loss": 3.3967, + "theoretical_loss": 5.521062687257016, + "tokens_seen": 38666240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007986038114343029, + "loss": 3.3977, + "theoretical_loss": 5.519600823057332, + "tokens_seen": 38731776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985877632898697, + "loss": 3.4202, + "theoretical_loss": 5.518142121569613, + "tokens_seen": 38797312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985717151454363, + "loss": 3.3039, + "theoretical_loss": 5.516686570626293, + "tokens_seen": 38862848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985556670010031, + "loss": 3.3701, + "theoretical_loss": 5.515234158127032, + "tokens_seen": 38928384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985396188565697, + "loss": 3.3696, + "theoretical_loss": 5.5137848720382365, + "tokens_seen": 38993920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985235707121365, + "loss": 3.4991, + "theoretical_loss": 5.5123387003925775, + "tokens_seen": 39059456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007985075225677031, + "loss": 3.3928, + "theoretical_loss": 5.510895631288515, + "tokens_seen": 39124992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984914744232699, + "loss": 3.3664, + "theoretical_loss": 5.50945565288983, + "tokens_seen": 39190528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984754262788365, + "loss": 3.3201, + "theoretical_loss": 5.508018753425152, + "tokens_seen": 39256064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 22534, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 3.435593366622925, + "objective/train/theoretical_loss": 5.506584921187503, + "objective/train/tokens_used": 59781600, + "theoretical_loss": 5.506584921187503, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984593781344033, + "loss": 3.4356, + "theoretical_loss": 5.506584921187503, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984433299899699, + "loss": 3.3481, + "theoretical_loss": 5.505154144533835, + "tokens_seen": 39387136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984272818455367, + "loss": 3.2633, + "theoretical_loss": 5.503726411884568, + "tokens_seen": 39452672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007984112337011033, + "loss": 3.3036, + "theoretical_loss": 5.502301711723156, + "tokens_seen": 39518208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983951855566701, + "loss": 3.2297, + "theoretical_loss": 5.5008800325956235, + "tokens_seen": 39583744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983791374122367, + "loss": 3.3255, + "theoretical_loss": 5.499461363110136, + "tokens_seen": 39649280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983630892678035, + "loss": 3.3713, + "theoretical_loss": 5.498045691936548, + "tokens_seen": 39714816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983470411233701, + "loss": 3.2234, + "theoretical_loss": 5.496633007805977, + "tokens_seen": 39780352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983309929789369, + "loss": 3.1825, + "theoretical_loss": 5.495223299510372, + "tokens_seen": 39845888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007983149448345036, + "loss": 3.355, + "theoretical_loss": 5.493816555902079, + "tokens_seen": 39911424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982988966900703, + "loss": 3.3841, + "theoretical_loss": 5.492412765893421, + "tokens_seen": 39976960 + }, + { + "epoch": 0.01, + "learning_rate": 0.000798282848545637, + "loss": 3.3008, + "theoretical_loss": 5.49101191845628, + "tokens_seen": 40042496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982668004012037, + "loss": 3.2659, + "theoretical_loss": 5.489614002621677, + "tokens_seen": 40108032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982507522567704, + "loss": 3.3211, + "theoretical_loss": 5.4882190074793575, + "tokens_seen": 40173568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982347041123371, + "loss": 3.2794, + "theoretical_loss": 5.486826922177387, + "tokens_seen": 40239104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982186559679038, + "loss": 3.4319, + "theoretical_loss": 5.4854377359217406, + "tokens_seen": 40304640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007982026078234705, + "loss": 3.2653, + "theoretical_loss": 5.484051437975901, + "tokens_seen": 40370176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981865596790372, + "loss": 3.3589, + "theoretical_loss": 5.482668017660462, + "tokens_seen": 40435712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981705115346039, + "loss": 3.2239, + "theoretical_loss": 5.481287464352727, + "tokens_seen": 40501248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981544633901706, + "loss": 3.2691, + "theoretical_loss": 5.479909767486326, + "tokens_seen": 40566784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981384152457373, + "loss": 3.2705, + "theoretical_loss": 5.478534916550817, + "tokens_seen": 40632320 + }, + { + "epoch": 0.01, + "learning_rate": 0.000798122367101304, + "loss": 3.2859, + "theoretical_loss": 5.477162901091306, + "tokens_seen": 40697856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007981063189568707, + "loss": 3.4215, + "theoretical_loss": 5.475793710708061, + "tokens_seen": 40763392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980902708124374, + "loss": 3.1642, + "theoretical_loss": 5.474427335056138, + "tokens_seen": 40828928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980742226680041, + "loss": 3.1913, + "theoretical_loss": 5.473063763844998, + "tokens_seen": 40894464 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980581745235708, + "loss": 3.2397, + "theoretical_loss": 5.471702986838139, + "tokens_seen": 40960000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980421263791375, + "loss": 3.1662, + "theoretical_loss": 5.470344993852729, + "tokens_seen": 41025536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980260782347042, + "loss": 3.0606, + "theoretical_loss": 5.46898977475923, + "tokens_seen": 41091072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007980100300902709, + "loss": 3.3902, + "theoretical_loss": 5.467637319481042, + "tokens_seen": 41156608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007979939819458376, + "loss": 3.0364, + "theoretical_loss": 5.466287617994144, + "tokens_seen": 41222144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007979779338014043, + "loss": 3.224, + "theoretical_loss": 5.464940660326731, + "tokens_seen": 41287680 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797961885656971, + "loss": 3.2479, + "theoretical_loss": 5.463596436558861, + "tokens_seen": 41353216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007979458375125376, + "loss": 3.139, + "theoretical_loss": 5.462254936822105, + "tokens_seen": 41418752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007979297893681044, + "loss": 3.2887, + "theoretical_loss": 5.4609161512992, + "tokens_seen": 41484288 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797913741223671, + "loss": 3.1795, + "theoretical_loss": 5.4595800702237005, + "tokens_seen": 41549824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978976930792378, + "loss": 3.2645, + "theoretical_loss": 5.458246683879636, + "tokens_seen": 41615360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978816449348044, + "loss": 3.2391, + "theoretical_loss": 5.456915982601173, + "tokens_seen": 41680896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978655967903712, + "loss": 3.1832, + "theoretical_loss": 5.455587956772277, + "tokens_seen": 41746432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978495486459378, + "loss": 3.1946, + "theoretical_loss": 5.454262596826377, + "tokens_seen": 41811968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978335005015046, + "loss": 3.1349, + "theoretical_loss": 5.452939893246034, + "tokens_seen": 41877504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978174523570712, + "loss": 3.2418, + "theoretical_loss": 5.451619836562617, + "tokens_seen": 41943040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007978014042126379, + "loss": 3.162, + "theoretical_loss": 5.450302417355973, + "tokens_seen": 42008576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977853560682046, + "loss": 3.08, + "theoretical_loss": 5.448987626254097, + "tokens_seen": 42074112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977693079237713, + "loss": 3.1325, + "theoretical_loss": 5.447675453932829, + "tokens_seen": 42139648 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797753259779338, + "loss": 3.2414, + "theoretical_loss": 5.446365891115516, + "tokens_seen": 42205184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977372116349047, + "loss": 3.3154, + "theoretical_loss": 5.445058928572712, + "tokens_seen": 42270720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977211634904714, + "loss": 3.2803, + "theoretical_loss": 5.443754557121853, + "tokens_seen": 42336256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007977051153460381, + "loss": 3.2213, + "theoretical_loss": 5.442452767626957, + "tokens_seen": 42401792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976890672016048, + "loss": 3.1288, + "theoretical_loss": 5.441153550998311, + "tokens_seen": 42467328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976730190571715, + "loss": 3.1906, + "theoretical_loss": 5.439856898192163, + "tokens_seen": 42532864 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976569709127382, + "loss": 3.305, + "theoretical_loss": 5.438562800210427, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976409227683049, + "loss": 3.19, + "theoretical_loss": 5.437271248100375, + "tokens_seen": 42663936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976248746238716, + "loss": 3.2208, + "theoretical_loss": 5.435982232954342, + "tokens_seen": 42729472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007976088264794383, + "loss": 3.1801, + "theoretical_loss": 5.434695745909433, + "tokens_seen": 42795008 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797592778335005, + "loss": 3.1413, + "theoretical_loss": 5.433411778147226, + "tokens_seen": 42860544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975767301905718, + "loss": 3.2043, + "theoretical_loss": 5.432130320893483, + "tokens_seen": 42926080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975606820461384, + "loss": 3.1159, + "theoretical_loss": 5.430851365417862, + "tokens_seen": 42991616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975446339017052, + "loss": 3.252, + "theoretical_loss": 5.429574903033627, + "tokens_seen": 43057152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975285857572718, + "loss": 3.0309, + "theoretical_loss": 5.428300925097373, + "tokens_seen": 43122688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007975125376128386, + "loss": 3.0413, + "theoretical_loss": 5.427029423008732, + "tokens_seen": 43188224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974964894684052, + "loss": 3.1815, + "theoretical_loss": 5.425760388210108, + "tokens_seen": 43253760 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797480441323972, + "loss": 3.0532, + "theoretical_loss": 5.424493812186386, + "tokens_seen": 43319296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974643931795386, + "loss": 3.0778, + "theoretical_loss": 5.423229686464667, + "tokens_seen": 43384832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974483450351054, + "loss": 3.1643, + "theoretical_loss": 5.421968002613992, + "tokens_seen": 43450368 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797432296890672, + "loss": 3.0246, + "theoretical_loss": 5.420708752245071, + "tokens_seen": 43515904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974162487462388, + "loss": 3.084, + "theoretical_loss": 5.419451927010014, + "tokens_seen": 43581440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007974002006018054, + "loss": 2.9828, + "theoretical_loss": 5.418197518602068, + "tokens_seen": 43646976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973841524573722, + "loss": 3.2811, + "theoretical_loss": 5.41694551875535, + "tokens_seen": 43712512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973681043129388, + "loss": 3.0724, + "theoretical_loss": 5.415695919244586, + "tokens_seen": 43778048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973520561685056, + "loss": 3.2274, + "theoretical_loss": 5.41444871188485, + "tokens_seen": 43843584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973360080240722, + "loss": 3.0714, + "theoretical_loss": 5.413203888531308, + "tokens_seen": 43909120 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797319959879639, + "loss": 3.1786, + "theoretical_loss": 5.411961441078963, + "tokens_seen": 43974656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007973039117352056, + "loss": 3.0765, + "theoretical_loss": 5.410721361462395, + "tokens_seen": 44040192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972878635907724, + "loss": 2.9274, + "theoretical_loss": 5.409483641655516, + "tokens_seen": 44105728 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797271815446339, + "loss": 2.9688, + "theoretical_loss": 5.408248273671317, + "tokens_seen": 44171264 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972557673019058, + "loss": 3.1094, + "theoretical_loss": 5.407015249561624, + "tokens_seen": 44236800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972397191574724, + "loss": 3.0363, + "theoretical_loss": 5.405784561416844, + "tokens_seen": 44302336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972236710130392, + "loss": 3.1509, + "theoretical_loss": 5.404556201365727, + "tokens_seen": 44367872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007972076228686058, + "loss": 3.1461, + "theoretical_loss": 5.403330161575123, + "tokens_seen": 44433408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007971915747241726, + "loss": 3.2156, + "theoretical_loss": 5.402106434249742, + "tokens_seen": 44498944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007971755265797392, + "loss": 2.9571, + "theoretical_loss": 5.400885011631912, + "tokens_seen": 44564480 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797159478435306, + "loss": 3.115, + "theoretical_loss": 5.399665886001349, + "tokens_seen": 44630016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007971434302908726, + "loss": 3.095, + "theoretical_loss": 5.398449049674916, + "tokens_seen": 44695552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007971273821464394, + "loss": 2.8581, + "theoretical_loss": 5.397234495006392, + "tokens_seen": 44761088 + }, + { + "epoch": 0.01, + "learning_rate": 0.000797111334002006, + "loss": 3.0027, + "theoretical_loss": 5.396022214386246, + "tokens_seen": 44826624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970952858575728, + "loss": 3.0842, + "theoretical_loss": 5.394812200241393, + "tokens_seen": 44892160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970792377131394, + "loss": 3.1089, + "theoretical_loss": 5.393604445034988, + "tokens_seen": 44957696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970631895687062, + "loss": 2.9782, + "theoretical_loss": 5.392398941266183, + "tokens_seen": 45023232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970471414242728, + "loss": 2.982, + "theoretical_loss": 5.391195681469908, + "tokens_seen": 45088768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970310932798396, + "loss": 3.0419, + "theoretical_loss": 5.389994658216651, + "tokens_seen": 45154304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007970150451354062, + "loss": 3.0054, + "theoretical_loss": 5.388795864112235, + "tokens_seen": 45219840 + }, + { + "epoch": 0.01, + "learning_rate": 0.000796998996990973, + "loss": 2.9654, + "theoretical_loss": 5.387599291797599, + "tokens_seen": 45285376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969829488465396, + "loss": 2.9248, + "theoretical_loss": 5.386404933948584, + "tokens_seen": 45350912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969669007021064, + "loss": 2.9716, + "theoretical_loss": 5.38521278327571, + "tokens_seen": 45416448 + }, + { + "epoch": 0.01, + "learning_rate": 0.000796950852557673, + "loss": 2.9865, + "theoretical_loss": 5.384022832523971, + "tokens_seen": 45481984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969348044132398, + "loss": 3.0551, + "theoretical_loss": 5.382835074472616, + "tokens_seen": 45547520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969187562688065, + "loss": 3.0242, + "theoretical_loss": 5.381649501934939, + "tokens_seen": 45613056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007969027081243732, + "loss": 3.1321, + "theoretical_loss": 5.3804661077580755, + "tokens_seen": 45678592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968866599799399, + "loss": 2.9587, + "theoretical_loss": 5.379284884822787, + "tokens_seen": 45744128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968706118355066, + "loss": 3.1394, + "theoretical_loss": 5.37810582604326, + "tokens_seen": 45809664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 24846, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 3.0651895999908447, + "objective/train/theoretical_loss": 5.376928924366901, + "objective/train/tokens_used": 66335200, + "theoretical_loss": 5.376928924366901, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968545636910733, + "loss": 3.0652, + "theoretical_loss": 5.376928924366901, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00079683851554664, + "loss": 2.9256, + "theoretical_loss": 5.37575417277413, + "tokens_seen": 45940736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968224674022067, + "loss": 2.9612, + "theoretical_loss": 5.374581564278184, + "tokens_seen": 46006272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007968064192577734, + "loss": 2.9538, + "theoretical_loss": 5.373411091924918, + "tokens_seen": 46071808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967903711133401, + "loss": 2.986, + "theoretical_loss": 5.372242748792598, + "tokens_seen": 46137344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967743229689068, + "loss": 2.8112, + "theoretical_loss": 5.371076527991711, + "tokens_seen": 46202880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967582748244735, + "loss": 2.935, + "theoretical_loss": 5.36991242266477, + "tokens_seen": 46268416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967422266800402, + "loss": 2.9079, + "theoretical_loss": 5.36875042598612, + "tokens_seen": 46333952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967261785356069, + "loss": 3.0976, + "theoretical_loss": 5.3675905311617385, + "tokens_seen": 46399488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007967101303911736, + "loss": 2.9532, + "theoretical_loss": 5.366432731429054, + "tokens_seen": 46465024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966940822467403, + "loss": 3.0112, + "theoretical_loss": 5.365277020056752, + "tokens_seen": 46530560 + }, + { + "epoch": 0.01, + "learning_rate": 0.000796678034102307, + "loss": 3.0015, + "theoretical_loss": 5.364123390344582, + "tokens_seen": 46596096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966619859578737, + "loss": 2.9189, + "theoretical_loss": 5.362971835623181, + "tokens_seen": 46661632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966459378134404, + "loss": 2.8917, + "theoretical_loss": 5.361822349253881, + "tokens_seen": 46727168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966298896690071, + "loss": 2.9613, + "theoretical_loss": 5.360674924628524, + "tokens_seen": 46792704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007966138415245738, + "loss": 2.9993, + "theoretical_loss": 5.359529555169287, + "tokens_seen": 46858240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965977933801405, + "loss": 2.9576, + "theoretical_loss": 5.358386234328487, + "tokens_seen": 46923776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965817452357072, + "loss": 2.8915, + "theoretical_loss": 5.357244955588417, + "tokens_seen": 46989312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965656970912739, + "loss": 2.8678, + "theoretical_loss": 5.356105712461154, + "tokens_seen": 47054848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965496489468405, + "loss": 2.9256, + "theoretical_loss": 5.354968498488391, + "tokens_seen": 47120384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965336008024073, + "loss": 2.9371, + "theoretical_loss": 5.353833307241256, + "tokens_seen": 47185920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965175526579739, + "loss": 3.0885, + "theoretical_loss": 5.352700132320135, + "tokens_seen": 47251456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007965015045135407, + "loss": 2.8375, + "theoretical_loss": 5.351568967354504, + "tokens_seen": 47316992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964854563691073, + "loss": 2.8839, + "theoretical_loss": 5.350439806002757, + "tokens_seen": 47382528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964694082246741, + "loss": 2.999, + "theoretical_loss": 5.349312641952025, + "tokens_seen": 47448064 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964533600802407, + "loss": 2.9293, + "theoretical_loss": 5.348187468918022, + "tokens_seen": 47513600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964373119358075, + "loss": 2.7788, + "theoretical_loss": 5.347064280644861, + "tokens_seen": 47579136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964212637913741, + "loss": 2.8874, + "theoretical_loss": 5.345943070904897, + "tokens_seen": 47644672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007964052156469408, + "loss": 2.9413, + "theoretical_loss": 5.344823833498557, + "tokens_seen": 47710208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963891675025075, + "loss": 2.9782, + "theoretical_loss": 5.343706562254177, + "tokens_seen": 47775744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963731193580743, + "loss": 2.9903, + "theoretical_loss": 5.342591251027833, + "tokens_seen": 47841280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963570712136409, + "loss": 2.8307, + "theoretical_loss": 5.3414778937031855, + "tokens_seen": 47906816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963410230692077, + "loss": 2.9094, + "theoretical_loss": 5.340366484191316, + "tokens_seen": 47972352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963249749247743, + "loss": 2.8458, + "theoretical_loss": 5.339257016430563, + "tokens_seen": 48037888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007963089267803411, + "loss": 2.814, + "theoretical_loss": 5.338149484386367, + "tokens_seen": 48103424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962928786359077, + "loss": 2.9672, + "theoretical_loss": 5.337043882051111, + "tokens_seen": 48168960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962768304914745, + "loss": 2.9521, + "theoretical_loss": 5.3359402034439665, + "tokens_seen": 48234496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962607823470411, + "loss": 2.863, + "theoretical_loss": 5.334838442610732, + "tokens_seen": 48300032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962447342026079, + "loss": 2.7467, + "theoretical_loss": 5.333738593623686, + "tokens_seen": 48365568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962286860581745, + "loss": 3.0181, + "theoretical_loss": 5.332640650581424, + "tokens_seen": 48431104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007962126379137413, + "loss": 2.9079, + "theoretical_loss": 5.331544607608718, + "tokens_seen": 48496640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961965897693079, + "loss": 2.9096, + "theoretical_loss": 5.330450458856356, + "tokens_seen": 48562176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961805416248747, + "loss": 2.8218, + "theoretical_loss": 5.329358198500994, + "tokens_seen": 48627712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961644934804413, + "loss": 2.7594, + "theoretical_loss": 5.328267820745011, + "tokens_seen": 48693248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961484453360081, + "loss": 2.8122, + "theoretical_loss": 5.327179319816354, + "tokens_seen": 48758784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961323971915747, + "loss": 2.8653, + "theoretical_loss": 5.326092689968397, + "tokens_seen": 48824320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961163490471415, + "loss": 2.8692, + "theoretical_loss": 5.3250079254797935, + "tokens_seen": 48889856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007961003009027081, + "loss": 2.8527, + "theoretical_loss": 5.323925020654329, + "tokens_seen": 48955392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960842527582749, + "loss": 2.8064, + "theoretical_loss": 5.322843969820781, + "tokens_seen": 49020928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960682046138415, + "loss": 2.7689, + "theoretical_loss": 5.321764767332772, + "tokens_seen": 49086464 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960521564694083, + "loss": 2.7609, + "theoretical_loss": 5.320687407568633, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960361083249749, + "loss": 2.8155, + "theoretical_loss": 5.3196118849312555, + "tokens_seen": 49217536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960200601805417, + "loss": 2.911, + "theoretical_loss": 5.31853819384796, + "tokens_seen": 49283072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007960040120361083, + "loss": 2.7449, + "theoretical_loss": 5.31746632877035, + "tokens_seen": 49348608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007959879638916751, + "loss": 2.8588, + "theoretical_loss": 5.316396284174177, + "tokens_seen": 49414144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0007959719157472417, + "loss": 2.7736, + "theoretical_loss": 5.3153280545592025, + "tokens_seen": 49479680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007959558676028085, + "loss": 2.966, + "theoretical_loss": 5.314261634449066, + "tokens_seen": 49545216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007959398194583751, + "loss": 2.9891, + "theoretical_loss": 5.313197018391143, + "tokens_seen": 49610752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007959237713139419, + "loss": 2.8049, + "theoretical_loss": 5.312134200956416, + "tokens_seen": 49676288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007959077231695085, + "loss": 2.818, + "theoretical_loss": 5.3110731767393435, + "tokens_seen": 49741824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958916750250753, + "loss": 2.742, + "theoretical_loss": 5.31001394035772, + "tokens_seen": 49807360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958756268806419, + "loss": 2.8855, + "theoretical_loss": 5.308956486452549, + "tokens_seen": 49872896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958595787362087, + "loss": 2.6922, + "theoretical_loss": 5.307900809687913, + "tokens_seen": 49938432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958435305917753, + "loss": 2.7624, + "theoretical_loss": 5.306846904750847, + "tokens_seen": 50003968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958274824473421, + "loss": 2.7657, + "theoretical_loss": 5.305794766351202, + "tokens_seen": 50069504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007958114343029087, + "loss": 2.6639, + "theoretical_loss": 5.304744389221523, + "tokens_seen": 50135040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957953861584755, + "loss": 2.762, + "theoretical_loss": 5.303695768116921, + "tokens_seen": 50200576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957793380140421, + "loss": 2.815, + "theoretical_loss": 5.302648897814946, + "tokens_seen": 50266112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957632898696089, + "loss": 2.7472, + "theoretical_loss": 5.301603773115465, + "tokens_seen": 50331648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957472417251755, + "loss": 2.7874, + "theoretical_loss": 5.300560388840532, + "tokens_seen": 50397184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957311935807423, + "loss": 2.776, + "theoretical_loss": 5.299518739834273, + "tokens_seen": 50462720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007957151454363089, + "loss": 2.7818, + "theoretical_loss": 5.298478820962751, + "tokens_seen": 50528256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956990972918757, + "loss": 2.7797, + "theoretical_loss": 5.297440627113859, + "tokens_seen": 50593792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956830491474423, + "loss": 2.7686, + "theoretical_loss": 5.296404153197189, + "tokens_seen": 50659328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956670010030091, + "loss": 2.7342, + "theoretical_loss": 5.295369394143913, + "tokens_seen": 50724864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956509528585757, + "loss": 2.7897, + "theoretical_loss": 5.294336344906668, + "tokens_seen": 50790400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956349047141425, + "loss": 2.8069, + "theoretical_loss": 5.293305000459435, + "tokens_seen": 50855936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956188565697091, + "loss": 2.6506, + "theoretical_loss": 5.292275355797421, + "tokens_seen": 50921472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007956028084252759, + "loss": 2.8034, + "theoretical_loss": 5.291247405936941, + "tokens_seen": 50987008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955867602808426, + "loss": 2.7252, + "theoretical_loss": 5.290221145915307, + "tokens_seen": 51052544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955707121364093, + "loss": 2.7591, + "theoretical_loss": 5.289196570790708, + "tokens_seen": 51118080 + }, + { + "epoch": 0.02, + "learning_rate": 0.000795554663991976, + "loss": 2.6543, + "theoretical_loss": 5.288173675642094, + "tokens_seen": 51183616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955386158475427, + "loss": 2.6381, + "theoretical_loss": 5.287152455569069, + "tokens_seen": 51249152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955225677031094, + "loss": 2.7299, + "theoretical_loss": 5.286132905691774, + "tokens_seen": 51314688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007955065195586761, + "loss": 2.4976, + "theoretical_loss": 5.285115021150775, + "tokens_seen": 51380224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954904714142428, + "loss": 2.6586, + "theoretical_loss": 5.284098797106948, + "tokens_seen": 51445760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954744232698095, + "loss": 2.7922, + "theoretical_loss": 5.283084228741377, + "tokens_seen": 51511296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954583751253762, + "loss": 2.6488, + "theoretical_loss": 5.282071311255235, + "tokens_seen": 51576832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954423269809429, + "loss": 2.8, + "theoretical_loss": 5.2810600398696845, + "tokens_seen": 51642368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954262788365096, + "loss": 2.6972, + "theoretical_loss": 5.2800504098257575, + "tokens_seen": 51707904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007954102306920763, + "loss": 2.6349, + "theoretical_loss": 5.279042416384257, + "tokens_seen": 51773440 + }, + { + "epoch": 0.02, + "learning_rate": 0.000795394182547643, + "loss": 2.6639, + "theoretical_loss": 5.278036054825645, + "tokens_seen": 51838976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953781344032097, + "loss": 2.6419, + "theoretical_loss": 5.277031320449941, + "tokens_seen": 51904512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953620862587764, + "loss": 2.7968, + "theoretical_loss": 5.27602820857661, + "tokens_seen": 51970048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953460381143431, + "loss": 2.6677, + "theoretical_loss": 5.275026714544463, + "tokens_seen": 52035584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953299899699098, + "loss": 2.6115, + "theoretical_loss": 5.27402683371155, + "tokens_seen": 52101120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007953139418254765, + "loss": 2.6203, + "theoretical_loss": 5.273028561455056, + "tokens_seen": 52166656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952978936810432, + "loss": 2.7309, + "theoretical_loss": 5.272031893171203, + "tokens_seen": 52232192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952818455366099, + "loss": 2.7589, + "theoretical_loss": 5.27103682427514, + "tokens_seen": 52297728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952657973921766, + "loss": 2.7783, + "theoretical_loss": 5.270043350200851, + "tokens_seen": 52363264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 27101, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 2.5798048973083496, + "objective/train/theoretical_loss": 5.269051466401043, + "objective/train/tokens_used": 72888800, + "theoretical_loss": 5.269051466401043, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952497492477433, + "loss": 2.5798, + "theoretical_loss": 5.269051466401043, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.00079523370110331, + "loss": 2.748, + "theoretical_loss": 5.268061168347058, + "tokens_seen": 52494336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952176529588767, + "loss": 2.5742, + "theoretical_loss": 5.267072451528764, + "tokens_seen": 52559872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007952016048144434, + "loss": 2.4865, + "theoretical_loss": 5.266085311454461, + "tokens_seen": 52625408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951855566700101, + "loss": 2.7233, + "theoretical_loss": 5.265099743650779, + "tokens_seen": 52690944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951695085255768, + "loss": 2.6638, + "theoretical_loss": 5.264115743662588, + "tokens_seen": 52756480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951534603811434, + "loss": 2.7692, + "theoretical_loss": 5.26313330705289, + "tokens_seen": 52822016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951374122367102, + "loss": 2.5976, + "theoretical_loss": 5.262152429402734, + "tokens_seen": 52887552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951213640922768, + "loss": 2.7364, + "theoretical_loss": 5.261173106311116, + "tokens_seen": 52953088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007951053159478436, + "loss": 2.5708, + "theoretical_loss": 5.2601953333948765, + "tokens_seen": 53018624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007950892678034102, + "loss": 2.6982, + "theoretical_loss": 5.25921910628862, + "tokens_seen": 53084160 + }, + { + "epoch": 0.02, + "learning_rate": 0.000795073219658977, + "loss": 2.6809, + "theoretical_loss": 5.258244420644611, + "tokens_seen": 53149696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007950571715145436, + "loss": 2.7632, + "theoretical_loss": 5.257271272132685, + "tokens_seen": 53215232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007950411233701104, + "loss": 2.7068, + "theoretical_loss": 5.256299656440154, + "tokens_seen": 53280768 + }, + { + "epoch": 0.02, + "learning_rate": 0.000795025075225677, + "loss": 2.7195, + "theoretical_loss": 5.255329569271717, + "tokens_seen": 53346304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007950090270812438, + "loss": 2.5318, + "theoretical_loss": 5.254361006349367, + "tokens_seen": 53411840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949929789368104, + "loss": 2.6181, + "theoretical_loss": 5.2533939634122975, + "tokens_seen": 53477376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949769307923772, + "loss": 2.7352, + "theoretical_loss": 5.252428436216821, + "tokens_seen": 53542912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949608826479438, + "loss": 2.6394, + "theoretical_loss": 5.251464420536269, + "tokens_seen": 53608448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949448345035106, + "loss": 2.668, + "theoretical_loss": 5.250501912160911, + "tokens_seen": 53673984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007949287863590772, + "loss": 2.7109, + "theoretical_loss": 5.24954090689786, + "tokens_seen": 53739520 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794912738214644, + "loss": 2.5298, + "theoretical_loss": 5.248581400570989, + "tokens_seen": 53805056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948966900702106, + "loss": 2.6028, + "theoretical_loss": 5.247623389020841, + "tokens_seen": 53870592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948806419257774, + "loss": 2.5502, + "theoretical_loss": 5.246666868104547, + "tokens_seen": 53936128 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794864593781344, + "loss": 2.5574, + "theoretical_loss": 5.245711833695731, + "tokens_seen": 54001664 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948485456369108, + "loss": 2.6273, + "theoretical_loss": 5.244758281684433, + "tokens_seen": 54067200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948324974924774, + "loss": 2.5492, + "theoretical_loss": 5.24380620797702, + "tokens_seen": 54132736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948164493480442, + "loss": 2.6207, + "theoretical_loss": 5.242855608496102, + "tokens_seen": 54198272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007948004012036108, + "loss": 2.5862, + "theoretical_loss": 5.2419064791804475, + "tokens_seen": 54263808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007947843530591776, + "loss": 2.6388, + "theoretical_loss": 5.240958815984901, + "tokens_seen": 54329344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007947683049147442, + "loss": 2.5906, + "theoretical_loss": 5.2400126148803, + "tokens_seen": 54394880 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794752256770311, + "loss": 2.5325, + "theoretical_loss": 5.23906787185339, + "tokens_seen": 54460416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007947362086258776, + "loss": 2.5763, + "theoretical_loss": 5.238124582906748, + "tokens_seen": 54525952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007947201604814444, + "loss": 2.8481, + "theoretical_loss": 5.237182744058696, + "tokens_seen": 54591488 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794704112337011, + "loss": 2.4253, + "theoretical_loss": 5.236242351343224, + "tokens_seen": 54657024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946880641925778, + "loss": 2.5232, + "theoretical_loss": 5.235303400809904, + "tokens_seen": 54722560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946720160481444, + "loss": 2.6427, + "theoretical_loss": 5.2343658885238185, + "tokens_seen": 54788096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946559679037112, + "loss": 2.5511, + "theoretical_loss": 5.2334298105654735, + "tokens_seen": 54853632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946399197592778, + "loss": 2.5845, + "theoretical_loss": 5.232495163030723, + "tokens_seen": 54919168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946238716148446, + "loss": 2.572, + "theoretical_loss": 5.231561942030694, + "tokens_seen": 54984704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007946078234704112, + "loss": 2.6078, + "theoretical_loss": 5.230630143691698, + "tokens_seen": 55050240 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794591775325978, + "loss": 2.5558, + "theoretical_loss": 5.22969976415517, + "tokens_seen": 55115776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007945757271815446, + "loss": 2.6206, + "theoretical_loss": 5.228770799577576, + "tokens_seen": 55181312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007945596790371114, + "loss": 2.4544, + "theoretical_loss": 5.227843246130343, + "tokens_seen": 55246848 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794543630892678, + "loss": 2.5025, + "theoretical_loss": 5.2269170999997865, + "tokens_seen": 55312384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007945275827482448, + "loss": 2.5459, + "theoretical_loss": 5.225992357387031, + "tokens_seen": 55377920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007945115346038114, + "loss": 2.5993, + "theoretical_loss": 5.225069014507933, + "tokens_seen": 55443456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944954864593782, + "loss": 2.5679, + "theoretical_loss": 5.224147067593012, + "tokens_seen": 55508992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944794383149448, + "loss": 2.5954, + "theoretical_loss": 5.223226512887375, + "tokens_seen": 55574528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944633901705116, + "loss": 2.5244, + "theoretical_loss": 5.222307346650636, + "tokens_seen": 55640064 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944473420260782, + "loss": 2.4612, + "theoretical_loss": 5.221389565156855, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794431293881645, + "loss": 2.5992, + "theoretical_loss": 5.2204731646944555, + "tokens_seen": 55771136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007944152457372116, + "loss": 2.4763, + "theoretical_loss": 5.219558141566156, + "tokens_seen": 55836672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943991975927784, + "loss": 2.5833, + "theoretical_loss": 5.218644492088899, + "tokens_seen": 55902208 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794383149448345, + "loss": 2.5174, + "theoretical_loss": 5.217732212593778, + "tokens_seen": 55967744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943671013039118, + "loss": 2.4826, + "theoretical_loss": 5.216821299425968, + "tokens_seen": 56033280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943510531594784, + "loss": 2.6457, + "theoretical_loss": 5.215911748944652, + "tokens_seen": 56098816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943350050150452, + "loss": 2.5952, + "theoretical_loss": 5.215003557522959, + "tokens_seen": 56164352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943189568706118, + "loss": 2.4906, + "theoretical_loss": 5.21409672154788, + "tokens_seen": 56229888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007943029087261786, + "loss": 2.5894, + "theoretical_loss": 5.213191237420215, + "tokens_seen": 56295424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007942868605817452, + "loss": 2.5959, + "theoretical_loss": 5.212287101554496, + "tokens_seen": 56360960 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794270812437312, + "loss": 2.5346, + "theoretical_loss": 5.211384310378916, + "tokens_seen": 56426496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007942547642928786, + "loss": 2.5443, + "theoretical_loss": 5.210482860335269, + "tokens_seen": 56492032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007942387161484454, + "loss": 2.5812, + "theoretical_loss": 5.2095827478788745, + "tokens_seen": 56557568 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794222668004012, + "loss": 2.4374, + "theoretical_loss": 5.208683969478518, + "tokens_seen": 56623104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007942066198595788, + "loss": 2.4504, + "theoretical_loss": 5.2077865216163755, + "tokens_seen": 56688640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941905717151455, + "loss": 2.4962, + "theoretical_loss": 5.206890400787959, + "tokens_seen": 56754176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941745235707122, + "loss": 2.4319, + "theoretical_loss": 5.205995603502039, + "tokens_seen": 56819712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941584754262789, + "loss": 2.4834, + "theoretical_loss": 5.205102126280586, + "tokens_seen": 56885248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941424272818456, + "loss": 2.5599, + "theoretical_loss": 5.204209965658704, + "tokens_seen": 56950784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007941263791374123, + "loss": 2.5048, + "theoretical_loss": 5.203319118184565, + "tokens_seen": 57016320 + }, + { + "epoch": 0.02, + "learning_rate": 0.000794110330992979, + "loss": 2.604, + "theoretical_loss": 5.202429580419343, + "tokens_seen": 57081856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940942828485457, + "loss": 2.4432, + "theoretical_loss": 5.201541348937157, + "tokens_seen": 57147392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940782347041124, + "loss": 2.5292, + "theoretical_loss": 5.200654420324998, + "tokens_seen": 57212928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940621865596791, + "loss": 2.4261, + "theoretical_loss": 5.199768791182677, + "tokens_seen": 57278464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940461384152458, + "loss": 2.4407, + "theoretical_loss": 5.198884458122748, + "tokens_seen": 57344000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940300902708125, + "loss": 2.3966, + "theoretical_loss": 5.198001417770461, + "tokens_seen": 57409536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007940140421263792, + "loss": 2.5058, + "theoretical_loss": 5.1971196667636885, + "tokens_seen": 57475072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939979939819459, + "loss": 2.5757, + "theoretical_loss": 5.196239201752871, + "tokens_seen": 57540608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939819458375126, + "loss": 2.4193, + "theoretical_loss": 5.19536001940095, + "tokens_seen": 57606144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939658976930794, + "loss": 2.4897, + "theoretical_loss": 5.1944821163833135, + "tokens_seen": 57671680 + }, + { + "epoch": 0.02, + "learning_rate": 0.000793949849548646, + "loss": 2.4132, + "theoretical_loss": 5.19360548938773, + "tokens_seen": 57737216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939338014042128, + "loss": 2.4693, + "theoretical_loss": 5.192730135114292, + "tokens_seen": 57802752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939177532597794, + "loss": 2.4497, + "theoretical_loss": 5.191856050275353, + "tokens_seen": 57868288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007939017051153462, + "loss": 2.4771, + "theoretical_loss": 5.190983231595473, + "tokens_seen": 57933824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938856569709128, + "loss": 2.4195, + "theoretical_loss": 5.1901116758113535, + "tokens_seen": 57999360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938696088264795, + "loss": 2.4006, + "theoretical_loss": 5.189241379671782, + "tokens_seen": 58064896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938535606820462, + "loss": 2.5972, + "theoretical_loss": 5.188372339937576, + "tokens_seen": 58130432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938375125376129, + "loss": 2.4239, + "theoretical_loss": 5.187504553381521, + "tokens_seen": 58195968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938214643931796, + "loss": 2.6121, + "theoretical_loss": 5.1866380167883115, + "tokens_seen": 58261504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007938054162487463, + "loss": 2.5954, + "theoretical_loss": 5.1857727269545, + "tokens_seen": 58327040 + }, + { + "epoch": 0.02, + "learning_rate": 0.000793789368104313, + "loss": 2.4094, + "theoretical_loss": 5.184908680688437, + "tokens_seen": 58392576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937733199598797, + "loss": 2.4097, + "theoretical_loss": 5.18404587481021, + "tokens_seen": 58458112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937572718154463, + "loss": 2.4062, + "theoretical_loss": 5.183184306151595, + "tokens_seen": 58523648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937412236710131, + "loss": 2.3971, + "theoretical_loss": 5.182323971555995, + "tokens_seen": 58589184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937251755265797, + "loss": 2.477, + "theoretical_loss": 5.181464867878384, + "tokens_seen": 58654720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007937091273821465, + "loss": 2.6, + "theoretical_loss": 5.1806069919852575, + "tokens_seen": 58720256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936930792377131, + "loss": 2.5417, + "theoretical_loss": 5.17975034075457, + "tokens_seen": 58785792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936770310932799, + "loss": 2.3662, + "theoretical_loss": 5.1788949110756874, + "tokens_seen": 58851328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936609829488465, + "loss": 2.4413, + "theoretical_loss": 5.178040699849327, + "tokens_seen": 58916864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 29491, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 2.4602999687194824, + "objective/train/theoretical_loss": 5.1771877039875065, + "objective/train/tokens_used": 79442400, + "theoretical_loss": 5.1771877039875065, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936449348044133, + "loss": 2.4603, + "theoretical_loss": 5.1771877039875065, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936288866599799, + "loss": 2.4153, + "theoretical_loss": 5.176335920413489, + "tokens_seen": 59047936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007936128385155467, + "loss": 2.6104, + "theoretical_loss": 5.175485346061732, + "tokens_seen": 59113472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935967903711133, + "loss": 2.4952, + "theoretical_loss": 5.174635977877831, + "tokens_seen": 59179008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935807422266801, + "loss": 2.3725, + "theoretical_loss": 5.173787812818468, + "tokens_seen": 59244544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935646940822467, + "loss": 2.2532, + "theoretical_loss": 5.172940847851359, + "tokens_seen": 59310080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935486459378135, + "loss": 2.4305, + "theoretical_loss": 5.172095079955204, + "tokens_seen": 59375616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935325977933801, + "loss": 2.4633, + "theoretical_loss": 5.171250506119634, + "tokens_seen": 59441152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935165496489469, + "loss": 2.4208, + "theoretical_loss": 5.170407123345155, + "tokens_seen": 59506688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007935005015045135, + "loss": 2.4468, + "theoretical_loss": 5.169564928643103, + "tokens_seen": 59572224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934844533600803, + "loss": 2.3946, + "theoretical_loss": 5.168723919035591, + "tokens_seen": 59637760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934684052156469, + "loss": 2.4687, + "theoretical_loss": 5.167884091555456, + "tokens_seen": 59703296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934523570712137, + "loss": 2.4246, + "theoretical_loss": 5.167045443246213, + "tokens_seen": 59768832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934363089267803, + "loss": 2.4423, + "theoretical_loss": 5.166207971162001, + "tokens_seen": 59834368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934202607823471, + "loss": 2.3983, + "theoretical_loss": 5.1653716723675345, + "tokens_seen": 59899904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007934042126379137, + "loss": 2.4609, + "theoretical_loss": 5.164536543938056, + "tokens_seen": 59965440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933881644934805, + "loss": 2.3967, + "theoretical_loss": 5.163702582959283, + "tokens_seen": 60030976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933721163490471, + "loss": 2.3973, + "theoretical_loss": 5.162869786527363, + "tokens_seen": 60096512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933560682046139, + "loss": 2.4105, + "theoretical_loss": 5.1620381517488205, + "tokens_seen": 60162048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933400200601805, + "loss": 2.4944, + "theoretical_loss": 5.161207675740515, + "tokens_seen": 60227584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933239719157473, + "loss": 2.3914, + "theoretical_loss": 5.160378355629584, + "tokens_seen": 60293120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007933079237713139, + "loss": 2.2923, + "theoretical_loss": 5.159550188553405, + "tokens_seen": 60358656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932918756268807, + "loss": 2.4562, + "theoretical_loss": 5.15872317165954, + "tokens_seen": 60424192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932758274824473, + "loss": 2.2974, + "theoretical_loss": 5.1578973021056935, + "tokens_seen": 60489728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932597793380141, + "loss": 2.3379, + "theoretical_loss": 5.157072577059662, + "tokens_seen": 60555264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932437311935807, + "loss": 2.4035, + "theoretical_loss": 5.15624899369929, + "tokens_seen": 60620800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932276830491475, + "loss": 2.4439, + "theoretical_loss": 5.155426549212422, + "tokens_seen": 60686336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007932116349047141, + "loss": 2.1819, + "theoretical_loss": 5.154605240796855, + "tokens_seen": 60751872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931955867602809, + "loss": 2.3471, + "theoretical_loss": 5.153785065660296, + "tokens_seen": 60817408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931795386158475, + "loss": 2.4269, + "theoretical_loss": 5.152966021020315, + "tokens_seen": 60882944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931634904714143, + "loss": 2.3903, + "theoretical_loss": 5.152148104104297, + "tokens_seen": 60948480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931474423269809, + "loss": 2.2739, + "theoretical_loss": 5.1513313121494, + "tokens_seen": 61014016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931313941825477, + "loss": 2.3234, + "theoretical_loss": 5.15051564240251, + "tokens_seen": 61079552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007931153460381143, + "loss": 2.3041, + "theoretical_loss": 5.149701092120194, + "tokens_seen": 61145088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930992978936811, + "loss": 2.3438, + "theoretical_loss": 5.148887658568658, + "tokens_seen": 61210624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930832497492477, + "loss": 2.4146, + "theoretical_loss": 5.148075339023699, + "tokens_seen": 61276160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930672016048145, + "loss": 2.3163, + "theoretical_loss": 5.147264130770671, + "tokens_seen": 61341696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930511534603811, + "loss": 2.3089, + "theoretical_loss": 5.146454031104426, + "tokens_seen": 61407232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930351053159479, + "loss": 2.445, + "theoretical_loss": 5.1456450373292855, + "tokens_seen": 61472768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930190571715145, + "loss": 2.404, + "theoretical_loss": 5.144837146758987, + "tokens_seen": 61538304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007930030090270813, + "loss": 2.4489, + "theoretical_loss": 5.144030356716646, + "tokens_seen": 61603840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929869608826479, + "loss": 2.5282, + "theoretical_loss": 5.1432246645347135, + "tokens_seen": 61669376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929709127382147, + "loss": 2.317, + "theoretical_loss": 5.14242006755493, + "tokens_seen": 61734912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929548645937813, + "loss": 2.4457, + "theoretical_loss": 5.141616563128288, + "tokens_seen": 61800448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929388164493481, + "loss": 2.2941, + "theoretical_loss": 5.140814148614985, + "tokens_seen": 61865984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929227683049147, + "loss": 2.4758, + "theoretical_loss": 5.1400128213843885, + "tokens_seen": 61931520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007929067201604815, + "loss": 2.3312, + "theoretical_loss": 5.139212578814986, + "tokens_seen": 61997056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928906720160481, + "loss": 2.4919, + "theoretical_loss": 5.13841341829435, + "tokens_seen": 62062592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928746238716149, + "loss": 2.4333, + "theoretical_loss": 5.137615337219098, + "tokens_seen": 62128128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928585757271815, + "loss": 2.3636, + "theoretical_loss": 5.136818332994844, + "tokens_seen": 62193664 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928425275827483, + "loss": 2.4269, + "theoretical_loss": 5.136022403036167, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792826479438315, + "loss": 2.3482, + "theoretical_loss": 5.135227544766565, + "tokens_seen": 62324736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007928104312938817, + "loss": 2.2758, + "theoretical_loss": 5.134433755618414, + "tokens_seen": 62390272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927943831494484, + "loss": 2.3603, + "theoretical_loss": 5.1336410330329345, + "tokens_seen": 62455808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927783350050151, + "loss": 2.4681, + "theoretical_loss": 5.132849374460145, + "tokens_seen": 62521344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927622868605819, + "loss": 2.2553, + "theoretical_loss": 5.132058777358827, + "tokens_seen": 62586880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927462387161485, + "loss": 2.4522, + "theoretical_loss": 5.1312692391964845, + "tokens_seen": 62652416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927301905717153, + "loss": 2.3626, + "theoretical_loss": 5.1304807574493, + "tokens_seen": 62717952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007927141424272819, + "loss": 2.3232, + "theoretical_loss": 5.129693329602108, + "tokens_seen": 62783488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926980942828487, + "loss": 2.3389, + "theoretical_loss": 5.128906953148342, + "tokens_seen": 62849024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926820461384153, + "loss": 2.487, + "theoretical_loss": 5.128121625590007, + "tokens_seen": 62914560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926659979939821, + "loss": 2.4006, + "theoretical_loss": 5.127337344437633, + "tokens_seen": 62980096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926499498495487, + "loss": 2.4503, + "theoretical_loss": 5.126554107210245, + "tokens_seen": 63045632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926339017051155, + "loss": 2.2779, + "theoretical_loss": 5.125771911435321, + "tokens_seen": 63111168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926178535606821, + "loss": 2.2988, + "theoretical_loss": 5.124990754648754, + "tokens_seen": 63176704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007926018054162489, + "loss": 2.3609, + "theoretical_loss": 5.124210634394814, + "tokens_seen": 63242240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925857572718155, + "loss": 2.3058, + "theoretical_loss": 5.1234315482261135, + "tokens_seen": 63307776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925697091273823, + "loss": 2.3137, + "theoretical_loss": 5.122653493703571, + "tokens_seen": 63373312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925536609829489, + "loss": 2.3022, + "theoretical_loss": 5.121876468396373, + "tokens_seen": 63438848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925376128385157, + "loss": 2.4712, + "theoretical_loss": 5.121100469881933, + "tokens_seen": 63504384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925215646940823, + "loss": 2.2307, + "theoretical_loss": 5.120325495745863, + "tokens_seen": 63569920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007925055165496491, + "loss": 2.4119, + "theoretical_loss": 5.119551543581933, + "tokens_seen": 63635456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924894684052157, + "loss": 2.296, + "theoretical_loss": 5.118778610992038, + "tokens_seen": 63700992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924734202607824, + "loss": 2.2566, + "theoretical_loss": 5.118006695586154, + "tokens_seen": 63766528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924573721163491, + "loss": 2.3311, + "theoretical_loss": 5.1172357949823155, + "tokens_seen": 63832064 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924413239719158, + "loss": 2.3055, + "theoretical_loss": 5.116465906806567, + "tokens_seen": 63897600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924252758274825, + "loss": 2.313, + "theoretical_loss": 5.11569702869294, + "tokens_seen": 63963136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007924092276830492, + "loss": 2.3058, + "theoretical_loss": 5.114929158283407, + "tokens_seen": 64028672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923931795386159, + "loss": 2.3471, + "theoretical_loss": 5.114162293227853, + "tokens_seen": 64094208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923771313941826, + "loss": 2.3048, + "theoretical_loss": 5.113396431184043, + "tokens_seen": 64159744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923610832497493, + "loss": 2.3641, + "theoretical_loss": 5.112631569817578, + "tokens_seen": 64225280 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792345035105316, + "loss": 2.3404, + "theoretical_loss": 5.111867706801872, + "tokens_seen": 64290816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923289869608826, + "loss": 2.4302, + "theoretical_loss": 5.111104839818113, + "tokens_seen": 64356352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007923129388164494, + "loss": 2.4105, + "theoretical_loss": 5.1103429665552245, + "tokens_seen": 64421888 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792296890672016, + "loss": 2.2838, + "theoretical_loss": 5.109582084709841, + "tokens_seen": 64487424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922808425275828, + "loss": 2.3176, + "theoretical_loss": 5.108822191986269, + "tokens_seen": 64552960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922647943831494, + "loss": 2.32, + "theoretical_loss": 5.108063286096453, + "tokens_seen": 64618496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922487462387162, + "loss": 2.2686, + "theoretical_loss": 5.107305364759943, + "tokens_seen": 64684032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922326980942828, + "loss": 2.2104, + "theoretical_loss": 5.106548425703869, + "tokens_seen": 64749568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922166499498496, + "loss": 2.3018, + "theoretical_loss": 5.1057924666628915, + "tokens_seen": 64815104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007922006018054162, + "loss": 2.2446, + "theoretical_loss": 5.105037485379189, + "tokens_seen": 64880640 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792184553660983, + "loss": 2.1876, + "theoretical_loss": 5.104283479602407, + "tokens_seen": 64946176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007921685055165496, + "loss": 2.3042, + "theoretical_loss": 5.103530447089642, + "tokens_seen": 65011712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007921524573721164, + "loss": 2.2763, + "theoretical_loss": 5.102778385605397, + "tokens_seen": 65077248 + }, + { + "epoch": 0.02, + "learning_rate": 0.000792136409227683, + "loss": 2.3011, + "theoretical_loss": 5.102027292921557, + "tokens_seen": 65142784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007921203610832498, + "loss": 2.2809, + "theoretical_loss": 5.101277166817351, + "tokens_seen": 65208320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007921043129388164, + "loss": 2.4468, + "theoretical_loss": 5.100528005079328, + "tokens_seen": 65273856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920882647943832, + "loss": 2.348, + "theoretical_loss": 5.099779805501318, + "tokens_seen": 65339392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920722166499498, + "loss": 2.4444, + "theoretical_loss": 5.099032565884409, + "tokens_seen": 65404928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920561685055166, + "loss": 2.2229, + "theoretical_loss": 5.098286284036908, + "tokens_seen": 65470464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 32172, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 2.2884788513183594, + "objective/train/theoretical_loss": 5.097540957774312, + "objective/train/tokens_used": 85996000, + "theoretical_loss": 5.097540957774312, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920401203610832, + "loss": 2.2885, + "theoretical_loss": 5.097540957774312, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.00079202407221665, + "loss": 2.144, + "theoretical_loss": 5.096796584919282, + "tokens_seen": 65601536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007920080240722166, + "loss": 2.236, + "theoretical_loss": 5.096053163301608, + "tokens_seen": 65667072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919919759277834, + "loss": 2.2372, + "theoretical_loss": 5.095310690758177, + "tokens_seen": 65732608 + }, + { + "epoch": 0.02, + "learning_rate": 0.00079197592778335, + "loss": 2.2302, + "theoretical_loss": 5.094569165132949, + "tokens_seen": 65798144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919598796389168, + "loss": 2.1925, + "theoretical_loss": 5.09382858427692, + "tokens_seen": 65863680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919438314944834, + "loss": 2.3218, + "theoretical_loss": 5.0930889460481, + "tokens_seen": 65929216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919277833500502, + "loss": 2.3288, + "theoretical_loss": 5.092350248311471, + "tokens_seen": 65994752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007919117352056168, + "loss": 2.2272, + "theoretical_loss": 5.091612488938971, + "tokens_seen": 66060288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007918956870611836, + "loss": 2.3599, + "theoretical_loss": 5.090875665809454, + "tokens_seen": 66125824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007918796389167502, + "loss": 2.2867, + "theoretical_loss": 5.09013977680867, + "tokens_seen": 66191360 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791863590772317, + "loss": 2.3893, + "theoretical_loss": 5.089404819829225, + "tokens_seen": 66256896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007918475426278836, + "loss": 2.3538, + "theoretical_loss": 5.088670792770563, + "tokens_seen": 66322432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007918314944834504, + "loss": 2.3108, + "theoretical_loss": 5.087937693538927, + "tokens_seen": 66387968 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791815446339017, + "loss": 2.3665, + "theoretical_loss": 5.0872055200473385, + "tokens_seen": 66453504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917993981945838, + "loss": 2.3964, + "theoretical_loss": 5.0864742702155645, + "tokens_seen": 66519040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917833500501504, + "loss": 2.1746, + "theoretical_loss": 5.0857439419700885, + "tokens_seen": 66584576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917673019057172, + "loss": 2.2295, + "theoretical_loss": 5.085014533244088, + "tokens_seen": 66650112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917512537612838, + "loss": 2.4937, + "theoretical_loss": 5.084286041977396, + "tokens_seen": 66715648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917352056168506, + "loss": 2.3194, + "theoretical_loss": 5.083558466116487, + "tokens_seen": 66781184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007917191574724172, + "loss": 2.1903, + "theoretical_loss": 5.082831803614433, + "tokens_seen": 66846720 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791703109327984, + "loss": 2.2532, + "theoretical_loss": 5.082106052430891, + "tokens_seen": 66912256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916870611835506, + "loss": 2.3094, + "theoretical_loss": 5.081381210532063, + "tokens_seen": 66977792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916710130391174, + "loss": 2.4095, + "theoretical_loss": 5.08065727589068, + "tokens_seen": 67043328 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791654964894684, + "loss": 2.406, + "theoretical_loss": 5.079934246485963, + "tokens_seen": 67108864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916389167502508, + "loss": 2.1762, + "theoretical_loss": 5.079212120303606, + "tokens_seen": 67174400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916228686058174, + "loss": 2.3653, + "theoretical_loss": 5.078490895335744, + "tokens_seen": 67239936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007916068204613842, + "loss": 2.1304, + "theoretical_loss": 5.077770569580924, + "tokens_seen": 67305472 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791590772316951, + "loss": 2.3093, + "theoretical_loss": 5.077051141044086, + "tokens_seen": 67371008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007915747241725176, + "loss": 2.2845, + "theoretical_loss": 5.076332607736527, + "tokens_seen": 67436544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007915586760280844, + "loss": 2.3291, + "theoretical_loss": 5.0756149676758815, + "tokens_seen": 67502080 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791542627883651, + "loss": 2.2732, + "theoretical_loss": 5.0748982188860925, + "tokens_seen": 67567616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007915265797392178, + "loss": 2.2529, + "theoretical_loss": 5.074182359397387, + "tokens_seen": 67633152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007915105315947844, + "loss": 2.3085, + "theoretical_loss": 5.073467387246247, + "tokens_seen": 67698688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914944834503512, + "loss": 2.2992, + "theoretical_loss": 5.072753300475385, + "tokens_seen": 67764224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914784353059178, + "loss": 2.2994, + "theoretical_loss": 5.072040097133718, + "tokens_seen": 67829760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914623871614846, + "loss": 2.3977, + "theoretical_loss": 5.0713277752763455, + "tokens_seen": 67895296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914463390170512, + "loss": 2.2032, + "theoretical_loss": 5.070616332964518, + "tokens_seen": 67960832 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791430290872618, + "loss": 2.282, + "theoretical_loss": 5.069905768265615, + "tokens_seen": 68026368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007914142427281846, + "loss": 2.2396, + "theoretical_loss": 5.069196079253119, + "tokens_seen": 68091904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913981945837514, + "loss": 2.2792, + "theoretical_loss": 5.068487264006592, + "tokens_seen": 68157440 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791382146439318, + "loss": 2.255, + "theoretical_loss": 5.0677793206116455, + "tokens_seen": 68222976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913660982948848, + "loss": 2.3527, + "theoretical_loss": 5.067072247159923, + "tokens_seen": 68288512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913500501504514, + "loss": 2.2779, + "theoretical_loss": 5.06636604174907, + "tokens_seen": 68354048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913340020060182, + "loss": 2.25, + "theoretical_loss": 5.065660702482711, + "tokens_seen": 68419584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913179538615848, + "loss": 2.3612, + "theoretical_loss": 5.064956227470425, + "tokens_seen": 68485120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007913019057171516, + "loss": 2.243, + "theoretical_loss": 5.06425261482772, + "tokens_seen": 68550656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007912858575727182, + "loss": 2.221, + "theoretical_loss": 5.063549862676011, + "tokens_seen": 68616192 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791269809428285, + "loss": 2.1482, + "theoretical_loss": 5.062847969142591, + "tokens_seen": 68681728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007912537612838516, + "loss": 2.2824, + "theoretical_loss": 5.062146932360616, + "tokens_seen": 68747264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007912377131394184, + "loss": 2.2794, + "theoretical_loss": 5.061446750469072, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791221664994985, + "loss": 2.3805, + "theoretical_loss": 5.060747421612758, + "tokens_seen": 68878336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007912056168505518, + "loss": 2.1662, + "theoretical_loss": 5.060048943942254, + "tokens_seen": 68943872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911895687061184, + "loss": 2.1614, + "theoretical_loss": 5.059351315613908, + "tokens_seen": 69009408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911735205616852, + "loss": 2.3213, + "theoretical_loss": 5.058654534789804, + "tokens_seen": 69074944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911574724172518, + "loss": 2.3061, + "theoretical_loss": 5.057958599637746, + "tokens_seen": 69140480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911414242728186, + "loss": 2.2007, + "theoretical_loss": 5.057263508331225, + "tokens_seen": 69206016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007911253761283852, + "loss": 2.2452, + "theoretical_loss": 5.0565692590494065, + "tokens_seen": 69271552 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791109327983952, + "loss": 2.1786, + "theoretical_loss": 5.055875849977101, + "tokens_seen": 69337088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910932798395186, + "loss": 2.262, + "theoretical_loss": 5.055183279304744, + "tokens_seen": 69402624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910772316950853, + "loss": 2.1715, + "theoretical_loss": 5.0544915452283705, + "tokens_seen": 69468160 + }, + { + "epoch": 0.02, + "learning_rate": 0.000791061183550652, + "loss": 2.2834, + "theoretical_loss": 5.053800645949597, + "tokens_seen": 69533696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910451354062187, + "loss": 2.2832, + "theoretical_loss": 5.053110579675593, + "tokens_seen": 69599232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910290872617854, + "loss": 2.2676, + "theoretical_loss": 5.052421344619064, + "tokens_seen": 69664768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007910130391173521, + "loss": 2.243, + "theoretical_loss": 5.051732938998227, + "tokens_seen": 69730304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909969909729188, + "loss": 2.2622, + "theoretical_loss": 5.051045361036787, + "tokens_seen": 69795840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909809428284855, + "loss": 2.1861, + "theoretical_loss": 5.0503586089639185, + "tokens_seen": 69861376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909648946840522, + "loss": 2.2282, + "theoretical_loss": 5.04967268101424, + "tokens_seen": 69926912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909488465396189, + "loss": 2.3254, + "theoretical_loss": 5.048987575427793, + "tokens_seen": 69992448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909327983951855, + "loss": 2.3074, + "theoretical_loss": 5.048303290450023, + "tokens_seen": 70057984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909167502507523, + "loss": 2.3328, + "theoretical_loss": 5.047619824331753, + "tokens_seen": 70123520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007909007021063189, + "loss": 2.1674, + "theoretical_loss": 5.046937175329167, + "tokens_seen": 70189056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908846539618857, + "loss": 2.3524, + "theoretical_loss": 5.046255341703786, + "tokens_seen": 70254592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908686058174523, + "loss": 2.1512, + "theoretical_loss": 5.045574321722445, + "tokens_seen": 70320128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908525576730191, + "loss": 2.3023, + "theoretical_loss": 5.0448941136572785, + "tokens_seen": 70385664 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908365095285857, + "loss": 2.1874, + "theoretical_loss": 5.044214715785689, + "tokens_seen": 70451200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908204613841525, + "loss": 2.3012, + "theoretical_loss": 5.043536126390334, + "tokens_seen": 70516736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007908044132397191, + "loss": 2.265, + "theoretical_loss": 5.042858343759107, + "tokens_seen": 70582272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907883650952859, + "loss": 2.2598, + "theoretical_loss": 5.0421813661851065, + "tokens_seen": 70647808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907723169508525, + "loss": 2.2182, + "theoretical_loss": 5.041505191966625, + "tokens_seen": 70713344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907562688064193, + "loss": 2.1663, + "theoretical_loss": 5.040829819407123, + "tokens_seen": 70778880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907402206619859, + "loss": 2.2501, + "theoretical_loss": 5.040155246815212, + "tokens_seen": 70844416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907241725175527, + "loss": 2.3264, + "theoretical_loss": 5.039481472504633, + "tokens_seen": 70909952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007907081243731193, + "loss": 2.257, + "theoretical_loss": 5.038808494794232, + "tokens_seen": 70975488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906920762286861, + "loss": 2.2255, + "theoretical_loss": 5.038136312007948, + "tokens_seen": 71041024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906760280842527, + "loss": 2.2399, + "theoretical_loss": 5.037464922474786, + "tokens_seen": 71106560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906599799398195, + "loss": 2.208, + "theoretical_loss": 5.036794324528802, + "tokens_seen": 71172096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906439317953861, + "loss": 2.2584, + "theoretical_loss": 5.036124516509076, + "tokens_seen": 71237632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906278836509529, + "loss": 2.1656, + "theoretical_loss": 5.035455496759701, + "tokens_seen": 71303168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007906118355065195, + "loss": 2.214, + "theoretical_loss": 5.034787263629758, + "tokens_seen": 71368704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905957873620863, + "loss": 2.2408, + "theoretical_loss": 5.0341198154732965, + "tokens_seen": 71434240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905797392176529, + "loss": 2.2403, + "theoretical_loss": 5.033453150649318, + "tokens_seen": 71499776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905636910732197, + "loss": 2.299, + "theoretical_loss": 5.032787267521752, + "tokens_seen": 71565312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905476429287863, + "loss": 2.1793, + "theoretical_loss": 5.032122164459443, + "tokens_seen": 71630848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905315947843531, + "loss": 2.1469, + "theoretical_loss": 5.031457839836122, + "tokens_seen": 71696384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007905155466399197, + "loss": 2.2367, + "theoretical_loss": 5.0307942920304, + "tokens_seen": 71761920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904994984954865, + "loss": 2.1601, + "theoretical_loss": 5.030131519425737, + "tokens_seen": 71827456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904834503510531, + "loss": 2.1401, + "theoretical_loss": 5.02946952041043, + "tokens_seen": 71892992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904674022066199, + "loss": 2.4711, + "theoretical_loss": 5.028808293377589, + "tokens_seen": 71958528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904513540621865, + "loss": 2.2271, + "theoretical_loss": 5.028147836725127, + "tokens_seen": 72024064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 34719, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 2.1790566444396973, + "objective/train/theoretical_loss": 5.0274881488557295, + "objective/train/tokens_used": 92549600, + "theoretical_loss": 5.0274881488557295, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904353059177533, + "loss": 2.1791, + "theoretical_loss": 5.0274881488557295, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904192577733199, + "loss": 2.295, + "theoretical_loss": 5.026829228176848, + "tokens_seen": 72155136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007904032096288867, + "loss": 2.2253, + "theoretical_loss": 5.026171073100672, + "tokens_seen": 72220672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903871614844534, + "loss": 2.1531, + "theoretical_loss": 5.025513682044114, + "tokens_seen": 72286208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903711133400201, + "loss": 2.1894, + "theoretical_loss": 5.024857053428793, + "tokens_seen": 72351744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903550651955868, + "loss": 2.1588, + "theoretical_loss": 5.024201185681015, + "tokens_seen": 72417280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903390170511535, + "loss": 2.2089, + "theoretical_loss": 5.023546077231752, + "tokens_seen": 72482816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903229689067202, + "loss": 2.1823, + "theoretical_loss": 5.0228917265166295, + "tokens_seen": 72548352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007903069207622869, + "loss": 2.1929, + "theoretical_loss": 5.022238131975905, + "tokens_seen": 72613888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902908726178537, + "loss": 2.1978, + "theoretical_loss": 5.021585292054446, + "tokens_seen": 72679424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902748244734203, + "loss": 2.0794, + "theoretical_loss": 5.020933205201725, + "tokens_seen": 72744960 + }, + { + "epoch": 0.02, + "learning_rate": 0.000790258776328987, + "loss": 2.1309, + "theoretical_loss": 5.020281869871787, + "tokens_seen": 72810496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902427281845537, + "loss": 2.3304, + "theoretical_loss": 5.019631284523241, + "tokens_seen": 72876032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902266800401205, + "loss": 2.2437, + "theoretical_loss": 5.01898144761924, + "tokens_seen": 72941568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007902106318956871, + "loss": 2.1908, + "theoretical_loss": 5.018332357627462, + "tokens_seen": 73007104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901945837512539, + "loss": 2.1411, + "theoretical_loss": 5.017684013020096, + "tokens_seen": 73072640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901785356068205, + "loss": 2.2339, + "theoretical_loss": 5.017036412273823, + "tokens_seen": 73138176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901624874623873, + "loss": 2.254, + "theoretical_loss": 5.016389553869796, + "tokens_seen": 73203712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901464393179539, + "loss": 2.2574, + "theoretical_loss": 5.0157434362936275, + "tokens_seen": 73269248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901303911735207, + "loss": 2.2642, + "theoretical_loss": 5.01509805803537, + "tokens_seen": 73334784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007901143430290873, + "loss": 2.2578, + "theoretical_loss": 5.0144534175894995, + "tokens_seen": 73400320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900982948846541, + "loss": 2.196, + "theoretical_loss": 5.013809513454898, + "tokens_seen": 73465856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900822467402207, + "loss": 2.2533, + "theoretical_loss": 5.013166344134838, + "tokens_seen": 73531392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900661985957875, + "loss": 2.1913, + "theoretical_loss": 5.012523908136963, + "tokens_seen": 73596928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900501504513541, + "loss": 2.2703, + "theoretical_loss": 5.011882203973277, + "tokens_seen": 73662464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900341023069209, + "loss": 2.1812, + "theoretical_loss": 5.011241230160119, + "tokens_seen": 73728000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900180541624875, + "loss": 2.2361, + "theoretical_loss": 5.0106009852181534, + "tokens_seen": 73793536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007900020060180543, + "loss": 2.2004, + "theoretical_loss": 5.009961467672353, + "tokens_seen": 73859072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899859578736209, + "loss": 2.1511, + "theoretical_loss": 5.009322676051976, + "tokens_seen": 73924608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899699097291877, + "loss": 2.2886, + "theoretical_loss": 5.008684608890562, + "tokens_seen": 73990144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899538615847543, + "loss": 2.2065, + "theoretical_loss": 5.008047264725901, + "tokens_seen": 74055680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899378134403211, + "loss": 2.2599, + "theoretical_loss": 5.007410642100032, + "tokens_seen": 74121216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899217652958877, + "loss": 2.0565, + "theoretical_loss": 5.006774739559216, + "tokens_seen": 74186752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007899057171514545, + "loss": 2.161, + "theoretical_loss": 5.00613955565392, + "tokens_seen": 74252288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898896690070211, + "loss": 2.0608, + "theoretical_loss": 5.005505088938815, + "tokens_seen": 74317824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898736208625879, + "loss": 2.1718, + "theoretical_loss": 5.004871337972742, + "tokens_seen": 74383360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898575727181545, + "loss": 2.1034, + "theoretical_loss": 5.004238301318706, + "tokens_seen": 74448896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898415245737213, + "loss": 2.2283, + "theoretical_loss": 5.003605977543861, + "tokens_seen": 74514432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898254764292879, + "loss": 2.1599, + "theoretical_loss": 5.00297436521949, + "tokens_seen": 74579968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007898094282848547, + "loss": 2.2186, + "theoretical_loss": 5.002343462920992, + "tokens_seen": 74645504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897933801404213, + "loss": 2.2726, + "theoretical_loss": 5.001713269227867, + "tokens_seen": 74711040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897773319959881, + "loss": 2.1977, + "theoretical_loss": 5.0010837827237005, + "tokens_seen": 74776576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897612838515547, + "loss": 2.2277, + "theoretical_loss": 5.000455001996143, + "tokens_seen": 74842112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897452357071215, + "loss": 2.2989, + "theoretical_loss": 4.999826925636905, + "tokens_seen": 74907648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897291875626881, + "loss": 2.2514, + "theoretical_loss": 4.999199552241732, + "tokens_seen": 74973184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007897131394182549, + "loss": 2.312, + "theoretical_loss": 4.998572880410396, + "tokens_seen": 75038720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896970912738215, + "loss": 2.1228, + "theoretical_loss": 4.997946908746677, + "tokens_seen": 75104256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896810431293882, + "loss": 2.2409, + "theoretical_loss": 4.997321635858347, + "tokens_seen": 75169792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896649949849549, + "loss": 2.2001, + "theoretical_loss": 4.9966970603571585, + "tokens_seen": 75235328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896489468405216, + "loss": 2.2675, + "theoretical_loss": 4.996073180858831, + "tokens_seen": 75300864 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896328986960883, + "loss": 2.1907, + "theoretical_loss": 4.995449995983028, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789616850551655, + "loss": 2.2062, + "theoretical_loss": 4.994827504353353, + "tokens_seen": 75431936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007896008024072217, + "loss": 2.183, + "theoretical_loss": 4.994205704597325, + "tokens_seen": 75497472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895847542627884, + "loss": 2.1929, + "theoretical_loss": 4.993584595346373, + "tokens_seen": 75563008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895687061183551, + "loss": 2.2374, + "theoretical_loss": 4.992964175235814, + "tokens_seen": 75628544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895526579739218, + "loss": 2.1921, + "theoretical_loss": 4.992344442904843, + "tokens_seen": 75694080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895366098294884, + "loss": 2.1265, + "theoretical_loss": 4.991725396996515, + "tokens_seen": 75759616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895205616850552, + "loss": 2.028, + "theoretical_loss": 4.991107036157738, + "tokens_seen": 75825152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007895045135406218, + "loss": 2.1131, + "theoretical_loss": 4.990489359039248, + "tokens_seen": 75890688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007894884653961886, + "loss": 2.1755, + "theoretical_loss": 4.989872364295605, + "tokens_seen": 75956224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007894724172517552, + "loss": 2.1793, + "theoretical_loss": 4.989256050585174, + "tokens_seen": 76021760 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789456369107322, + "loss": 2.2634, + "theoretical_loss": 4.9886404165701075, + "tokens_seen": 76087296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007894403209628886, + "loss": 2.1688, + "theoretical_loss": 4.98802546091634, + "tokens_seen": 76152832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007894242728184554, + "loss": 2.2535, + "theoretical_loss": 4.9874111822935685, + "tokens_seen": 76218368 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789408224674022, + "loss": 2.0708, + "theoretical_loss": 4.986797579375239, + "tokens_seen": 76283904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893921765295888, + "loss": 2.1895, + "theoretical_loss": 4.986184650838532, + "tokens_seen": 76349440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893761283851554, + "loss": 2.1738, + "theoretical_loss": 4.985572395364354, + "tokens_seen": 76414976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893600802407222, + "loss": 2.0693, + "theoretical_loss": 4.98496081163732, + "tokens_seen": 76480512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893440320962888, + "loss": 2.0786, + "theoretical_loss": 4.984349898345735, + "tokens_seen": 76546048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893279839518556, + "loss": 2.147, + "theoretical_loss": 4.98373965418159, + "tokens_seen": 76611584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007893119358074222, + "loss": 2.2609, + "theoretical_loss": 4.983130077840543, + "tokens_seen": 76677120 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789295887662989, + "loss": 2.2384, + "theoretical_loss": 4.982521168021906, + "tokens_seen": 76742656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007892798395185556, + "loss": 2.0189, + "theoretical_loss": 4.981912923428634, + "tokens_seen": 76808192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007892637913741224, + "loss": 2.2331, + "theoretical_loss": 4.981305342767307, + "tokens_seen": 76873728 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789247743229689, + "loss": 2.0864, + "theoretical_loss": 4.980698424748122, + "tokens_seen": 76939264 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007892316950852558, + "loss": 2.2626, + "theoretical_loss": 4.980092168084878, + "tokens_seen": 77004800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007892156469408224, + "loss": 2.1412, + "theoretical_loss": 4.97948657149496, + "tokens_seen": 77070336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891995987963892, + "loss": 2.1168, + "theoretical_loss": 4.978881633699331, + "tokens_seen": 77135872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891835506519559, + "loss": 2.2427, + "theoretical_loss": 4.9782773534225155, + "tokens_seen": 77201408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891675025075226, + "loss": 2.1032, + "theoretical_loss": 4.977673729392586, + "tokens_seen": 77266944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891514543630893, + "loss": 2.1723, + "theoretical_loss": 4.977070760341153, + "tokens_seen": 77332480 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789135406218656, + "loss": 2.0192, + "theoretical_loss": 4.976468445003352, + "tokens_seen": 77398016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891193580742227, + "loss": 2.0722, + "theoretical_loss": 4.975866782117828, + "tokens_seen": 77463552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007891033099297894, + "loss": 2.2401, + "theoretical_loss": 4.975265770426724, + "tokens_seen": 77529088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890872617853561, + "loss": 2.1938, + "theoretical_loss": 4.974665408675667, + "tokens_seen": 77594624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890712136409228, + "loss": 2.2931, + "theoretical_loss": 4.974065695613763, + "tokens_seen": 77660160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890551654964895, + "loss": 2.1322, + "theoretical_loss": 4.973466629993574, + "tokens_seen": 77725696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890391173520562, + "loss": 2.2289, + "theoretical_loss": 4.972868210571109, + "tokens_seen": 77791232 + }, + { + "epoch": 0.02, + "learning_rate": 0.000789023069207623, + "loss": 2.2627, + "theoretical_loss": 4.972270436105815, + "tokens_seen": 77856768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007890070210631896, + "loss": 2.2138, + "theoretical_loss": 4.971673305360563, + "tokens_seen": 77922304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889909729187563, + "loss": 2.0407, + "theoretical_loss": 4.971076817101633, + "tokens_seen": 77987840 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788974924774323, + "loss": 2.1262, + "theoretical_loss": 4.970480970098703, + "tokens_seen": 78053376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889588766298897, + "loss": 2.1401, + "theoretical_loss": 4.96988576312484, + "tokens_seen": 78118912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889428284854564, + "loss": 2.1337, + "theoretical_loss": 4.969291194956485, + "tokens_seen": 78184448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889267803410232, + "loss": 2.2153, + "theoretical_loss": 4.968697264373438, + "tokens_seen": 78249984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007889107321965898, + "loss": 2.2326, + "theoretical_loss": 4.968103970158854, + "tokens_seen": 78315520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007888946840521566, + "loss": 2.1139, + "theoretical_loss": 4.967511311099223, + "tokens_seen": 78381056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007888786359077232, + "loss": 2.2219, + "theoretical_loss": 4.96691928598436, + "tokens_seen": 78446592 + }, + { + "epoch": 0.02, + "learning_rate": 0.00078886258776329, + "loss": 2.1414, + "theoretical_loss": 4.966327893607398, + "tokens_seen": 78512128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007888465396188566, + "loss": 2.2232, + "theoretical_loss": 4.96573713276477, + "tokens_seen": 78577664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 36940, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 2.0884451866149902, + "objective/train/theoretical_loss": 4.965147002256201, + "objective/train/tokens_used": 99103200, + "theoretical_loss": 4.965147002256201, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007888304914744234, + "loss": 2.0884, + "theoretical_loss": 4.965147002256201, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.00078881444332999, + "loss": 2.0536, + "theoretical_loss": 4.964557500884695, + "tokens_seen": 78708736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887983951855568, + "loss": 2.1919, + "theoretical_loss": 4.963968627456523, + "tokens_seen": 78774272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887823470411234, + "loss": 2.2003, + "theoretical_loss": 4.96338038078121, + "tokens_seen": 78839808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887662988966902, + "loss": 2.0346, + "theoretical_loss": 4.962792759671528, + "tokens_seen": 78905344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887502507522568, + "loss": 2.1342, + "theoretical_loss": 4.962205762943478, + "tokens_seen": 78970880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887342026078236, + "loss": 2.2052, + "theoretical_loss": 4.961619389416288, + "tokens_seen": 79036416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007887181544633902, + "loss": 2.2535, + "theoretical_loss": 4.96103363791239, + "tokens_seen": 79101952 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788702106318957, + "loss": 2.1753, + "theoretical_loss": 4.960448507257416, + "tokens_seen": 79167488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886860581745236, + "loss": 2.1496, + "theoretical_loss": 4.959863996280188, + "tokens_seen": 79233024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886700100300904, + "loss": 2.0766, + "theoretical_loss": 4.959280103812697, + "tokens_seen": 79298560 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788653961885657, + "loss": 2.0173, + "theoretical_loss": 4.958696828690105, + "tokens_seen": 79364096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886379137412238, + "loss": 2.0692, + "theoretical_loss": 4.958114169750726, + "tokens_seen": 79429632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886218655967904, + "loss": 2.206, + "theoretical_loss": 4.957532125836012, + "tokens_seen": 79495168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007886058174523572, + "loss": 2.0888, + "theoretical_loss": 4.956950695790551, + "tokens_seen": 79560704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885897693079238, + "loss": 2.1318, + "theoretical_loss": 4.95636987846205, + "tokens_seen": 79626240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885737211634906, + "loss": 2.1276, + "theoretical_loss": 4.955789672701323, + "tokens_seen": 79691776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885576730190572, + "loss": 2.1568, + "theoretical_loss": 4.955210077362285, + "tokens_seen": 79757312 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788541624874624, + "loss": 2.2067, + "theoretical_loss": 4.954631091301934, + "tokens_seen": 79822848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885255767301906, + "loss": 2.2183, + "theoretical_loss": 4.954052713380349, + "tokens_seen": 79888384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007885095285857574, + "loss": 2.2139, + "theoretical_loss": 4.953474942460673, + "tokens_seen": 79953920 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788493480441324, + "loss": 2.3113, + "theoretical_loss": 4.952897777409104, + "tokens_seen": 80019456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884774322968908, + "loss": 2.2662, + "theoretical_loss": 4.952321217094882, + "tokens_seen": 80084992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884613841524574, + "loss": 2.102, + "theoretical_loss": 4.951745260390284, + "tokens_seen": 80150528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884453360080242, + "loss": 2.2117, + "theoretical_loss": 4.951169906170607, + "tokens_seen": 80216064 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884292878635908, + "loss": 2.1562, + "theoretical_loss": 4.950595153314161, + "tokens_seen": 80281600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007884132397191576, + "loss": 2.164, + "theoretical_loss": 4.950021000702261, + "tokens_seen": 80347136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883971915747242, + "loss": 2.2053, + "theoretical_loss": 4.949447447219208, + "tokens_seen": 80412672 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788381143430291, + "loss": 2.2245, + "theoretical_loss": 4.948874491752287, + "tokens_seen": 80478208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883650952858576, + "loss": 2.1905, + "theoretical_loss": 4.9483021331917545, + "tokens_seen": 80543744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883490471414244, + "loss": 2.142, + "theoretical_loss": 4.9477303704308255, + "tokens_seen": 80609280 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788332998996991, + "loss": 2.1618, + "theoretical_loss": 4.947159202365664, + "tokens_seen": 80674816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883169508525578, + "loss": 2.1577, + "theoretical_loss": 4.946588627895375, + "tokens_seen": 80740352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007883009027081244, + "loss": 2.0829, + "theoretical_loss": 4.946018645921994, + "tokens_seen": 80805888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882848545636911, + "loss": 2.1016, + "theoretical_loss": 4.945449255350473, + "tokens_seen": 80871424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882688064192578, + "loss": 2.1309, + "theoretical_loss": 4.944880455088677, + "tokens_seen": 80936960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882527582748245, + "loss": 2.1288, + "theoretical_loss": 4.944312244047365, + "tokens_seen": 81002496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882367101303912, + "loss": 2.1433, + "theoretical_loss": 4.943744621140189, + "tokens_seen": 81068032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882206619859579, + "loss": 2.1942, + "theoretical_loss": 4.94317758528368, + "tokens_seen": 81133568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007882046138415246, + "loss": 2.1101, + "theoretical_loss": 4.942611135397234, + "tokens_seen": 81199104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881885656970913, + "loss": 2.1531, + "theoretical_loss": 4.942045270403112, + "tokens_seen": 81264640 + }, + { + "epoch": 0.02, + "learning_rate": 0.000788172517552658, + "loss": 2.2471, + "theoretical_loss": 4.941479989226421, + "tokens_seen": 81330176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881564694082247, + "loss": 2.2154, + "theoretical_loss": 4.940915290795106, + "tokens_seen": 81395712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881404212637913, + "loss": 2.1522, + "theoretical_loss": 4.940351174039947, + "tokens_seen": 81461248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881243731193581, + "loss": 2.168, + "theoretical_loss": 4.939787637894537, + "tokens_seen": 81526784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007881083249749247, + "loss": 2.1257, + "theoretical_loss": 4.939224681295286, + "tokens_seen": 81592320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880922768304915, + "loss": 2.0687, + "theoretical_loss": 4.938662303181399, + "tokens_seen": 81657856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880762286860581, + "loss": 2.0866, + "theoretical_loss": 4.938100502494878, + "tokens_seen": 81723392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880601805416249, + "loss": 2.1731, + "theoretical_loss": 4.9375392781805, + "tokens_seen": 81788928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880441323971915, + "loss": 2.1461, + "theoretical_loss": 4.936978629185818, + "tokens_seen": 81854464 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880280842527583, + "loss": 2.1646, + "theoretical_loss": 4.936418554461147, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007880120361083249, + "loss": 2.0447, + "theoretical_loss": 4.935859052959554, + "tokens_seen": 81985536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879959879638917, + "loss": 2.2203, + "theoretical_loss": 4.935300123636851, + "tokens_seen": 82051072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879799398194584, + "loss": 2.1977, + "theoretical_loss": 4.93474176545158, + "tokens_seen": 82116608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879638916750251, + "loss": 2.1775, + "theoretical_loss": 4.934183977365015, + "tokens_seen": 82182144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879478435305918, + "loss": 2.1124, + "theoretical_loss": 4.9336267583411395, + "tokens_seen": 82247680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879317953861585, + "loss": 2.0796, + "theoretical_loss": 4.933070107346644, + "tokens_seen": 82313216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007879157472417252, + "loss": 2.0991, + "theoretical_loss": 4.932514023350921, + "tokens_seen": 82378752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0007878996990972919, + "loss": 2.1563, + "theoretical_loss": 4.931958505326042, + "tokens_seen": 82444288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878836509528586, + "loss": 2.0317, + "theoretical_loss": 4.931403552246769, + "tokens_seen": 82509824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878676028084253, + "loss": 2.135, + "theoretical_loss": 4.930849163090521, + "tokens_seen": 82575360 + }, + { + "epoch": 0.03, + "learning_rate": 0.000787851554663992, + "loss": 2.1251, + "theoretical_loss": 4.930295336837387, + "tokens_seen": 82640896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878355065195587, + "loss": 2.105, + "theoretical_loss": 4.9297420724701055, + "tokens_seen": 82706432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878194583751254, + "loss": 2.1526, + "theoretical_loss": 4.929189368974054, + "tokens_seen": 82771968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007878034102306921, + "loss": 2.1333, + "theoretical_loss": 4.928637225337246, + "tokens_seen": 82837504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877873620862588, + "loss": 2.0942, + "theoretical_loss": 4.9280856405503215, + "tokens_seen": 82903040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877713139418255, + "loss": 2.0241, + "theoretical_loss": 4.927534613606534, + "tokens_seen": 82968576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877552657973922, + "loss": 2.1364, + "theoretical_loss": 4.926984143501745, + "tokens_seen": 83034112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877392176529589, + "loss": 2.2187, + "theoretical_loss": 4.926434229234411, + "tokens_seen": 83099648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877231695085256, + "loss": 2.1759, + "theoretical_loss": 4.925884869805586, + "tokens_seen": 83165184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007877071213640923, + "loss": 2.1543, + "theoretical_loss": 4.925336064218893, + "tokens_seen": 83230720 + }, + { + "epoch": 0.03, + "learning_rate": 0.000787691073219659, + "loss": 2.1695, + "theoretical_loss": 4.924787811480538, + "tokens_seen": 83296256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876750250752257, + "loss": 2.1016, + "theoretical_loss": 4.9242401105992855, + "tokens_seen": 83361792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876589769307924, + "loss": 1.9869, + "theoretical_loss": 4.923692960586452, + "tokens_seen": 83427328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876429287863591, + "loss": 1.9533, + "theoretical_loss": 4.923146360455906, + "tokens_seen": 83492864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876268806419258, + "loss": 2.0596, + "theoretical_loss": 4.92260030922405, + "tokens_seen": 83558400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007876108324974925, + "loss": 2.1809, + "theoretical_loss": 4.922054805909816, + "tokens_seen": 83623936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875947843530592, + "loss": 2.0021, + "theoretical_loss": 4.921509849534658, + "tokens_seen": 83689472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875787362086259, + "loss": 2.1222, + "theoretical_loss": 4.920965439122538, + "tokens_seen": 83755008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875626880641927, + "loss": 2.2289, + "theoretical_loss": 4.9204215736999295, + "tokens_seen": 83820544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875466399197593, + "loss": 2.2235, + "theoretical_loss": 4.919878252295793, + "tokens_seen": 83886080 + }, + { + "epoch": 0.03, + "learning_rate": 0.000787530591775326, + "loss": 2.0148, + "theoretical_loss": 4.919335473941581, + "tokens_seen": 83951616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007875145436308927, + "loss": 2.1073, + "theoretical_loss": 4.918793237671224, + "tokens_seen": 84017152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874984954864595, + "loss": 2.2016, + "theoretical_loss": 4.918251542521121, + "tokens_seen": 84082688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874824473420261, + "loss": 2.1669, + "theoretical_loss": 4.9177103875301365, + "tokens_seen": 84148224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874663991975929, + "loss": 2.0046, + "theoretical_loss": 4.917169771739586, + "tokens_seen": 84213760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874503510531595, + "loss": 1.9713, + "theoretical_loss": 4.916629694193233, + "tokens_seen": 84279296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874343029087263, + "loss": 2.1506, + "theoretical_loss": 4.916090153937276, + "tokens_seen": 84344832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874182547642929, + "loss": 2.2232, + "theoretical_loss": 4.915551150020346, + "tokens_seen": 84410368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007874022066198597, + "loss": 2.0884, + "theoretical_loss": 4.915012681493495, + "tokens_seen": 84475904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873861584754263, + "loss": 2.0906, + "theoretical_loss": 4.914474747410187, + "tokens_seen": 84541440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873701103309931, + "loss": 2.0496, + "theoretical_loss": 4.913937346826293, + "tokens_seen": 84606976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873540621865597, + "loss": 2.0105, + "theoretical_loss": 4.91340047880008, + "tokens_seen": 84672512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873380140421265, + "loss": 2.0424, + "theoretical_loss": 4.9128641423922055, + "tokens_seen": 84738048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873219658976931, + "loss": 1.9974, + "theoretical_loss": 4.912328336665711, + "tokens_seen": 84803584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007873059177532599, + "loss": 2.1345, + "theoretical_loss": 4.911793060686007, + "tokens_seen": 84869120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872898696088265, + "loss": 2.1816, + "theoretical_loss": 4.9112583135208725, + "tokens_seen": 84934656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872738214643933, + "loss": 2.2291, + "theoretical_loss": 4.910724094240447, + "tokens_seen": 85000192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872577733199599, + "loss": 2.0991, + "theoretical_loss": 4.910190401917214, + "tokens_seen": 85065728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872417251755267, + "loss": 2.018, + "theoretical_loss": 4.909657235626009, + "tokens_seen": 85131264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 39240, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 2.064126491546631, + "objective/train/theoretical_loss": 4.909124594443993, + "objective/train/tokens_used": 105656800, + "theoretical_loss": 4.909124594443993, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872256770310933, + "loss": 2.0641, + "theoretical_loss": 4.909124594443993, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007872096288866601, + "loss": 2.1136, + "theoretical_loss": 4.908592477450659, + "tokens_seen": 85262336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871935807422267, + "loss": 2.0321, + "theoretical_loss": 4.908060883727821, + "tokens_seen": 85327872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871775325977935, + "loss": 2.1324, + "theoretical_loss": 4.9075298123596, + "tokens_seen": 85393408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871614844533601, + "loss": 2.1192, + "theoretical_loss": 4.906999262432427, + "tokens_seen": 85458944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871454363089269, + "loss": 2.2101, + "theoretical_loss": 4.906469233035025, + "tokens_seen": 85524480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871293881644935, + "loss": 2.1805, + "theoretical_loss": 4.905939723258408, + "tokens_seen": 85590016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007871133400200603, + "loss": 2.2154, + "theoretical_loss": 4.905410732195875, + "tokens_seen": 85655552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870972918756269, + "loss": 2.2694, + "theoretical_loss": 4.904882258942994, + "tokens_seen": 85721088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870812437311937, + "loss": 2.0957, + "theoretical_loss": 4.904354302597603, + "tokens_seen": 85786624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870651955867603, + "loss": 1.9985, + "theoretical_loss": 4.903826862259798, + "tokens_seen": 85852160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870491474423271, + "loss": 1.8715, + "theoretical_loss": 4.90329993703193, + "tokens_seen": 85917696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870330992978937, + "loss": 1.9849, + "theoretical_loss": 4.902773526018591, + "tokens_seen": 85983232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870170511534605, + "loss": 2.2081, + "theoretical_loss": 4.902247628326613, + "tokens_seen": 86048768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007870010030090271, + "loss": 2.134, + "theoretical_loss": 4.9017222430650556, + "tokens_seen": 86114304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869849548645939, + "loss": 2.1433, + "theoretical_loss": 4.901197369345205, + "tokens_seen": 86179840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869689067201605, + "loss": 2.0677, + "theoretical_loss": 4.9006730062805595, + "tokens_seen": 86245376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869528585757273, + "loss": 2.1004, + "theoretical_loss": 4.9001491529868275, + "tokens_seen": 86310912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869368104312939, + "loss": 2.1944, + "theoretical_loss": 4.899625808581918, + "tokens_seen": 86376448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869207622868607, + "loss": 2.0342, + "theoretical_loss": 4.899102972185936, + "tokens_seen": 86441984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007869047141424273, + "loss": 2.0419, + "theoretical_loss": 4.898580642921171, + "tokens_seen": 86507520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868886659979941, + "loss": 2.0886, + "theoretical_loss": 4.898058819912093, + "tokens_seen": 86573056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868726178535607, + "loss": 2.0689, + "theoretical_loss": 4.897537502285347, + "tokens_seen": 86638592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868565697091274, + "loss": 2.1105, + "theoretical_loss": 4.897016689169741, + "tokens_seen": 86704128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868405215646941, + "loss": 2.1364, + "theoretical_loss": 4.896496379696243, + "tokens_seen": 86769664 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868244734202608, + "loss": 2.1053, + "theoretical_loss": 4.895976572997973, + "tokens_seen": 86835200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007868084252758275, + "loss": 2.2204, + "theoretical_loss": 4.895457268210196, + "tokens_seen": 86900736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867923771313942, + "loss": 2.0587, + "theoretical_loss": 4.8949384644703136, + "tokens_seen": 86966272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867763289869609, + "loss": 2.1282, + "theoretical_loss": 4.894420160917861, + "tokens_seen": 87031808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867602808425276, + "loss": 2.0678, + "theoretical_loss": 4.893902356694495, + "tokens_seen": 87097344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867442326980943, + "loss": 2.1591, + "theoretical_loss": 4.8933850509439925, + "tokens_seen": 87162880 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786728184553661, + "loss": 2.0862, + "theoretical_loss": 4.89286824281224, + "tokens_seen": 87228416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007867121364092277, + "loss": 2.2062, + "theoretical_loss": 4.892351931447224, + "tokens_seen": 87293952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866960882647944, + "loss": 2.0411, + "theoretical_loss": 4.891836115999034, + "tokens_seen": 87359488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866800401203611, + "loss": 2.1868, + "theoretical_loss": 4.891320795619847, + "tokens_seen": 87425024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866639919759278, + "loss": 2.0745, + "theoretical_loss": 4.890805969463923, + "tokens_seen": 87490560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866479438314945, + "loss": 2.0923, + "theoretical_loss": 4.8902916366876, + "tokens_seen": 87556096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866318956870612, + "loss": 2.0783, + "theoretical_loss": 4.889777796449285, + "tokens_seen": 87621632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007866158475426279, + "loss": 2.037, + "theoretical_loss": 4.889264447909451, + "tokens_seen": 87687168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865997993981946, + "loss": 2.1086, + "theoretical_loss": 4.888751590230626, + "tokens_seen": 87752704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865837512537613, + "loss": 2.154, + "theoretical_loss": 4.888239222577389, + "tokens_seen": 87818240 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786567703109328, + "loss": 2.1473, + "theoretical_loss": 4.887727344116364, + "tokens_seen": 87883776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865516549648947, + "loss": 2.2003, + "theoretical_loss": 4.8872159540162095, + "tokens_seen": 87949312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865356068204614, + "loss": 2.0546, + "theoretical_loss": 4.88670505144762, + "tokens_seen": 88014848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865195586760281, + "loss": 2.1056, + "theoretical_loss": 4.8861946355833075, + "tokens_seen": 88080384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007865035105315948, + "loss": 2.1079, + "theoretical_loss": 4.885684705598009, + "tokens_seen": 88145920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864874623871615, + "loss": 2.0428, + "theoretical_loss": 4.885175260668468, + "tokens_seen": 88211456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864714142427282, + "loss": 2.1482, + "theoretical_loss": 4.884666299973436, + "tokens_seen": 88276992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864553660982949, + "loss": 1.985, + "theoretical_loss": 4.884157822693663, + "tokens_seen": 88342528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864393179538616, + "loss": 2.1221, + "theoretical_loss": 4.88364982801189, + "tokens_seen": 88408064 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007864232698094283, + "loss": 2.1343, + "theoretical_loss": 4.8831423151128455, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786407221664995, + "loss": 1.9906, + "theoretical_loss": 4.8826352831832365, + "tokens_seen": 88539136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863911735205617, + "loss": 2.0461, + "theoretical_loss": 4.882128731411745, + "tokens_seen": 88604672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863751253761284, + "loss": 1.9914, + "theoretical_loss": 4.881622658989022, + "tokens_seen": 88670208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863590772316951, + "loss": 2.0667, + "theoretical_loss": 4.881117065107675, + "tokens_seen": 88735744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863430290872618, + "loss": 2.055, + "theoretical_loss": 4.880611948962271, + "tokens_seen": 88801280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863269809428285, + "loss": 2.2267, + "theoretical_loss": 4.880107309749322, + "tokens_seen": 88866816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007863109327983952, + "loss": 2.1405, + "theoretical_loss": 4.879603146667284, + "tokens_seen": 88932352 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786294884653962, + "loss": 2.175, + "theoretical_loss": 4.879099458916553, + "tokens_seen": 88997888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007862788365095286, + "loss": 2.1198, + "theoretical_loss": 4.878596245699449, + "tokens_seen": 89063424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007862627883650953, + "loss": 2.1215, + "theoretical_loss": 4.878093506220221, + "tokens_seen": 89128960 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786246740220662, + "loss": 2.0613, + "theoretical_loss": 4.877591239685035, + "tokens_seen": 89194496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007862306920762287, + "loss": 2.0941, + "theoretical_loss": 4.877089445301969, + "tokens_seen": 89260032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007862146439317954, + "loss": 2.0956, + "theoretical_loss": 4.876588122281007, + "tokens_seen": 89325568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861985957873622, + "loss": 2.0404, + "theoretical_loss": 4.876087269834034, + "tokens_seen": 89391104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861825476429288, + "loss": 2.195, + "theoretical_loss": 4.875586887174828, + "tokens_seen": 89456640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861664994984956, + "loss": 2.0452, + "theoretical_loss": 4.875086973519058, + "tokens_seen": 89522176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861504513540622, + "loss": 2.0918, + "theoretical_loss": 4.874587528084271, + "tokens_seen": 89587712 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786134403209629, + "loss": 2.0711, + "theoretical_loss": 4.874088550089896, + "tokens_seen": 89653248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861183550651956, + "loss": 2.018, + "theoretical_loss": 4.873590038757229, + "tokens_seen": 89718784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007861023069207624, + "loss": 1.9862, + "theoretical_loss": 4.87309199330943, + "tokens_seen": 89784320 + }, + { + "epoch": 0.03, + "learning_rate": 0.000786086258776329, + "loss": 2.1764, + "theoretical_loss": 4.872594412971521, + "tokens_seen": 89849856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860702106318958, + "loss": 2.0663, + "theoretical_loss": 4.872097296970376, + "tokens_seen": 89915392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860541624874624, + "loss": 1.9736, + "theoretical_loss": 4.871600644534717, + "tokens_seen": 89980928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860381143430292, + "loss": 2.2087, + "theoretical_loss": 4.871104454895106, + "tokens_seen": 90046464 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860220661985958, + "loss": 2.1285, + "theoretical_loss": 4.870608727283942, + "tokens_seen": 90112000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007860060180541626, + "loss": 2.2043, + "theoretical_loss": 4.870113460935455, + "tokens_seen": 90177536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007859899699097292, + "loss": 2.1125, + "theoretical_loss": 4.869618655085697, + "tokens_seen": 90243072 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785973921765296, + "loss": 2.1254, + "theoretical_loss": 4.869124308972543, + "tokens_seen": 90308608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007859578736208626, + "loss": 2.1154, + "theoretical_loss": 4.868630421835677, + "tokens_seen": 90374144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007859418254764294, + "loss": 2.0145, + "theoretical_loss": 4.868136992916593, + "tokens_seen": 90439680 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785925777331996, + "loss": 2.2243, + "theoretical_loss": 4.867644021458585, + "tokens_seen": 90505216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007859097291875628, + "loss": 2.0343, + "theoretical_loss": 4.8671515067067475, + "tokens_seen": 90570752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858936810431294, + "loss": 2.0246, + "theoretical_loss": 4.866659447907962, + "tokens_seen": 90636288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858776328986962, + "loss": 2.0421, + "theoretical_loss": 4.866167844310896, + "tokens_seen": 90701824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858615847542628, + "loss": 2.0609, + "theoretical_loss": 4.865676695165996, + "tokens_seen": 90767360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858455366098296, + "loss": 2.0328, + "theoretical_loss": 4.865185999725487, + "tokens_seen": 90832896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007858294884653962, + "loss": 2.0952, + "theoretical_loss": 4.864695757243359, + "tokens_seen": 90898432 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785813440320963, + "loss": 2.0305, + "theoretical_loss": 4.864205966975364, + "tokens_seen": 90963968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857973921765296, + "loss": 2.1004, + "theoretical_loss": 4.8637166281790165, + "tokens_seen": 91029504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857813440320964, + "loss": 2.016, + "theoretical_loss": 4.86322774011358, + "tokens_seen": 91095040 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785765295887663, + "loss": 2.1677, + "theoretical_loss": 4.862739302040067, + "tokens_seen": 91160576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857492477432298, + "loss": 2.1866, + "theoretical_loss": 4.862251313221231, + "tokens_seen": 91226112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857331995987964, + "loss": 2.0291, + "theoretical_loss": 4.861763772921561, + "tokens_seen": 91291648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857171514543632, + "loss": 2.1311, + "theoretical_loss": 4.861276680407279, + "tokens_seen": 91357184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007857011033099298, + "loss": 2.0395, + "theoretical_loss": 4.860790034946331, + "tokens_seen": 91422720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007856850551654966, + "loss": 2.1172, + "theoretical_loss": 4.860303835808384, + "tokens_seen": 91488256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007856690070210632, + "loss": 2.0382, + "theoretical_loss": 4.859818082264822, + "tokens_seen": 91553792 + }, + { + "epoch": 0.03, + "learning_rate": 0.00078565295887663, + "loss": 2.1008, + "theoretical_loss": 4.859332773588736, + "tokens_seen": 91619328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007856369107321966, + "loss": 2.0718, + "theoretical_loss": 4.858847909054923, + "tokens_seen": 91684864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 41581, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 2.044309139251709, + "objective/train/theoretical_loss": 4.858363487939879, + "objective/train/tokens_used": 112210400, + "theoretical_loss": 4.858363487939879, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007856208625877634, + "loss": 2.0443, + "theoretical_loss": 4.858363487939879, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.00078560481444333, + "loss": 2.0722, + "theoretical_loss": 4.857879509521796, + "tokens_seen": 91815936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855887662988968, + "loss": 1.9915, + "theoretical_loss": 4.857395973080552, + "tokens_seen": 91881472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855727181544634, + "loss": 2.1225, + "theoretical_loss": 4.856912877897713, + "tokens_seen": 91947008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855566700100302, + "loss": 2.079, + "theoretical_loss": 4.85643022325652, + "tokens_seen": 92012544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855406218655968, + "loss": 2.1277, + "theoretical_loss": 4.855948008441891, + "tokens_seen": 92078080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855245737211636, + "loss": 2.0752, + "theoretical_loss": 4.85546623274041, + "tokens_seen": 92143616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007855085255767302, + "loss": 1.9623, + "theoretical_loss": 4.854984895440325, + "tokens_seen": 92209152 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785492477432297, + "loss": 2.0694, + "theoretical_loss": 4.854503995831545, + "tokens_seen": 92274688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007854764292878636, + "loss": 2.137, + "theoretical_loss": 4.854023533205627, + "tokens_seen": 92340224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007854603811434303, + "loss": 1.9219, + "theoretical_loss": 4.853543506855784, + "tokens_seen": 92405760 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785444332998997, + "loss": 2.0269, + "theoretical_loss": 4.853063916076865, + "tokens_seen": 92471296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007854282848545637, + "loss": 2.0388, + "theoretical_loss": 4.852584760165362, + "tokens_seen": 92536832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007854122367101304, + "loss": 2.0979, + "theoretical_loss": 4.8521060384194, + "tokens_seen": 92602368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853961885656971, + "loss": 2.0948, + "theoretical_loss": 4.851627750138731, + "tokens_seen": 92667904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853801404212638, + "loss": 2.0571, + "theoretical_loss": 4.851149894624731, + "tokens_seen": 92733440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853640922768305, + "loss": 2.0777, + "theoretical_loss": 4.8506724711803955, + "tokens_seen": 92798976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853480441323972, + "loss": 2.0432, + "theoretical_loss": 4.850195479110333, + "tokens_seen": 92864512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853319959879639, + "loss": 2.0506, + "theoretical_loss": 4.849718917720762, + "tokens_seen": 92930048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007853159478435306, + "loss": 2.1005, + "theoretical_loss": 4.849242786319503, + "tokens_seen": 92995584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852998996990973, + "loss": 2.1595, + "theoretical_loss": 4.848767084215979, + "tokens_seen": 93061120 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785283851554664, + "loss": 2.0331, + "theoretical_loss": 4.848291810721204, + "tokens_seen": 93126656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852678034102307, + "loss": 1.9728, + "theoretical_loss": 4.847816965147783, + "tokens_seen": 93192192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852517552657974, + "loss": 2.0968, + "theoretical_loss": 4.847342546809907, + "tokens_seen": 93257728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852357071213641, + "loss": 2.0359, + "theoretical_loss": 4.846868555023345, + "tokens_seen": 93323264 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852196589769308, + "loss": 2.0387, + "theoretical_loss": 4.846394989105444, + "tokens_seen": 93388800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007852036108324975, + "loss": 2.1412, + "theoretical_loss": 4.845921848375118, + "tokens_seen": 93454336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851875626880642, + "loss": 2.0493, + "theoretical_loss": 4.845449132152851, + "tokens_seen": 93519872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851715145436309, + "loss": 2.1466, + "theoretical_loss": 4.844976839760683, + "tokens_seen": 93585408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851554663991976, + "loss": 2.0749, + "theoretical_loss": 4.844504970522217, + "tokens_seen": 93650944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851394182547643, + "loss": 2.1461, + "theoretical_loss": 4.844033523762603, + "tokens_seen": 93716480 + }, + { + "epoch": 0.03, + "learning_rate": 0.000785123370110331, + "loss": 2.0977, + "theoretical_loss": 4.843562498808538, + "tokens_seen": 93782016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007851073219658977, + "loss": 1.9856, + "theoretical_loss": 4.843091894988265, + "tokens_seen": 93847552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850912738214644, + "loss": 2.1281, + "theoretical_loss": 4.842621711631564, + "tokens_seen": 93913088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850752256770311, + "loss": 1.9901, + "theoretical_loss": 4.842151948069745, + "tokens_seen": 93978624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850591775325978, + "loss": 2.1158, + "theoretical_loss": 4.84168260363565, + "tokens_seen": 94044160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850431293881645, + "loss": 1.9949, + "theoretical_loss": 4.841213677663648, + "tokens_seen": 94109696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850270812437312, + "loss": 2.1107, + "theoretical_loss": 4.840745169489621, + "tokens_seen": 94175232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007850110330992979, + "loss": 2.0583, + "theoretical_loss": 4.840277078450969, + "tokens_seen": 94240768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849949849548646, + "loss": 1.8914, + "theoretical_loss": 4.839809403886606, + "tokens_seen": 94306304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849789368104313, + "loss": 2.0857, + "theoretical_loss": 4.839342145136947, + "tokens_seen": 94371840 + }, + { + "epoch": 0.03, + "learning_rate": 0.000784962888665998, + "loss": 1.991, + "theoretical_loss": 4.838875301543913, + "tokens_seen": 94437376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849468405215647, + "loss": 2.0393, + "theoretical_loss": 4.838408872450919, + "tokens_seen": 94502912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849307923771314, + "loss": 1.9876, + "theoretical_loss": 4.837942857202876, + "tokens_seen": 94568448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007849147442326981, + "loss": 2.1477, + "theoretical_loss": 4.837477255146178, + "tokens_seen": 94633984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848986960882648, + "loss": 2.0716, + "theoretical_loss": 4.837012065628711, + "tokens_seen": 94699520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848826479438315, + "loss": 2.0637, + "theoretical_loss": 4.836547287999833, + "tokens_seen": 94765056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848665997993982, + "loss": 2.0833, + "theoretical_loss": 4.836082921610381, + "tokens_seen": 94830592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848505516549649, + "loss": 2.0517, + "theoretical_loss": 4.835618965812664, + "tokens_seen": 94896128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848345035105317, + "loss": 2.0187, + "theoretical_loss": 4.835155419960453, + "tokens_seen": 94961664 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007848184553660983, + "loss": 1.9685, + "theoretical_loss": 4.8346922834089865, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.000784802407221665, + "loss": 2.0483, + "theoretical_loss": 4.834229555514956, + "tokens_seen": 95092736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847863590772317, + "loss": 2.021, + "theoretical_loss": 4.833767235636511, + "tokens_seen": 95158272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847703109327985, + "loss": 1.9857, + "theoretical_loss": 4.833305323133246, + "tokens_seen": 95223808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847542627883651, + "loss": 2.1295, + "theoretical_loss": 4.832843817366204, + "tokens_seen": 95289344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847382146439319, + "loss": 2.0209, + "theoretical_loss": 4.832382717697866, + "tokens_seen": 95354880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847221664994985, + "loss": 2.1253, + "theoretical_loss": 4.831922023492153, + "tokens_seen": 95420416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007847061183550653, + "loss": 2.1372, + "theoretical_loss": 4.831461734114415, + "tokens_seen": 95485952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846900702106319, + "loss": 1.971, + "theoretical_loss": 4.831001848931429, + "tokens_seen": 95551488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846740220661987, + "loss": 1.9966, + "theoretical_loss": 4.830542367311402, + "tokens_seen": 95617024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846579739217653, + "loss": 2.0957, + "theoretical_loss": 4.830083288623955, + "tokens_seen": 95682560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846419257773321, + "loss": 2.0049, + "theoretical_loss": 4.829624612240126, + "tokens_seen": 95748096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846258776328987, + "loss": 2.0561, + "theoretical_loss": 4.829166337532367, + "tokens_seen": 95813632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007846098294884655, + "loss": 1.9718, + "theoretical_loss": 4.828708463874532, + "tokens_seen": 95879168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845937813440321, + "loss": 2.0241, + "theoretical_loss": 4.828250990641886, + "tokens_seen": 95944704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845777331995989, + "loss": 2.1065, + "theoretical_loss": 4.827793917211083, + "tokens_seen": 96010240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845616850551655, + "loss": 2.0386, + "theoretical_loss": 4.827337242960181, + "tokens_seen": 96075776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845456369107323, + "loss": 1.9958, + "theoretical_loss": 4.8268809672686235, + "tokens_seen": 96141312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845295887662989, + "loss": 2.136, + "theoretical_loss": 4.826425089517245, + "tokens_seen": 96206848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007845135406218657, + "loss": 1.9563, + "theoretical_loss": 4.825969609088258, + "tokens_seen": 96272384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844974924774323, + "loss": 1.9684, + "theoretical_loss": 4.825514525365258, + "tokens_seen": 96337920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844814443329991, + "loss": 2.0392, + "theoretical_loss": 4.825059837733214, + "tokens_seen": 96403456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844653961885657, + "loss": 2.0169, + "theoretical_loss": 4.824605545578463, + "tokens_seen": 96468992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844493480441325, + "loss": 1.9456, + "theoretical_loss": 4.824151648288714, + "tokens_seen": 96534528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844332998996991, + "loss": 2.1419, + "theoretical_loss": 4.823698145253035, + "tokens_seen": 96600064 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844172517552659, + "loss": 2.0515, + "theoretical_loss": 4.823245035861852, + "tokens_seen": 96665600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007844012036108325, + "loss": 2.0123, + "theoretical_loss": 4.822792319506952, + "tokens_seen": 96731136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843851554663993, + "loss": 2.0603, + "theoretical_loss": 4.822339995581466, + "tokens_seen": 96796672 + }, + { + "epoch": 0.03, + "learning_rate": 0.000784369107321966, + "loss": 2.0218, + "theoretical_loss": 4.821888063479874, + "tokens_seen": 96862208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843530591775327, + "loss": 2.084, + "theoretical_loss": 4.8214365225980025, + "tokens_seen": 96927744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843370110330994, + "loss": 1.9555, + "theoretical_loss": 4.8209853723330145, + "tokens_seen": 96993280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843209628886661, + "loss": 2.02, + "theoretical_loss": 4.820534612083408, + "tokens_seen": 97058816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007843049147442328, + "loss": 2.1019, + "theoretical_loss": 4.820084241249013, + "tokens_seen": 97124352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842888665997995, + "loss": 2.0529, + "theoretical_loss": 4.819634259230991, + "tokens_seen": 97189888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842728184553662, + "loss": 1.9951, + "theoretical_loss": 4.819184665431821, + "tokens_seen": 97255424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842567703109329, + "loss": 2.0711, + "theoretical_loss": 4.818735459255306, + "tokens_seen": 97320960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842407221664996, + "loss": 1.9041, + "theoretical_loss": 4.818286640106564, + "tokens_seen": 97386496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842246740220663, + "loss": 2.0159, + "theoretical_loss": 4.817838207392027, + "tokens_seen": 97452032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007842086258776329, + "loss": 2.109, + "theoretical_loss": 4.817390160519435, + "tokens_seen": 97517568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841925777331997, + "loss": 2.0206, + "theoretical_loss": 4.816942498897831, + "tokens_seen": 97583104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841765295887663, + "loss": 1.9728, + "theoretical_loss": 4.816495221937563, + "tokens_seen": 97648640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841604814443331, + "loss": 1.9432, + "theoretical_loss": 4.816048329050273, + "tokens_seen": 97714176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841444332998997, + "loss": 2.0888, + "theoretical_loss": 4.815601819648897, + "tokens_seen": 97779712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841283851554665, + "loss": 2.1234, + "theoretical_loss": 4.815155693147666, + "tokens_seen": 97845248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007841123370110331, + "loss": 2.1408, + "theoretical_loss": 4.81470994896209, + "tokens_seen": 97910784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840962888665999, + "loss": 2.0632, + "theoretical_loss": 4.8142645865089655, + "tokens_seen": 97976320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840802407221665, + "loss": 1.9535, + "theoretical_loss": 4.813819605206369, + "tokens_seen": 98041856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840641925777332, + "loss": 2.0469, + "theoretical_loss": 4.813375004473651, + "tokens_seen": 98107392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840481444332999, + "loss": 2.0781, + "theoretical_loss": 4.812930783731433, + "tokens_seen": 98172928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840320962888666, + "loss": 2.0, + "theoretical_loss": 4.8124869424016055, + "tokens_seen": 98238464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 43774, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 2.0564637184143066, + "objective/train/theoretical_loss": 4.812043479907323, + "objective/train/tokens_used": 118764000, + "theoretical_loss": 4.812043479907323, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007840160481444333, + "loss": 2.0565, + "theoretical_loss": 4.812043479907323, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.000784, + "loss": 2.0398, + "theoretical_loss": 4.8116003956730005, + "tokens_seen": 98369536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839839518555667, + "loss": 2.028, + "theoretical_loss": 4.811157689124311, + "tokens_seen": 98435072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839679037111334, + "loss": 2.0112, + "theoretical_loss": 4.8107153596881815, + "tokens_seen": 98500608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839518555667001, + "loss": 1.9807, + "theoretical_loss": 4.810273406792788, + "tokens_seen": 98566144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839358074222668, + "loss": 2.0177, + "theoretical_loss": 4.809831829867555, + "tokens_seen": 98631680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839197592778335, + "loss": 2.0595, + "theoretical_loss": 4.809390628343146, + "tokens_seen": 98697216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007839037111334002, + "loss": 2.0136, + "theoretical_loss": 4.808949801651468, + "tokens_seen": 98762752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838876629889669, + "loss": 2.0394, + "theoretical_loss": 4.8085093492256625, + "tokens_seen": 98828288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838716148445336, + "loss": 2.0705, + "theoretical_loss": 4.808069270500102, + "tokens_seen": 98893824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838555667001003, + "loss": 2.061, + "theoretical_loss": 4.80762956491039, + "tokens_seen": 98959360 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783839518555667, + "loss": 1.9414, + "theoretical_loss": 4.807190231893355, + "tokens_seen": 99024896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838234704112337, + "loss": 1.9911, + "theoretical_loss": 4.806751270887048, + "tokens_seen": 99090432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007838074222668004, + "loss": 2.0794, + "theoretical_loss": 4.806312681330736, + "tokens_seen": 99155968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837913741223671, + "loss": 2.0249, + "theoretical_loss": 4.805874462664903, + "tokens_seen": 99221504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837753259779338, + "loss": 2.051, + "theoretical_loss": 4.805436614331247, + "tokens_seen": 99287040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837592778335005, + "loss": 1.8966, + "theoretical_loss": 4.804999135772668, + "tokens_seen": 99352576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837432296890672, + "loss": 2.1574, + "theoretical_loss": 4.804562026433278, + "tokens_seen": 99418112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837271815446339, + "loss": 2.0045, + "theoretical_loss": 4.804125285758383, + "tokens_seen": 99483648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007837111334002006, + "loss": 2.0258, + "theoretical_loss": 4.803688913194495, + "tokens_seen": 99549184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836950852557673, + "loss": 2.1233, + "theoretical_loss": 4.8032529081893145, + "tokens_seen": 99614720 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783679037111334, + "loss": 1.9813, + "theoretical_loss": 4.802817270191736, + "tokens_seen": 99680256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836629889669007, + "loss": 1.9358, + "theoretical_loss": 4.802381998651841, + "tokens_seen": 99745792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836469408224674, + "loss": 2.0028, + "theoretical_loss": 4.801947093020896, + "tokens_seen": 99811328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836308926780341, + "loss": 2.0859, + "theoretical_loss": 4.801512552751349, + "tokens_seen": 99876864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007836148445336008, + "loss": 2.0154, + "theoretical_loss": 4.801078377296827, + "tokens_seen": 99942400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835987963891675, + "loss": 1.8705, + "theoretical_loss": 4.800644566112128, + "tokens_seen": 100007936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835827482447342, + "loss": 1.9481, + "theoretical_loss": 4.800211118653225, + "tokens_seen": 100073472 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783566700100301, + "loss": 2.0314, + "theoretical_loss": 4.799778034377258, + "tokens_seen": 100139008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835506519558676, + "loss": 1.9933, + "theoretical_loss": 4.799345312742531, + "tokens_seen": 100204544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835346038114343, + "loss": 1.993, + "theoretical_loss": 4.798912953208511, + "tokens_seen": 100270080 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783518555667001, + "loss": 2.149, + "theoretical_loss": 4.798480955235822, + "tokens_seen": 100335616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007835025075225677, + "loss": 2.0972, + "theoretical_loss": 4.798049318286243, + "tokens_seen": 100401152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834864593781344, + "loss": 1.946, + "theoretical_loss": 4.797618041822707, + "tokens_seen": 100466688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834704112337011, + "loss": 1.9969, + "theoretical_loss": 4.797187125309293, + "tokens_seen": 100532224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834543630892678, + "loss": 2.0673, + "theoretical_loss": 4.796756568211227, + "tokens_seen": 100597760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834383149448346, + "loss": 1.9801, + "theoretical_loss": 4.796326369994876, + "tokens_seen": 100663296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007834222668004012, + "loss": 1.964, + "theoretical_loss": 4.795896530127747, + "tokens_seen": 100728832 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783406218655968, + "loss": 1.99, + "theoretical_loss": 4.795467048078484, + "tokens_seen": 100794368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833901705115346, + "loss": 2.089, + "theoretical_loss": 4.79503792331686, + "tokens_seen": 100859904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833741223671014, + "loss": 2.0026, + "theoretical_loss": 4.794609155313782, + "tokens_seen": 100925440 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783358074222668, + "loss": 2.1178, + "theoretical_loss": 4.79418074354128, + "tokens_seen": 100990976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833420260782348, + "loss": 2.0955, + "theoretical_loss": 4.79375268747251, + "tokens_seen": 101056512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833259779338014, + "loss": 1.859, + "theoretical_loss": 4.793324986581746, + "tokens_seen": 101122048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007833099297893682, + "loss": 2.0182, + "theoretical_loss": 4.79289764034438, + "tokens_seen": 101187584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832938816449348, + "loss": 2.0961, + "theoretical_loss": 4.792470648236918, + "tokens_seen": 101253120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832778335005016, + "loss": 2.0372, + "theoretical_loss": 4.792044009736978, + "tokens_seen": 101318656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832617853560682, + "loss": 1.8907, + "theoretical_loss": 4.791617724323283, + "tokens_seen": 101384192 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783245737211635, + "loss": 2.1526, + "theoretical_loss": 4.791191791475663, + "tokens_seen": 101449728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832296890672016, + "loss": 2.0196, + "theoretical_loss": 4.79076621067505, + "tokens_seen": 101515264 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007832136409227684, + "loss": 1.9194, + "theoretical_loss": 4.790340981403474, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783197592778335, + "loss": 2.0683, + "theoretical_loss": 4.7899161031440585, + "tokens_seen": 101646336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831815446339018, + "loss": 2.0078, + "theoretical_loss": 4.7894915753810245, + "tokens_seen": 101711872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831654964894685, + "loss": 2.0629, + "theoretical_loss": 4.789067397599677, + "tokens_seen": 101777408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831494483450352, + "loss": 2.1398, + "theoretical_loss": 4.788643569286411, + "tokens_seen": 101842944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831334002006019, + "loss": 1.9818, + "theoretical_loss": 4.788220089928705, + "tokens_seen": 101908480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831173520561686, + "loss": 1.9772, + "theoretical_loss": 4.787796959015118, + "tokens_seen": 101974016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007831013039117353, + "loss": 2.0421, + "theoretical_loss": 4.787374176035287, + "tokens_seen": 102039552 + }, + { + "epoch": 0.03, + "learning_rate": 0.000783085255767302, + "loss": 2.0502, + "theoretical_loss": 4.78695174047992, + "tokens_seen": 102105088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830692076228687, + "loss": 2.0433, + "theoretical_loss": 4.786529651840802, + "tokens_seen": 102170624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830531594784354, + "loss": 1.8912, + "theoretical_loss": 4.786107909610784, + "tokens_seen": 102236160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830371113340021, + "loss": 2.0267, + "theoretical_loss": 4.785686513283783, + "tokens_seen": 102301696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830210631895688, + "loss": 2.0968, + "theoretical_loss": 4.785265462354779, + "tokens_seen": 102367232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007830050150451355, + "loss": 2.0695, + "theoretical_loss": 4.7848447563198135, + "tokens_seen": 102432768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829889669007022, + "loss": 1.9965, + "theoretical_loss": 4.784424394675983, + "tokens_seen": 102498304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829729187562689, + "loss": 2.0899, + "theoretical_loss": 4.784004376921439, + "tokens_seen": 102563840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829568706118356, + "loss": 1.9928, + "theoretical_loss": 4.783584702555387, + "tokens_seen": 102629376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829408224674023, + "loss": 2.0884, + "theoretical_loss": 4.783165371078075, + "tokens_seen": 102694912 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782924774322969, + "loss": 2.0653, + "theoretical_loss": 4.782746381990803, + "tokens_seen": 102760448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007829087261785357, + "loss": 2.0589, + "theoretical_loss": 4.78232773479591, + "tokens_seen": 102825984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828926780341024, + "loss": 2.0893, + "theoretical_loss": 4.7819094289967765, + "tokens_seen": 102891520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828766298896691, + "loss": 2.0153, + "theoretical_loss": 4.781491464097819, + "tokens_seen": 102957056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828605817452358, + "loss": 2.0343, + "theoretical_loss": 4.781073839604488, + "tokens_seen": 103022592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828445336008025, + "loss": 1.9041, + "theoretical_loss": 4.780656555023268, + "tokens_seen": 103088128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828284854563692, + "loss": 2.0899, + "theoretical_loss": 4.780239609861669, + "tokens_seen": 103153664 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007828124373119358, + "loss": 2.0505, + "theoretical_loss": 4.77982300362823, + "tokens_seen": 103219200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827963891675026, + "loss": 1.9479, + "theoretical_loss": 4.779406735832509, + "tokens_seen": 103284736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827803410230692, + "loss": 2.0336, + "theoretical_loss": 4.778990805985089, + "tokens_seen": 103350272 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782764292878636, + "loss": 1.9833, + "theoretical_loss": 4.778575213597568, + "tokens_seen": 103415808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827482447342026, + "loss": 1.949, + "theoretical_loss": 4.778159958182557, + "tokens_seen": 103481344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827321965897694, + "loss": 1.9087, + "theoretical_loss": 4.777745039253682, + "tokens_seen": 103546880 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782716148445336, + "loss": 2.1781, + "theoretical_loss": 4.777330456325579, + "tokens_seen": 103612416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007827001003009028, + "loss": 1.9407, + "theoretical_loss": 4.776916208913887, + "tokens_seen": 103677952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826840521564694, + "loss": 1.9856, + "theoretical_loss": 4.776502296535252, + "tokens_seen": 103743488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826680040120361, + "loss": 2.1527, + "theoretical_loss": 4.776088718707317, + "tokens_seen": 103809024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826519558676028, + "loss": 1.965, + "theoretical_loss": 4.77567547494873, + "tokens_seen": 103874560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826359077231695, + "loss": 1.9659, + "theoretical_loss": 4.775262564779128, + "tokens_seen": 103940096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826198595787362, + "loss": 1.9975, + "theoretical_loss": 4.774849987719145, + "tokens_seen": 104005632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007826038114343029, + "loss": 1.8441, + "theoretical_loss": 4.774437743290404, + "tokens_seen": 104071168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825877632898696, + "loss": 1.9205, + "theoretical_loss": 4.774025831015517, + "tokens_seen": 104136704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825717151454363, + "loss": 2.0538, + "theoretical_loss": 4.7736142504180785, + "tokens_seen": 104202240 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782555667001003, + "loss": 2.0038, + "theoretical_loss": 4.773203001022668, + "tokens_seen": 104267776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825396188565697, + "loss": 2.0319, + "theoretical_loss": 4.772792082354843, + "tokens_seen": 104333312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825235707121364, + "loss": 1.9619, + "theoretical_loss": 4.77238149394114, + "tokens_seen": 104398848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007825075225677031, + "loss": 2.1084, + "theoretical_loss": 4.771971235309065, + "tokens_seen": 104464384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824914744232698, + "loss": 1.9957, + "theoretical_loss": 4.771561305987104, + "tokens_seen": 104529920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824754262788365, + "loss": 1.953, + "theoretical_loss": 4.771151705504707, + "tokens_seen": 104595456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824593781344032, + "loss": 2.0034, + "theoretical_loss": 4.770742433392289, + "tokens_seen": 104660992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824433299899699, + "loss": 2.0302, + "theoretical_loss": 4.770333489181233, + "tokens_seen": 104726528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824272818455366, + "loss": 2.0035, + "theoretical_loss": 4.769924872403883, + "tokens_seen": 104792064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 46458, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.9015421867370605, + "objective/train/theoretical_loss": 4.769516582593539, + "objective/train/tokens_used": 125317600, + "theoretical_loss": 4.769516582593539, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007824112337011033, + "loss": 1.9015, + "theoretical_loss": 4.769516582593539, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.00078239518555667, + "loss": 1.8893, + "theoretical_loss": 4.769108619284461, + "tokens_seen": 104923136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823791374122367, + "loss": 1.978, + "theoretical_loss": 4.768700982011859, + "tokens_seen": 104988672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823630892678034, + "loss": 1.8873, + "theoretical_loss": 4.768293670311899, + "tokens_seen": 105054208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823470411233701, + "loss": 2.0086, + "theoretical_loss": 4.7678866837216916, + "tokens_seen": 105119744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823309929789368, + "loss": 1.969, + "theoretical_loss": 4.767480021779294, + "tokens_seen": 105185280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007823149448345035, + "loss": 2.1185, + "theoretical_loss": 4.76707368402371, + "tokens_seen": 105250816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822988966900702, + "loss": 2.0132, + "theoretical_loss": 4.766667669994881, + "tokens_seen": 105316352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822828485456369, + "loss": 2.0492, + "theoretical_loss": 4.76626197923369, + "tokens_seen": 105381888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822668004012036, + "loss": 2.0807, + "theoretical_loss": 4.765856611281953, + "tokens_seen": 105447424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822507522567703, + "loss": 1.9707, + "theoretical_loss": 4.765451565682423, + "tokens_seen": 105512960 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782234704112337, + "loss": 1.9635, + "theoretical_loss": 4.765046841978783, + "tokens_seen": 105578496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822186559679037, + "loss": 2.0984, + "theoretical_loss": 4.764642439715642, + "tokens_seen": 105644032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007822026078234704, + "loss": 2.1008, + "theoretical_loss": 4.764238358438539, + "tokens_seen": 105709568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821865596790371, + "loss": 1.9651, + "theoretical_loss": 4.763834597693935, + "tokens_seen": 105775104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821705115346038, + "loss": 1.9968, + "theoretical_loss": 4.763431157029213, + "tokens_seen": 105840640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821544633901705, + "loss": 1.9752, + "theoretical_loss": 4.763028035992675, + "tokens_seen": 105906176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821384152457372, + "loss": 2.1189, + "theoretical_loss": 4.762625234133537, + "tokens_seen": 105971712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821223671013039, + "loss": 2.0155, + "theoretical_loss": 4.762222751001933, + "tokens_seen": 106037248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007821063189568706, + "loss": 1.9496, + "theoretical_loss": 4.761820586148906, + "tokens_seen": 106102784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820902708124373, + "loss": 1.951, + "theoretical_loss": 4.761418739126407, + "tokens_seen": 106168320 + }, + { + "epoch": 0.03, + "learning_rate": 0.000782074222668004, + "loss": 1.8936, + "theoretical_loss": 4.761017209487298, + "tokens_seen": 106233856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820581745235707, + "loss": 1.8837, + "theoretical_loss": 4.760615996785342, + "tokens_seen": 106299392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820421263791375, + "loss": 2.0782, + "theoretical_loss": 4.760215100575204, + "tokens_seen": 106364928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820260782347041, + "loss": 1.9477, + "theoretical_loss": 4.759814520412451, + "tokens_seen": 106430464 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007820100300902709, + "loss": 2.0783, + "theoretical_loss": 4.759414255853547, + "tokens_seen": 106496000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819939819458375, + "loss": 2.0302, + "theoretical_loss": 4.759014306455848, + "tokens_seen": 106561536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819779338014043, + "loss": 1.9217, + "theoretical_loss": 4.758614671777604, + "tokens_seen": 106627072 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781961885656971, + "loss": 2.0256, + "theoretical_loss": 4.758215351377956, + "tokens_seen": 106692608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819458375125377, + "loss": 2.076, + "theoretical_loss": 4.757816344816933, + "tokens_seen": 106758144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819297893681044, + "loss": 1.9322, + "theoretical_loss": 4.75741765165545, + "tokens_seen": 106823680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007819137412236711, + "loss": 1.942, + "theoretical_loss": 4.7570192714553015, + "tokens_seen": 106889216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818976930792378, + "loss": 1.8512, + "theoretical_loss": 4.756621203779169, + "tokens_seen": 106954752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818816449348045, + "loss": 2.0647, + "theoretical_loss": 4.756223448190605, + "tokens_seen": 107020288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818655967903712, + "loss": 1.9946, + "theoretical_loss": 4.755826004254046, + "tokens_seen": 107085824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818495486459379, + "loss": 1.9706, + "theoretical_loss": 4.755428871534798, + "tokens_seen": 107151360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818335005015046, + "loss": 1.98, + "theoretical_loss": 4.755032049599039, + "tokens_seen": 107216896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007818174523570713, + "loss": 1.994, + "theoretical_loss": 4.754635538013817, + "tokens_seen": 107282432 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781801404212638, + "loss": 1.8982, + "theoretical_loss": 4.754239336347048, + "tokens_seen": 107347968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817853560682047, + "loss": 2.0406, + "theoretical_loss": 4.753843444167513, + "tokens_seen": 107413504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817693079237714, + "loss": 1.9653, + "theoretical_loss": 4.753447861044853, + "tokens_seen": 107479040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817532597793381, + "loss": 1.8986, + "theoretical_loss": 4.753052586549572, + "tokens_seen": 107544576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817372116349048, + "loss": 2.018, + "theoretical_loss": 4.75265762025303, + "tokens_seen": 107610112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817211634904715, + "loss": 2.1007, + "theoretical_loss": 4.752262961727446, + "tokens_seen": 107675648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007817051153460382, + "loss": 1.9031, + "theoretical_loss": 4.75186861054589, + "tokens_seen": 107741184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816890672016049, + "loss": 1.9357, + "theoretical_loss": 4.751474566282285, + "tokens_seen": 107806720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816730190571716, + "loss": 1.9696, + "theoretical_loss": 4.751080828511402, + "tokens_seen": 107872256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816569709127383, + "loss": 1.9687, + "theoretical_loss": 4.750687396808859, + "tokens_seen": 107937792 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781640922768305, + "loss": 2.0962, + "theoretical_loss": 4.7502942707511195, + "tokens_seen": 108003328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816248746238717, + "loss": 2.0112, + "theoretical_loss": 4.749901449915489, + "tokens_seen": 108068864 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007816088264794384, + "loss": 1.9628, + "theoretical_loss": 4.749508933880117, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815927783350051, + "loss": 2.0048, + "theoretical_loss": 4.7491167222239845, + "tokens_seen": 108199936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815767301905718, + "loss": 1.9553, + "theoretical_loss": 4.7487248145269145, + "tokens_seen": 108265472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815606820461385, + "loss": 1.9705, + "theoretical_loss": 4.7483332103695615, + "tokens_seen": 108331008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815446339017052, + "loss": 1.9298, + "theoretical_loss": 4.747941909333411, + "tokens_seen": 108396544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815285857572719, + "loss": 1.9251, + "theoretical_loss": 4.747550911000781, + "tokens_seen": 108462080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007815125376128386, + "loss": 1.9817, + "theoretical_loss": 4.747160214954813, + "tokens_seen": 108527616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814964894684053, + "loss": 2.0413, + "theoretical_loss": 4.746769820779477, + "tokens_seen": 108593152 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781480441323972, + "loss": 1.931, + "theoretical_loss": 4.746379728059566, + "tokens_seen": 108658688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814643931795387, + "loss": 1.9666, + "theoretical_loss": 4.745989936380692, + "tokens_seen": 108724224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814483450351054, + "loss": 1.9137, + "theoretical_loss": 4.745600445329288, + "tokens_seen": 108789760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814322968906721, + "loss": 2.0553, + "theoretical_loss": 4.745211254492602, + "tokens_seen": 108855296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814162487462388, + "loss": 2.0251, + "theoretical_loss": 4.744822363458699, + "tokens_seen": 108920832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007814002006018055, + "loss": 1.8646, + "theoretical_loss": 4.744433771816455, + "tokens_seen": 108986368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813841524573721, + "loss": 1.913, + "theoretical_loss": 4.744045479155557, + "tokens_seen": 109051904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813681043129389, + "loss": 1.8624, + "theoretical_loss": 4.7436574850665005, + "tokens_seen": 109117440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813520561685055, + "loss": 2.0835, + "theoretical_loss": 4.743269789140587, + "tokens_seen": 109182976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813360080240723, + "loss": 1.8438, + "theoretical_loss": 4.742882390969925, + "tokens_seen": 109248512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813199598796389, + "loss": 2.0004, + "theoretical_loss": 4.742495290147421, + "tokens_seen": 109314048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007813039117352057, + "loss": 1.9366, + "theoretical_loss": 4.742108486266783, + "tokens_seen": 109379584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812878635907723, + "loss": 2.0586, + "theoretical_loss": 4.74172197892252, + "tokens_seen": 109445120 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781271815446339, + "loss": 1.9156, + "theoretical_loss": 4.741335767709934, + "tokens_seen": 109510656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812557673019057, + "loss": 1.9783, + "theoretical_loss": 4.740949852225122, + "tokens_seen": 109576192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812397191574724, + "loss": 1.9158, + "theoretical_loss": 4.740564232064973, + "tokens_seen": 109641728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812236710130391, + "loss": 2.059, + "theoretical_loss": 4.7401789068271665, + "tokens_seen": 109707264 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007812076228686058, + "loss": 2.0115, + "theoretical_loss": 4.73979387611017, + "tokens_seen": 109772800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811915747241725, + "loss": 1.8627, + "theoretical_loss": 4.739409139513235, + "tokens_seen": 109838336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811755265797392, + "loss": 1.9969, + "theoretical_loss": 4.739024696636401, + "tokens_seen": 109903872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811594784353059, + "loss": 2.0016, + "theoretical_loss": 4.738640547080485, + "tokens_seen": 109969408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811434302908726, + "loss": 1.9306, + "theoretical_loss": 4.738256690447088, + "tokens_seen": 110034944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007811273821464393, + "loss": 1.9452, + "theoretical_loss": 4.7378731263385845, + "tokens_seen": 110100480 + }, + { + "epoch": 0.03, + "learning_rate": 0.000781111334002006, + "loss": 2.0136, + "theoretical_loss": 4.737489854358129, + "tokens_seen": 110166016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810952858575727, + "loss": 2.14, + "theoretical_loss": 4.737106874109649, + "tokens_seen": 110231552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810792377131394, + "loss": 1.9898, + "theoretical_loss": 4.736724185197844, + "tokens_seen": 110297088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810631895687061, + "loss": 1.8838, + "theoretical_loss": 4.7363417872281826, + "tokens_seen": 110362624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810471414242728, + "loss": 1.9304, + "theoretical_loss": 4.735959679806903, + "tokens_seen": 110428160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810310932798395, + "loss": 1.8655, + "theoretical_loss": 4.735577862541007, + "tokens_seen": 110493696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007810150451354062, + "loss": 1.8707, + "theoretical_loss": 4.735196335038264, + "tokens_seen": 110559232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809989969909729, + "loss": 1.905, + "theoretical_loss": 4.734815096907203, + "tokens_seen": 110624768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809829488465396, + "loss": 1.9189, + "theoretical_loss": 4.734434147757117, + "tokens_seen": 110690304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809669007021063, + "loss": 1.991, + "theoretical_loss": 4.734053487198054, + "tokens_seen": 110755840 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780950852557673, + "loss": 1.8235, + "theoretical_loss": 4.733673114840817, + "tokens_seen": 110821376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809348044132397, + "loss": 1.8794, + "theoretical_loss": 4.73329303029697, + "tokens_seen": 110886912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809187562688064, + "loss": 1.9225, + "theoretical_loss": 4.732913233178825, + "tokens_seen": 110952448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007809027081243731, + "loss": 2.0501, + "theoretical_loss": 4.732533723099444, + "tokens_seen": 111017984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808866599799398, + "loss": 1.9909, + "theoretical_loss": 4.732154499672642, + "tokens_seen": 111083520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808706118355065, + "loss": 1.9695, + "theoretical_loss": 4.731775562512977, + "tokens_seen": 111149056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808545636910732, + "loss": 2.0358, + "theoretical_loss": 4.7313969112357555, + "tokens_seen": 111214592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808385155466399, + "loss": 2.052, + "theoretical_loss": 4.731018545457024, + "tokens_seen": 111280128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808224674022066, + "loss": 1.9492, + "theoretical_loss": 4.730640464793574, + "tokens_seen": 111345664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 48676, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 2.0233168601989746, + "objective/train/theoretical_loss": 4.730262668862931, + "objective/train/tokens_used": 131871200, + "theoretical_loss": 4.730262668862931, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007808064192577733, + "loss": 2.0233, + "theoretical_loss": 4.730262668862931, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.00078079037111334, + "loss": 1.9551, + "theoretical_loss": 4.729885157283366, + "tokens_seen": 111476736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807743229689067, + "loss": 1.9509, + "theoretical_loss": 4.729507929673879, + "tokens_seen": 111542272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807582748244735, + "loss": 2.0183, + "theoretical_loss": 4.729130985654208, + "tokens_seen": 111607808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807422266800401, + "loss": 1.9095, + "theoretical_loss": 4.72875432484482, + "tokens_seen": 111673344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807261785356069, + "loss": 2.0049, + "theoretical_loss": 4.728377946866915, + "tokens_seen": 111738880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007807101303911736, + "loss": 1.9661, + "theoretical_loss": 4.72800185134242, + "tokens_seen": 111804416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806940822467403, + "loss": 1.9616, + "theoretical_loss": 4.727626037893989, + "tokens_seen": 111869952 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780678034102307, + "loss": 1.9889, + "theoretical_loss": 4.727250506145001, + "tokens_seen": 111935488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806619859578737, + "loss": 1.9736, + "theoretical_loss": 4.726875255719557, + "tokens_seen": 112001024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806459378134404, + "loss": 1.7773, + "theoretical_loss": 4.726500286242482, + "tokens_seen": 112066560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806298896690071, + "loss": 1.9239, + "theoretical_loss": 4.7261255973393155, + "tokens_seen": 112132096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007806138415245738, + "loss": 1.9951, + "theoretical_loss": 4.725751188636319, + "tokens_seen": 112197632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805977933801405, + "loss": 1.957, + "theoretical_loss": 4.725377059760468, + "tokens_seen": 112263168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805817452357072, + "loss": 1.9312, + "theoretical_loss": 4.725003210339452, + "tokens_seen": 112328704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805656970912739, + "loss": 1.9269, + "theoretical_loss": 4.724629640001672, + "tokens_seen": 112394240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805496489468406, + "loss": 1.9711, + "theoretical_loss": 4.724256348376242, + "tokens_seen": 112459776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805336008024073, + "loss": 1.9213, + "theoretical_loss": 4.723883335092982, + "tokens_seen": 112525312 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780517552657974, + "loss": 1.9006, + "theoretical_loss": 4.723510599782419, + "tokens_seen": 112590848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007805015045135407, + "loss": 1.955, + "theoretical_loss": 4.723138142075788, + "tokens_seen": 112656384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804854563691074, + "loss": 1.995, + "theoretical_loss": 4.722765961605024, + "tokens_seen": 112721920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804694082246741, + "loss": 1.8787, + "theoretical_loss": 4.722394058002765, + "tokens_seen": 112787456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804533600802408, + "loss": 1.9327, + "theoretical_loss": 4.7220224309023475, + "tokens_seen": 112852992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804373119358075, + "loss": 1.9779, + "theoretical_loss": 4.721651079937811, + "tokens_seen": 112918528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804212637913742, + "loss": 1.9749, + "theoretical_loss": 4.7212800047438845, + "tokens_seen": 112984064 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007804052156469409, + "loss": 2.0586, + "theoretical_loss": 4.720909204955996, + "tokens_seen": 113049600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803891675025076, + "loss": 2.066, + "theoretical_loss": 4.720538680210268, + "tokens_seen": 113115136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803731193580743, + "loss": 2.0196, + "theoretical_loss": 4.720168430143508, + "tokens_seen": 113180672 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780357071213641, + "loss": 1.9345, + "theoretical_loss": 4.719798454393219, + "tokens_seen": 113246208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803410230692077, + "loss": 1.973, + "theoretical_loss": 4.719428752597588, + "tokens_seen": 113311744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803249749247744, + "loss": 1.9379, + "theoretical_loss": 4.719059324395493, + "tokens_seen": 113377280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007803089267803411, + "loss": 1.9317, + "theoretical_loss": 4.71869016942649, + "tokens_seen": 113442816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802928786359078, + "loss": 2.0539, + "theoretical_loss": 4.718321287330823, + "tokens_seen": 113508352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802768304914745, + "loss": 2.0419, + "theoretical_loss": 4.7179526777494125, + "tokens_seen": 113573888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802607823470412, + "loss": 1.9361, + "theoretical_loss": 4.717584340323864, + "tokens_seen": 113639424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802447342026079, + "loss": 1.9626, + "theoretical_loss": 4.7172162746964545, + "tokens_seen": 113704960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802286860581746, + "loss": 2.1289, + "theoretical_loss": 4.716848480510142, + "tokens_seen": 113770496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007802126379137413, + "loss": 1.8971, + "theoretical_loss": 4.716480957408556, + "tokens_seen": 113836032 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780196589769308, + "loss": 1.9621, + "theoretical_loss": 4.716113705035999, + "tokens_seen": 113901568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801805416248747, + "loss": 1.9462, + "theoretical_loss": 4.715746723037448, + "tokens_seen": 113967104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801644934804414, + "loss": 1.9668, + "theoretical_loss": 4.715380011058542, + "tokens_seen": 114032640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801484453360081, + "loss": 2.0023, + "theoretical_loss": 4.715013568745597, + "tokens_seen": 114098176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801323971915748, + "loss": 1.967, + "theoretical_loss": 4.714647395745587, + "tokens_seen": 114163712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801163490471415, + "loss": 1.9684, + "theoretical_loss": 4.714281491706154, + "tokens_seen": 114229248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007801003009027082, + "loss": 2.0977, + "theoretical_loss": 4.713915856275603, + "tokens_seen": 114294784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800842527582749, + "loss": 1.8655, + "theoretical_loss": 4.713550489102901, + "tokens_seen": 114360320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800682046138416, + "loss": 1.8451, + "theoretical_loss": 4.7131853898376725, + "tokens_seen": 114425856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800521564694083, + "loss": 1.8491, + "theoretical_loss": 4.7128205581302005, + "tokens_seen": 114491392 + }, + { + "epoch": 0.03, + "learning_rate": 0.000780036108324975, + "loss": 1.8477, + "theoretical_loss": 4.712455993631425, + "tokens_seen": 114556928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800200601805417, + "loss": 2.0076, + "theoretical_loss": 4.712091695992941, + "tokens_seen": 114622464 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007800040120361084, + "loss": 1.9494, + "theoretical_loss": 4.711727664866995, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.000779987963891675, + "loss": 1.9673, + "theoretical_loss": 4.7113638999064875, + "tokens_seen": 114753536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799719157472418, + "loss": 2.0366, + "theoretical_loss": 4.711000400764967, + "tokens_seen": 114819072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799558676028084, + "loss": 2.0538, + "theoretical_loss": 4.71063716709663, + "tokens_seen": 114884608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799398194583752, + "loss": 2.0796, + "theoretical_loss": 4.7102741985563235, + "tokens_seen": 114950144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799237713139418, + "loss": 1.9856, + "theoretical_loss": 4.709911494799533, + "tokens_seen": 115015680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007799077231695086, + "loss": 1.9479, + "theoretical_loss": 4.709549055482395, + "tokens_seen": 115081216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007798916750250752, + "loss": 1.8631, + "theoretical_loss": 4.709186880261683, + "tokens_seen": 115146752 + }, + { + "epoch": 0.03, + "learning_rate": 0.000779875626880642, + "loss": 1.8449, + "theoretical_loss": 4.708824968794813, + "tokens_seen": 115212288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007798595787362086, + "loss": 1.8909, + "theoretical_loss": 4.70846332073984, + "tokens_seen": 115277824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007798435305917753, + "loss": 2.0191, + "theoretical_loss": 4.708101935755455, + "tokens_seen": 115343360 + }, + { + "epoch": 0.03, + "learning_rate": 0.000779827482447342, + "loss": 1.9506, + "theoretical_loss": 4.707740813500988, + "tokens_seen": 115408896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0007798114343029087, + "loss": 1.9725, + "theoretical_loss": 4.707379953636399, + "tokens_seen": 115474432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797953861584754, + "loss": 1.9368, + "theoretical_loss": 4.7070193558222835, + "tokens_seen": 115539968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797793380140421, + "loss": 1.8776, + "theoretical_loss": 4.7066590197198686, + "tokens_seen": 115605504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797632898696088, + "loss": 1.9702, + "theoretical_loss": 4.706298944991008, + "tokens_seen": 115671040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797472417251755, + "loss": 2.0099, + "theoretical_loss": 4.705939131298189, + "tokens_seen": 115736576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797311935807422, + "loss": 1.9403, + "theoretical_loss": 4.705579578304519, + "tokens_seen": 115802112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007797151454363089, + "loss": 2.0244, + "theoretical_loss": 4.705220285673736, + "tokens_seen": 115867648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796990972918756, + "loss": 1.9738, + "theoretical_loss": 4.704861253070198, + "tokens_seen": 115933184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796830491474423, + "loss": 1.8945, + "theoretical_loss": 4.704502480158887, + "tokens_seen": 115998720 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779667001003009, + "loss": 1.8195, + "theoretical_loss": 4.704143966605406, + "tokens_seen": 116064256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796509528585757, + "loss": 1.9645, + "theoretical_loss": 4.703785712075973, + "tokens_seen": 116129792 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796349047141424, + "loss": 1.9542, + "theoretical_loss": 4.703427716237432, + "tokens_seen": 116195328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796188565697091, + "loss": 1.9322, + "theoretical_loss": 4.703069978757233, + "tokens_seen": 116260864 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007796028084252758, + "loss": 1.82, + "theoretical_loss": 4.702712499303448, + "tokens_seen": 116326400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007795867602808425, + "loss": 1.9836, + "theoretical_loss": 4.7023552775447595, + "tokens_seen": 116391936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007795707121364092, + "loss": 1.939, + "theoretical_loss": 4.701998313150462, + "tokens_seen": 116457472 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779554663991976, + "loss": 2.086, + "theoretical_loss": 4.701641605790459, + "tokens_seen": 116523008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007795386158475426, + "loss": 1.7689, + "theoretical_loss": 4.701285155135265, + "tokens_seen": 116588544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007795225677031094, + "loss": 2.0352, + "theoretical_loss": 4.700928960856, + "tokens_seen": 116654080 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779506519558676, + "loss": 2.0243, + "theoretical_loss": 4.70057302262439, + "tokens_seen": 116719616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794904714142428, + "loss": 1.945, + "theoretical_loss": 4.700217340112767, + "tokens_seen": 116785152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794744232698094, + "loss": 1.9355, + "theoretical_loss": 4.699861912994063, + "tokens_seen": 116850688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794583751253762, + "loss": 1.7788, + "theoretical_loss": 4.699506740941813, + "tokens_seen": 116916224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794423269809428, + "loss": 2.1069, + "theoretical_loss": 4.699151823630155, + "tokens_seen": 116981760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794262788365096, + "loss": 1.904, + "theoretical_loss": 4.698797160733818, + "tokens_seen": 117047296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007794102306920762, + "loss": 1.9836, + "theoretical_loss": 4.6984427519281375, + "tokens_seen": 117112832 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779394182547643, + "loss": 1.9304, + "theoretical_loss": 4.698088596889037, + "tokens_seen": 117178368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007793781344032096, + "loss": 2.0516, + "theoretical_loss": 4.697734695293039, + "tokens_seen": 117243904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007793620862587764, + "loss": 1.9003, + "theoretical_loss": 4.697381046817256, + "tokens_seen": 117309440 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779346038114343, + "loss": 1.8785, + "theoretical_loss": 4.697027651139393, + "tokens_seen": 117374976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007793299899699098, + "loss": 2.0907, + "theoretical_loss": 4.696674507937746, + "tokens_seen": 117440512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007793139418254765, + "loss": 1.8739, + "theoretical_loss": 4.696321616891199, + "tokens_seen": 117506048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792978936810432, + "loss": 1.9546, + "theoretical_loss": 4.695968977679221, + "tokens_seen": 117571584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792818455366099, + "loss": 1.9477, + "theoretical_loss": 4.695616589981871, + "tokens_seen": 117637120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792657973921766, + "loss": 2.028, + "theoretical_loss": 4.695264453479789, + "tokens_seen": 117702656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792497492477433, + "loss": 1.9423, + "theoretical_loss": 4.6949125678542, + "tokens_seen": 117768192 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077923370110331, + "loss": 1.9417, + "theoretical_loss": 4.69456093278691, + "tokens_seen": 117833728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792176529588767, + "loss": 1.9821, + "theoretical_loss": 4.694209547960304, + "tokens_seen": 117899264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 50996, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.9989361763000488, + "objective/train/theoretical_loss": 4.693858413057347, + "objective/train/tokens_used": 138424800, + "theoretical_loss": 4.693858413057347, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007792016048144434, + "loss": 1.9989, + "theoretical_loss": 4.693858413057347, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791855566700101, + "loss": 1.8493, + "theoretical_loss": 4.693507527761584, + "tokens_seen": 118030336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791695085255768, + "loss": 1.9184, + "theoretical_loss": 4.6931568917571305, + "tokens_seen": 118095872 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791534603811435, + "loss": 2.0155, + "theoretical_loss": 4.692806504728683, + "tokens_seen": 118161408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791374122367102, + "loss": 2.0563, + "theoretical_loss": 4.692456366361506, + "tokens_seen": 118226944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791213640922769, + "loss": 1.853, + "theoretical_loss": 4.692106476341438, + "tokens_seen": 118292480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007791053159478436, + "loss": 2.0302, + "theoretical_loss": 4.691756834354891, + "tokens_seen": 118358016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790892678034103, + "loss": 1.9261, + "theoretical_loss": 4.691407440088843, + "tokens_seen": 118423552 + }, + { + "epoch": 0.04, + "learning_rate": 0.000779073219658977, + "loss": 1.8879, + "theoretical_loss": 4.69105829323084, + "tokens_seen": 118489088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790571715145437, + "loss": 2.0205, + "theoretical_loss": 4.690709393468995, + "tokens_seen": 118554624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790411233701104, + "loss": 1.976, + "theoretical_loss": 4.690360740491989, + "tokens_seen": 118620160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790250752256771, + "loss": 1.9956, + "theoretical_loss": 4.690012333989063, + "tokens_seen": 118685696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007790090270812438, + "loss": 2.083, + "theoretical_loss": 4.689664173650023, + "tokens_seen": 118751232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789929789368105, + "loss": 1.83, + "theoretical_loss": 4.689316259165235, + "tokens_seen": 118816768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789769307923772, + "loss": 1.9149, + "theoretical_loss": 4.6889685902256275, + "tokens_seen": 118882304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789608826479439, + "loss": 1.9617, + "theoretical_loss": 4.688621166522684, + "tokens_seen": 118947840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789448345035106, + "loss": 2.0881, + "theoretical_loss": 4.688273987748448, + "tokens_seen": 119013376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007789287863590773, + "loss": 1.9944, + "theoretical_loss": 4.687927053595518, + "tokens_seen": 119078912 + }, + { + "epoch": 0.04, + "learning_rate": 0.000778912738214644, + "loss": 1.9076, + "theoretical_loss": 4.687580363757048, + "tokens_seen": 119144448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788966900702107, + "loss": 1.8912, + "theoretical_loss": 4.6872339179267435, + "tokens_seen": 119209984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788806419257774, + "loss": 1.9355, + "theoretical_loss": 4.686887715798864, + "tokens_seen": 119275520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788645937813441, + "loss": 2.0869, + "theoretical_loss": 4.6865417570682215, + "tokens_seen": 119341056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788485456369108, + "loss": 1.9936, + "theoretical_loss": 4.686196041430172, + "tokens_seen": 119406592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788324974924775, + "loss": 1.8853, + "theoretical_loss": 4.685850568580624, + "tokens_seen": 119472128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788164493480442, + "loss": 1.9053, + "theoretical_loss": 4.685505338216032, + "tokens_seen": 119537664 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007788004012036109, + "loss": 2.103, + "theoretical_loss": 4.685160350033398, + "tokens_seen": 119603200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787843530591776, + "loss": 2.0069, + "theoretical_loss": 4.684815603730263, + "tokens_seen": 119668736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787683049147443, + "loss": 1.9858, + "theoretical_loss": 4.684471099004718, + "tokens_seen": 119734272 + }, + { + "epoch": 0.04, + "learning_rate": 0.000778752256770311, + "loss": 2.022, + "theoretical_loss": 4.684126835555389, + "tokens_seen": 119799808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787362086258777, + "loss": 2.0067, + "theoretical_loss": 4.683782813081448, + "tokens_seen": 119865344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787201604814444, + "loss": 1.9334, + "theoretical_loss": 4.683439031282606, + "tokens_seen": 119930880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007787041123370111, + "loss": 1.8767, + "theoretical_loss": 4.683095489859107, + "tokens_seen": 119996416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786880641925778, + "loss": 1.937, + "theoretical_loss": 4.682752188511737, + "tokens_seen": 120061952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786720160481445, + "loss": 1.9756, + "theoretical_loss": 4.682409126941815, + "tokens_seen": 120127488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786559679037112, + "loss": 2.0309, + "theoretical_loss": 4.682066304851195, + "tokens_seen": 120193024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786399197592779, + "loss": 1.9196, + "theoretical_loss": 4.681723721942264, + "tokens_seen": 120258560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786238716148446, + "loss": 1.9544, + "theoretical_loss": 4.6813813779179405, + "tokens_seen": 120324096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007786078234704113, + "loss": 2.0535, + "theoretical_loss": 4.681039272481674, + "tokens_seen": 120389632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785917753259779, + "loss": 2.1047, + "theoretical_loss": 4.680697405337443, + "tokens_seen": 120455168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785757271815447, + "loss": 1.9193, + "theoretical_loss": 4.680355776189755, + "tokens_seen": 120520704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785596790371113, + "loss": 1.7992, + "theoretical_loss": 4.680014384743641, + "tokens_seen": 120586240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785436308926781, + "loss": 1.9377, + "theoretical_loss": 4.679673230704664, + "tokens_seen": 120651776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785275827482447, + "loss": 1.8548, + "theoretical_loss": 4.679332313778904, + "tokens_seen": 120717312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007785115346038115, + "loss": 2.0185, + "theoretical_loss": 4.678991633672968, + "tokens_seen": 120782848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784954864593781, + "loss": 1.8658, + "theoretical_loss": 4.678651190093988, + "tokens_seen": 120848384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784794383149449, + "loss": 1.9428, + "theoretical_loss": 4.67831098274961, + "tokens_seen": 120913920 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784633901705115, + "loss": 1.9332, + "theoretical_loss": 4.677971011348005, + "tokens_seen": 120979456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784473420260782, + "loss": 1.9761, + "theoretical_loss": 4.677631275597859, + "tokens_seen": 121044992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784312938816449, + "loss": 1.9033, + "theoretical_loss": 4.677291775208378, + "tokens_seen": 121110528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007784152457372116, + "loss": 1.9176, + "theoretical_loss": 4.676952509889281, + "tokens_seen": 121176064 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783991975927783, + "loss": 1.9518, + "theoretical_loss": 4.676613479350804, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.000778383149448345, + "loss": 1.8744, + "theoretical_loss": 4.6762746833036966, + "tokens_seen": 121307136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783671013039117, + "loss": 1.9279, + "theoretical_loss": 4.675936121459218, + "tokens_seen": 121372672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783510531594785, + "loss": 1.8694, + "theoretical_loss": 4.675597793529143, + "tokens_seen": 121438208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783350050150451, + "loss": 1.8999, + "theoretical_loss": 4.675259699225752, + "tokens_seen": 121503744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783189568706119, + "loss": 1.8602, + "theoretical_loss": 4.674921838261836, + "tokens_seen": 121569280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007783029087261785, + "loss": 1.9503, + "theoretical_loss": 4.674584210350695, + "tokens_seen": 121634816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782868605817453, + "loss": 1.8664, + "theoretical_loss": 4.674246815206136, + "tokens_seen": 121700352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782708124373119, + "loss": 1.8057, + "theoretical_loss": 4.673909652542466, + "tokens_seen": 121765888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782547642928787, + "loss": 1.981, + "theoretical_loss": 4.673572722074502, + "tokens_seen": 121831424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782387161484453, + "loss": 2.042, + "theoretical_loss": 4.673236023517562, + "tokens_seen": 121896960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782226680040121, + "loss": 1.8831, + "theoretical_loss": 4.672899556587465, + "tokens_seen": 121962496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007782066198595787, + "loss": 1.9124, + "theoretical_loss": 4.672563321000533, + "tokens_seen": 122028032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781905717151455, + "loss": 1.9745, + "theoretical_loss": 4.672227316473584, + "tokens_seen": 122093568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781745235707121, + "loss": 1.9028, + "theoretical_loss": 4.671891542723937, + "tokens_seen": 122159104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781584754262789, + "loss": 2.0556, + "theoretical_loss": 4.671555999469408, + "tokens_seen": 122224640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781424272818455, + "loss": 1.988, + "theoretical_loss": 4.671220686428308, + "tokens_seen": 122290176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781263791374123, + "loss": 1.9089, + "theoretical_loss": 4.670885603319445, + "tokens_seen": 122355712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007781103309929789, + "loss": 1.9169, + "theoretical_loss": 4.670550749862119, + "tokens_seen": 122421248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780942828485457, + "loss": 1.8623, + "theoretical_loss": 4.670216125776122, + "tokens_seen": 122486784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780782347041123, + "loss": 1.8273, + "theoretical_loss": 4.669881730781741, + "tokens_seen": 122552320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780621865596791, + "loss": 1.9365, + "theoretical_loss": 4.66954756459975, + "tokens_seen": 122617856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780461384152457, + "loss": 1.9484, + "theoretical_loss": 4.669213626951413, + "tokens_seen": 122683392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780300902708125, + "loss": 2.0991, + "theoretical_loss": 4.668879917558484, + "tokens_seen": 122748928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007780140421263791, + "loss": 1.9432, + "theoretical_loss": 4.6685464361432025, + "tokens_seen": 122814464 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779979939819459, + "loss": 1.9753, + "theoretical_loss": 4.668213182428295, + "tokens_seen": 122880000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779819458375126, + "loss": 1.8774, + "theoretical_loss": 4.667880156136971, + "tokens_seen": 122945536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779658976930793, + "loss": 1.9131, + "theoretical_loss": 4.667547356992927, + "tokens_seen": 123011072 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777949849548646, + "loss": 1.9159, + "theoretical_loss": 4.667214784720338, + "tokens_seen": 123076608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779338014042127, + "loss": 1.9548, + "theoretical_loss": 4.666882439043865, + "tokens_seen": 123142144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779177532597794, + "loss": 1.9843, + "theoretical_loss": 4.666550319688646, + "tokens_seen": 123207680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007779017051153461, + "loss": 1.8915, + "theoretical_loss": 4.6662184263803, + "tokens_seen": 123273216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778856569709128, + "loss": 1.966, + "theoretical_loss": 4.665886758844923, + "tokens_seen": 123338752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778696088264795, + "loss": 1.9598, + "theoretical_loss": 4.66555531680909, + "tokens_seen": 123404288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778535606820462, + "loss": 1.8788, + "theoretical_loss": 4.665224099999852, + "tokens_seen": 123469824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778375125376129, + "loss": 1.8986, + "theoretical_loss": 4.664893108144735, + "tokens_seen": 123535360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778214643931796, + "loss": 1.7917, + "theoretical_loss": 4.664562340971735, + "tokens_seen": 123600896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007778054162487463, + "loss": 2.1133, + "theoretical_loss": 4.664231798209327, + "tokens_seen": 123666432 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777789368104313, + "loss": 1.7635, + "theoretical_loss": 4.663901479586454, + "tokens_seen": 123731968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777733199598797, + "loss": 2.0078, + "theoretical_loss": 4.663571384832531, + "tokens_seen": 123797504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777572718154464, + "loss": 1.9385, + "theoretical_loss": 4.663241513677443, + "tokens_seen": 123863040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777412236710131, + "loss": 1.8945, + "theoretical_loss": 4.662911865851541, + "tokens_seen": 123928576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777251755265798, + "loss": 2.0032, + "theoretical_loss": 4.662582441085648, + "tokens_seen": 123994112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007777091273821465, + "loss": 1.9572, + "theoretical_loss": 4.662253239111049, + "tokens_seen": 124059648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776930792377132, + "loss": 1.9845, + "theoretical_loss": 4.661924259659497, + "tokens_seen": 124125184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776770310932799, + "loss": 1.9505, + "theoretical_loss": 4.661595502463211, + "tokens_seen": 124190720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776609829488466, + "loss": 1.9152, + "theoretical_loss": 4.661266967254868, + "tokens_seen": 124256256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776449348044133, + "loss": 1.9791, + "theoretical_loss": 4.660938653767614, + "tokens_seen": 124321792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077762888665998, + "loss": 1.8655, + "theoretical_loss": 4.660610561735049, + "tokens_seen": 124387328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007776128385155467, + "loss": 2.0847, + "theoretical_loss": 4.66028269089124, + "tokens_seen": 124452864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 53273, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 2.0250937938690186, + "objective/train/theoretical_loss": 4.659955040970709, + "objective/train/tokens_used": 144978400, + "theoretical_loss": 4.659955040970709, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775967903711134, + "loss": 2.0251, + "theoretical_loss": 4.659955040970709, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775807422266801, + "loss": 1.8376, + "theoretical_loss": 4.659627611708437, + "tokens_seen": 124583936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775646940822468, + "loss": 1.9538, + "theoretical_loss": 4.659300402839863, + "tokens_seen": 124649472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775486459378135, + "loss": 1.8889, + "theoretical_loss": 4.658973414100881, + "tokens_seen": 124715008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775325977933802, + "loss": 2.039, + "theoretical_loss": 4.658646645227842, + "tokens_seen": 124780544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775165496489469, + "loss": 1.8919, + "theoretical_loss": 4.6583200959575475, + "tokens_seen": 124846080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007775005015045136, + "loss": 2.011, + "theoretical_loss": 4.657993766027255, + "tokens_seen": 124911616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774844533600803, + "loss": 1.9573, + "theoretical_loss": 4.657667655174674, + "tokens_seen": 124977152 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777468405215647, + "loss": 1.8056, + "theoretical_loss": 4.657341763137963, + "tokens_seen": 125042688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774523570712137, + "loss": 1.942, + "theoretical_loss": 4.657016089655732, + "tokens_seen": 125108224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774363089267804, + "loss": 1.9043, + "theoretical_loss": 4.65669063446704, + "tokens_seen": 125173760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774202607823471, + "loss": 1.7944, + "theoretical_loss": 4.656365397311393, + "tokens_seen": 125239296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007774042126379138, + "loss": 1.8253, + "theoretical_loss": 4.656040377928743, + "tokens_seen": 125304832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773881644934805, + "loss": 1.7155, + "theoretical_loss": 4.655715576059493, + "tokens_seen": 125370368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773721163490472, + "loss": 1.9427, + "theoretical_loss": 4.655390991444484, + "tokens_seen": 125435904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773560682046139, + "loss": 1.9742, + "theoretical_loss": 4.6550666238250065, + "tokens_seen": 125501440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773400200601806, + "loss": 1.8937, + "theoretical_loss": 4.65474247294279, + "tokens_seen": 125566976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007773239719157473, + "loss": 1.9277, + "theoretical_loss": 4.654418538540009, + "tokens_seen": 125632512 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777307923771314, + "loss": 2.006, + "theoretical_loss": 4.654094820359278, + "tokens_seen": 125698048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772918756268807, + "loss": 2.005, + "theoretical_loss": 4.653771318143651, + "tokens_seen": 125763584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772758274824474, + "loss": 1.7953, + "theoretical_loss": 4.65344803163662, + "tokens_seen": 125829120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772597793380141, + "loss": 1.863, + "theoretical_loss": 4.653124960582118, + "tokens_seen": 125894656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772437311935808, + "loss": 2.0596, + "theoretical_loss": 4.652802104724513, + "tokens_seen": 125960192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772276830491475, + "loss": 1.96, + "theoretical_loss": 4.652479463808609, + "tokens_seen": 126025728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007772116349047142, + "loss": 1.9092, + "theoretical_loss": 4.652157037579647, + "tokens_seen": 126091264 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771955867602808, + "loss": 1.9429, + "theoretical_loss": 4.6518348257833, + "tokens_seen": 126156800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771795386158476, + "loss": 1.962, + "theoretical_loss": 4.651512828165675, + "tokens_seen": 126222336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771634904714142, + "loss": 1.9165, + "theoretical_loss": 4.651191044473311, + "tokens_seen": 126287872 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777147442326981, + "loss": 2.0334, + "theoretical_loss": 4.65086947445318, + "tokens_seen": 126353408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771313941825476, + "loss": 1.9604, + "theoretical_loss": 4.650548117852681, + "tokens_seen": 126418944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007771153460381144, + "loss": 1.8603, + "theoretical_loss": 4.650226974419646, + "tokens_seen": 126484480 + }, + { + "epoch": 0.04, + "learning_rate": 0.000777099297893681, + "loss": 1.9798, + "theoretical_loss": 4.649906043902332, + "tokens_seen": 126550016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770832497492478, + "loss": 2.0097, + "theoretical_loss": 4.6495853260494275, + "tokens_seen": 126615552 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770672016048144, + "loss": 1.9388, + "theoretical_loss": 4.649264820610043, + "tokens_seen": 126681088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770511534603812, + "loss": 2.0125, + "theoretical_loss": 4.648944527333717, + "tokens_seen": 126746624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770351053159478, + "loss": 1.8029, + "theoretical_loss": 4.648624445970414, + "tokens_seen": 126812160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770190571715146, + "loss": 1.9509, + "theoretical_loss": 4.648304576270519, + "tokens_seen": 126877696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007770030090270812, + "loss": 1.9193, + "theoretical_loss": 4.647984917984841, + "tokens_seen": 126943232 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776986960882648, + "loss": 1.9608, + "theoretical_loss": 4.647665470864613, + "tokens_seen": 127008768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007769709127382146, + "loss": 1.9046, + "theoretical_loss": 4.6473462346614856, + "tokens_seen": 127074304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007769548645937814, + "loss": 1.8915, + "theoretical_loss": 4.647027209127531, + "tokens_seen": 127139840 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776938816449348, + "loss": 1.9559, + "theoretical_loss": 4.646708394015242, + "tokens_seen": 127205376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007769227683049148, + "loss": 2.0008, + "theoretical_loss": 4.646389789077524, + "tokens_seen": 127270912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007769067201604814, + "loss": 2.0062, + "theoretical_loss": 4.646071394067707, + "tokens_seen": 127336448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768906720160482, + "loss": 1.9537, + "theoretical_loss": 4.645753208739533, + "tokens_seen": 127401984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768746238716148, + "loss": 1.9244, + "theoretical_loss": 4.645435232847159, + "tokens_seen": 127467520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768585757271816, + "loss": 1.8396, + "theoretical_loss": 4.645117466145157, + "tokens_seen": 127533056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768425275827482, + "loss": 1.7292, + "theoretical_loss": 4.6447999083885145, + "tokens_seen": 127598592 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776826479438315, + "loss": 1.732, + "theoretical_loss": 4.644482559332629, + "tokens_seen": 127664128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007768104312938816, + "loss": 1.9047, + "theoretical_loss": 4.644165418733312, + "tokens_seen": 127729664 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767943831494484, + "loss": 1.8288, + "theoretical_loss": 4.6438484863467835, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776778335005015, + "loss": 1.986, + "theoretical_loss": 4.643531761929674, + "tokens_seen": 127860736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767622868605818, + "loss": 1.8109, + "theoretical_loss": 4.643215245239025, + "tokens_seen": 127926272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767462387161484, + "loss": 1.8901, + "theoretical_loss": 4.642898936032285, + "tokens_seen": 127991808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767301905717152, + "loss": 1.8135, + "theoretical_loss": 4.6425828340673085, + "tokens_seen": 128057344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007767141424272818, + "loss": 1.9875, + "theoretical_loss": 4.642266939102357, + "tokens_seen": 128122880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766980942828486, + "loss": 1.9042, + "theoretical_loss": 4.641951250896098, + "tokens_seen": 128188416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766820461384152, + "loss": 2.1037, + "theoretical_loss": 4.641635769207603, + "tokens_seen": 128253952 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776665997993982, + "loss": 1.8608, + "theoretical_loss": 4.641320493796349, + "tokens_seen": 128319488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766499498495486, + "loss": 1.9688, + "theoretical_loss": 4.641005424422213, + "tokens_seen": 128385024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766339017051154, + "loss": 1.8788, + "theoretical_loss": 4.640690560845474, + "tokens_seen": 128450560 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776617853560682, + "loss": 1.9964, + "theoretical_loss": 4.640375902826815, + "tokens_seen": 128516096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007766018054162488, + "loss": 1.9816, + "theoretical_loss": 4.640061450127316, + "tokens_seen": 128581632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765857572718155, + "loss": 1.9421, + "theoretical_loss": 4.639747202508459, + "tokens_seen": 128647168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765697091273822, + "loss": 1.9786, + "theoretical_loss": 4.639433159732121, + "tokens_seen": 128712704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765536609829489, + "loss": 1.9394, + "theoretical_loss": 4.639119321560581, + "tokens_seen": 128778240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765376128385156, + "loss": 1.8522, + "theoretical_loss": 4.638805687756511, + "tokens_seen": 128843776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007765215646940823, + "loss": 2.0082, + "theoretical_loss": 4.63849225808298, + "tokens_seen": 128909312 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776505516549649, + "loss": 1.8465, + "theoretical_loss": 4.6381790323034515, + "tokens_seen": 128974848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764894684052157, + "loss": 1.8826, + "theoretical_loss": 4.637866010181784, + "tokens_seen": 129040384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764734202607824, + "loss": 1.9264, + "theoretical_loss": 4.63755319148223, + "tokens_seen": 129105920 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764573721163491, + "loss": 1.917, + "theoretical_loss": 4.637240575969432, + "tokens_seen": 129171456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764413239719158, + "loss": 1.8722, + "theoretical_loss": 4.636928163408426, + "tokens_seen": 129236992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764252758274825, + "loss": 1.9663, + "theoretical_loss": 4.636615953564637, + "tokens_seen": 129302528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007764092276830492, + "loss": 1.8886, + "theoretical_loss": 4.636303946203881, + "tokens_seen": 129368064 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763931795386159, + "loss": 1.9165, + "theoretical_loss": 4.635992141092364, + "tokens_seen": 129433600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763771313941826, + "loss": 1.8921, + "theoretical_loss": 4.635680537996679, + "tokens_seen": 129499136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763610832497493, + "loss": 1.9513, + "theoretical_loss": 4.635369136683805, + "tokens_seen": 129564672 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776345035105316, + "loss": 1.9293, + "theoretical_loss": 4.6350579369211085, + "tokens_seen": 129630208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763289869608827, + "loss": 1.8728, + "theoretical_loss": 4.634746938476344, + "tokens_seen": 129695744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007763129388164494, + "loss": 1.9293, + "theoretical_loss": 4.634436141117645, + "tokens_seen": 129761280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762968906720161, + "loss": 1.9717, + "theoretical_loss": 4.634125544613536, + "tokens_seen": 129826816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762808425275828, + "loss": 1.9701, + "theoretical_loss": 4.63381514873292, + "tokens_seen": 129892352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762647943831495, + "loss": 1.886, + "theoretical_loss": 4.633504953245085, + "tokens_seen": 129957888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762487462387162, + "loss": 1.8814, + "theoretical_loss": 4.633194957919697, + "tokens_seen": 130023424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762326980942829, + "loss": 1.9399, + "theoretical_loss": 4.632885162526806, + "tokens_seen": 130088960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762166499498496, + "loss": 1.8572, + "theoretical_loss": 4.63257556683684, + "tokens_seen": 130154496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007762006018054163, + "loss": 1.8296, + "theoretical_loss": 4.632266170620607, + "tokens_seen": 130220032 + }, + { + "epoch": 0.04, + "learning_rate": 0.000776184553660983, + "loss": 1.8973, + "theoretical_loss": 4.631956973649293, + "tokens_seen": 130285568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761685055165497, + "loss": 1.9464, + "theoretical_loss": 4.631647975694461, + "tokens_seen": 130351104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761524573721164, + "loss": 1.9565, + "theoretical_loss": 4.631339176528051, + "tokens_seen": 130416640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761364092276831, + "loss": 1.9187, + "theoretical_loss": 4.631030575922379, + "tokens_seen": 130482176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761203610832498, + "loss": 2.0334, + "theoretical_loss": 4.630722173650135, + "tokens_seen": 130547712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007761043129388165, + "loss": 1.8334, + "theoretical_loss": 4.630413969484383, + "tokens_seen": 130613248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760882647943832, + "loss": 2.0124, + "theoretical_loss": 4.630105963198563, + "tokens_seen": 130678784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760722166499499, + "loss": 1.997, + "theoretical_loss": 4.629798154566485, + "tokens_seen": 130744320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760561685055166, + "loss": 1.9289, + "theoretical_loss": 4.62949054336233, + "tokens_seen": 130809856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760401203610833, + "loss": 1.889, + "theoretical_loss": 4.629183129360653, + "tokens_seen": 130875392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077602407221665, + "loss": 1.9031, + "theoretical_loss": 4.628875912336378, + "tokens_seen": 130940928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007760080240722167, + "loss": 1.9547, + "theoretical_loss": 4.628568892064797, + "tokens_seen": 131006464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 55660, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.9429093599319458, + "objective/train/theoretical_loss": 4.62826206832157, + "objective/train/tokens_used": 151532000, + "theoretical_loss": 4.62826206832157, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759919759277834, + "loss": 1.9429, + "theoretical_loss": 4.62826206832157, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759759277833502, + "loss": 1.951, + "theoretical_loss": 4.6279554408827295, + "tokens_seen": 131137536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759598796389168, + "loss": 1.8921, + "theoretical_loss": 4.62764900952467, + "tokens_seen": 131203072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759438314944836, + "loss": 1.9007, + "theoretical_loss": 4.627342774024155, + "tokens_seen": 131268608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759277833500502, + "loss": 2.032, + "theoretical_loss": 4.6270367341583105, + "tokens_seen": 131334144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007759117352056169, + "loss": 1.9691, + "theoretical_loss": 4.626730889704632, + "tokens_seen": 131399680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758956870611836, + "loss": 1.911, + "theoretical_loss": 4.626425240440973, + "tokens_seen": 131465216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758796389167503, + "loss": 1.9496, + "theoretical_loss": 4.626119786145557, + "tokens_seen": 131530752 + }, + { + "epoch": 0.04, + "learning_rate": 0.000775863590772317, + "loss": 1.9619, + "theoretical_loss": 4.625814526596962, + "tokens_seen": 131596288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758475426278837, + "loss": 1.9118, + "theoretical_loss": 4.625509461574135, + "tokens_seen": 131661824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758314944834504, + "loss": 1.8598, + "theoretical_loss": 4.625204590856377, + "tokens_seen": 131727360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007758154463390171, + "loss": 1.9388, + "theoretical_loss": 4.624899914223355, + "tokens_seen": 131792896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757993981945837, + "loss": 1.7842, + "theoretical_loss": 4.624595431455091, + "tokens_seen": 131858432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757833500501505, + "loss": 1.9605, + "theoretical_loss": 4.62429114233197, + "tokens_seen": 131923968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757673019057171, + "loss": 2.0652, + "theoretical_loss": 4.623987046634729, + "tokens_seen": 131989504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757512537612839, + "loss": 1.8835, + "theoretical_loss": 4.623683144144465, + "tokens_seen": 132055040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757352056168505, + "loss": 1.9783, + "theoretical_loss": 4.623379434642632, + "tokens_seen": 132120576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757191574724173, + "loss": 1.8827, + "theoretical_loss": 4.623075917911039, + "tokens_seen": 132186112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007757031093279839, + "loss": 1.8191, + "theoretical_loss": 4.622772593731848, + "tokens_seen": 132251648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756870611835507, + "loss": 1.8235, + "theoretical_loss": 4.622469461887577, + "tokens_seen": 132317184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756710130391173, + "loss": 1.9826, + "theoretical_loss": 4.622166522161096, + "tokens_seen": 132382720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756549648946841, + "loss": 2.0123, + "theoretical_loss": 4.621863774335628, + "tokens_seen": 132448256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756389167502507, + "loss": 1.8155, + "theoretical_loss": 4.621561218194747, + "tokens_seen": 132513792 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756228686058175, + "loss": 1.9544, + "theoretical_loss": 4.62125885352238, + "tokens_seen": 132579328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007756068204613841, + "loss": 1.9323, + "theoretical_loss": 4.620956680102801, + "tokens_seen": 132644864 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755907723169509, + "loss": 1.8998, + "theoretical_loss": 4.620654697720637, + "tokens_seen": 132710400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755747241725175, + "loss": 1.845, + "theoretical_loss": 4.620352906160861, + "tokens_seen": 132775936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755586760280843, + "loss": 1.8736, + "theoretical_loss": 4.6200513052087935, + "tokens_seen": 132841472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755426278836509, + "loss": 1.9324, + "theoretical_loss": 4.6197498946501065, + "tokens_seen": 132907008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755265797392177, + "loss": 1.8713, + "theoretical_loss": 4.6194486742708145, + "tokens_seen": 132972544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007755105315947843, + "loss": 2.0236, + "theoretical_loss": 4.6191476438572785, + "tokens_seen": 133038080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754944834503511, + "loss": 1.9157, + "theoretical_loss": 4.618846803196206, + "tokens_seen": 133103616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754784353059177, + "loss": 1.8248, + "theoretical_loss": 4.618546152074648, + "tokens_seen": 133169152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754623871614845, + "loss": 1.8383, + "theoretical_loss": 4.61824569028, + "tokens_seen": 133234688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754463390170511, + "loss": 1.8905, + "theoretical_loss": 4.617945417599998, + "tokens_seen": 133300224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754302908726179, + "loss": 1.8923, + "theoretical_loss": 4.617645333822722, + "tokens_seen": 133365760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007754142427281845, + "loss": 1.8889, + "theoretical_loss": 4.617345438736594, + "tokens_seen": 133431296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753981945837513, + "loss": 1.9894, + "theoretical_loss": 4.617045732130375, + "tokens_seen": 133496832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753821464393179, + "loss": 2.1688, + "theoretical_loss": 4.616746213793167, + "tokens_seen": 133562368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753660982948847, + "loss": 1.9595, + "theoretical_loss": 4.616446883514412, + "tokens_seen": 133627904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753500501504513, + "loss": 1.8683, + "theoretical_loss": 4.616147741083891, + "tokens_seen": 133693440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753340020060181, + "loss": 1.8758, + "theoretical_loss": 4.6158487862917195, + "tokens_seen": 133758976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753179538615847, + "loss": 1.9103, + "theoretical_loss": 4.615550018928353, + "tokens_seen": 133824512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007753019057171515, + "loss": 1.8714, + "theoretical_loss": 4.615251438784584, + "tokens_seen": 133890048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752858575727181, + "loss": 1.8895, + "theoretical_loss": 4.61495304565154, + "tokens_seen": 133955584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752698094282849, + "loss": 1.892, + "theoretical_loss": 4.614654839320682, + "tokens_seen": 134021120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752537612838515, + "loss": 2.0136, + "theoretical_loss": 4.614356819583808, + "tokens_seen": 134086656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752377131394183, + "loss": 1.9087, + "theoretical_loss": 4.614058986233047, + "tokens_seen": 134152192 + }, + { + "epoch": 0.04, + "learning_rate": 0.000775221664994985, + "loss": 1.9696, + "theoretical_loss": 4.613761339060863, + "tokens_seen": 134217728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007752056168505517, + "loss": 1.9087, + "theoretical_loss": 4.613463877860051, + "tokens_seen": 134283264 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751895687061184, + "loss": 2.06, + "theoretical_loss": 4.61316660242374, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751735205616851, + "loss": 2.0039, + "theoretical_loss": 4.6128695125453865, + "tokens_seen": 134414336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751574724172518, + "loss": 1.9339, + "theoretical_loss": 4.612572608018779, + "tokens_seen": 134479872 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751414242728185, + "loss": 2.2059, + "theoretical_loss": 4.612275888638036, + "tokens_seen": 134545408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751253761283852, + "loss": 2.2559, + "theoretical_loss": 4.611979354197603, + "tokens_seen": 134610944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007751093279839519, + "loss": 1.9838, + "theoretical_loss": 4.6116830044922565, + "tokens_seen": 134676480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750932798395186, + "loss": 2.212, + "theoretical_loss": 4.611386839317097, + "tokens_seen": 134742016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750772316950853, + "loss": 2.0549, + "theoretical_loss": 4.611090858467554, + "tokens_seen": 134807552 + }, + { + "epoch": 0.04, + "learning_rate": 0.000775061183550652, + "loss": 2.469, + "theoretical_loss": 4.610795061739384, + "tokens_seen": 134873088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750451354062187, + "loss": 2.051, + "theoretical_loss": 4.610499448928666, + "tokens_seen": 134938624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750290872617854, + "loss": 2.0906, + "theoretical_loss": 4.610204019831805, + "tokens_seen": 135004160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007750130391173521, + "loss": 2.0623, + "theoretical_loss": 4.609908774245532, + "tokens_seen": 135069696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749969909729188, + "loss": 2.146, + "theoretical_loss": 4.609613711966898, + "tokens_seen": 135135232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749809428284855, + "loss": 2.0754, + "theoretical_loss": 4.60931883279328, + "tokens_seen": 135200768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749648946840522, + "loss": 2.0812, + "theoretical_loss": 4.6090241365223745, + "tokens_seen": 135266304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749488465396189, + "loss": 2.1798, + "theoretical_loss": 4.6087296229522, + "tokens_seen": 135331840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749327983951856, + "loss": 2.2384, + "theoretical_loss": 4.608435291881099, + "tokens_seen": 135397376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007749167502507523, + "loss": 2.0114, + "theoretical_loss": 4.608141143107728, + "tokens_seen": 135462912 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774900702106319, + "loss": 2.032, + "theoretical_loss": 4.607847176431067, + "tokens_seen": 135528448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748846539618857, + "loss": 2.1132, + "theoretical_loss": 4.6075533916504146, + "tokens_seen": 135593984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748686058174524, + "loss": 2.045, + "theoretical_loss": 4.607259788565385, + "tokens_seen": 135659520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748525576730191, + "loss": 1.8809, + "theoretical_loss": 4.606966366975914, + "tokens_seen": 135725056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748365095285858, + "loss": 2.0153, + "theoretical_loss": 4.606673126682249, + "tokens_seen": 135790592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748204613841525, + "loss": 1.9686, + "theoretical_loss": 4.6063800674849595, + "tokens_seen": 135856128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007748044132397192, + "loss": 2.1016, + "theoretical_loss": 4.606087189184924, + "tokens_seen": 135921664 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747883650952859, + "loss": 1.929, + "theoretical_loss": 4.6057944915833415, + "tokens_seen": 135987200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747723169508527, + "loss": 1.9082, + "theoretical_loss": 4.6055019744817205, + "tokens_seen": 136052736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747562688064193, + "loss": 2.0086, + "theoretical_loss": 4.605209637681886, + "tokens_seen": 136118272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747402206619861, + "loss": 1.8842, + "theoretical_loss": 4.604917480985975, + "tokens_seen": 136183808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747241725175527, + "loss": 1.9044, + "theoretical_loss": 4.604625504196438, + "tokens_seen": 136249344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007747081243731195, + "loss": 1.9976, + "theoretical_loss": 4.604333707116033, + "tokens_seen": 136314880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746920762286861, + "loss": 1.87, + "theoretical_loss": 4.604042089547834, + "tokens_seen": 136380416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746760280842529, + "loss": 2.0676, + "theoretical_loss": 4.603750651295222, + "tokens_seen": 136445952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746599799398195, + "loss": 2.0567, + "theoretical_loss": 4.603459392161888, + "tokens_seen": 136511488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746439317953863, + "loss": 1.9025, + "theoretical_loss": 4.603168311951835, + "tokens_seen": 136577024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746278836509529, + "loss": 1.9117, + "theoretical_loss": 4.60287741046937, + "tokens_seen": 136642560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007746118355065197, + "loss": 2.0292, + "theoretical_loss": 4.602586687519111, + "tokens_seen": 136708096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745957873620863, + "loss": 2.0501, + "theoretical_loss": 4.602296142905981, + "tokens_seen": 136773632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745797392176531, + "loss": 1.9745, + "theoretical_loss": 4.602005776435211, + "tokens_seen": 136839168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745636910732197, + "loss": 1.9776, + "theoretical_loss": 4.601715587912338, + "tokens_seen": 136904704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745476429287865, + "loss": 2.0835, + "theoretical_loss": 4.601425577143203, + "tokens_seen": 136970240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745315947843531, + "loss": 2.0722, + "theoretical_loss": 4.601135743933952, + "tokens_seen": 137035776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007745155466399198, + "loss": 2.1287, + "theoretical_loss": 4.600846088091036, + "tokens_seen": 137101312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744994984954865, + "loss": 1.9373, + "theoretical_loss": 4.600556609421206, + "tokens_seen": 137166848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744834503510532, + "loss": 2.0686, + "theoretical_loss": 4.600267307731521, + "tokens_seen": 137232384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744674022066199, + "loss": 2.0877, + "theoretical_loss": 4.5999781828293385, + "tokens_seen": 137297920 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744513540621866, + "loss": 1.9723, + "theoretical_loss": 4.5996892345223195, + "tokens_seen": 137363456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744353059177533, + "loss": 2.0593, + "theoretical_loss": 4.599400462618423, + "tokens_seen": 137428992 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077441925777332, + "loss": 1.9818, + "theoretical_loss": 4.59911186692591, + "tokens_seen": 137494528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007744032096288867, + "loss": 1.9381, + "theoretical_loss": 4.598823447253343, + "tokens_seen": 137560064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 58515, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.963025689125061, + "objective/train/theoretical_loss": 4.598535203409582, + "objective/train/tokens_used": 158085600, + "theoretical_loss": 4.598535203409582, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743871614844534, + "loss": 1.963, + "theoretical_loss": 4.598535203409582, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00077437111334002, + "loss": 2.0808, + "theoretical_loss": 4.598247135203783, + "tokens_seen": 137691136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743550651955868, + "loss": 2.07, + "theoretical_loss": 4.5979592424454045, + "tokens_seen": 137756672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743390170511534, + "loss": 2.0288, + "theoretical_loss": 4.597671524944198, + "tokens_seen": 137822208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743229689067202, + "loss": 2.1158, + "theoretical_loss": 4.597383982510216, + "tokens_seen": 137887744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007743069207622868, + "loss": 1.8745, + "theoretical_loss": 4.597096614953802, + "tokens_seen": 137953280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007742908726178536, + "loss": 2.2142, + "theoretical_loss": 4.596809422085599, + "tokens_seen": 138018816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007742748244734202, + "loss": 1.9296, + "theoretical_loss": 4.596522403716542, + "tokens_seen": 138084352 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774258776328987, + "loss": 2.0163, + "theoretical_loss": 4.596235559657863, + "tokens_seen": 138149888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007742427281845536, + "loss": 2.0199, + "theoretical_loss": 4.595948889721086, + "tokens_seen": 138215424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007742266800401204, + "loss": 2.0205, + "theoretical_loss": 4.595662393718026, + "tokens_seen": 138280960 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774210631895687, + "loss": 2.0998, + "theoretical_loss": 4.595376071460796, + "tokens_seen": 138346496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741945837512538, + "loss": 2.0387, + "theoretical_loss": 4.595089922761796, + "tokens_seen": 138412032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741785356068204, + "loss": 2.1723, + "theoretical_loss": 4.594803947433717, + "tokens_seen": 138477568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741624874623872, + "loss": 1.961, + "theoretical_loss": 4.594518145289545, + "tokens_seen": 138543104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741464393179538, + "loss": 2.0687, + "theoretical_loss": 4.594232516142553, + "tokens_seen": 138608640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741303911735206, + "loss": 1.9523, + "theoretical_loss": 4.593947059806302, + "tokens_seen": 138674176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007741143430290872, + "loss": 2.2408, + "theoretical_loss": 4.593661776094647, + "tokens_seen": 138739712 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774098294884654, + "loss": 2.0517, + "theoretical_loss": 4.593376664821726, + "tokens_seen": 138805248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740822467402206, + "loss": 2.0091, + "theoretical_loss": 4.593091725801967, + "tokens_seen": 138870784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740661985957874, + "loss": 1.9329, + "theoretical_loss": 4.592806958850087, + "tokens_seen": 138936320 + }, + { + "epoch": 0.04, + "learning_rate": 0.000774050150451354, + "loss": 2.0677, + "theoretical_loss": 4.592522363781086, + "tokens_seen": 139001856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740341023069208, + "loss": 1.8435, + "theoretical_loss": 4.592237940410252, + "tokens_seen": 139067392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740180541624874, + "loss": 2.0432, + "theoretical_loss": 4.591953688553159, + "tokens_seen": 139132928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007740020060180542, + "loss": 2.1285, + "theoretical_loss": 4.591669608025663, + "tokens_seen": 139198464 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739859578736208, + "loss": 2.0844, + "theoretical_loss": 4.591385698643908, + "tokens_seen": 139264000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739699097291876, + "loss": 1.9827, + "theoretical_loss": 4.591101960224321, + "tokens_seen": 139329536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739538615847542, + "loss": 1.9665, + "theoretical_loss": 4.590818392583608, + "tokens_seen": 139395072 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773937813440321, + "loss": 2.0399, + "theoretical_loss": 4.590534995538762, + "tokens_seen": 139460608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739217652958876, + "loss": 1.9692, + "theoretical_loss": 4.590251768907057, + "tokens_seen": 139526144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007739057171514544, + "loss": 1.9305, + "theoretical_loss": 4.5899687125060495, + "tokens_seen": 139591680 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773889669007021, + "loss": 2.0723, + "theoretical_loss": 4.589685826153573, + "tokens_seen": 139657216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738736208625878, + "loss": 1.9322, + "theoretical_loss": 4.589403109667746, + "tokens_seen": 139722752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738575727181545, + "loss": 1.917, + "theoretical_loss": 4.589120562866961, + "tokens_seen": 139788288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738415245737212, + "loss": 2.0315, + "theoretical_loss": 4.588838185569897, + "tokens_seen": 139853824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738254764292879, + "loss": 2.0167, + "theoretical_loss": 4.588555977595506, + "tokens_seen": 139919360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007738094282848546, + "loss": 2.0811, + "theoretical_loss": 4.58827393876302, + "tokens_seen": 139984896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737933801404213, + "loss": 1.9253, + "theoretical_loss": 4.587992068891948, + "tokens_seen": 140050432 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773777331995988, + "loss": 1.9362, + "theoretical_loss": 4.587710367802076, + "tokens_seen": 140115968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737612838515547, + "loss": 2.0221, + "theoretical_loss": 4.587428835313469, + "tokens_seen": 140181504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737452357071214, + "loss": 2.0653, + "theoretical_loss": 4.587147471246463, + "tokens_seen": 140247040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737291875626881, + "loss": 1.927, + "theoretical_loss": 4.586866275421673, + "tokens_seen": 140312576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007737131394182548, + "loss": 1.9966, + "theoretical_loss": 4.586585247659986, + "tokens_seen": 140378112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736970912738215, + "loss": 1.9139, + "theoretical_loss": 4.586304387782567, + "tokens_seen": 140443648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736810431293882, + "loss": 1.973, + "theoretical_loss": 4.5860236956108515, + "tokens_seen": 140509184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736649949849549, + "loss": 1.8935, + "theoretical_loss": 4.585743170966548, + "tokens_seen": 140574720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736489468405216, + "loss": 2.0073, + "theoretical_loss": 4.58546281367164, + "tokens_seen": 140640256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736328986960883, + "loss": 1.968, + "theoretical_loss": 4.585182623548381, + "tokens_seen": 140705792 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773616850551655, + "loss": 2.0452, + "theoretical_loss": 4.584902600419295, + "tokens_seen": 140771328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007736008024072217, + "loss": 1.8525, + "theoretical_loss": 4.584622744107181, + "tokens_seen": 140836864 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735847542627884, + "loss": 1.9866, + "theoretical_loss": 4.584343054435103, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735687061183552, + "loss": 1.9483, + "theoretical_loss": 4.5840635312264, + "tokens_seen": 140967936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735526579739218, + "loss": 1.9608, + "theoretical_loss": 4.583784174304677, + "tokens_seen": 141033472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735366098294886, + "loss": 1.8675, + "theoretical_loss": 4.583504983493809, + "tokens_seen": 141099008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007735205616850552, + "loss": 1.8178, + "theoretical_loss": 4.583225958617938, + "tokens_seen": 141164544 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773504513540622, + "loss": 1.8741, + "theoretical_loss": 4.582947099501475, + "tokens_seen": 141230080 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734884653961886, + "loss": 1.9559, + "theoretical_loss": 4.5826684059691, + "tokens_seen": 141295616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734724172517554, + "loss": 1.9883, + "theoretical_loss": 4.582389877845754, + "tokens_seen": 141361152 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773456369107322, + "loss": 1.9825, + "theoretical_loss": 4.58211151495665, + "tokens_seen": 141426688 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734403209628888, + "loss": 1.7169, + "theoretical_loss": 4.581833317127263, + "tokens_seen": 141492224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734242728184554, + "loss": 1.9222, + "theoretical_loss": 4.581555284183335, + "tokens_seen": 141557760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007734082246740222, + "loss": 1.9835, + "theoretical_loss": 4.581277415950872, + "tokens_seen": 141623296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733921765295888, + "loss": 2.1773, + "theoretical_loss": 4.580999712256142, + "tokens_seen": 141688832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733761283851556, + "loss": 1.8872, + "theoretical_loss": 4.5807221729256815, + "tokens_seen": 141754368 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733600802407222, + "loss": 1.8931, + "theoretical_loss": 4.580444797786283, + "tokens_seen": 141819904 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773344032096289, + "loss": 1.9273, + "theoretical_loss": 4.5801675866650085, + "tokens_seen": 141885440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733279839518556, + "loss": 2.0059, + "theoretical_loss": 4.579890539389177, + "tokens_seen": 141950976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007733119358074224, + "loss": 2.1157, + "theoretical_loss": 4.579613655786369, + "tokens_seen": 142016512 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773295887662989, + "loss": 1.873, + "theoretical_loss": 4.579336935684431, + "tokens_seen": 142082048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732798395185558, + "loss": 2.0334, + "theoretical_loss": 4.579060378911464, + "tokens_seen": 142147584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732637913741224, + "loss": 1.9416, + "theoretical_loss": 4.578783985295832, + "tokens_seen": 142213120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732477432296892, + "loss": 2.013, + "theoretical_loss": 4.578507754666157, + "tokens_seen": 142278656 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732316950852558, + "loss": 1.9329, + "theoretical_loss": 4.57823168685132, + "tokens_seen": 142344192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007732156469408226, + "loss": 1.9049, + "theoretical_loss": 4.577955781680462, + "tokens_seen": 142409728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731995987963892, + "loss": 1.9542, + "theoretical_loss": 4.57768003898298, + "tokens_seen": 142475264 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773183550651956, + "loss": 2.1182, + "theoretical_loss": 4.577404458588529, + "tokens_seen": 142540800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731675025075226, + "loss": 1.9606, + "theoretical_loss": 4.577129040327021, + "tokens_seen": 142606336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731514543630894, + "loss": 1.851, + "theoretical_loss": 4.576853784028622, + "tokens_seen": 142671872 + }, + { + "epoch": 0.04, + "learning_rate": 0.000773135406218656, + "loss": 1.8816, + "theoretical_loss": 4.576578689523759, + "tokens_seen": 142737408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731193580742227, + "loss": 1.9601, + "theoretical_loss": 4.576303756643108, + "tokens_seen": 142802944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007731033099297894, + "loss": 1.7964, + "theoretical_loss": 4.576028985217606, + "tokens_seen": 142868480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730872617853561, + "loss": 2.0734, + "theoretical_loss": 4.5757543750784375, + "tokens_seen": 142934016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730712136409228, + "loss": 1.9743, + "theoretical_loss": 4.5754799260570485, + "tokens_seen": 142999552 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730551654964895, + "loss": 1.979, + "theoretical_loss": 4.575205637985132, + "tokens_seen": 143065088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730391173520562, + "loss": 1.945, + "theoretical_loss": 4.574931510694635, + "tokens_seen": 143130624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730230692076229, + "loss": 1.9358, + "theoretical_loss": 4.574657544017761, + "tokens_seen": 143196160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007730070210631896, + "loss": 2.0871, + "theoretical_loss": 4.57438373778696, + "tokens_seen": 143261696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729909729187563, + "loss": 1.8903, + "theoretical_loss": 4.574110091834936, + "tokens_seen": 143327232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729749247743229, + "loss": 2.0474, + "theoretical_loss": 4.573836605994645, + "tokens_seen": 143392768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729588766298897, + "loss": 2.0572, + "theoretical_loss": 4.573563280099288, + "tokens_seen": 143458304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729428284854563, + "loss": 1.8948, + "theoretical_loss": 4.573290113982324, + "tokens_seen": 143523840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729267803410231, + "loss": 1.9941, + "theoretical_loss": 4.573017107477455, + "tokens_seen": 143589376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007729107321965897, + "loss": 1.9212, + "theoretical_loss": 4.572744260418633, + "tokens_seen": 143654912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728946840521565, + "loss": 1.918, + "theoretical_loss": 4.572471572640059, + "tokens_seen": 143720448 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728786359077231, + "loss": 2.0237, + "theoretical_loss": 4.572199043976185, + "tokens_seen": 143785984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728625877632899, + "loss": 1.8364, + "theoretical_loss": 4.571926674261705, + "tokens_seen": 143851520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728465396188565, + "loss": 1.9238, + "theoretical_loss": 4.571654463331563, + "tokens_seen": 143917056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728304914744233, + "loss": 1.9978, + "theoretical_loss": 4.571382411020949, + "tokens_seen": 143982592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007728144433299899, + "loss": 1.8705, + "theoretical_loss": 4.5711105171653, + "tokens_seen": 144048128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727983951855567, + "loss": 1.8143, + "theoretical_loss": 4.570838781600296, + "tokens_seen": 144113664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 60961, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.9238232374191284, + "objective/train/theoretical_loss": 4.570567204161865, + "objective/train/tokens_used": 164639200, + "theoretical_loss": 4.570567204161865, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727823470411233, + "loss": 1.9238, + "theoretical_loss": 4.570567204161865, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727662988966901, + "loss": 1.8309, + "theoretical_loss": 4.570295784686176, + "tokens_seen": 144244736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727502507522567, + "loss": 1.9888, + "theoretical_loss": 4.570024523009646, + "tokens_seen": 144310272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727342026078235, + "loss": 1.9715, + "theoretical_loss": 4.569753418968933, + "tokens_seen": 144375808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727181544633901, + "loss": 1.9952, + "theoretical_loss": 4.569482472400939, + "tokens_seen": 144441344 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007727021063189569, + "loss": 1.8836, + "theoretical_loss": 4.5692116831428065, + "tokens_seen": 144506880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726860581745235, + "loss": 2.0555, + "theoretical_loss": 4.568941051031925, + "tokens_seen": 144572416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726700100300903, + "loss": 1.951, + "theoretical_loss": 4.568670575905921, + "tokens_seen": 144637952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726539618856569, + "loss": 1.8794, + "theoretical_loss": 4.568400257602666, + "tokens_seen": 144703488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726379137412237, + "loss": 1.9484, + "theoretical_loss": 4.568130095960268, + "tokens_seen": 144769024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726218655967903, + "loss": 1.9693, + "theoretical_loss": 4.56786009081708, + "tokens_seen": 144834560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007726058174523571, + "loss": 1.8785, + "theoretical_loss": 4.567590242011692, + "tokens_seen": 144900096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725897693079237, + "loss": 1.9614, + "theoretical_loss": 4.567320549382931, + "tokens_seen": 144965632 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725737211634905, + "loss": 1.9111, + "theoretical_loss": 4.56705101276987, + "tokens_seen": 145031168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725576730190571, + "loss": 1.9446, + "theoretical_loss": 4.566781632011814, + "tokens_seen": 145096704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725416248746239, + "loss": 2.0029, + "theoretical_loss": 4.566512406948309, + "tokens_seen": 145162240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725255767301905, + "loss": 1.7963, + "theoretical_loss": 4.566243337419137, + "tokens_seen": 145227776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007725095285857573, + "loss": 2.0048, + "theoretical_loss": 4.5659744232643185, + "tokens_seen": 145293312 + }, + { + "epoch": 0.04, + "learning_rate": 0.000772493480441324, + "loss": 2.0227, + "theoretical_loss": 4.565705664324111, + "tokens_seen": 145358848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724774322968907, + "loss": 1.9297, + "theoretical_loss": 4.565437060439006, + "tokens_seen": 145424384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724613841524574, + "loss": 1.9079, + "theoretical_loss": 4.565168611449732, + "tokens_seen": 145489920 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724453360080241, + "loss": 1.8954, + "theoretical_loss": 4.564900317197251, + "tokens_seen": 145555456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724292878635908, + "loss": 1.8184, + "theoretical_loss": 4.564632177522766, + "tokens_seen": 145620992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007724132397191575, + "loss": 1.9318, + "theoretical_loss": 4.564364192267705, + "tokens_seen": 145686528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723971915747242, + "loss": 1.8754, + "theoretical_loss": 4.564096361273736, + "tokens_seen": 145752064 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723811434302909, + "loss": 1.993, + "theoretical_loss": 4.563828684382761, + "tokens_seen": 145817600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723650952858577, + "loss": 1.9158, + "theoretical_loss": 4.563561161436912, + "tokens_seen": 145883136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723490471414243, + "loss": 1.9014, + "theoretical_loss": 4.563293792278554, + "tokens_seen": 145948672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723329989969911, + "loss": 2.0462, + "theoretical_loss": 4.563026576750286, + "tokens_seen": 146014208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723169508525577, + "loss": 1.8942, + "theoretical_loss": 4.562759514694935, + "tokens_seen": 146079744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007723009027081245, + "loss": 1.9372, + "theoretical_loss": 4.5624926059555655, + "tokens_seen": 146145280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722848545636911, + "loss": 1.8486, + "theoretical_loss": 4.562225850375466, + "tokens_seen": 146210816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722688064192579, + "loss": 2.0066, + "theoretical_loss": 4.561959247798159, + "tokens_seen": 146276352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722527582748245, + "loss": 1.8802, + "theoretical_loss": 4.561692798067396, + "tokens_seen": 146341888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722367101303913, + "loss": 1.9621, + "theoretical_loss": 4.561426501027158, + "tokens_seen": 146407424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722206619859579, + "loss": 1.9914, + "theoretical_loss": 4.561160356521656, + "tokens_seen": 146472960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007722046138415247, + "loss": 1.9165, + "theoretical_loss": 4.5608943643953275, + "tokens_seen": 146538496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721885656970913, + "loss": 1.9507, + "theoretical_loss": 4.56062852449284, + "tokens_seen": 146604032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721725175526581, + "loss": 1.876, + "theoretical_loss": 4.5603628366590865, + "tokens_seen": 146669568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721564694082247, + "loss": 1.8305, + "theoretical_loss": 4.560097300739191, + "tokens_seen": 146735104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721404212637915, + "loss": 2.0827, + "theoretical_loss": 4.5598319165785, + "tokens_seen": 146800640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721243731193581, + "loss": 1.7736, + "theoretical_loss": 4.55956668402259, + "tokens_seen": 146866176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007721083249749249, + "loss": 1.904, + "theoretical_loss": 4.559301602917261, + "tokens_seen": 146931712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720922768304915, + "loss": 1.9652, + "theoretical_loss": 4.55903667310854, + "tokens_seen": 146997248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720762286860583, + "loss": 1.8459, + "theoretical_loss": 4.558771894442677, + "tokens_seen": 147062784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720601805416249, + "loss": 1.8826, + "theoretical_loss": 4.558507266766151, + "tokens_seen": 147128320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720441323971917, + "loss": 1.9618, + "theoretical_loss": 4.558242789925659, + "tokens_seen": 147193856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720280842527583, + "loss": 1.867, + "theoretical_loss": 4.557978463768127, + "tokens_seen": 147259392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007720120361083251, + "loss": 1.8419, + "theoretical_loss": 4.557714288140703, + "tokens_seen": 147324928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719959879638917, + "loss": 1.8418, + "theoretical_loss": 4.557450262890757, + "tokens_seen": 147390464 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719799398194585, + "loss": 1.8463, + "theoretical_loss": 4.557186387865883, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719638916750251, + "loss": 1.9147, + "theoretical_loss": 4.556922662913895, + "tokens_seen": 147521536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719478435305919, + "loss": 1.8544, + "theoretical_loss": 4.55665908788283, + "tokens_seen": 147587072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719317953861585, + "loss": 1.8465, + "theoretical_loss": 4.556395662620945, + "tokens_seen": 147652608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007719157472417253, + "loss": 1.7813, + "theoretical_loss": 4.5561323869767225, + "tokens_seen": 147718144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718996990972919, + "loss": 2.1008, + "theoretical_loss": 4.555869260798859, + "tokens_seen": 147783680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718836509528587, + "loss": 1.9972, + "theoretical_loss": 4.5556062839362745, + "tokens_seen": 147849216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718676028084253, + "loss": 1.9089, + "theoretical_loss": 4.555343456238109, + "tokens_seen": 147914752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718515546639921, + "loss": 1.9051, + "theoretical_loss": 4.55508077755372, + "tokens_seen": 147980288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718355065195587, + "loss": 1.9815, + "theoretical_loss": 4.554818247732685, + "tokens_seen": 148045824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718194583751255, + "loss": 1.8505, + "theoretical_loss": 4.554555866624798, + "tokens_seen": 148111360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007718034102306921, + "loss": 1.8926, + "theoretical_loss": 4.554293634080074, + "tokens_seen": 148176896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007717873620862589, + "loss": 1.8781, + "theoretical_loss": 4.554031549948743, + "tokens_seen": 148242432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007717713139418255, + "loss": 1.8525, + "theoretical_loss": 4.5537696140812525, + "tokens_seen": 148307968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007717552657973923, + "loss": 1.8953, + "theoretical_loss": 4.553507826328268, + "tokens_seen": 148373504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0007717392176529589, + "loss": 1.9935, + "theoretical_loss": 4.553246186540671, + "tokens_seen": 148439040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007717231695085256, + "loss": 2.0621, + "theoretical_loss": 4.552984694569555, + "tokens_seen": 148504576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007717071213640923, + "loss": 1.8323, + "theoretical_loss": 4.552723350266236, + "tokens_seen": 148570112 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771691073219659, + "loss": 1.8755, + "theoretical_loss": 4.552462153482239, + "tokens_seen": 148635648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716750250752257, + "loss": 1.9182, + "theoretical_loss": 4.552201104069307, + "tokens_seen": 148701184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716589769307924, + "loss": 1.9216, + "theoretical_loss": 4.551940201879396, + "tokens_seen": 148766720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716429287863591, + "loss": 1.8752, + "theoretical_loss": 4.551679446764677, + "tokens_seen": 148832256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716268806419258, + "loss": 1.936, + "theoretical_loss": 4.551418838577531, + "tokens_seen": 148897792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007716108324974925, + "loss": 1.8786, + "theoretical_loss": 4.551158377170557, + "tokens_seen": 148963328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715947843530592, + "loss": 1.9279, + "theoretical_loss": 4.5508980623965645, + "tokens_seen": 149028864 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715787362086258, + "loss": 1.8514, + "theoretical_loss": 4.550637894108572, + "tokens_seen": 149094400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715626880641926, + "loss": 1.8439, + "theoretical_loss": 4.5503778721598165, + "tokens_seen": 149159936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715466399197592, + "loss": 2.1313, + "theoretical_loss": 4.55011799640374, + "tokens_seen": 149225472 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771530591775326, + "loss": 1.9196, + "theoretical_loss": 4.549858266694001, + "tokens_seen": 149291008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007715145436308926, + "loss": 1.8477, + "theoretical_loss": 4.549598682884463, + "tokens_seen": 149356544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714984954864594, + "loss": 1.9181, + "theoretical_loss": 4.549339244829204, + "tokens_seen": 149422080 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771482447342026, + "loss": 1.9003, + "theoretical_loss": 4.549079952382511, + "tokens_seen": 149487616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714663991975928, + "loss": 1.8811, + "theoretical_loss": 4.548820805398881, + "tokens_seen": 149553152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714503510531594, + "loss": 1.8629, + "theoretical_loss": 4.548561803733017, + "tokens_seen": 149618688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714343029087262, + "loss": 1.9023, + "theoretical_loss": 4.548302947239834, + "tokens_seen": 149684224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714182547642928, + "loss": 1.8714, + "theoretical_loss": 4.548044235774454, + "tokens_seen": 149749760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007714022066198596, + "loss": 1.9741, + "theoretical_loss": 4.547785669192207, + "tokens_seen": 149815296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007713861584754262, + "loss": 1.804, + "theoretical_loss": 4.547527247348631, + "tokens_seen": 149880832 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771370110330993, + "loss": 1.7849, + "theoretical_loss": 4.547268970099471, + "tokens_seen": 149946368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007713540621865596, + "loss": 1.8523, + "theoretical_loss": 4.547010837300677, + "tokens_seen": 150011904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007713380140421264, + "loss": 1.8931, + "theoretical_loss": 4.546752848808408, + "tokens_seen": 150077440 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771321965897693, + "loss": 1.9961, + "theoretical_loss": 4.546495004479027, + "tokens_seen": 150142976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007713059177532598, + "loss": 1.9581, + "theoretical_loss": 4.546237304169104, + "tokens_seen": 150208512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712898696088264, + "loss": 1.944, + "theoretical_loss": 4.545979747735413, + "tokens_seen": 150274048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712738214643932, + "loss": 1.8109, + "theoretical_loss": 4.5457223350349345, + "tokens_seen": 150339584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712577733199598, + "loss": 1.8735, + "theoretical_loss": 4.545465065924851, + "tokens_seen": 150405120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712417251755266, + "loss": 1.8665, + "theoretical_loss": 4.54520794026255, + "tokens_seen": 150470656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007712256770310932, + "loss": 1.9122, + "theoretical_loss": 4.544950957905625, + "tokens_seen": 150536192 + }, + { + "epoch": 0.05, + "learning_rate": 0.00077120962888666, + "loss": 1.9116, + "theoretical_loss": 4.54469411871187, + "tokens_seen": 150601728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711935807422266, + "loss": 1.946, + "theoretical_loss": 4.544437422539282, + "tokens_seen": 150667264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 63294, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.7721097469329834, + "objective/train/theoretical_loss": 4.544180869246063, + "objective/train/tokens_used": 171192800, + "theoretical_loss": 4.544180869246063, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711775325977934, + "loss": 1.7721, + "theoretical_loss": 4.544180869246063, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711614844533602, + "loss": 1.9401, + "theoretical_loss": 4.543924458690612, + "tokens_seen": 150798336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711454363089268, + "loss": 1.9516, + "theoretical_loss": 4.5436681907315375, + "tokens_seen": 150863872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711293881644936, + "loss": 1.834, + "theoretical_loss": 4.543412065227642, + "tokens_seen": 150929408 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007711133400200602, + "loss": 1.8954, + "theoretical_loss": 4.5431560820379335, + "tokens_seen": 150994944 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771097291875627, + "loss": 1.9046, + "theoretical_loss": 4.54290024102162, + "tokens_seen": 151060480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710812437311936, + "loss": 1.7943, + "theoretical_loss": 4.542644542038108, + "tokens_seen": 151126016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710651955867604, + "loss": 2.0143, + "theoretical_loss": 4.542388984947005, + "tokens_seen": 151191552 + }, + { + "epoch": 0.05, + "learning_rate": 0.000771049147442327, + "loss": 1.9111, + "theoretical_loss": 4.5421335696081195, + "tokens_seen": 151257088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710330992978938, + "loss": 1.8796, + "theoretical_loss": 4.541878295881456, + "tokens_seen": 151322624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710170511534604, + "loss": 1.9437, + "theoretical_loss": 4.541623163627221, + "tokens_seen": 151388160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007710010030090272, + "loss": 1.9952, + "theoretical_loss": 4.541368172705817, + "tokens_seen": 151453696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709849548645938, + "loss": 1.9175, + "theoretical_loss": 4.541113322977846, + "tokens_seen": 151519232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709689067201606, + "loss": 1.9967, + "theoretical_loss": 4.540858614304108, + "tokens_seen": 151584768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709528585757272, + "loss": 1.8233, + "theoretical_loss": 4.5406040465455995, + "tokens_seen": 151650304 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770936810431294, + "loss": 1.9011, + "theoretical_loss": 4.5403496195635125, + "tokens_seen": 151715840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709207622868606, + "loss": 2.0133, + "theoretical_loss": 4.54009533321924, + "tokens_seen": 151781376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007709047141424274, + "loss": 1.892, + "theoretical_loss": 4.539841187374366, + "tokens_seen": 151846912 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770888665997994, + "loss": 1.8707, + "theoretical_loss": 4.539587181890674, + "tokens_seen": 151912448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708726178535608, + "loss": 1.9512, + "theoretical_loss": 4.539333316630143, + "tokens_seen": 151977984 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708565697091274, + "loss": 1.8424, + "theoretical_loss": 4.539079591454943, + "tokens_seen": 152043520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708405215646942, + "loss": 1.8447, + "theoretical_loss": 4.5388260062274455, + "tokens_seen": 152109056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708244734202608, + "loss": 1.9066, + "theoretical_loss": 4.538572560810211, + "tokens_seen": 152174592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007708084252758276, + "loss": 1.8271, + "theoretical_loss": 4.538319255065995, + "tokens_seen": 152240128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007707923771313942, + "loss": 1.9183, + "theoretical_loss": 4.53806608885775, + "tokens_seen": 152305664 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770776328986961, + "loss": 1.8153, + "theoretical_loss": 4.537813062048618, + "tokens_seen": 152371200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007707602808425276, + "loss": 1.9209, + "theoretical_loss": 4.537560174501936, + "tokens_seen": 152436736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007707442326980944, + "loss": 1.8581, + "theoretical_loss": 4.537307426081232, + "tokens_seen": 152502272 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770728184553661, + "loss": 1.9426, + "theoretical_loss": 4.53705481665023, + "tokens_seen": 152567808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007707121364092278, + "loss": 1.8507, + "theoretical_loss": 4.536802346072841, + "tokens_seen": 152633344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706960882647944, + "loss": 1.8256, + "theoretical_loss": 4.536550014213171, + "tokens_seen": 152698880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706800401203612, + "loss": 1.9372, + "theoretical_loss": 4.536297820935517, + "tokens_seen": 152764416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706639919759278, + "loss": 2.0163, + "theoretical_loss": 4.536045766104366, + "tokens_seen": 152829952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706479438314946, + "loss": 1.8985, + "theoretical_loss": 4.535793849584396, + "tokens_seen": 152895488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007706318956870612, + "loss": 1.9097, + "theoretical_loss": 4.535542071240473, + "tokens_seen": 152961024 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770615847542628, + "loss": 1.9219, + "theoretical_loss": 4.535290430937657, + "tokens_seen": 153026560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705997993981946, + "loss": 1.9989, + "theoretical_loss": 4.535038928541194, + "tokens_seen": 153092096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705837512537614, + "loss": 1.8063, + "theoretical_loss": 4.534787563916522, + "tokens_seen": 153157632 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770567703109328, + "loss": 1.9977, + "theoretical_loss": 4.534536336929264, + "tokens_seen": 153223168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705516549648948, + "loss": 1.8801, + "theoretical_loss": 4.534285247445236, + "tokens_seen": 153288704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705356068204614, + "loss": 1.843, + "theoretical_loss": 4.534034295330439, + "tokens_seen": 153354240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705195586760282, + "loss": 1.8133, + "theoretical_loss": 4.5337834804510635, + "tokens_seen": 153419776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007705035105315948, + "loss": 1.8428, + "theoretical_loss": 4.533532802673484, + "tokens_seen": 153485312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007704874623871616, + "loss": 1.7653, + "theoretical_loss": 4.533282261864267, + "tokens_seen": 153550848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007704714142427282, + "loss": 1.7916, + "theoretical_loss": 4.533031857890164, + "tokens_seen": 153616384 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770455366098295, + "loss": 1.8959, + "theoretical_loss": 4.532781590618109, + "tokens_seen": 153681920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007704393179538616, + "loss": 1.8476, + "theoretical_loss": 4.532531459915229, + "tokens_seen": 153747456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007704232698094284, + "loss": 1.9944, + "theoretical_loss": 4.532281465648831, + "tokens_seen": 153812992 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770407221664995, + "loss": 1.8107, + "theoretical_loss": 4.532031607686411, + "tokens_seen": 153878528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703911735205618, + "loss": 1.8925, + "theoretical_loss": 4.531781885895647, + "tokens_seen": 153944064 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703751253761284, + "loss": 1.8457, + "theoretical_loss": 4.531532300144404, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703590772316952, + "loss": 1.9029, + "theoretical_loss": 4.531282850300731, + "tokens_seen": 154075136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703430290872618, + "loss": 1.8317, + "theoretical_loss": 4.53103353623286, + "tokens_seen": 154140672 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703269809428285, + "loss": 1.8707, + "theoretical_loss": 4.530784357809209, + "tokens_seen": 154206208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007703109327983952, + "loss": 1.8097, + "theoretical_loss": 4.5305353148983745, + "tokens_seen": 154271744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702948846539619, + "loss": 1.7925, + "theoretical_loss": 4.530286407369143, + "tokens_seen": 154337280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702788365095286, + "loss": 1.7986, + "theoretical_loss": 4.530037635090476, + "tokens_seen": 154402816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702627883650953, + "loss": 1.7938, + "theoretical_loss": 4.529788997931526, + "tokens_seen": 154468352 + }, + { + "epoch": 0.05, + "learning_rate": 0.000770246740220662, + "loss": 1.8068, + "theoretical_loss": 4.52954049576162, + "tokens_seen": 154533888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702306920762287, + "loss": 1.9102, + "theoretical_loss": 4.529292128450271, + "tokens_seen": 154599424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007702146439317954, + "loss": 1.917, + "theoretical_loss": 4.52904389586717, + "tokens_seen": 154664960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701985957873621, + "loss": 1.8685, + "theoretical_loss": 4.528795797882193, + "tokens_seen": 154730496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701825476429287, + "loss": 1.9085, + "theoretical_loss": 4.528547834365394, + "tokens_seen": 154796032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701664994984955, + "loss": 1.8218, + "theoretical_loss": 4.52830000518701, + "tokens_seen": 154861568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701504513540621, + "loss": 1.7771, + "theoretical_loss": 4.528052310217453, + "tokens_seen": 154927104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701344032096289, + "loss": 1.9205, + "theoretical_loss": 4.527804749327321, + "tokens_seen": 154992640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701183550651955, + "loss": 1.7947, + "theoretical_loss": 4.527557322387387, + "tokens_seen": 155058176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007701023069207623, + "loss": 1.8499, + "theoretical_loss": 4.527310029268604, + "tokens_seen": 155123712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700862587763289, + "loss": 2.0212, + "theoretical_loss": 4.527062869842107, + "tokens_seen": 155189248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700702106318957, + "loss": 1.8781, + "theoretical_loss": 4.526815843979204, + "tokens_seen": 155254784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700541624874623, + "loss": 1.7566, + "theoretical_loss": 4.5265689515513845, + "tokens_seen": 155320320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700381143430291, + "loss": 1.7649, + "theoretical_loss": 4.526322192430316, + "tokens_seen": 155385856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700220661985957, + "loss": 1.8284, + "theoretical_loss": 4.526075566487842, + "tokens_seen": 155451392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007700060180541625, + "loss": 1.9143, + "theoretical_loss": 4.525829073595984, + "tokens_seen": 155516928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699899699097291, + "loss": 1.8158, + "theoretical_loss": 4.5255827136269415, + "tokens_seen": 155582464 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699739217652959, + "loss": 1.7568, + "theoretical_loss": 4.525336486453088, + "tokens_seen": 155648000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699578736208627, + "loss": 1.763, + "theoretical_loss": 4.525090391946974, + "tokens_seen": 155713536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699418254764293, + "loss": 1.8462, + "theoretical_loss": 4.524844429981327, + "tokens_seen": 155779072 + }, + { + "epoch": 0.05, + "learning_rate": 0.000769925777331996, + "loss": 1.8713, + "theoretical_loss": 4.52459860042905, + "tokens_seen": 155844608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007699097291875627, + "loss": 1.9221, + "theoretical_loss": 4.524352903163219, + "tokens_seen": 155910144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698936810431295, + "loss": 1.9221, + "theoretical_loss": 4.524107338057089, + "tokens_seen": 155975680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698776328986961, + "loss": 1.7941, + "theoretical_loss": 4.523861904984086, + "tokens_seen": 156041216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698615847542629, + "loss": 1.7618, + "theoretical_loss": 4.5236166038178105, + "tokens_seen": 156106752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698455366098295, + "loss": 1.9284, + "theoretical_loss": 4.52337143443204, + "tokens_seen": 156172288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698294884653963, + "loss": 1.8633, + "theoretical_loss": 4.523126396700722, + "tokens_seen": 156237824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007698134403209629, + "loss": 2.0108, + "theoretical_loss": 4.522881490497981, + "tokens_seen": 156303360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697973921765297, + "loss": 1.8716, + "theoretical_loss": 4.522636715698111, + "tokens_seen": 156368896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697813440320963, + "loss": 1.9278, + "theoretical_loss": 4.522392072175582, + "tokens_seen": 156434432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697652958876631, + "loss": 1.8369, + "theoretical_loss": 4.522147559805034, + "tokens_seen": 156499968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697492477432297, + "loss": 1.8796, + "theoretical_loss": 4.5219031784612795, + "tokens_seen": 156565504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697331995987965, + "loss": 1.853, + "theoretical_loss": 4.521658928019305, + "tokens_seen": 156631040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697171514543631, + "loss": 1.7639, + "theoretical_loss": 4.521414808354266, + "tokens_seen": 156696576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007697011033099299, + "loss": 1.9447, + "theoretical_loss": 4.52117081934149, + "tokens_seen": 156762112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696850551654965, + "loss": 1.8389, + "theoretical_loss": 4.520926960856476, + "tokens_seen": 156827648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696690070210633, + "loss": 1.9491, + "theoretical_loss": 4.520683232774893, + "tokens_seen": 156893184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696529588766299, + "loss": 1.9055, + "theoretical_loss": 4.520439634972579, + "tokens_seen": 156958720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696369107321967, + "loss": 1.9063, + "theoretical_loss": 4.520196167325546, + "tokens_seen": 157024256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696208625877633, + "loss": 1.8827, + "theoretical_loss": 4.519952829709972, + "tokens_seen": 157089792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007696048144433301, + "loss": 1.7906, + "theoretical_loss": 4.519709622002205, + "tokens_seen": 157155328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695887662988967, + "loss": 1.808, + "theoretical_loss": 4.519466544078762, + "tokens_seen": 157220864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 65757, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.8944233655929565, + "objective/train/theoretical_loss": 4.519223595816332, + "objective/train/tokens_used": 177746400, + "theoretical_loss": 4.519223595816332, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695727181544635, + "loss": 1.8944, + "theoretical_loss": 4.519223595816332, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695566700100301, + "loss": 1.8895, + "theoretical_loss": 4.518980777091768, + "tokens_seen": 157351936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695406218655969, + "loss": 1.8229, + "theoretical_loss": 4.518738087782093, + "tokens_seen": 157417472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695245737211635, + "loss": 1.8728, + "theoretical_loss": 4.5184955277644985, + "tokens_seen": 157483008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007695085255767303, + "loss": 2.0235, + "theoretical_loss": 4.518253096916341, + "tokens_seen": 157548544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694924774322969, + "loss": 1.869, + "theoretical_loss": 4.518010795115151, + "tokens_seen": 157614080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694764292878637, + "loss": 1.8485, + "theoretical_loss": 4.517768622238616, + "tokens_seen": 157679616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694603811434303, + "loss": 1.6952, + "theoretical_loss": 4.5175265781646, + "tokens_seen": 157745152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694443329989971, + "loss": 1.8528, + "theoretical_loss": 4.517284662771125, + "tokens_seen": 157810688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694282848545637, + "loss": 1.8879, + "theoretical_loss": 4.5170428759363865, + "tokens_seen": 157876224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007694122367101305, + "loss": 2.0272, + "theoretical_loss": 4.516801217538739, + "tokens_seen": 157941760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693961885656971, + "loss": 1.9991, + "theoretical_loss": 4.516559687456708, + "tokens_seen": 158007296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693801404212639, + "loss": 1.8421, + "theoretical_loss": 4.516318285568982, + "tokens_seen": 158072832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693640922768305, + "loss": 1.7922, + "theoretical_loss": 4.516077011754414, + "tokens_seen": 158138368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693480441323973, + "loss": 2.0169, + "theoretical_loss": 4.51583586589202, + "tokens_seen": 158203904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693319959879639, + "loss": 1.864, + "theoretical_loss": 4.515594847860985, + "tokens_seen": 158269440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007693159478435307, + "loss": 1.9233, + "theoretical_loss": 4.515353957540656, + "tokens_seen": 158334976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692998996990973, + "loss": 1.9782, + "theoretical_loss": 4.515113194810541, + "tokens_seen": 158400512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692838515546641, + "loss": 1.9078, + "theoretical_loss": 4.514872559550316, + "tokens_seen": 158466048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692678034102307, + "loss": 1.8511, + "theoretical_loss": 4.514632051639815, + "tokens_seen": 158531584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692517552657975, + "loss": 1.8825, + "theoretical_loss": 4.51439167095904, + "tokens_seen": 158597120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692357071213641, + "loss": 1.827, + "theoretical_loss": 4.5141514173881525, + "tokens_seen": 158662656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692196589769309, + "loss": 1.8861, + "theoretical_loss": 4.513911290807478, + "tokens_seen": 158728192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007692036108324975, + "loss": 1.7447, + "theoretical_loss": 4.513671291097502, + "tokens_seen": 158793728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691875626880643, + "loss": 1.916, + "theoretical_loss": 4.513431418138874, + "tokens_seen": 158859264 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691715145436309, + "loss": 1.9695, + "theoretical_loss": 4.513191671812404, + "tokens_seen": 158924800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691554663991977, + "loss": 1.9074, + "theoretical_loss": 4.512952051999061, + "tokens_seen": 158990336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691394182547643, + "loss": 1.8326, + "theoretical_loss": 4.512712558579979, + "tokens_seen": 159055872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691233701103311, + "loss": 1.8922, + "theoretical_loss": 4.512473191436451, + "tokens_seen": 159121408 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007691073219658977, + "loss": 1.9429, + "theoretical_loss": 4.512233950449927, + "tokens_seen": 159186944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690912738214645, + "loss": 1.7524, + "theoretical_loss": 4.511994835502023, + "tokens_seen": 159252480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690752256770311, + "loss": 1.879, + "theoretical_loss": 4.511755846474511, + "tokens_seen": 159318016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690591775325979, + "loss": 2.0729, + "theoretical_loss": 4.511516983249322, + "tokens_seen": 159383552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690431293881645, + "loss": 1.867, + "theoretical_loss": 4.511278245708548, + "tokens_seen": 159449088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690270812437313, + "loss": 1.9098, + "theoretical_loss": 4.511039633734439, + "tokens_seen": 159514624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007690110330992979, + "loss": 1.8243, + "theoretical_loss": 4.5108011472094045, + "tokens_seen": 159580160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689949849548647, + "loss": 1.8814, + "theoretical_loss": 4.510562786016012, + "tokens_seen": 159645696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689789368104313, + "loss": 1.8576, + "theoretical_loss": 4.5103245500369855, + "tokens_seen": 159711232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689628886659981, + "loss": 1.8567, + "theoretical_loss": 4.510086439155209, + "tokens_seen": 159776768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689468405215647, + "loss": 1.9073, + "theoretical_loss": 4.509848453253722, + "tokens_seen": 159842304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689307923771315, + "loss": 1.9951, + "theoretical_loss": 4.509610592215726, + "tokens_seen": 159907840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007689147442326981, + "loss": 1.8949, + "theoretical_loss": 4.509372855924571, + "tokens_seen": 159973376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688986960882648, + "loss": 1.7646, + "theoretical_loss": 4.5091352442637715, + "tokens_seen": 160038912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688826479438315, + "loss": 1.8691, + "theoretical_loss": 4.508897757116995, + "tokens_seen": 160104448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688665997993982, + "loss": 1.8943, + "theoretical_loss": 4.508660394368066, + "tokens_seen": 160169984 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688505516549649, + "loss": 1.9748, + "theoretical_loss": 4.508423155900964, + "tokens_seen": 160235520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688345035105316, + "loss": 1.7981, + "theoretical_loss": 4.508186041599823, + "tokens_seen": 160301056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007688184553660983, + "loss": 1.7237, + "theoretical_loss": 4.5079490513489375, + "tokens_seen": 160366592 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768802407221665, + "loss": 1.8996, + "theoretical_loss": 4.50771218503275, + "tokens_seen": 160432128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687863590772316, + "loss": 2.0224, + "theoretical_loss": 4.507475442535863, + "tokens_seen": 160497664 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687703109327984, + "loss": 1.9483, + "theoretical_loss": 4.507238823743033, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687542627883651, + "loss": 1.8003, + "theoretical_loss": 4.507002328539167, + "tokens_seen": 160628736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687382146439318, + "loss": 1.7646, + "theoretical_loss": 4.506765956809331, + "tokens_seen": 160694272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687221664994985, + "loss": 1.8767, + "theoretical_loss": 4.50652970843874, + "tokens_seen": 160759808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007687061183550652, + "loss": 1.8529, + "theoretical_loss": 4.506293583312766, + "tokens_seen": 160825344 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768690070210632, + "loss": 1.6486, + "theoretical_loss": 4.506057581316933, + "tokens_seen": 160890880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007686740220661986, + "loss": 1.81, + "theoretical_loss": 4.505821702336917, + "tokens_seen": 160956416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007686579739217653, + "loss": 1.8689, + "theoretical_loss": 4.505585946258547, + "tokens_seen": 161021952 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768641925777332, + "loss": 1.842, + "theoretical_loss": 4.505350312967805, + "tokens_seen": 161087488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007686258776328987, + "loss": 1.7522, + "theoretical_loss": 4.5051148023508265, + "tokens_seen": 161153024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007686098294884654, + "loss": 1.9213, + "theoretical_loss": 4.504879414293896, + "tokens_seen": 161218560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685937813440322, + "loss": 1.9442, + "theoretical_loss": 4.50464414868345, + "tokens_seen": 161284096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685777331995988, + "loss": 1.8515, + "theoretical_loss": 4.504409005406078, + "tokens_seen": 161349632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685616850551656, + "loss": 1.8907, + "theoretical_loss": 4.504173984348521, + "tokens_seen": 161415168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685456369107322, + "loss": 1.8962, + "theoretical_loss": 4.503939085397667, + "tokens_seen": 161480704 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768529588766299, + "loss": 1.888, + "theoretical_loss": 4.5037043084405575, + "tokens_seen": 161546240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007685135406218656, + "loss": 1.8349, + "theoretical_loss": 4.503469653364386, + "tokens_seen": 161611776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684974924774324, + "loss": 1.9325, + "theoretical_loss": 4.503235120056491, + "tokens_seen": 161677312 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768481444332999, + "loss": 1.9023, + "theoretical_loss": 4.503000708404365, + "tokens_seen": 161742848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684653961885658, + "loss": 1.7732, + "theoretical_loss": 4.502766418295647, + "tokens_seen": 161808384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684493480441324, + "loss": 1.8676, + "theoretical_loss": 4.502532249618129, + "tokens_seen": 161873920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684332998996992, + "loss": 1.6695, + "theoretical_loss": 4.502298202259748, + "tokens_seen": 161939456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684172517552658, + "loss": 1.8236, + "theoretical_loss": 4.502064276108593, + "tokens_seen": 162004992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007684012036108326, + "loss": 1.8506, + "theoretical_loss": 4.501830471052898, + "tokens_seen": 162070528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007683851554663992, + "loss": 1.8639, + "theoretical_loss": 4.501596786981047, + "tokens_seen": 162136064 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768369107321966, + "loss": 1.8099, + "theoretical_loss": 4.501363223781574, + "tokens_seen": 162201600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007683530591775326, + "loss": 1.8228, + "theoretical_loss": 4.501129781343157, + "tokens_seen": 162267136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007683370110330994, + "loss": 1.818, + "theoretical_loss": 4.500896459554624, + "tokens_seen": 162332672 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768320962888666, + "loss": 1.8882, + "theoretical_loss": 4.500663258304948, + "tokens_seen": 162398208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007683049147442328, + "loss": 1.8502, + "theoretical_loss": 4.500430177483253, + "tokens_seen": 162463744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682888665997994, + "loss": 1.7494, + "theoretical_loss": 4.500197216978804, + "tokens_seen": 162529280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682728184553662, + "loss": 1.7342, + "theoretical_loss": 4.499964376681016, + "tokens_seen": 162594816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682567703109328, + "loss": 1.8732, + "theoretical_loss": 4.49973165647945, + "tokens_seen": 162660352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682407221664996, + "loss": 1.8919, + "theoretical_loss": 4.4994990562638115, + "tokens_seen": 162725888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007682246740220662, + "loss": 1.9732, + "theoretical_loss": 4.499266575923954, + "tokens_seen": 162791424 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768208625877633, + "loss": 1.8033, + "theoretical_loss": 4.499034215349875, + "tokens_seen": 162856960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681925777331996, + "loss": 1.754, + "theoretical_loss": 4.498801974431714, + "tokens_seen": 162922496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681765295887664, + "loss": 1.8017, + "theoretical_loss": 4.498569853059762, + "tokens_seen": 162988032 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768160481444333, + "loss": 1.8504, + "theoretical_loss": 4.49833785112445, + "tokens_seen": 163053568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681444332998998, + "loss": 1.8278, + "theoretical_loss": 4.498105968516354, + "tokens_seen": 163119104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681283851554664, + "loss": 1.8672, + "theoretical_loss": 4.497874205126195, + "tokens_seen": 163184640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007681123370110332, + "loss": 1.8862, + "theoretical_loss": 4.497642560844838, + "tokens_seen": 163250176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680962888665998, + "loss": 2.042, + "theoretical_loss": 4.497411035563291, + "tokens_seen": 163315712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680802407221666, + "loss": 1.8039, + "theoretical_loss": 4.497179629172704, + "tokens_seen": 163381248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680641925777332, + "loss": 1.8527, + "theoretical_loss": 4.4969483415643765, + "tokens_seen": 163446784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680481444333, + "loss": 1.8565, + "theoretical_loss": 4.496717172629742, + "tokens_seen": 163512320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680320962888666, + "loss": 1.9447, + "theoretical_loss": 4.496486122260382, + "tokens_seen": 163577856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007680160481444334, + "loss": 1.7381, + "theoretical_loss": 4.4962551903480215, + "tokens_seen": 163643392 + }, + { + "epoch": 0.05, + "learning_rate": 0.000768, + "loss": 1.8522, + "theoretical_loss": 4.496024376784522, + "tokens_seen": 163708928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679839518555668, + "loss": 1.7952, + "theoretical_loss": 4.495793681461894, + "tokens_seen": 163774464 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 68034, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.8746311664581299, + "objective/train/theoretical_loss": 4.495563104272286, + "objective/train/tokens_used": 184300000, + "theoretical_loss": 4.495563104272286, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679679037111334, + "loss": 1.8746, + "theoretical_loss": 4.495563104272286, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679518555667002, + "loss": 1.7779, + "theoretical_loss": 4.495332645107985, + "tokens_seen": 163905536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679358074222668, + "loss": 1.8592, + "theoretical_loss": 4.495102303861426, + "tokens_seen": 163971072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679197592778336, + "loss": 1.8209, + "theoretical_loss": 4.4948720804251785, + "tokens_seen": 164036608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007679037111334002, + "loss": 1.7162, + "theoretical_loss": 4.494641974691959, + "tokens_seen": 164102144 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767887662988967, + "loss": 1.8027, + "theoretical_loss": 4.494411986554619, + "tokens_seen": 164167680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007678716148445336, + "loss": 1.8185, + "theoretical_loss": 4.494182115906153, + "tokens_seen": 164233216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007678555667001004, + "loss": 1.713, + "theoretical_loss": 4.493952362639694, + "tokens_seen": 164298752 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767839518555667, + "loss": 1.9033, + "theoretical_loss": 4.493722726648519, + "tokens_seen": 164364288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007678234704112338, + "loss": 1.9633, + "theoretical_loss": 4.493493207826037, + "tokens_seen": 164429824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007678074222668004, + "loss": 1.7328, + "theoretical_loss": 4.493263806065803, + "tokens_seen": 164495360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677913741223672, + "loss": 1.7692, + "theoretical_loss": 4.493034521261509, + "tokens_seen": 164560896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677753259779338, + "loss": 1.9106, + "theoretical_loss": 4.492805353306984, + "tokens_seen": 164626432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677592778335006, + "loss": 1.8737, + "theoretical_loss": 4.492576302096199, + "tokens_seen": 164691968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677432296890672, + "loss": 1.7918, + "theoretical_loss": 4.49234736752326, + "tokens_seen": 164757504 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767727181544634, + "loss": 1.871, + "theoretical_loss": 4.492118549482413, + "tokens_seen": 164823040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007677111334002006, + "loss": 1.8635, + "theoretical_loss": 4.491889847868041, + "tokens_seen": 164888576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676950852557674, + "loss": 1.8765, + "theoretical_loss": 4.491661262574665, + "tokens_seen": 164954112 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767679037111334, + "loss": 1.895, + "theoretical_loss": 4.491432793496945, + "tokens_seen": 165019648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676629889669008, + "loss": 1.8275, + "theoretical_loss": 4.4912044405296765, + "tokens_seen": 165085184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676469408224674, + "loss": 1.8488, + "theoretical_loss": 4.49097620356779, + "tokens_seen": 165150720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676308926780342, + "loss": 1.8425, + "theoretical_loss": 4.490748082506356, + "tokens_seen": 165216256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007676148445336008, + "loss": 1.8183, + "theoretical_loss": 4.490520077240582, + "tokens_seen": 165281792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675987963891676, + "loss": 1.7612, + "theoretical_loss": 4.490292187665808, + "tokens_seen": 165347328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675827482447342, + "loss": 1.8528, + "theoretical_loss": 4.490064413677514, + "tokens_seen": 165412864 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767566700100301, + "loss": 1.9439, + "theoretical_loss": 4.4898367551713125, + "tokens_seen": 165478400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675506519558676, + "loss": 1.8569, + "theoretical_loss": 4.489609212042954, + "tokens_seen": 165543936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675346038114344, + "loss": 1.9445, + "theoretical_loss": 4.489381784188322, + "tokens_seen": 165609472 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767518555667001, + "loss": 1.9607, + "theoretical_loss": 4.489154471503438, + "tokens_seen": 165675008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007675025075225677, + "loss": 1.9154, + "theoretical_loss": 4.488927273884456, + "tokens_seen": 165740544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674864593781344, + "loss": 1.793, + "theoretical_loss": 4.488700191227665, + "tokens_seen": 165806080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674704112337011, + "loss": 1.77, + "theoretical_loss": 4.488473223429491, + "tokens_seen": 165871616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674543630892678, + "loss": 1.7784, + "theoretical_loss": 4.48824637038649, + "tokens_seen": 165937152 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674383149448345, + "loss": 1.757, + "theoretical_loss": 4.488019631995354, + "tokens_seen": 166002688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674222668004012, + "loss": 1.9577, + "theoretical_loss": 4.48779300815291, + "tokens_seen": 166068224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007674062186559679, + "loss": 1.7849, + "theoretical_loss": 4.4875664987561175, + "tokens_seen": 166133760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673901705115346, + "loss": 1.9536, + "theoretical_loss": 4.487340103702068, + "tokens_seen": 166199296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673741223671013, + "loss": 1.7424, + "theoretical_loss": 4.487113822887988, + "tokens_seen": 166264832 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767358074222668, + "loss": 1.7993, + "theoretical_loss": 4.486887656211234, + "tokens_seen": 166330368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673420260782347, + "loss": 1.7571, + "theoretical_loss": 4.486661603569301, + "tokens_seen": 166395904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673259779338014, + "loss": 1.7829, + "theoretical_loss": 4.486435664859809, + "tokens_seen": 166461440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007673099297893681, + "loss": 1.809, + "theoretical_loss": 4.486209839980516, + "tokens_seen": 166526976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672938816449348, + "loss": 1.8188, + "theoretical_loss": 4.485984128829307, + "tokens_seen": 166592512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672778335005015, + "loss": 1.8781, + "theoretical_loss": 4.485758531304205, + "tokens_seen": 166658048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672617853560682, + "loss": 1.9506, + "theoretical_loss": 4.485533047303358, + "tokens_seen": 166723584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672457372116349, + "loss": 1.9245, + "theoretical_loss": 4.485307676725048, + "tokens_seen": 166789120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672296890672016, + "loss": 1.8347, + "theoretical_loss": 4.485082419467691, + "tokens_seen": 166854656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007672136409227683, + "loss": 1.8, + "theoretical_loss": 4.484857275429828, + "tokens_seen": 166920192 + }, + { + "epoch": 0.05, + "learning_rate": 0.000767197592778335, + "loss": 1.7967, + "theoretical_loss": 4.484632244510136, + "tokens_seen": 166985728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671815446339017, + "loss": 1.7989, + "theoretical_loss": 4.4844073266074185, + "tokens_seen": 167051264 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671654964894685, + "loss": 1.8184, + "theoretical_loss": 4.484182521620611, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671494483450351, + "loss": 1.7635, + "theoretical_loss": 4.48395782944878, + "tokens_seen": 167182336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671334002006019, + "loss": 1.8308, + "theoretical_loss": 4.483733249991118, + "tokens_seen": 167247872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671173520561685, + "loss": 1.8299, + "theoretical_loss": 4.483508783146952, + "tokens_seen": 167313408 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007671013039117353, + "loss": 1.8609, + "theoretical_loss": 4.483284428815735, + "tokens_seen": 167378944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670852557673019, + "loss": 1.8029, + "theoretical_loss": 4.483060186897049, + "tokens_seen": 167444480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670692076228687, + "loss": 1.8604, + "theoretical_loss": 4.482836057290607, + "tokens_seen": 167510016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670531594784353, + "loss": 1.8782, + "theoretical_loss": 4.4826120398962495, + "tokens_seen": 167575552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670371113340021, + "loss": 1.7944, + "theoretical_loss": 4.482388134613943, + "tokens_seen": 167641088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670210631895687, + "loss": 1.8592, + "theoretical_loss": 4.482164341343788, + "tokens_seen": 167706624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007670050150451355, + "loss": 1.7592, + "theoretical_loss": 4.481940659986008, + "tokens_seen": 167772160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669889669007021, + "loss": 1.7657, + "theoretical_loss": 4.481717090440956, + "tokens_seen": 167837696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669729187562689, + "loss": 1.7377, + "theoretical_loss": 4.481493632609112, + "tokens_seen": 167903232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669568706118355, + "loss": 1.8152, + "theoretical_loss": 4.481270286391084, + "tokens_seen": 167968768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669408224674023, + "loss": 1.7121, + "theoretical_loss": 4.481047051687609, + "tokens_seen": 168034304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669247743229689, + "loss": 1.9053, + "theoretical_loss": 4.480823928399547, + "tokens_seen": 168099840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007669087261785357, + "loss": 1.81, + "theoretical_loss": 4.480600916427887, + "tokens_seen": 168165376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668926780341023, + "loss": 1.8638, + "theoretical_loss": 4.480378015673745, + "tokens_seen": 168230912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668766298896691, + "loss": 1.8954, + "theoretical_loss": 4.480155226038362, + "tokens_seen": 168296448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668605817452357, + "loss": 1.7743, + "theoretical_loss": 4.479932547423107, + "tokens_seen": 168361984 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668445336008025, + "loss": 1.8612, + "theoretical_loss": 4.479709979729472, + "tokens_seen": 168427520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668284854563691, + "loss": 1.8205, + "theoretical_loss": 4.479487522859078, + "tokens_seen": 168493056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007668124373119359, + "loss": 1.768, + "theoretical_loss": 4.4792651767136675, + "tokens_seen": 168558592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667963891675025, + "loss": 1.796, + "theoretical_loss": 4.479042941195113, + "tokens_seen": 168624128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667803410230693, + "loss": 1.8316, + "theoretical_loss": 4.478820816205408, + "tokens_seen": 168689664 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667642928786359, + "loss": 1.7718, + "theoretical_loss": 4.4785988016466725, + "tokens_seen": 168755200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667482447342027, + "loss": 1.8341, + "theoretical_loss": 4.4783768974211515, + "tokens_seen": 168820736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667321965897693, + "loss": 1.9364, + "theoretical_loss": 4.478155103431215, + "tokens_seen": 168886272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667161484453361, + "loss": 1.7932, + "theoretical_loss": 4.477933419579356, + "tokens_seen": 168951808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007667001003009027, + "loss": 1.8589, + "theoretical_loss": 4.477711845768191, + "tokens_seen": 169017344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666840521564695, + "loss": 1.8692, + "theoretical_loss": 4.477490381900461, + "tokens_seen": 169082880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666680040120361, + "loss": 1.8408, + "theoretical_loss": 4.477269027879032, + "tokens_seen": 169148416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666519558676029, + "loss": 1.8432, + "theoretical_loss": 4.477047783606892, + "tokens_seen": 169213952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666359077231695, + "loss": 1.9238, + "theoretical_loss": 4.476826648987151, + "tokens_seen": 169279488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666198595787363, + "loss": 1.8107, + "theoretical_loss": 4.476605623923044, + "tokens_seen": 169345024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007666038114343029, + "loss": 1.7872, + "theoretical_loss": 4.47638470831793, + "tokens_seen": 169410560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665877632898697, + "loss": 1.9428, + "theoretical_loss": 4.476163902075286, + "tokens_seen": 169476096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665717151454363, + "loss": 1.8615, + "theoretical_loss": 4.475943205098716, + "tokens_seen": 169541632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665556670010031, + "loss": 1.8295, + "theoretical_loss": 4.475722617291944, + "tokens_seen": 169607168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665396188565697, + "loss": 1.9984, + "theoretical_loss": 4.4755021385588165, + "tokens_seen": 169672704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665235707121365, + "loss": 1.8964, + "theoretical_loss": 4.475281768803301, + "tokens_seen": 169738240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007665075225677031, + "loss": 1.7493, + "theoretical_loss": 4.475061507929487, + "tokens_seen": 169803776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664914744232699, + "loss": 1.9898, + "theoretical_loss": 4.474841355841587, + "tokens_seen": 169869312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664754262788365, + "loss": 1.822, + "theoretical_loss": 4.474621312443932, + "tokens_seen": 169934848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664593781344033, + "loss": 1.9682, + "theoretical_loss": 4.474401377640977, + "tokens_seen": 170000384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664433299899699, + "loss": 1.8189, + "theoretical_loss": 4.474181551337294, + "tokens_seen": 170065920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664272818455367, + "loss": 1.8791, + "theoretical_loss": 4.473961833437579, + "tokens_seen": 170131456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007664112337011033, + "loss": 1.9552, + "theoretical_loss": 4.473742223846646, + "tokens_seen": 170196992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663951855566701, + "loss": 1.8052, + "theoretical_loss": 4.473522722469431, + "tokens_seen": 170262528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663791374122367, + "loss": 1.9086, + "theoretical_loss": 4.47330332921099, + "tokens_seen": 170328064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 70789, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.8844119310379028, + "objective/train/theoretical_loss": 4.473084043976495, + "objective/train/tokens_used": 190853600, + "theoretical_loss": 4.473084043976495, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663630892678035, + "loss": 1.8844, + "theoretical_loss": 4.473084043976495, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663470411233702, + "loss": 1.9333, + "theoretical_loss": 4.472864866671245, + "tokens_seen": 170459136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663309929789369, + "loss": 1.8575, + "theoretical_loss": 4.472645797200651, + "tokens_seen": 170524672 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007663149448345036, + "loss": 1.8499, + "theoretical_loss": 4.472426835470247, + "tokens_seen": 170590208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662988966900703, + "loss": 1.881, + "theoretical_loss": 4.472207981385686, + "tokens_seen": 170655744 + }, + { + "epoch": 0.05, + "learning_rate": 0.000766282848545637, + "loss": 1.8488, + "theoretical_loss": 4.471989234852737, + "tokens_seen": 170721280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662668004012037, + "loss": 1.8342, + "theoretical_loss": 4.471770595777292, + "tokens_seen": 170786816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662507522567703, + "loss": 1.8127, + "theoretical_loss": 4.4715520640653565, + "tokens_seen": 170852352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662347041123371, + "loss": 1.8033, + "theoretical_loss": 4.471333639623059, + "tokens_seen": 170917888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662186559679037, + "loss": 1.892, + "theoretical_loss": 4.471115322356643, + "tokens_seen": 170983424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007662026078234705, + "loss": 1.7726, + "theoretical_loss": 4.470897112172471, + "tokens_seen": 171048960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661865596790371, + "loss": 2.0568, + "theoretical_loss": 4.470679008977022, + "tokens_seen": 171114496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661705115346039, + "loss": 1.7931, + "theoretical_loss": 4.470461012676894, + "tokens_seen": 171180032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661544633901705, + "loss": 1.6596, + "theoretical_loss": 4.470243123178802, + "tokens_seen": 171245568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661384152457373, + "loss": 1.8412, + "theoretical_loss": 4.4700253403895776, + "tokens_seen": 171311104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661223671013039, + "loss": 1.8774, + "theoretical_loss": 4.469807664216168, + "tokens_seen": 171376640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007661063189568706, + "loss": 1.8796, + "theoretical_loss": 4.469590094565638, + "tokens_seen": 171442176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660902708124373, + "loss": 1.9763, + "theoretical_loss": 4.469372631345172, + "tokens_seen": 171507712 + }, + { + "epoch": 0.05, + "learning_rate": 0.000766074222668004, + "loss": 1.737, + "theoretical_loss": 4.469155274462064, + "tokens_seen": 171573248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660581745235707, + "loss": 1.7833, + "theoretical_loss": 4.468938023823732, + "tokens_seen": 171638784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660421263791374, + "loss": 1.8698, + "theoretical_loss": 4.468720879337704, + "tokens_seen": 171704320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660260782347041, + "loss": 1.8787, + "theoretical_loss": 4.468503840911627, + "tokens_seen": 171769856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007660100300902708, + "loss": 1.8694, + "theoretical_loss": 4.46828690845326, + "tokens_seen": 171835392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007659939819458375, + "loss": 1.9094, + "theoretical_loss": 4.468070081870483, + "tokens_seen": 171900928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007659779338014042, + "loss": 1.7438, + "theoretical_loss": 4.467853361071284, + "tokens_seen": 171966464 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765961885656971, + "loss": 1.8563, + "theoretical_loss": 4.467636745963772, + "tokens_seen": 172032000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007659458375125376, + "loss": 1.7819, + "theoretical_loss": 4.467420236456169, + "tokens_seen": 172097536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007659297893681043, + "loss": 1.6783, + "theoretical_loss": 4.46720383245681, + "tokens_seen": 172163072 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765913741223671, + "loss": 1.9597, + "theoretical_loss": 4.466987533874145, + "tokens_seen": 172228608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658976930792377, + "loss": 1.8961, + "theoretical_loss": 4.4667713406167415, + "tokens_seen": 172294144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658816449348044, + "loss": 1.8189, + "theoretical_loss": 4.466555252593277, + "tokens_seen": 172359680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658655967903711, + "loss": 1.9351, + "theoretical_loss": 4.4663392697125435, + "tokens_seen": 172425216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658495486459378, + "loss": 1.7525, + "theoretical_loss": 4.466123391883448, + "tokens_seen": 172490752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658335005015046, + "loss": 1.864, + "theoretical_loss": 4.4659076190150095, + "tokens_seen": 172556288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007658174523570712, + "loss": 1.8967, + "theoretical_loss": 4.465691951016364, + "tokens_seen": 172621824 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765801404212638, + "loss": 1.766, + "theoretical_loss": 4.465476387796755, + "tokens_seen": 172687360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657853560682046, + "loss": 1.7432, + "theoretical_loss": 4.465260929265543, + "tokens_seen": 172752896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657693079237714, + "loss": 1.8648, + "theoretical_loss": 4.4650455753322005, + "tokens_seen": 172818432 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765753259779338, + "loss": 1.7019, + "theoretical_loss": 4.464830325906312, + "tokens_seen": 172883968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657372116349048, + "loss": 1.7988, + "theoretical_loss": 4.464615180897574, + "tokens_seen": 172949504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657211634904714, + "loss": 1.6652, + "theoretical_loss": 4.464400140215796, + "tokens_seen": 173015040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007657051153460382, + "loss": 1.775, + "theoretical_loss": 4.4641852037708984, + "tokens_seen": 173080576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656890672016048, + "loss": 1.7437, + "theoretical_loss": 4.463970371472916, + "tokens_seen": 173146112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656730190571716, + "loss": 1.8064, + "theoretical_loss": 4.463755643231992, + "tokens_seen": 173211648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656569709127382, + "loss": 1.8278, + "theoretical_loss": 4.463541018958384, + "tokens_seen": 173277184 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765640922768305, + "loss": 1.9779, + "theoretical_loss": 4.463326498562459, + "tokens_seen": 173342720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656248746238716, + "loss": 1.8041, + "theoretical_loss": 4.463112081954696, + "tokens_seen": 173408256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007656088264794384, + "loss": 1.8698, + "theoretical_loss": 4.462897769045684, + "tokens_seen": 173473792 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765592778335005, + "loss": 1.7609, + "theoretical_loss": 4.462683559746123, + "tokens_seen": 173539328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655767301905718, + "loss": 1.8934, + "theoretical_loss": 4.4624694539668255, + "tokens_seen": 173604864 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655606820461384, + "loss": 1.79, + "theoretical_loss": 4.462255451618713, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655446339017052, + "loss": 1.7272, + "theoretical_loss": 4.462041552612815, + "tokens_seen": 173735936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655285857572718, + "loss": 1.8072, + "theoretical_loss": 4.461827756860275, + "tokens_seen": 173801472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007655125376128386, + "loss": 1.8277, + "theoretical_loss": 4.461614064272345, + "tokens_seen": 173867008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654964894684052, + "loss": 1.8084, + "theoretical_loss": 4.461400474760383, + "tokens_seen": 173932544 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765480441323972, + "loss": 1.7361, + "theoretical_loss": 4.461186988235864, + "tokens_seen": 173998080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654643931795386, + "loss": 1.7529, + "theoretical_loss": 4.460973604610366, + "tokens_seen": 174063616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654483450351054, + "loss": 1.8618, + "theoretical_loss": 4.4607603237955775, + "tokens_seen": 174129152 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765432296890672, + "loss": 1.7613, + "theoretical_loss": 4.460547145703298, + "tokens_seen": 174194688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654162487462388, + "loss": 1.8339, + "theoretical_loss": 4.460334070245436, + "tokens_seen": 174260224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007654002006018054, + "loss": 1.9052, + "theoretical_loss": 4.460121097334004, + "tokens_seen": 174325760 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653841524573722, + "loss": 1.843, + "theoretical_loss": 4.4599082268811285, + "tokens_seen": 174391296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653681043129388, + "loss": 1.8093, + "theoretical_loss": 4.459695458799041, + "tokens_seen": 174456832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653520561685056, + "loss": 1.9478, + "theoretical_loss": 4.459482793000083, + "tokens_seen": 174522368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653360080240722, + "loss": 1.8297, + "theoretical_loss": 4.459270229396701, + "tokens_seen": 174587904 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765319959879639, + "loss": 1.8508, + "theoretical_loss": 4.459057767901454, + "tokens_seen": 174653440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007653039117352056, + "loss": 1.8418, + "theoretical_loss": 4.458845408427004, + "tokens_seen": 174718976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652878635907724, + "loss": 1.722, + "theoretical_loss": 4.458633150886124, + "tokens_seen": 174784512 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765271815446339, + "loss": 1.7616, + "theoretical_loss": 4.458420995191691, + "tokens_seen": 174850048 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652557673019058, + "loss": 1.6859, + "theoretical_loss": 4.458208941256691, + "tokens_seen": 174915584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652397191574724, + "loss": 1.8105, + "theoretical_loss": 4.457996988994218, + "tokens_seen": 174981120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652236710130392, + "loss": 1.762, + "theoretical_loss": 4.457785138317469, + "tokens_seen": 175046656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007652076228686058, + "loss": 1.8061, + "theoretical_loss": 4.4575733891397515, + "tokens_seen": 175112192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651915747241726, + "loss": 1.7592, + "theoretical_loss": 4.457361741374477, + "tokens_seen": 175177728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651755265797392, + "loss": 1.7275, + "theoretical_loss": 4.457150194935165, + "tokens_seen": 175243264 + }, + { + "epoch": 0.05, + "learning_rate": 0.000765159478435306, + "loss": 1.9058, + "theoretical_loss": 4.456938749735439, + "tokens_seen": 175308800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651434302908727, + "loss": 1.8365, + "theoretical_loss": 4.45672740568903, + "tokens_seen": 175374336 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651273821464394, + "loss": 1.8521, + "theoretical_loss": 4.456516162709774, + "tokens_seen": 175439872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007651113340020061, + "loss": 1.8275, + "theoretical_loss": 4.456305020711613, + "tokens_seen": 175505408 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650952858575728, + "loss": 1.8127, + "theoretical_loss": 4.456093979608594, + "tokens_seen": 175570944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650792377131395, + "loss": 1.7682, + "theoretical_loss": 4.455883039314869, + "tokens_seen": 175636480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650631895687062, + "loss": 1.8201, + "theoretical_loss": 4.455672199744695, + "tokens_seen": 175702016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650471414242729, + "loss": 1.8676, + "theoretical_loss": 4.455461460812435, + "tokens_seen": 175767552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650310932798396, + "loss": 1.8545, + "theoretical_loss": 4.455250822432556, + "tokens_seen": 175833088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007650150451354063, + "loss": 1.8366, + "theoretical_loss": 4.4550402845196295, + "tokens_seen": 175898624 + }, + { + "epoch": 0.05, + "learning_rate": 0.000764998996990973, + "loss": 1.7392, + "theoretical_loss": 4.45482984698833, + "tokens_seen": 175964160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649829488465397, + "loss": 1.777, + "theoretical_loss": 4.454619509753439, + "tokens_seen": 176029696 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649669007021064, + "loss": 1.7912, + "theoretical_loss": 4.45440927272984, + "tokens_seen": 176095232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649508525576731, + "loss": 1.7614, + "theoretical_loss": 4.454199135832521, + "tokens_seen": 176160768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649348044132398, + "loss": 1.793, + "theoretical_loss": 4.4539890989765745, + "tokens_seen": 176226304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649187562688065, + "loss": 1.7368, + "theoretical_loss": 4.453779162077193, + "tokens_seen": 176291840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007649027081243732, + "loss": 1.719, + "theoretical_loss": 4.453569325049679, + "tokens_seen": 176357376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648866599799399, + "loss": 1.9351, + "theoretical_loss": 4.453359587809431, + "tokens_seen": 176422912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648706118355066, + "loss": 1.818, + "theoretical_loss": 4.453149950271956, + "tokens_seen": 176488448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648545636910732, + "loss": 1.8517, + "theoretical_loss": 4.452940412352861, + "tokens_seen": 176553984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00076483851554664, + "loss": 1.7958, + "theoretical_loss": 4.452730973967856, + "tokens_seen": 176619520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648224674022066, + "loss": 1.9261, + "theoretical_loss": 4.452521635032754, + "tokens_seen": 176685056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007648064192577734, + "loss": 1.861, + "theoretical_loss": 4.452312395463473, + "tokens_seen": 176750592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00076479037111334, + "loss": 1.6451, + "theoretical_loss": 4.452103255176027, + "tokens_seen": 176816128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647743229689068, + "loss": 1.8513, + "theoretical_loss": 4.451894214086537, + "tokens_seen": 176881664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 73013, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.7999730110168457, + "objective/train/theoretical_loss": 4.4516852721112254, + "objective/train/tokens_used": 197407200, + "theoretical_loss": 4.4516852721112254, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647582748244734, + "loss": 1.8, + "theoretical_loss": 4.4516852721112254, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647422266800402, + "loss": 1.6797, + "theoretical_loss": 4.451476429166416, + "tokens_seen": 177012736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647261785356068, + "loss": 1.7962, + "theoretical_loss": 4.451267685168532, + "tokens_seen": 177078272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007647101303911735, + "loss": 1.9107, + "theoretical_loss": 4.451059040034102, + "tokens_seen": 177143808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646940822467402, + "loss": 1.8403, + "theoretical_loss": 4.450850493679752, + "tokens_seen": 177209344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646780341023069, + "loss": 1.7199, + "theoretical_loss": 4.450642046022211, + "tokens_seen": 177274880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646619859578736, + "loss": 1.8501, + "theoretical_loss": 4.45043369697831, + "tokens_seen": 177340416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646459378134403, + "loss": 1.7564, + "theoretical_loss": 4.450225446464978, + "tokens_seen": 177405952 + }, + { + "epoch": 0.05, + "learning_rate": 0.000764629889669007, + "loss": 1.8507, + "theoretical_loss": 4.450017294399247, + "tokens_seen": 177471488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007646138415245737, + "loss": 1.8915, + "theoretical_loss": 4.449809240698249, + "tokens_seen": 177537024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645977933801404, + "loss": 1.8548, + "theoretical_loss": 4.449601285279215, + "tokens_seen": 177602560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645817452357071, + "loss": 1.7469, + "theoretical_loss": 4.449393428059478, + "tokens_seen": 177668096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645656970912738, + "loss": 1.7879, + "theoretical_loss": 4.449185668956469, + "tokens_seen": 177733632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645496489468405, + "loss": 1.7461, + "theoretical_loss": 4.448978007887721, + "tokens_seen": 177799168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645336008024072, + "loss": 1.8203, + "theoretical_loss": 4.448770444770865, + "tokens_seen": 177864704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645175526579739, + "loss": 1.7898, + "theoretical_loss": 4.448562979523633, + "tokens_seen": 177930240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007645015045135406, + "loss": 1.8204, + "theoretical_loss": 4.448355612063855, + "tokens_seen": 177995776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644854563691073, + "loss": 1.6813, + "theoretical_loss": 4.4481483423094605, + "tokens_seen": 178061312 + }, + { + "epoch": 0.05, + "learning_rate": 0.000764469408224674, + "loss": 1.8498, + "theoretical_loss": 4.447941170178478, + "tokens_seen": 178126848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644533600802407, + "loss": 1.7561, + "theoretical_loss": 4.447734095589036, + "tokens_seen": 178192384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644373119358075, + "loss": 1.8036, + "theoretical_loss": 4.4475271184593606, + "tokens_seen": 178257920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644212637913741, + "loss": 1.7504, + "theoretical_loss": 4.447320238707777, + "tokens_seen": 178323456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007644052156469409, + "loss": 1.9131, + "theoretical_loss": 4.447113456252707, + "tokens_seen": 178388992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643891675025075, + "loss": 1.7592, + "theoretical_loss": 4.446906771012674, + "tokens_seen": 178454528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643731193580743, + "loss": 1.8133, + "theoretical_loss": 4.446700182906298, + "tokens_seen": 178520064 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643570712136409, + "loss": 1.7284, + "theoretical_loss": 4.446493691852297, + "tokens_seen": 178585600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643410230692077, + "loss": 1.8666, + "theoretical_loss": 4.4462872977694845, + "tokens_seen": 178651136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643249749247743, + "loss": 1.7361, + "theoretical_loss": 4.4460810005767755, + "tokens_seen": 178716672 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007643089267803411, + "loss": 1.8114, + "theoretical_loss": 4.445874800193181, + "tokens_seen": 178782208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642928786359077, + "loss": 1.8329, + "theoretical_loss": 4.445668696537808, + "tokens_seen": 178847744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642768304914745, + "loss": 1.8223, + "theoretical_loss": 4.4454626895298635, + "tokens_seen": 178913280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642607823470411, + "loss": 1.7408, + "theoretical_loss": 4.4452567790886475, + "tokens_seen": 178978816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642447342026079, + "loss": 1.7559, + "theoretical_loss": 4.445050965133561, + "tokens_seen": 179044352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642286860581745, + "loss": 1.6963, + "theoretical_loss": 4.4448452475841, + "tokens_seen": 179109888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007642126379137413, + "loss": 1.7475, + "theoretical_loss": 4.444639626359856, + "tokens_seen": 179175424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641965897693079, + "loss": 1.8811, + "theoretical_loss": 4.444434101380519, + "tokens_seen": 179240960 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641805416248747, + "loss": 1.8442, + "theoretical_loss": 4.444228672565874, + "tokens_seen": 179306496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641644934804413, + "loss": 1.8744, + "theoretical_loss": 4.444023339835803, + "tokens_seen": 179372032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641484453360081, + "loss": 1.7885, + "theoretical_loss": 4.443818103110283, + "tokens_seen": 179437568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641323971915747, + "loss": 1.81, + "theoretical_loss": 4.443612962309387, + "tokens_seen": 179503104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641163490471415, + "loss": 1.7993, + "theoretical_loss": 4.4434079173532846, + "tokens_seen": 179568640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007641003009027081, + "loss": 1.8098, + "theoretical_loss": 4.4432029681622405, + "tokens_seen": 179634176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640842527582749, + "loss": 1.7731, + "theoretical_loss": 4.442998114656614, + "tokens_seen": 179699712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640682046138415, + "loss": 1.6623, + "theoretical_loss": 4.442793356756861, + "tokens_seen": 179765248 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640521564694083, + "loss": 1.8102, + "theoretical_loss": 4.442588694383532, + "tokens_seen": 179830784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640361083249749, + "loss": 1.759, + "theoretical_loss": 4.44238412745727, + "tokens_seen": 179896320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640200601805417, + "loss": 1.7714, + "theoretical_loss": 4.442179655898819, + "tokens_seen": 179961856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007640040120361083, + "loss": 1.8469, + "theoretical_loss": 4.44197527962901, + "tokens_seen": 180027392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639879638916751, + "loss": 1.7636, + "theoretical_loss": 4.441770998568773, + "tokens_seen": 180092928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639719157472417, + "loss": 1.7409, + "theoretical_loss": 4.441566812639133, + "tokens_seen": 180158464 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639558676028085, + "loss": 1.7341, + "theoretical_loss": 4.441362721761207, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639398194583752, + "loss": 1.8142, + "theoretical_loss": 4.441158725856207, + "tokens_seen": 180289536 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639237713139419, + "loss": 1.8437, + "theoretical_loss": 4.440954824845438, + "tokens_seen": 180355072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007639077231695086, + "loss": 1.9326, + "theoretical_loss": 4.4407510186503, + "tokens_seen": 180420608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638916750250753, + "loss": 1.809, + "theoretical_loss": 4.440547307192286, + "tokens_seen": 180486144 + }, + { + "epoch": 0.05, + "learning_rate": 0.000763875626880642, + "loss": 1.9295, + "theoretical_loss": 4.440343690392984, + "tokens_seen": 180551680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638595787362087, + "loss": 1.8522, + "theoretical_loss": 4.4401401681740715, + "tokens_seen": 180617216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638435305917754, + "loss": 1.7237, + "theoretical_loss": 4.439936740457323, + "tokens_seen": 180682752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638274824473421, + "loss": 1.789, + "theoretical_loss": 4.439733407164605, + "tokens_seen": 180748288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007638114343029088, + "loss": 1.9288, + "theoretical_loss": 4.439530168217875, + "tokens_seen": 180813824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637953861584755, + "loss": 1.8781, + "theoretical_loss": 4.439327023539187, + "tokens_seen": 180879360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637793380140422, + "loss": 1.7994, + "theoretical_loss": 4.439123973050684, + "tokens_seen": 180944896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637632898696089, + "loss": 1.7935, + "theoretical_loss": 4.438921016674603, + "tokens_seen": 181010432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637472417251756, + "loss": 1.7576, + "theoretical_loss": 4.438718154333274, + "tokens_seen": 181075968 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007637311935807423, + "loss": 1.7834, + "theoretical_loss": 4.438515385949118, + "tokens_seen": 181141504 + }, + { + "epoch": 0.05, + "learning_rate": 0.000763715145436309, + "loss": 1.7537, + "theoretical_loss": 4.438312711444647, + "tokens_seen": 181207040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007636990972918757, + "loss": 1.8203, + "theoretical_loss": 4.438110130742469, + "tokens_seen": 181272576 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007636830491474424, + "loss": 1.7772, + "theoretical_loss": 4.437907643765281, + "tokens_seen": 181338112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007636670010030091, + "loss": 1.7507, + "theoretical_loss": 4.43770525043587, + "tokens_seen": 181403648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0007636509528585758, + "loss": 1.8583, + "theoretical_loss": 4.437502950677116, + "tokens_seen": 181469184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007636349047141425, + "loss": 1.7524, + "theoretical_loss": 4.437300744411992, + "tokens_seen": 181534720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007636188565697092, + "loss": 1.6643, + "theoretical_loss": 4.43709863156356, + "tokens_seen": 181600256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007636028084252759, + "loss": 1.8245, + "theoretical_loss": 4.436896612054975, + "tokens_seen": 181665792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635867602808426, + "loss": 1.8651, + "theoretical_loss": 4.43669468580948, + "tokens_seen": 181731328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635707121364093, + "loss": 1.8246, + "theoretical_loss": 4.43649285275041, + "tokens_seen": 181796864 + }, + { + "epoch": 0.06, + "learning_rate": 0.000763554663991976, + "loss": 1.9091, + "theoretical_loss": 4.436291112801193, + "tokens_seen": 181862400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635386158475427, + "loss": 1.8584, + "theoretical_loss": 4.436089465885345, + "tokens_seen": 181927936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635225677031094, + "loss": 1.724, + "theoretical_loss": 4.435887911926472, + "tokens_seen": 181993472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007635065195586761, + "loss": 1.6499, + "theoretical_loss": 4.435686450848273, + "tokens_seen": 182059008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634904714142428, + "loss": 1.7099, + "theoretical_loss": 4.435485082574533, + "tokens_seen": 182124544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634744232698095, + "loss": 1.699, + "theoretical_loss": 4.43528380702913, + "tokens_seen": 182190080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634583751253762, + "loss": 1.7324, + "theoretical_loss": 4.435082624136031, + "tokens_seen": 182255616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634423269809429, + "loss": 1.747, + "theoretical_loss": 4.4348815338192935, + "tokens_seen": 182321152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634262788365095, + "loss": 1.8122, + "theoretical_loss": 4.434680536003062, + "tokens_seen": 182386688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007634102306920763, + "loss": 1.7842, + "theoretical_loss": 4.434479630611573, + "tokens_seen": 182452224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633941825476429, + "loss": 1.8103, + "theoretical_loss": 4.43427881756915, + "tokens_seen": 182517760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633781344032097, + "loss": 1.8227, + "theoretical_loss": 4.434078096800209, + "tokens_seen": 182583296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633620862587763, + "loss": 1.7468, + "theoretical_loss": 4.433877468229251, + "tokens_seen": 182648832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633460381143431, + "loss": 1.764, + "theoretical_loss": 4.433676931780868, + "tokens_seen": 182714368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633299899699097, + "loss": 1.6694, + "theoretical_loss": 4.43347648737974, + "tokens_seen": 182779904 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007633139418254764, + "loss": 1.8151, + "theoretical_loss": 4.433276134950637, + "tokens_seen": 182845440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632978936810431, + "loss": 1.7672, + "theoretical_loss": 4.4330758744184156, + "tokens_seen": 182910976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632818455366098, + "loss": 1.952, + "theoretical_loss": 4.432875705708021, + "tokens_seen": 182976512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632657973921765, + "loss": 1.7285, + "theoretical_loss": 4.432675628744488, + "tokens_seen": 183042048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632497492477432, + "loss": 1.8432, + "theoretical_loss": 4.432475643452937, + "tokens_seen": 183107584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632337011033099, + "loss": 1.7405, + "theoretical_loss": 4.432275749758579, + "tokens_seen": 183173120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632176529588766, + "loss": 1.8195, + "theoretical_loss": 4.432075947586711, + "tokens_seen": 183238656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007632016048144433, + "loss": 1.8257, + "theoretical_loss": 4.431876236862717, + "tokens_seen": 183304192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00076318555667001, + "loss": 1.6803, + "theoretical_loss": 4.43167661751207, + "tokens_seen": 183369728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631695085255767, + "loss": 1.905, + "theoretical_loss": 4.431477089460331, + "tokens_seen": 183435264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 75209, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.7620269060134888, + "objective/train/theoretical_loss": 4.431277652633145, + "objective/train/tokens_used": 203960800, + "theoretical_loss": 4.431277652633145, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631534603811434, + "loss": 1.762, + "theoretical_loss": 4.431277652633145, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631374122367101, + "loss": 1.8916, + "theoretical_loss": 4.431078306956248, + "tokens_seen": 183566336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631213640922768, + "loss": 1.9074, + "theoretical_loss": 4.43087905235546, + "tokens_seen": 183631872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007631053159478436, + "loss": 1.7626, + "theoretical_loss": 4.43067988875669, + "tokens_seen": 183697408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007630892678034102, + "loss": 1.828, + "theoretical_loss": 4.430480816085931, + "tokens_seen": 183762944 + }, + { + "epoch": 0.06, + "learning_rate": 0.000763073219658977, + "loss": 1.8253, + "theoretical_loss": 4.430281834269265, + "tokens_seen": 183828480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007630571715145436, + "loss": 1.7543, + "theoretical_loss": 4.4300829432328594, + "tokens_seen": 183894016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007630411233701104, + "loss": 1.7862, + "theoretical_loss": 4.429884142902968, + "tokens_seen": 183959552 + }, + { + "epoch": 0.06, + "learning_rate": 0.000763025075225677, + "loss": 1.6709, + "theoretical_loss": 4.429685433205931, + "tokens_seen": 184025088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007630090270812438, + "loss": 1.798, + "theoretical_loss": 4.429486814068174, + "tokens_seen": 184090624 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629929789368104, + "loss": 1.8637, + "theoretical_loss": 4.4292882854162094, + "tokens_seen": 184156160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629769307923772, + "loss": 1.7704, + "theoretical_loss": 4.429089847176634, + "tokens_seen": 184221696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629608826479438, + "loss": 1.7683, + "theoretical_loss": 4.428891499276132, + "tokens_seen": 184287232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629448345035106, + "loss": 1.8815, + "theoretical_loss": 4.42869324164147, + "tokens_seen": 184352768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007629287863590772, + "loss": 1.7091, + "theoretical_loss": 4.428495074199505, + "tokens_seen": 184418304 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762912738214644, + "loss": 1.6927, + "theoretical_loss": 4.428296996877176, + "tokens_seen": 184483840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628966900702106, + "loss": 1.7418, + "theoretical_loss": 4.428099009601506, + "tokens_seen": 184549376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628806419257774, + "loss": 1.8684, + "theoretical_loss": 4.4279011122996055, + "tokens_seen": 184614912 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762864593781344, + "loss": 1.7133, + "theoretical_loss": 4.427703304898668, + "tokens_seen": 184680448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628485456369108, + "loss": 1.7586, + "theoretical_loss": 4.427505587325974, + "tokens_seen": 184745984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628324974924774, + "loss": 1.7689, + "theoretical_loss": 4.427307959508885, + "tokens_seen": 184811520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628164493480442, + "loss": 1.8452, + "theoretical_loss": 4.4271104213748504, + "tokens_seen": 184877056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007628004012036108, + "loss": 1.7734, + "theoretical_loss": 4.426912972851404, + "tokens_seen": 184942592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627843530591776, + "loss": 1.9269, + "theoretical_loss": 4.426715613866159, + "tokens_seen": 185008128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627683049147442, + "loss": 1.783, + "theoretical_loss": 4.426518344346819, + "tokens_seen": 185073664 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762752256770311, + "loss": 1.7716, + "theoretical_loss": 4.426321164221168, + "tokens_seen": 185139200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627362086258777, + "loss": 1.7277, + "theoretical_loss": 4.426124073417074, + "tokens_seen": 185204736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627201604814444, + "loss": 1.8053, + "theoretical_loss": 4.42592707186249, + "tokens_seen": 185270272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007627041123370111, + "loss": 1.6595, + "theoretical_loss": 4.42573015948545, + "tokens_seen": 185335808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626880641925778, + "loss": 1.8182, + "theoretical_loss": 4.425533336214076, + "tokens_seen": 185401344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626720160481445, + "loss": 1.7558, + "theoretical_loss": 4.425336601976568, + "tokens_seen": 185466880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626559679037112, + "loss": 1.7747, + "theoretical_loss": 4.425139956701214, + "tokens_seen": 185532416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626399197592779, + "loss": 1.7903, + "theoretical_loss": 4.42494340031638, + "tokens_seen": 185597952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626238716148446, + "loss": 1.7132, + "theoretical_loss": 4.424746932750521, + "tokens_seen": 185663488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007626078234704113, + "loss": 1.9077, + "theoretical_loss": 4.4245505539321694, + "tokens_seen": 185729024 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762591775325978, + "loss": 1.9028, + "theoretical_loss": 4.424354263789943, + "tokens_seen": 185794560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625757271815447, + "loss": 1.8168, + "theoretical_loss": 4.424158062252541, + "tokens_seen": 185860096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625596790371114, + "loss": 1.8349, + "theoretical_loss": 4.423961949248748, + "tokens_seen": 185925632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625436308926781, + "loss": 1.8333, + "theoretical_loss": 4.423765924707427, + "tokens_seen": 185991168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625275827482448, + "loss": 1.739, + "theoretical_loss": 4.4235699885575235, + "tokens_seen": 186056704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007625115346038115, + "loss": 1.6889, + "theoretical_loss": 4.423374140728068, + "tokens_seen": 186122240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624954864593782, + "loss": 1.746, + "theoretical_loss": 4.423178381148172, + "tokens_seen": 186187776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624794383149449, + "loss": 1.8359, + "theoretical_loss": 4.422982709747026, + "tokens_seen": 186253312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624633901705116, + "loss": 1.7539, + "theoretical_loss": 4.4227871264539065, + "tokens_seen": 186318848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624473420260783, + "loss": 1.6667, + "theoretical_loss": 4.422591631198167, + "tokens_seen": 186384384 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762431293881645, + "loss": 1.7127, + "theoretical_loss": 4.422396223909248, + "tokens_seen": 186449920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007624152457372117, + "loss": 1.8472, + "theoretical_loss": 4.422200904516666, + "tokens_seen": 186515456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623991975927784, + "loss": 1.7118, + "theoretical_loss": 4.4220056729500214, + "tokens_seen": 186580992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623831494483451, + "loss": 1.7744, + "theoretical_loss": 4.4218105291389955, + "tokens_seen": 186646528 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623671013039118, + "loss": 1.6852, + "theoretical_loss": 4.4216154730133495, + "tokens_seen": 186712064 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623510531594785, + "loss": 1.9056, + "theoretical_loss": 4.421420504502928, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623350050150452, + "loss": 1.7766, + "theoretical_loss": 4.421225623537652, + "tokens_seen": 186843136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623189568706119, + "loss": 1.6958, + "theoretical_loss": 4.421030830047528, + "tokens_seen": 186908672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007623029087261786, + "loss": 1.8023, + "theoretical_loss": 4.420836123962639, + "tokens_seen": 186974208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622868605817453, + "loss": 1.7609, + "theoretical_loss": 4.420641505213151, + "tokens_seen": 187039744 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762270812437312, + "loss": 1.8666, + "theoretical_loss": 4.42044697372931, + "tokens_seen": 187105280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622547642928787, + "loss": 1.6924, + "theoretical_loss": 4.420252529441439, + "tokens_seen": 187170816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622387161484454, + "loss": 1.7689, + "theoretical_loss": 4.420058172279945, + "tokens_seen": 187236352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622226680040121, + "loss": 1.7036, + "theoretical_loss": 4.419863902175313, + "tokens_seen": 187301888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007622066198595788, + "loss": 1.8167, + "theoretical_loss": 4.419669719058108, + "tokens_seen": 187367424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621905717151455, + "loss": 1.7143, + "theoretical_loss": 4.419475622858974, + "tokens_seen": 187432960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621745235707122, + "loss": 1.7541, + "theoretical_loss": 4.419281613508637, + "tokens_seen": 187498496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621584754262789, + "loss": 1.7815, + "theoretical_loss": 4.419087690937898, + "tokens_seen": 187564032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621424272818456, + "loss": 1.7691, + "theoretical_loss": 4.418893855077641, + "tokens_seen": 187629568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007621263791374123, + "loss": 1.7913, + "theoretical_loss": 4.4187001058588296, + "tokens_seen": 187695104 + }, + { + "epoch": 0.06, + "learning_rate": 0.000762110330992979, + "loss": 1.7181, + "theoretical_loss": 4.418506443212503, + "tokens_seen": 187760640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620942828485457, + "loss": 1.7297, + "theoretical_loss": 4.4183128670697815, + "tokens_seen": 187826176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620782347041124, + "loss": 1.7215, + "theoretical_loss": 4.418119377361863, + "tokens_seen": 187891712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620621865596791, + "loss": 1.7652, + "theoretical_loss": 4.417925974020028, + "tokens_seen": 187957248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620461384152458, + "loss": 1.8948, + "theoretical_loss": 4.4177326569756294, + "tokens_seen": 188022784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620300902708124, + "loss": 1.8231, + "theoretical_loss": 4.417539426160104, + "tokens_seen": 188088320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007620140421263792, + "loss": 1.7161, + "theoretical_loss": 4.417346281504963, + "tokens_seen": 188153856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619979939819458, + "loss": 1.8138, + "theoretical_loss": 4.417153222941798, + "tokens_seen": 188219392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619819458375126, + "loss": 1.9123, + "theoretical_loss": 4.416960250402279, + "tokens_seen": 188284928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619658976930792, + "loss": 1.7367, + "theoretical_loss": 4.416767363818153, + "tokens_seen": 188350464 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761949849548646, + "loss": 1.8215, + "theoretical_loss": 4.4165745631212445, + "tokens_seen": 188416000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619338014042126, + "loss": 1.7358, + "theoretical_loss": 4.416381848243456, + "tokens_seen": 188481536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007619177532597794, + "loss": 1.8557, + "theoretical_loss": 4.41618921911677, + "tokens_seen": 188547072 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761901705115346, + "loss": 1.7209, + "theoretical_loss": 4.415996675673242, + "tokens_seen": 188612608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618856569709127, + "loss": 1.8605, + "theoretical_loss": 4.415804217845009, + "tokens_seen": 188678144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618696088264794, + "loss": 1.8496, + "theoretical_loss": 4.415611845564284, + "tokens_seen": 188743680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618535606820461, + "loss": 1.775, + "theoretical_loss": 4.415419558763356, + "tokens_seen": 188809216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618375125376128, + "loss": 1.7735, + "theoretical_loss": 4.415227357374592, + "tokens_seen": 188874752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618214643931795, + "loss": 1.8272, + "theoretical_loss": 4.415035241330436, + "tokens_seen": 188940288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007618054162487462, + "loss": 1.7899, + "theoretical_loss": 4.41484321056341, + "tokens_seen": 189005824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617893681043129, + "loss": 1.7192, + "theoretical_loss": 4.4146512650061105, + "tokens_seen": 189071360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617733199598796, + "loss": 1.7036, + "theoretical_loss": 4.414459404591211, + "tokens_seen": 189136896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617572718154463, + "loss": 1.775, + "theoretical_loss": 4.414267629251463, + "tokens_seen": 189202432 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761741223671013, + "loss": 1.7385, + "theoretical_loss": 4.414075938919694, + "tokens_seen": 189267968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617251755265797, + "loss": 1.5763, + "theoretical_loss": 4.413884333528807, + "tokens_seen": 189333504 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007617091273821465, + "loss": 1.7538, + "theoretical_loss": 4.413692813011781, + "tokens_seen": 189399040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616930792377131, + "loss": 1.7807, + "theoretical_loss": 4.413501377301671, + "tokens_seen": 189464576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616770310932799, + "loss": 1.8343, + "theoretical_loss": 4.41331002633161, + "tokens_seen": 189530112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616609829488465, + "loss": 1.7837, + "theoretical_loss": 4.413118760034804, + "tokens_seen": 189595648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616449348044133, + "loss": 1.8891, + "theoretical_loss": 4.4129275783445365, + "tokens_seen": 189661184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616288866599799, + "loss": 1.7112, + "theoretical_loss": 4.412736481194166, + "tokens_seen": 189726720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007616128385155467, + "loss": 1.7991, + "theoretical_loss": 4.412545468517127, + "tokens_seen": 189792256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615967903711133, + "loss": 1.8096, + "theoretical_loss": 4.412354540246929, + "tokens_seen": 189857792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615807422266801, + "loss": 1.7634, + "theoretical_loss": 4.4121636963171555, + "tokens_seen": 189923328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615646940822467, + "loss": 1.7403, + "theoretical_loss": 4.411972936661467, + "tokens_seen": 189988864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 77536, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.6837046146392822, + "objective/train/theoretical_loss": 4.411782261213599, + "objective/train/tokens_used": 210514400, + "theoretical_loss": 4.411782261213599, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615486459378135, + "loss": 1.6837, + "theoretical_loss": 4.411782261213599, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615325977933802, + "loss": 1.7882, + "theoretical_loss": 4.41159166990736, + "tokens_seen": 190119936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615165496489469, + "loss": 1.7688, + "theoretical_loss": 4.411401162676636, + "tokens_seen": 190185472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007615005015045136, + "loss": 1.6323, + "theoretical_loss": 4.411210739455386, + "tokens_seen": 190251008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614844533600803, + "loss": 1.7742, + "theoretical_loss": 4.4110204001776445, + "tokens_seen": 190316544 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761468405215647, + "loss": 1.7357, + "theoretical_loss": 4.410830144777519, + "tokens_seen": 190382080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614523570712137, + "loss": 1.6707, + "theoretical_loss": 4.410639973189192, + "tokens_seen": 190447616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614363089267804, + "loss": 1.859, + "theoretical_loss": 4.410449885346922, + "tokens_seen": 190513152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614202607823471, + "loss": 1.7445, + "theoretical_loss": 4.4102598811850395, + "tokens_seen": 190578688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007614042126379138, + "loss": 1.78, + "theoretical_loss": 4.410069960637949, + "tokens_seen": 190644224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613881644934805, + "loss": 1.7216, + "theoretical_loss": 4.409880123640131, + "tokens_seen": 190709760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613721163490472, + "loss": 1.8239, + "theoretical_loss": 4.409690370126139, + "tokens_seen": 190775296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613560682046139, + "loss": 1.6882, + "theoretical_loss": 4.409500700030598, + "tokens_seen": 190840832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613400200601806, + "loss": 1.7236, + "theoretical_loss": 4.409311113288211, + "tokens_seen": 190906368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007613239719157473, + "loss": 1.8261, + "theoretical_loss": 4.409121609833749, + "tokens_seen": 190971904 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761307923771314, + "loss": 1.7441, + "theoretical_loss": 4.408932189602061, + "tokens_seen": 191037440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612918756268807, + "loss": 1.7432, + "theoretical_loss": 4.408742852528068, + "tokens_seen": 191102976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612758274824474, + "loss": 1.8753, + "theoretical_loss": 4.408553598546764, + "tokens_seen": 191168512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612597793380141, + "loss": 1.7225, + "theoretical_loss": 4.408364427593215, + "tokens_seen": 191234048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612437311935808, + "loss": 1.7486, + "theoretical_loss": 4.40817533960256, + "tokens_seen": 191299584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612276830491475, + "loss": 1.7881, + "theoretical_loss": 4.407986334510014, + "tokens_seen": 191365120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007612116349047142, + "loss": 1.7842, + "theoretical_loss": 4.407797412250861, + "tokens_seen": 191430656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611955867602809, + "loss": 1.8266, + "theoretical_loss": 4.40760857276046, + "tokens_seen": 191496192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611795386158476, + "loss": 1.7804, + "theoretical_loss": 4.407419815974242, + "tokens_seen": 191561728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611634904714143, + "loss": 1.7048, + "theoretical_loss": 4.407231141827708, + "tokens_seen": 191627264 + }, + { + "epoch": 0.06, + "learning_rate": 0.000761147442326981, + "loss": 1.6665, + "theoretical_loss": 4.407042550256436, + "tokens_seen": 191692800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611313941825477, + "loss": 1.7333, + "theoretical_loss": 4.406854041196072, + "tokens_seen": 191758336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007611153460381144, + "loss": 1.8141, + "theoretical_loss": 4.406665614582338, + "tokens_seen": 191823872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610992978936811, + "loss": 1.806, + "theoretical_loss": 4.406477270351024, + "tokens_seen": 191889408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610832497492478, + "loss": 1.7918, + "theoretical_loss": 4.4062890084379935, + "tokens_seen": 191954944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610672016048145, + "loss": 1.7733, + "theoretical_loss": 4.406100828779183, + "tokens_seen": 192020480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610511534603812, + "loss": 1.878, + "theoretical_loss": 4.4059127313106, + "tokens_seen": 192086016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610351053159479, + "loss": 1.7692, + "theoretical_loss": 4.405724715968323, + "tokens_seen": 192151552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610190571715146, + "loss": 1.7859, + "theoretical_loss": 4.405536782688502, + "tokens_seen": 192217088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007610030090270813, + "loss": 1.7671, + "theoretical_loss": 4.405348931407359, + "tokens_seen": 192282624 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760986960882648, + "loss": 1.8184, + "theoretical_loss": 4.405161162061187, + "tokens_seen": 192348160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609709127382147, + "loss": 1.7967, + "theoretical_loss": 4.404973474586351, + "tokens_seen": 192413696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609548645937814, + "loss": 1.7232, + "theoretical_loss": 4.404785868919285, + "tokens_seen": 192479232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609388164493481, + "loss": 1.8081, + "theoretical_loss": 4.404598344996497, + "tokens_seen": 192544768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609227683049148, + "loss": 1.724, + "theoretical_loss": 4.404410902754562, + "tokens_seen": 192610304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007609067201604815, + "loss": 1.7337, + "theoretical_loss": 4.404223542130129, + "tokens_seen": 192675840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608906720160482, + "loss": 1.5902, + "theoretical_loss": 4.404036263059917, + "tokens_seen": 192741376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608746238716149, + "loss": 1.7671, + "theoretical_loss": 4.403849065480714, + "tokens_seen": 192806912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608585757271816, + "loss": 1.7116, + "theoretical_loss": 4.403661949329381, + "tokens_seen": 192872448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608425275827483, + "loss": 1.7572, + "theoretical_loss": 4.403474914542848, + "tokens_seen": 192937984 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760826479438315, + "loss": 1.8424, + "theoretical_loss": 4.403287961058112, + "tokens_seen": 193003520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007608104312938817, + "loss": 1.9331, + "theoretical_loss": 4.403101088812248, + "tokens_seen": 193069056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607943831494484, + "loss": 1.7388, + "theoretical_loss": 4.4029142977423925, + "tokens_seen": 193134592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607783350050151, + "loss": 1.7969, + "theoretical_loss": 4.402727587785758, + "tokens_seen": 193200128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607622868605818, + "loss": 1.6588, + "theoretical_loss": 4.402540958879625, + "tokens_seen": 193265664 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607462387161485, + "loss": 1.7005, + "theoretical_loss": 4.402354410961341, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607301905717152, + "loss": 1.8124, + "theoretical_loss": 4.402167943968328, + "tokens_seen": 193396736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007607141424272819, + "loss": 1.8199, + "theoretical_loss": 4.4019815578380745, + "tokens_seen": 193462272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606980942828486, + "loss": 1.7504, + "theoretical_loss": 4.401795252508139, + "tokens_seen": 193527808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606820461384153, + "loss": 1.7858, + "theoretical_loss": 4.401609027916149, + "tokens_seen": 193593344 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760665997993982, + "loss": 1.7531, + "theoretical_loss": 4.4014228839998015, + "tokens_seen": 193658880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606499498495487, + "loss": 1.7594, + "theoretical_loss": 4.401236820696863, + "tokens_seen": 193724416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606339017051153, + "loss": 1.6804, + "theoretical_loss": 4.40105083794517, + "tokens_seen": 193789952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606178535606821, + "loss": 1.7331, + "theoretical_loss": 4.400864935682625, + "tokens_seen": 193855488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007606018054162487, + "loss": 1.7447, + "theoretical_loss": 4.400679113847202, + "tokens_seen": 193921024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605857572718155, + "loss": 1.7684, + "theoretical_loss": 4.400493372376943, + "tokens_seen": 193986560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605697091273821, + "loss": 1.7692, + "theoretical_loss": 4.400307711209957, + "tokens_seen": 194052096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605536609829489, + "loss": 1.7224, + "theoretical_loss": 4.400122130284426, + "tokens_seen": 194117632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605376128385155, + "loss": 1.7067, + "theoretical_loss": 4.399936629538595, + "tokens_seen": 194183168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605215646940823, + "loss": 1.7667, + "theoretical_loss": 4.3997512089107795, + "tokens_seen": 194248704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007605055165496489, + "loss": 1.72, + "theoretical_loss": 4.399565868339366, + "tokens_seen": 194314240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604894684052156, + "loss": 1.7202, + "theoretical_loss": 4.399380607762804, + "tokens_seen": 194379776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604734202607823, + "loss": 1.6807, + "theoretical_loss": 4.399195427119616, + "tokens_seen": 194445312 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760457372116349, + "loss": 1.7481, + "theoretical_loss": 4.399010326348388, + "tokens_seen": 194510848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604413239719157, + "loss": 1.7909, + "theoretical_loss": 4.398825305387778, + "tokens_seen": 194576384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604252758274824, + "loss": 1.8548, + "theoretical_loss": 4.39864036417651, + "tokens_seen": 194641920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007604092276830491, + "loss": 1.8049, + "theoretical_loss": 4.398455502653373, + "tokens_seen": 194707456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603931795386158, + "loss": 1.5683, + "theoretical_loss": 4.398270720757228, + "tokens_seen": 194772992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603771313941825, + "loss": 1.7704, + "theoretical_loss": 4.398086018427001, + "tokens_seen": 194838528 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603610832497492, + "loss": 1.8903, + "theoretical_loss": 4.397901395601687, + "tokens_seen": 194904064 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760345035105316, + "loss": 1.8512, + "theoretical_loss": 4.397716852220345, + "tokens_seen": 194969600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603289869608827, + "loss": 1.7635, + "theoretical_loss": 4.397532388222105, + "tokens_seen": 195035136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007603129388164494, + "loss": 1.7018, + "theoretical_loss": 4.397348003546161, + "tokens_seen": 195100672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602968906720161, + "loss": 1.7344, + "theoretical_loss": 4.397163698131776, + "tokens_seen": 195166208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602808425275828, + "loss": 1.831, + "theoretical_loss": 4.39697947191828, + "tokens_seen": 195231744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602647943831495, + "loss": 1.7557, + "theoretical_loss": 4.396795324845069, + "tokens_seen": 195297280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602487462387162, + "loss": 1.6197, + "theoretical_loss": 4.396611256851604, + "tokens_seen": 195362816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602326980942829, + "loss": 1.6612, + "theoretical_loss": 4.3964272678774154, + "tokens_seen": 195428352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602166499498496, + "loss": 1.7702, + "theoretical_loss": 4.396243357862099, + "tokens_seen": 195493888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007602006018054163, + "loss": 1.7023, + "theoretical_loss": 4.396059526745318, + "tokens_seen": 195559424 + }, + { + "epoch": 0.06, + "learning_rate": 0.000760184553660983, + "loss": 1.736, + "theoretical_loss": 4.395875774466799, + "tokens_seen": 195624960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601685055165497, + "loss": 1.7532, + "theoretical_loss": 4.395692100966338, + "tokens_seen": 195690496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601524573721164, + "loss": 1.6109, + "theoretical_loss": 4.395508506183796, + "tokens_seen": 195756032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601364092276831, + "loss": 1.7201, + "theoretical_loss": 4.395324990059098, + "tokens_seen": 195821568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601203610832498, + "loss": 1.6878, + "theoretical_loss": 4.395141552532239, + "tokens_seen": 195887104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007601043129388165, + "loss": 1.9185, + "theoretical_loss": 4.394958193543277, + "tokens_seen": 195952640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600882647943832, + "loss": 1.6981, + "theoretical_loss": 4.394774913032337, + "tokens_seen": 196018176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600722166499499, + "loss": 1.8504, + "theoretical_loss": 4.394591710939608, + "tokens_seen": 196083712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600561685055166, + "loss": 1.6923, + "theoretical_loss": 4.394408587205344, + "tokens_seen": 196149248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600401203610833, + "loss": 1.6863, + "theoretical_loss": 4.39422554176987, + "tokens_seen": 196214784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00076002407221665, + "loss": 1.6867, + "theoretical_loss": 4.39404257457357, + "tokens_seen": 196280320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007600080240722167, + "loss": 1.8805, + "theoretical_loss": 4.393859685556895, + "tokens_seen": 196345856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599919759277834, + "loss": 1.7739, + "theoretical_loss": 4.393676874660364, + "tokens_seen": 196411392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599759277833501, + "loss": 1.6739, + "theoretical_loss": 4.393494141824557, + "tokens_seen": 196476928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599598796389168, + "loss": 1.7102, + "theoretical_loss": 4.393311486990122, + "tokens_seen": 196542464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 79941, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.7070708274841309, + "objective/train/theoretical_loss": 4.39312891009777, + "objective/train/tokens_used": 217068000, + "theoretical_loss": 4.39312891009777, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599438314944835, + "loss": 1.7071, + "theoretical_loss": 4.39312891009777, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599277833500502, + "loss": 1.7248, + "theoretical_loss": 4.392946411088279, + "tokens_seen": 196673536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007599117352056169, + "loss": 1.767, + "theoretical_loss": 4.392763989902489, + "tokens_seen": 196739072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598956870611836, + "loss": 1.7185, + "theoretical_loss": 4.3925816464813074, + "tokens_seen": 196804608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598796389167503, + "loss": 1.8089, + "theoretical_loss": 4.392399380765704, + "tokens_seen": 196870144 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759863590772317, + "loss": 1.6937, + "theoretical_loss": 4.392217192696712, + "tokens_seen": 196935680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598475426278837, + "loss": 1.5537, + "theoretical_loss": 4.392035082215434, + "tokens_seen": 197001216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598314944834504, + "loss": 1.696, + "theoretical_loss": 4.391853049263032, + "tokens_seen": 197066752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007598154463390171, + "loss": 1.8234, + "theoretical_loss": 4.391671093780733, + "tokens_seen": 197132288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597993981945838, + "loss": 1.8006, + "theoretical_loss": 4.39148921570983, + "tokens_seen": 197197824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597833500501505, + "loss": 1.773, + "theoretical_loss": 4.391307414991678, + "tokens_seen": 197263360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597673019057172, + "loss": 1.721, + "theoretical_loss": 4.391125691567697, + "tokens_seen": 197328896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597512537612839, + "loss": 1.7165, + "theoretical_loss": 4.390944045379371, + "tokens_seen": 197394432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597352056168506, + "loss": 1.7357, + "theoretical_loss": 4.390762476368245, + "tokens_seen": 197459968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007597191574724173, + "loss": 1.8413, + "theoretical_loss": 4.390580984475932, + "tokens_seen": 197525504 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759703109327984, + "loss": 1.796, + "theoretical_loss": 4.3903995696441065, + "tokens_seen": 197591040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596870611835507, + "loss": 1.7677, + "theoretical_loss": 4.390218231814504, + "tokens_seen": 197656576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596710130391174, + "loss": 1.6409, + "theoretical_loss": 4.390036970928929, + "tokens_seen": 197722112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596549648946841, + "loss": 1.6503, + "theoretical_loss": 4.3898557869292425, + "tokens_seen": 197787648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596389167502508, + "loss": 1.8157, + "theoretical_loss": 4.389674679757374, + "tokens_seen": 197853184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596228686058175, + "loss": 1.6408, + "theoretical_loss": 4.389493649355314, + "tokens_seen": 197918720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007596068204613842, + "loss": 1.7022, + "theoretical_loss": 4.389312695665115, + "tokens_seen": 197984256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595907723169509, + "loss": 1.7141, + "theoretical_loss": 4.389131818628895, + "tokens_seen": 198049792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595747241725176, + "loss": 1.7267, + "theoretical_loss": 4.388951018188832, + "tokens_seen": 198115328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595586760280843, + "loss": 1.7026, + "theoretical_loss": 4.388770294287169, + "tokens_seen": 198180864 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759542627883651, + "loss": 1.9569, + "theoretical_loss": 4.38858964686621, + "tokens_seen": 198246400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595265797392177, + "loss": 1.7857, + "theoretical_loss": 4.388409075868323, + "tokens_seen": 198311936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007595105315947844, + "loss": 1.7623, + "theoretical_loss": 4.388228581235937, + "tokens_seen": 198377472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594944834503511, + "loss": 1.7654, + "theoretical_loss": 4.388048162911545, + "tokens_seen": 198443008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594784353059178, + "loss": 1.834, + "theoretical_loss": 4.3878678208377, + "tokens_seen": 198508544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594623871614845, + "loss": 1.6927, + "theoretical_loss": 4.38768755495702, + "tokens_seen": 198574080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594463390170512, + "loss": 1.7201, + "theoretical_loss": 4.387507365212182, + "tokens_seen": 198639616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594302908726179, + "loss": 1.8844, + "theoretical_loss": 4.387327251545927, + "tokens_seen": 198705152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007594142427281846, + "loss": 1.7649, + "theoretical_loss": 4.387147213901059, + "tokens_seen": 198770688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593981945837513, + "loss": 1.7685, + "theoretical_loss": 4.386967252220441, + "tokens_seen": 198836224 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759382146439318, + "loss": 1.6499, + "theoretical_loss": 4.386787366447, + "tokens_seen": 198901760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593660982948847, + "loss": 1.6753, + "theoretical_loss": 4.386607556523723, + "tokens_seen": 198967296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593500501504514, + "loss": 1.6749, + "theoretical_loss": 4.386427822393661, + "tokens_seen": 199032832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593340020060181, + "loss": 1.843, + "theoretical_loss": 4.386248163999923, + "tokens_seen": 199098368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593179538615848, + "loss": 1.8036, + "theoretical_loss": 4.3860685812856826, + "tokens_seen": 199163904 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007593019057171515, + "loss": 1.77, + "theoretical_loss": 4.385889074194173, + "tokens_seen": 199229440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007592858575727182, + "loss": 1.7111, + "theoretical_loss": 4.385709642668688, + "tokens_seen": 199294976 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759269809428285, + "loss": 1.7295, + "theoretical_loss": 4.385530286652586, + "tokens_seen": 199360512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007592537612838516, + "loss": 1.7991, + "theoretical_loss": 4.385351006089283, + "tokens_seen": 199426048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007592377131394182, + "loss": 1.7579, + "theoretical_loss": 4.385171800922257, + "tokens_seen": 199491584 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759221664994985, + "loss": 1.7003, + "theoretical_loss": 4.384992671095047, + "tokens_seen": 199557120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007592056168505516, + "loss": 1.7583, + "theoretical_loss": 4.384813616551252, + "tokens_seen": 199622656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591895687061184, + "loss": 1.8127, + "theoretical_loss": 4.384634637234534, + "tokens_seen": 199688192 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759173520561685, + "loss": 1.7827, + "theoretical_loss": 4.384455733088612, + "tokens_seen": 199753728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591574724172518, + "loss": 1.7641, + "theoretical_loss": 4.38427690405727, + "tokens_seen": 199819264 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591414242728184, + "loss": 1.7277, + "theoretical_loss": 4.384098150084349, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591253761283852, + "loss": 1.7378, + "theoretical_loss": 4.383919471113751, + "tokens_seen": 199950336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007591093279839518, + "loss": 1.7409, + "theoretical_loss": 4.383740867089441, + "tokens_seen": 200015872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007590932798395186, + "loss": 1.8297, + "theoretical_loss": 4.383562337955439, + "tokens_seen": 200081408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007590772316950852, + "loss": 1.804, + "theoretical_loss": 4.3833838836558305, + "tokens_seen": 200146944 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759061183550652, + "loss": 1.7346, + "theoretical_loss": 4.383205504134758, + "tokens_seen": 200212480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007590451354062186, + "loss": 1.7105, + "theoretical_loss": 4.383027199336424, + "tokens_seen": 200278016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007590290872617854, + "loss": 1.8286, + "theoretical_loss": 4.382848969205092, + "tokens_seen": 200343552 + }, + { + "epoch": 0.06, + "learning_rate": 0.000759013039117352, + "loss": 1.7716, + "theoretical_loss": 4.3826708136850865, + "tokens_seen": 200409088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589969909729188, + "loss": 1.7295, + "theoretical_loss": 4.382492732720786, + "tokens_seen": 200474624 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589809428284855, + "loss": 1.7726, + "theoretical_loss": 4.382314726256637, + "tokens_seen": 200540160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589648946840522, + "loss": 1.7471, + "theoretical_loss": 4.382136794237138, + "tokens_seen": 200605696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589488465396189, + "loss": 1.8189, + "theoretical_loss": 4.3819589366068525, + "tokens_seen": 200671232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589327983951856, + "loss": 1.7286, + "theoretical_loss": 4.381781153310399, + "tokens_seen": 200736768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007589167502507523, + "loss": 1.7286, + "theoretical_loss": 4.381603444292459, + "tokens_seen": 200802304 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758900702106319, + "loss": 1.769, + "theoretical_loss": 4.38142580949777, + "tokens_seen": 200867840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588846539618857, + "loss": 1.7038, + "theoretical_loss": 4.381248248871131, + "tokens_seen": 200933376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588686058174524, + "loss": 1.5798, + "theoretical_loss": 4.3810707623574, + "tokens_seen": 200998912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588525576730191, + "loss": 1.7143, + "theoretical_loss": 4.380893349901491, + "tokens_seen": 201064448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588365095285858, + "loss": 1.7735, + "theoretical_loss": 4.380716011448381, + "tokens_seen": 201129984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588204613841525, + "loss": 1.902, + "theoretical_loss": 4.380538746943103, + "tokens_seen": 201195520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007588044132397192, + "loss": 1.8396, + "theoretical_loss": 4.380361556330749, + "tokens_seen": 201261056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587883650952859, + "loss": 1.801, + "theoretical_loss": 4.380184439556472, + "tokens_seen": 201326592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587723169508526, + "loss": 1.6954, + "theoretical_loss": 4.38000739656548, + "tokens_seen": 201392128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587562688064193, + "loss": 1.7825, + "theoretical_loss": 4.379830427303041, + "tokens_seen": 201457664 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758740220661986, + "loss": 1.8515, + "theoretical_loss": 4.379653531714483, + "tokens_seen": 201523200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587241725175527, + "loss": 1.7923, + "theoretical_loss": 4.37947670974519, + "tokens_seen": 201588736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007587081243731194, + "loss": 1.7959, + "theoretical_loss": 4.379299961340605, + "tokens_seen": 201654272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586920762286861, + "loss": 1.829, + "theoretical_loss": 4.37912328644623, + "tokens_seen": 201719808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586760280842528, + "loss": 1.659, + "theoretical_loss": 4.378946685007623, + "tokens_seen": 201785344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586599799398195, + "loss": 1.7904, + "theoretical_loss": 4.378770156970403, + "tokens_seen": 201850880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586439317953862, + "loss": 1.7975, + "theoretical_loss": 4.378593702280243, + "tokens_seen": 201916416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586278836509529, + "loss": 1.6993, + "theoretical_loss": 4.378417320882879, + "tokens_seen": 201981952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007586118355065196, + "loss": 1.761, + "theoretical_loss": 4.378241012724099, + "tokens_seen": 202047488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585957873620863, + "loss": 1.7186, + "theoretical_loss": 4.378064777749753, + "tokens_seen": 202113024 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758579739217653, + "loss": 1.7332, + "theoretical_loss": 4.377888615905746, + "tokens_seen": 202178560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585636910732197, + "loss": 1.7773, + "theoretical_loss": 4.377712527138042, + "tokens_seen": 202244096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585476429287864, + "loss": 1.6866, + "theoretical_loss": 4.3775365113926625, + "tokens_seen": 202309632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585315947843531, + "loss": 1.6575, + "theoretical_loss": 4.377360568615685, + "tokens_seen": 202375168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007585155466399198, + "loss": 1.7092, + "theoretical_loss": 4.377184698753244, + "tokens_seen": 202440704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584994984954865, + "loss": 1.7431, + "theoretical_loss": 4.377008901751536, + "tokens_seen": 202506240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584834503510532, + "loss": 1.8526, + "theoretical_loss": 4.376833177556806, + "tokens_seen": 202571776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584674022066199, + "loss": 1.7593, + "theoretical_loss": 4.3766575261153635, + "tokens_seen": 202637312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584513540621866, + "loss": 1.6966, + "theoretical_loss": 4.3764819473735725, + "tokens_seen": 202702848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584353059177533, + "loss": 1.7525, + "theoretical_loss": 4.376306441277852, + "tokens_seen": 202768384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00075841925777332, + "loss": 1.6903, + "theoretical_loss": 4.376131007774681, + "tokens_seen": 202833920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007584032096288867, + "loss": 1.6677, + "theoretical_loss": 4.375955646810594, + "tokens_seen": 202899456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583871614844534, + "loss": 1.732, + "theoretical_loss": 4.37578035833218, + "tokens_seen": 202964992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583711133400201, + "loss": 1.849, + "theoretical_loss": 4.375605142286087, + "tokens_seen": 203030528 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583550651955868, + "loss": 1.8048, + "theoretical_loss": 4.37542999861902, + "tokens_seen": 203096064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 82204, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.6902105808258057, + "objective/train/theoretical_loss": 4.375254927277737, + "objective/train/tokens_used": 223621600, + "theoretical_loss": 4.375254927277737, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583390170511535, + "loss": 1.6902, + "theoretical_loss": 4.375254927277737, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583229689067202, + "loss": 1.7447, + "theoretical_loss": 4.375079928209058, + "tokens_seen": 203227136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007583069207622869, + "loss": 1.8088, + "theoretical_loss": 4.374905001359851, + "tokens_seen": 203292672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582908726178536, + "loss": 1.7274, + "theoretical_loss": 4.374730146677049, + "tokens_seen": 203358208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582748244734203, + "loss": 1.745, + "theoretical_loss": 4.374555364107635, + "tokens_seen": 203423744 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758258776328987, + "loss": 1.6991, + "theoretical_loss": 4.37438065359865, + "tokens_seen": 203489280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582427281845537, + "loss": 1.6572, + "theoretical_loss": 4.374206015097193, + "tokens_seen": 203554816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582266800401204, + "loss": 1.6793, + "theoretical_loss": 4.374031448550414, + "tokens_seen": 203620352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007582106318956871, + "loss": 1.7597, + "theoretical_loss": 4.373856953905523, + "tokens_seen": 203685888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581945837512538, + "loss": 1.7406, + "theoretical_loss": 4.373682531109785, + "tokens_seen": 203751424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581785356068205, + "loss": 1.7525, + "theoretical_loss": 4.37350818011052, + "tokens_seen": 203816960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581624874623872, + "loss": 1.7741, + "theoretical_loss": 4.3733339008551, + "tokens_seen": 203882496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581464393179539, + "loss": 1.5948, + "theoretical_loss": 4.373159693290961, + "tokens_seen": 203948032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581303911735206, + "loss": 1.5639, + "theoretical_loss": 4.372985557365586, + "tokens_seen": 204013568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007581143430290873, + "loss": 1.6878, + "theoretical_loss": 4.372811493026518, + "tokens_seen": 204079104 + }, + { + "epoch": 0.06, + "learning_rate": 0.000758098294884654, + "loss": 1.715, + "theoretical_loss": 4.3726375002213524, + "tokens_seen": 204144640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580822467402207, + "loss": 1.7636, + "theoretical_loss": 4.372463578897743, + "tokens_seen": 204210176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580661985957874, + "loss": 1.8306, + "theoretical_loss": 4.372289729003397, + "tokens_seen": 204275712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580501504513541, + "loss": 1.7944, + "theoretical_loss": 4.372115950486075, + "tokens_seen": 204341248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580341023069208, + "loss": 1.7497, + "theoretical_loss": 4.371942243293595, + "tokens_seen": 204406784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580180541624875, + "loss": 1.7654, + "theoretical_loss": 4.371768607373829, + "tokens_seen": 204472320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007580020060180542, + "loss": 1.7179, + "theoretical_loss": 4.371595042674704, + "tokens_seen": 204537856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579859578736209, + "loss": 1.8487, + "theoretical_loss": 4.371421549144202, + "tokens_seen": 204603392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579699097291876, + "loss": 1.6625, + "theoretical_loss": 4.371248126730357, + "tokens_seen": 204668928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579538615847543, + "loss": 1.7389, + "theoretical_loss": 4.3710747753812615, + "tokens_seen": 204734464 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757937813440321, + "loss": 1.6541, + "theoretical_loss": 4.370901495045061, + "tokens_seen": 204800000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579217652958877, + "loss": 1.7418, + "theoretical_loss": 4.370728285669954, + "tokens_seen": 204865536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007579057171514544, + "loss": 1.7529, + "theoretical_loss": 4.370555147204195, + "tokens_seen": 204931072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578896690070211, + "loss": 1.7492, + "theoretical_loss": 4.370382079596092, + "tokens_seen": 204996608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578736208625878, + "loss": 1.6434, + "theoretical_loss": 4.370209082794006, + "tokens_seen": 205062144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578575727181545, + "loss": 1.6672, + "theoretical_loss": 4.370036156746357, + "tokens_seen": 205127680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578415245737211, + "loss": 1.8261, + "theoretical_loss": 4.369863301401612, + "tokens_seen": 205193216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578254764292879, + "loss": 1.6499, + "theoretical_loss": 4.369690516708298, + "tokens_seen": 205258752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007578094282848545, + "loss": 1.7659, + "theoretical_loss": 4.3695178026149915, + "tokens_seen": 205324288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577933801404213, + "loss": 1.7906, + "theoretical_loss": 4.369345159070327, + "tokens_seen": 205389824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577773319959879, + "loss": 1.765, + "theoretical_loss": 4.369172586022989, + "tokens_seen": 205455360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577612838515547, + "loss": 1.6486, + "theoretical_loss": 4.369000083421716, + "tokens_seen": 205520896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577452357071213, + "loss": 1.6298, + "theoretical_loss": 4.368827651215304, + "tokens_seen": 205586432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577291875626881, + "loss": 1.7474, + "theoretical_loss": 4.368655289352598, + "tokens_seen": 205651968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007577131394182547, + "loss": 1.7744, + "theoretical_loss": 4.3684829977824995, + "tokens_seen": 205717504 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576970912738215, + "loss": 1.7353, + "theoretical_loss": 4.368310776453961, + "tokens_seen": 205783040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576810431293881, + "loss": 1.6938, + "theoretical_loss": 4.368138625315989, + "tokens_seen": 205848576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576649949849549, + "loss": 1.7471, + "theoretical_loss": 4.367966544317646, + "tokens_seen": 205914112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576489468405215, + "loss": 1.7376, + "theoretical_loss": 4.367794533408043, + "tokens_seen": 205979648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576328986960883, + "loss": 1.7466, + "theoretical_loss": 4.367622592536348, + "tokens_seen": 206045184 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757616850551655, + "loss": 1.6954, + "theoretical_loss": 4.36745072165178, + "tokens_seen": 206110720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007576008024072217, + "loss": 1.8167, + "theoretical_loss": 4.36727892070361, + "tokens_seen": 206176256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575847542627884, + "loss": 1.7401, + "theoretical_loss": 4.367107189641167, + "tokens_seen": 206241792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575687061183551, + "loss": 1.6569, + "theoretical_loss": 4.3669355284138245, + "tokens_seen": 206307328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575526579739218, + "loss": 1.7005, + "theoretical_loss": 4.366763936971017, + "tokens_seen": 206372864 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575366098294885, + "loss": 1.8048, + "theoretical_loss": 4.366592415262226, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575205616850552, + "loss": 1.821, + "theoretical_loss": 4.366420963236989, + "tokens_seen": 206503936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007575045135406219, + "loss": 1.751, + "theoretical_loss": 4.366249580844894, + "tokens_seen": 206569472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574884653961886, + "loss": 1.6118, + "theoretical_loss": 4.366078268035581, + "tokens_seen": 206635008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574724172517553, + "loss": 1.8002, + "theoretical_loss": 4.365907024758744, + "tokens_seen": 206700544 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757456369107322, + "loss": 1.6942, + "theoretical_loss": 4.365735850964131, + "tokens_seen": 206766080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574403209628887, + "loss": 1.717, + "theoretical_loss": 4.365564746601537, + "tokens_seen": 206831616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574242728184554, + "loss": 1.7408, + "theoretical_loss": 4.365393711620814, + "tokens_seen": 206897152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007574082246740221, + "loss": 1.6824, + "theoretical_loss": 4.365222745971863, + "tokens_seen": 206962688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573921765295888, + "loss": 1.8375, + "theoretical_loss": 4.365051849604638, + "tokens_seen": 207028224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573761283851555, + "loss": 1.7472, + "theoretical_loss": 4.364881022469148, + "tokens_seen": 207093760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573600802407222, + "loss": 1.7056, + "theoretical_loss": 4.364710264515448, + "tokens_seen": 207159296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573440320962889, + "loss": 1.8439, + "theoretical_loss": 4.36453957569365, + "tokens_seen": 207224832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573279839518556, + "loss": 1.7218, + "theoretical_loss": 4.364368955953913, + "tokens_seen": 207290368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007573119358074223, + "loss": 1.7204, + "theoretical_loss": 4.364198405246453, + "tokens_seen": 207355904 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757295887662989, + "loss": 1.7773, + "theoretical_loss": 4.364027923521535, + "tokens_seen": 207421440 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572798395185557, + "loss": 1.6877, + "theoretical_loss": 4.363857510729473, + "tokens_seen": 207486976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572637913741224, + "loss": 1.7402, + "theoretical_loss": 4.363687166820637, + "tokens_seen": 207552512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572477432296891, + "loss": 1.7111, + "theoretical_loss": 4.363516891745445, + "tokens_seen": 207618048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572316950852558, + "loss": 1.7395, + "theoretical_loss": 4.36334668545437, + "tokens_seen": 207683584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007572156469408225, + "loss": 1.847, + "theoretical_loss": 4.363176547897931, + "tokens_seen": 207749120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571995987963892, + "loss": 1.662, + "theoretical_loss": 4.363006479026702, + "tokens_seen": 207814656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571835506519559, + "loss": 1.6905, + "theoretical_loss": 4.3628364787913085, + "tokens_seen": 207880192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571675025075226, + "loss": 1.8538, + "theoretical_loss": 4.362666547142424, + "tokens_seen": 207945728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571514543630893, + "loss": 1.7348, + "theoretical_loss": 4.362496684030775, + "tokens_seen": 208011264 + }, + { + "epoch": 0.06, + "learning_rate": 0.000757135406218656, + "loss": 1.8124, + "theoretical_loss": 4.36232688940714, + "tokens_seen": 208076800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571193580742227, + "loss": 1.7501, + "theoretical_loss": 4.362157163222346, + "tokens_seen": 208142336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007571033099297894, + "loss": 1.7579, + "theoretical_loss": 4.361987505427272, + "tokens_seen": 208207872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570872617853561, + "loss": 1.751, + "theoretical_loss": 4.361817915972846, + "tokens_seen": 208273408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570712136409228, + "loss": 1.7616, + "theoretical_loss": 4.36164839481005, + "tokens_seen": 208338944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570551654964895, + "loss": 1.8296, + "theoretical_loss": 4.361478941889915, + "tokens_seen": 208404480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570391173520562, + "loss": 1.7435, + "theoretical_loss": 4.36130955716352, + "tokens_seen": 208470016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570230692076229, + "loss": 1.6989, + "theoretical_loss": 4.361140240581998, + "tokens_seen": 208535552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007570070210631896, + "loss": 1.8035, + "theoretical_loss": 4.36097099209653, + "tokens_seen": 208601088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569909729187563, + "loss": 1.7573, + "theoretical_loss": 4.36080181165835, + "tokens_seen": 208666624 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756974924774323, + "loss": 1.6855, + "theoretical_loss": 4.360632699218739, + "tokens_seen": 208732160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569588766298897, + "loss": 1.812, + "theoretical_loss": 4.360463654729029, + "tokens_seen": 208797696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569428284854564, + "loss": 1.7353, + "theoretical_loss": 4.360294678140605, + "tokens_seen": 208863232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569267803410231, + "loss": 1.6493, + "theoretical_loss": 4.360125769404897, + "tokens_seen": 208928768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007569107321965898, + "loss": 1.6855, + "theoretical_loss": 4.35995692847339, + "tokens_seen": 208994304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568946840521565, + "loss": 1.7107, + "theoretical_loss": 4.359788155297615, + "tokens_seen": 209059840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568786359077232, + "loss": 1.7624, + "theoretical_loss": 4.359619449829156, + "tokens_seen": 209125376 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568625877632899, + "loss": 1.8295, + "theoretical_loss": 4.359450812019643, + "tokens_seen": 209190912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568465396188566, + "loss": 1.6752, + "theoretical_loss": 4.359282241820761, + "tokens_seen": 209256448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007568304914744233, + "loss": 1.8464, + "theoretical_loss": 4.359113739184238, + "tokens_seen": 209321984 + }, + { + "epoch": 0.06, + "learning_rate": 0.00075681444332999, + "loss": 1.8006, + "theoretical_loss": 4.358945304061858, + "tokens_seen": 209387520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567983951855567, + "loss": 1.8592, + "theoretical_loss": 4.35877693640545, + "tokens_seen": 209453056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567823470411234, + "loss": 1.7009, + "theoretical_loss": 4.358608636166894, + "tokens_seen": 209518592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567662988966901, + "loss": 1.8587, + "theoretical_loss": 4.358440403298119, + "tokens_seen": 209584128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567502507522569, + "loss": 1.669, + "theoretical_loss": 4.3582722377511045, + "tokens_seen": 209649664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 84957, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.7557694911956787, + "objective/train/theoretical_loss": 4.358104139477878, + "objective/train/tokens_used": 230175200, + "theoretical_loss": 4.358104139477878, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567342026078235, + "loss": 1.7558, + "theoretical_loss": 4.358104139477878, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567181544633903, + "loss": 1.7169, + "theoretical_loss": 4.357936108430517, + "tokens_seen": 209780736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007567021063189569, + "loss": 1.6734, + "theoretical_loss": 4.357768144561146, + "tokens_seen": 209846272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566860581745237, + "loss": 1.8187, + "theoretical_loss": 4.357600247821942, + "tokens_seen": 209911808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566700100300903, + "loss": 1.6683, + "theoretical_loss": 4.3574324181651285, + "tokens_seen": 209977344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566539618856571, + "loss": 1.7935, + "theoretical_loss": 4.357264655542978, + "tokens_seen": 210042880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566379137412237, + "loss": 1.8475, + "theoretical_loss": 4.357096959907813, + "tokens_seen": 210108416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566218655967905, + "loss": 1.7322, + "theoretical_loss": 4.356929331212004, + "tokens_seen": 210173952 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007566058174523571, + "loss": 1.7798, + "theoretical_loss": 4.356761769407969, + "tokens_seen": 210239488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565897693079239, + "loss": 1.8022, + "theoretical_loss": 4.356594274448178, + "tokens_seen": 210305024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565737211634905, + "loss": 1.6858, + "theoretical_loss": 4.356426846285147, + "tokens_seen": 210370560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565576730190572, + "loss": 1.7772, + "theoretical_loss": 4.3562594848714395, + "tokens_seen": 210436096 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756541624874624, + "loss": 1.7201, + "theoretical_loss": 4.3560921901596705, + "tokens_seen": 210501632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565255767301906, + "loss": 1.677, + "theoretical_loss": 4.355924962102502, + "tokens_seen": 210567168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007565095285857573, + "loss": 1.6862, + "theoretical_loss": 4.3557578006526425, + "tokens_seen": 210632704 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756493480441324, + "loss": 1.7718, + "theoretical_loss": 4.355590705762852, + "tokens_seen": 210698240 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564774322968907, + "loss": 1.7889, + "theoretical_loss": 4.355423677385937, + "tokens_seen": 210763776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564613841524574, + "loss": 1.8241, + "theoretical_loss": 4.355256715474751, + "tokens_seen": 210829312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564453360080242, + "loss": 1.815, + "theoretical_loss": 4.355089819982198, + "tokens_seen": 210894848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564292878635908, + "loss": 1.7633, + "theoretical_loss": 4.354922990861229, + "tokens_seen": 210960384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007564132397191574, + "loss": 1.7236, + "theoretical_loss": 4.354756228064842, + "tokens_seen": 211025920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563971915747242, + "loss": 1.6991, + "theoretical_loss": 4.354589531546083, + "tokens_seen": 211091456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563811434302908, + "loss": 1.7559, + "theoretical_loss": 4.354422901258048, + "tokens_seen": 211156992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563650952858576, + "loss": 1.6282, + "theoretical_loss": 4.354256337153878, + "tokens_seen": 211222528 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563490471414242, + "loss": 1.7975, + "theoretical_loss": 4.354089839186763, + "tokens_seen": 211288064 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756332998996991, + "loss": 1.6835, + "theoretical_loss": 4.35392340730994, + "tokens_seen": 211353600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563169508525576, + "loss": 1.6923, + "theoretical_loss": 4.353757041476695, + "tokens_seen": 211419136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007563009027081244, + "loss": 1.748, + "theoretical_loss": 4.35359074164036, + "tokens_seen": 211484672 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756284854563691, + "loss": 1.7377, + "theoretical_loss": 4.3534245077543146, + "tokens_seen": 211550208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562688064192578, + "loss": 1.6532, + "theoretical_loss": 4.353258339771986, + "tokens_seen": 211615744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562527582748245, + "loss": 1.6354, + "theoretical_loss": 4.353092237646849, + "tokens_seen": 211681280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562367101303912, + "loss": 1.7168, + "theoretical_loss": 4.3529262013324255, + "tokens_seen": 211746816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562206619859579, + "loss": 1.6738, + "theoretical_loss": 4.352760230782284, + "tokens_seen": 211812352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007562046138415246, + "loss": 1.6526, + "theoretical_loss": 4.352594325950041, + "tokens_seen": 211877888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561885656970913, + "loss": 1.7619, + "theoretical_loss": 4.35242848678936, + "tokens_seen": 211943424 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756172517552658, + "loss": 1.6957, + "theoretical_loss": 4.35226271325395, + "tokens_seen": 212008960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561564694082247, + "loss": 1.7532, + "theoretical_loss": 4.352097005297569, + "tokens_seen": 212074496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561404212637914, + "loss": 1.5988, + "theoretical_loss": 4.351931362874019, + "tokens_seen": 212140032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561243731193581, + "loss": 1.7567, + "theoretical_loss": 4.351765785937154, + "tokens_seen": 212205568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007561083249749248, + "loss": 1.7913, + "theoretical_loss": 4.3516002744408695, + "tokens_seen": 212271104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560922768304915, + "loss": 1.7309, + "theoretical_loss": 4.351434828339109, + "tokens_seen": 212336640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560762286860582, + "loss": 1.8523, + "theoretical_loss": 4.351269447585864, + "tokens_seen": 212402176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560601805416249, + "loss": 1.8322, + "theoretical_loss": 4.3511041321351716, + "tokens_seen": 212467712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560441323971916, + "loss": 1.723, + "theoretical_loss": 4.350938881941116, + "tokens_seen": 212533248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007560280842527583, + "loss": 1.7828, + "theoretical_loss": 4.350773696957827, + "tokens_seen": 212598784 + }, + { + "epoch": 0.06, + "learning_rate": 0.000756012036108325, + "loss": 1.7081, + "theoretical_loss": 4.350608577139481, + "tokens_seen": 212664320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559959879638917, + "loss": 1.7537, + "theoretical_loss": 4.350443522440301, + "tokens_seen": 212729856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559799398194584, + "loss": 1.6697, + "theoretical_loss": 4.350278532814556, + "tokens_seen": 212795392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559638916750251, + "loss": 1.9349, + "theoretical_loss": 4.350113608216563, + "tokens_seen": 212860928 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559478435305918, + "loss": 1.7607, + "theoretical_loss": 4.3499487486006805, + "tokens_seen": 212926464 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559317953861585, + "loss": 1.7092, + "theoretical_loss": 4.349783953921318, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007559157472417252, + "loss": 1.852, + "theoretical_loss": 4.349619224132929, + "tokens_seen": 213057536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558996990972919, + "loss": 1.7047, + "theoretical_loss": 4.349454559190011, + "tokens_seen": 213123072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558836509528586, + "loss": 1.7372, + "theoretical_loss": 4.349289959047112, + "tokens_seen": 213188608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558676028084253, + "loss": 1.7609, + "theoretical_loss": 4.349125423658821, + "tokens_seen": 213254144 + }, + { + "epoch": 0.06, + "learning_rate": 0.000755851554663992, + "loss": 1.7236, + "theoretical_loss": 4.348960952979776, + "tokens_seen": 213319680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558355065195587, + "loss": 1.8124, + "theoretical_loss": 4.3487965469646594, + "tokens_seen": 213385216 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558194583751254, + "loss": 1.7797, + "theoretical_loss": 4.3486322055682, + "tokens_seen": 213450752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007558034102306921, + "loss": 1.6791, + "theoretical_loss": 4.348467928745171, + "tokens_seen": 213516288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557873620862588, + "loss": 1.6517, + "theoretical_loss": 4.348303716450391, + "tokens_seen": 213581824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557713139418255, + "loss": 1.7573, + "theoretical_loss": 4.348139568638727, + "tokens_seen": 213647360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557552657973922, + "loss": 1.8218, + "theoretical_loss": 4.347975485265087, + "tokens_seen": 213712896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557392176529589, + "loss": 1.8017, + "theoretical_loss": 4.347811466284428, + "tokens_seen": 213778432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557231695085256, + "loss": 1.7611, + "theoretical_loss": 4.34764751165175, + "tokens_seen": 213843968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007557071213640923, + "loss": 1.8309, + "theoretical_loss": 4.347483621322099, + "tokens_seen": 213909504 + }, + { + "epoch": 0.06, + "learning_rate": 0.000755691073219659, + "loss": 1.759, + "theoretical_loss": 4.347319795250567, + "tokens_seen": 213975040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556750250752257, + "loss": 1.7367, + "theoretical_loss": 4.347156033392291, + "tokens_seen": 214040576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556589769307924, + "loss": 1.7225, + "theoretical_loss": 4.346992335702451, + "tokens_seen": 214106112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556429287863591, + "loss": 1.7777, + "theoretical_loss": 4.346828702136273, + "tokens_seen": 214171648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556268806419258, + "loss": 1.7765, + "theoretical_loss": 4.34666513264903, + "tokens_seen": 214237184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007556108324974925, + "loss": 1.6971, + "theoretical_loss": 4.346501627196037, + "tokens_seen": 214302720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007555947843530592, + "loss": 1.7381, + "theoretical_loss": 4.346338185732655, + "tokens_seen": 214368256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007555787362086259, + "loss": 1.7924, + "theoretical_loss": 4.3461748082142915, + "tokens_seen": 214433792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0007555626880641926, + "loss": 1.8189, + "theoretical_loss": 4.346011494596395, + "tokens_seen": 214499328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007555466399197594, + "loss": 1.7618, + "theoretical_loss": 4.34584824483446, + "tokens_seen": 214564864 + }, + { + "epoch": 0.07, + "learning_rate": 0.000755530591775326, + "loss": 1.775, + "theoretical_loss": 4.345685058884029, + "tokens_seen": 214630400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007555145436308928, + "loss": 1.698, + "theoretical_loss": 4.345521936700684, + "tokens_seen": 214695936 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554984954864594, + "loss": 1.7327, + "theoretical_loss": 4.345358878240053, + "tokens_seen": 214761472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554824473420262, + "loss": 1.7759, + "theoretical_loss": 4.345195883457811, + "tokens_seen": 214827008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554663991975928, + "loss": 1.6508, + "theoretical_loss": 4.3450329523096745, + "tokens_seen": 214892544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554503510531596, + "loss": 1.8172, + "theoretical_loss": 4.344870084751406, + "tokens_seen": 214958080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554343029087262, + "loss": 1.8609, + "theoretical_loss": 4.34470728073881, + "tokens_seen": 215023616 + }, + { + "epoch": 0.07, + "learning_rate": 0.000755418254764293, + "loss": 1.7311, + "theoretical_loss": 4.344544540227737, + "tokens_seen": 215089152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007554022066198596, + "loss": 1.8145, + "theoretical_loss": 4.344381863174082, + "tokens_seen": 215154688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553861584754264, + "loss": 1.6288, + "theoretical_loss": 4.344219249533783, + "tokens_seen": 215220224 + }, + { + "epoch": 0.07, + "learning_rate": 0.000755370110330993, + "loss": 1.8157, + "theoretical_loss": 4.34405669926282, + "tokens_seen": 215285760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553540621865598, + "loss": 1.7173, + "theoretical_loss": 4.343894212317224, + "tokens_seen": 215351296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553380140421264, + "loss": 1.8164, + "theoretical_loss": 4.34373178865306, + "tokens_seen": 215416832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553219658976932, + "loss": 1.7982, + "theoretical_loss": 4.343569428226445, + "tokens_seen": 215482368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007553059177532598, + "loss": 1.7587, + "theoretical_loss": 4.343407130993536, + "tokens_seen": 215547904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007552898696088266, + "loss": 1.6608, + "theoretical_loss": 4.343244896910534, + "tokens_seen": 215613440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007552738214643932, + "loss": 1.5795, + "theoretical_loss": 4.343082725933684, + "tokens_seen": 215678976 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075525777331996, + "loss": 1.6416, + "theoretical_loss": 4.342920618019275, + "tokens_seen": 215744512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007552417251755266, + "loss": 1.6894, + "theoretical_loss": 4.342758573123639, + "tokens_seen": 215810048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007552256770310934, + "loss": 1.703, + "theoretical_loss": 4.342596591203153, + "tokens_seen": 215875584 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075520962888666, + "loss": 1.7567, + "theoretical_loss": 4.3424346722142335, + "tokens_seen": 215941120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551935807422268, + "loss": 1.7788, + "theoretical_loss": 4.342272816113344, + "tokens_seen": 216006656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551775325977934, + "loss": 1.693, + "theoretical_loss": 4.342111022856992, + "tokens_seen": 216072192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551614844533601, + "loss": 1.7425, + "theoretical_loss": 4.341949292401725, + "tokens_seen": 216137728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551454363089268, + "loss": 1.7275, + "theoretical_loss": 4.341787624704135, + "tokens_seen": 216203264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 87357, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.7541401386260986, + "objective/train/theoretical_loss": 4.341626019720858, + "objective/train/tokens_used": 236728800, + "theoretical_loss": 4.341626019720858, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551293881644935, + "loss": 1.7541, + "theoretical_loss": 4.341626019720858, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007551133400200602, + "loss": 1.8899, + "theoretical_loss": 4.341464477408572, + "tokens_seen": 216334336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550972918756269, + "loss": 1.6601, + "theoretical_loss": 4.341302997724, + "tokens_seen": 216399872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550812437311937, + "loss": 1.7421, + "theoretical_loss": 4.341141580623905, + "tokens_seen": 216465408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550651955867603, + "loss": 1.8931, + "theoretical_loss": 4.340980226065096, + "tokens_seen": 216530944 + }, + { + "epoch": 0.07, + "learning_rate": 0.000755049147442327, + "loss": 1.6918, + "theoretical_loss": 4.340818934004421, + "tokens_seen": 216596480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550330992978937, + "loss": 1.754, + "theoretical_loss": 4.340657704398774, + "tokens_seen": 216662016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550170511534603, + "loss": 1.6693, + "theoretical_loss": 4.340496537205093, + "tokens_seen": 216727552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007550010030090271, + "loss": 1.6629, + "theoretical_loss": 4.340335432380353, + "tokens_seen": 216793088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549849548645937, + "loss": 1.7004, + "theoretical_loss": 4.340174389881579, + "tokens_seen": 216858624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549689067201605, + "loss": 1.764, + "theoretical_loss": 4.340013409665833, + "tokens_seen": 216924160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549528585757271, + "loss": 1.7839, + "theoretical_loss": 4.339852491690221, + "tokens_seen": 216989696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549368104312939, + "loss": 1.6797, + "theoretical_loss": 4.339691635911892, + "tokens_seen": 217055232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549207622868605, + "loss": 1.7425, + "theoretical_loss": 4.339530842288037, + "tokens_seen": 217120768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007549047141424273, + "loss": 1.7092, + "theoretical_loss": 4.339370110775892, + "tokens_seen": 217186304 + }, + { + "epoch": 0.07, + "learning_rate": 0.000754888665997994, + "loss": 1.7533, + "theoretical_loss": 4.33920944133273, + "tokens_seen": 217251840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548726178535607, + "loss": 1.7142, + "theoretical_loss": 4.3390488339158715, + "tokens_seen": 217317376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548565697091274, + "loss": 1.7488, + "theoretical_loss": 4.338888288482675, + "tokens_seen": 217382912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548405215646941, + "loss": 1.7231, + "theoretical_loss": 4.338727804990544, + "tokens_seen": 217448448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548244734202608, + "loss": 1.6839, + "theoretical_loss": 4.338567383396924, + "tokens_seen": 217513984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007548084252758275, + "loss": 1.7208, + "theoretical_loss": 4.338407023659302, + "tokens_seen": 217579520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547923771313942, + "loss": 1.6647, + "theoretical_loss": 4.338246725735205, + "tokens_seen": 217645056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547763289869609, + "loss": 1.6131, + "theoretical_loss": 4.338086489582205, + "tokens_seen": 217710592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547602808425276, + "loss": 1.6504, + "theoretical_loss": 4.3379263151579135, + "tokens_seen": 217776128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547442326980943, + "loss": 1.6979, + "theoretical_loss": 4.337766202419986, + "tokens_seen": 217841664 + }, + { + "epoch": 0.07, + "learning_rate": 0.000754728184553661, + "loss": 1.7399, + "theoretical_loss": 4.337606151326119, + "tokens_seen": 217907200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007547121364092277, + "loss": 1.6977, + "theoretical_loss": 4.337446161834048, + "tokens_seen": 217972736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546960882647944, + "loss": 1.7266, + "theoretical_loss": 4.3372862339015565, + "tokens_seen": 218038272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546800401203611, + "loss": 1.7607, + "theoretical_loss": 4.337126367486462, + "tokens_seen": 218103808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546639919759278, + "loss": 1.8613, + "theoretical_loss": 4.33696656254663, + "tokens_seen": 218169344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546479438314945, + "loss": 1.7391, + "theoretical_loss": 4.336806819039962, + "tokens_seen": 218234880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546318956870612, + "loss": 1.7434, + "theoretical_loss": 4.336647136924407, + "tokens_seen": 218300416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007546158475426279, + "loss": 1.7183, + "theoretical_loss": 4.33648751615795, + "tokens_seen": 218365952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545997993981946, + "loss": 1.709, + "theoretical_loss": 4.33632795669862, + "tokens_seen": 218431488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545837512537613, + "loss": 1.8274, + "theoretical_loss": 4.336168458504486, + "tokens_seen": 218497024 + }, + { + "epoch": 0.07, + "learning_rate": 0.000754567703109328, + "loss": 1.5751, + "theoretical_loss": 4.336009021533661, + "tokens_seen": 218562560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545516549648947, + "loss": 1.8043, + "theoretical_loss": 4.335849645744295, + "tokens_seen": 218628096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545356068204614, + "loss": 1.6846, + "theoretical_loss": 4.3356903310945825, + "tokens_seen": 218693632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545195586760281, + "loss": 1.7287, + "theoretical_loss": 4.335531077542758, + "tokens_seen": 218759168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007545035105315948, + "loss": 1.7706, + "theoretical_loss": 4.335371885047096, + "tokens_seen": 218824704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544874623871615, + "loss": 1.7234, + "theoretical_loss": 4.335212753565914, + "tokens_seen": 218890240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544714142427282, + "loss": 1.7824, + "theoretical_loss": 4.33505368305757, + "tokens_seen": 218955776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544553660982949, + "loss": 1.7169, + "theoretical_loss": 4.33489467348046, + "tokens_seen": 219021312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544393179538616, + "loss": 1.7009, + "theoretical_loss": 4.334735724793026, + "tokens_seen": 219086848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007544232698094283, + "loss": 1.7544, + "theoretical_loss": 4.334576836953744, + "tokens_seen": 219152384 + }, + { + "epoch": 0.07, + "learning_rate": 0.000754407221664995, + "loss": 1.745, + "theoretical_loss": 4.334418009921139, + "tokens_seen": 219217920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543911735205617, + "loss": 1.8339, + "theoretical_loss": 4.334259243653769, + "tokens_seen": 219283456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543751253761284, + "loss": 1.7365, + "theoretical_loss": 4.334100538110237, + "tokens_seen": 219348992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543590772316951, + "loss": 1.7756, + "theoretical_loss": 4.333941893249184, + "tokens_seen": 219414528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543430290872619, + "loss": 1.6889, + "theoretical_loss": 4.333783309029296, + "tokens_seen": 219480064 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543269809428285, + "loss": 1.5964, + "theoretical_loss": 4.3336247854092935, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007543109327983953, + "loss": 1.8013, + "theoretical_loss": 4.333466322347942, + "tokens_seen": 219611136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542948846539619, + "loss": 1.7981, + "theoretical_loss": 4.333307919804045, + "tokens_seen": 219676672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542788365095287, + "loss": 1.754, + "theoretical_loss": 4.333149577736446, + "tokens_seen": 219742208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542627883650953, + "loss": 1.6121, + "theoretical_loss": 4.332991296104032, + "tokens_seen": 219807744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542467402206621, + "loss": 1.8843, + "theoretical_loss": 4.332833074865725, + "tokens_seen": 219873280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542306920762287, + "loss": 1.7242, + "theoretical_loss": 4.332674913980492, + "tokens_seen": 219938816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007542146439317955, + "loss": 1.7173, + "theoretical_loss": 4.332516813407337, + "tokens_seen": 220004352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541985957873621, + "loss": 1.8197, + "theoretical_loss": 4.332358773105307, + "tokens_seen": 220069888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541825476429289, + "loss": 1.8409, + "theoretical_loss": 4.332200793033485, + "tokens_seen": 220135424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541664994984955, + "loss": 1.7557, + "theoretical_loss": 4.332042873150997, + "tokens_seen": 220200960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541504513540623, + "loss": 1.7084, + "theoretical_loss": 4.3318850134170095, + "tokens_seen": 220266496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541344032096289, + "loss": 1.7834, + "theoretical_loss": 4.331727213790725, + "tokens_seen": 220332032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541183550651957, + "loss": 1.705, + "theoretical_loss": 4.331569474231388, + "tokens_seen": 220397568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007541023069207623, + "loss": 1.736, + "theoretical_loss": 4.331411794698286, + "tokens_seen": 220463104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540862587763291, + "loss": 1.7398, + "theoretical_loss": 4.33125417515074, + "tokens_seen": 220528640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540702106318957, + "loss": 1.6683, + "theoretical_loss": 4.331096615548115, + "tokens_seen": 220594176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540541624874625, + "loss": 1.6628, + "theoretical_loss": 4.330939115849814, + "tokens_seen": 220659712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540381143430291, + "loss": 1.675, + "theoretical_loss": 4.33078167601528, + "tokens_seen": 220725248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540220661985959, + "loss": 1.6805, + "theoretical_loss": 4.330624296003996, + "tokens_seen": 220790784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007540060180541625, + "loss": 1.8567, + "theoretical_loss": 4.330466975775483, + "tokens_seen": 220856320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539899699097293, + "loss": 1.6977, + "theoretical_loss": 4.330309715289301, + "tokens_seen": 220921856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539739217652959, + "loss": 1.686, + "theoretical_loss": 4.330152514505053, + "tokens_seen": 220987392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539578736208627, + "loss": 1.7442, + "theoretical_loss": 4.329995373382378, + "tokens_seen": 221052928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539418254764293, + "loss": 1.7602, + "theoretical_loss": 4.329838291880954, + "tokens_seen": 221118464 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539257773319961, + "loss": 1.8866, + "theoretical_loss": 4.329681269960499, + "tokens_seen": 221184000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007539097291875627, + "loss": 1.8567, + "theoretical_loss": 4.329524307580773, + "tokens_seen": 221249536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538936810431295, + "loss": 1.7735, + "theoretical_loss": 4.3293674047015696, + "tokens_seen": 221315072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538776328986961, + "loss": 1.7584, + "theoretical_loss": 4.329210561282726, + "tokens_seen": 221380608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538615847542629, + "loss": 1.6923, + "theoretical_loss": 4.329053777284115, + "tokens_seen": 221446144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538455366098295, + "loss": 1.7616, + "theoretical_loss": 4.328897052665653, + "tokens_seen": 221511680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007538294884653963, + "loss": 1.7035, + "theoretical_loss": 4.328740387387289, + "tokens_seen": 221577216 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753813440320963, + "loss": 1.825, + "theoretical_loss": 4.328583781409016, + "tokens_seen": 221642752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537973921765297, + "loss": 1.7146, + "theoretical_loss": 4.328427234690864, + "tokens_seen": 221708288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537813440320963, + "loss": 1.5788, + "theoretical_loss": 4.328270747192901, + "tokens_seen": 221773824 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753765295887663, + "loss": 1.6196, + "theoretical_loss": 4.328114318875236, + "tokens_seen": 221839360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537492477432297, + "loss": 1.694, + "theoretical_loss": 4.327957949698012, + "tokens_seen": 221904896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537331995987964, + "loss": 1.6947, + "theoretical_loss": 4.327801639621417, + "tokens_seen": 221970432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537171514543632, + "loss": 1.7786, + "theoretical_loss": 4.3276453886056725, + "tokens_seen": 222035968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007537011033099298, + "loss": 1.6427, + "theoretical_loss": 4.32748919661104, + "tokens_seen": 222101504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007536850551654966, + "loss": 1.7688, + "theoretical_loss": 4.327333063597821, + "tokens_seen": 222167040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007536690070210632, + "loss": 1.7061, + "theoretical_loss": 4.327176989526352, + "tokens_seen": 222232576 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075365295887663, + "loss": 1.7345, + "theoretical_loss": 4.327020974357012, + "tokens_seen": 222298112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007536369107321966, + "loss": 1.8073, + "theoretical_loss": 4.326865018050215, + "tokens_seen": 222363648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007536208625877632, + "loss": 1.7301, + "theoretical_loss": 4.326709120566415, + "tokens_seen": 222429184 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075360481444333, + "loss": 1.7917, + "theoretical_loss": 4.326553281866103, + "tokens_seen": 222494720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535887662988966, + "loss": 1.6926, + "theoretical_loss": 4.326397501909811, + "tokens_seen": 222560256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535727181544634, + "loss": 1.6728, + "theoretical_loss": 4.326241780658104, + "tokens_seen": 222625792 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075355667001003, + "loss": 1.7051, + "theoretical_loss": 4.32608611807159, + "tokens_seen": 222691328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535406218655968, + "loss": 1.6952, + "theoretical_loss": 4.325930514110912, + "tokens_seen": 222756864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 89737, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.6492441892623901, + "objective/train/theoretical_loss": 4.325774968736753, + "objective/train/tokens_used": 243282400, + "theoretical_loss": 4.325774968736753, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535245737211634, + "loss": 1.6492, + "theoretical_loss": 4.325774968736753, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007535085255767302, + "loss": 1.7581, + "theoretical_loss": 4.325619481909833, + "tokens_seen": 222887936 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534924774322969, + "loss": 1.7129, + "theoretical_loss": 4.325464053590908, + "tokens_seen": 222953472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534764292878636, + "loss": 1.7744, + "theoretical_loss": 4.325308683740776, + "tokens_seen": 223019008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534603811434303, + "loss": 1.7024, + "theoretical_loss": 4.325153372320269, + "tokens_seen": 223084544 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753444332998997, + "loss": 1.6814, + "theoretical_loss": 4.32499811929026, + "tokens_seen": 223150080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534282848545637, + "loss": 1.7055, + "theoretical_loss": 4.324842924611655, + "tokens_seen": 223215616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007534122367101304, + "loss": 1.841, + "theoretical_loss": 4.324687788245403, + "tokens_seen": 223281152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533961885656971, + "loss": 1.7495, + "theoretical_loss": 4.3245327101524875, + "tokens_seen": 223346688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533801404212638, + "loss": 1.6664, + "theoretical_loss": 4.324377690293929, + "tokens_seen": 223412224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533640922768305, + "loss": 1.7203, + "theoretical_loss": 4.324222728630787, + "tokens_seen": 223477760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533480441323972, + "loss": 1.7453, + "theoretical_loss": 4.324067825124159, + "tokens_seen": 223543296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533319959879639, + "loss": 1.6492, + "theoretical_loss": 4.323912979735178, + "tokens_seen": 223608832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007533159478435306, + "loss": 1.7522, + "theoretical_loss": 4.323758192425016, + "tokens_seen": 223674368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532998996990973, + "loss": 1.7921, + "theoretical_loss": 4.323603463154882, + "tokens_seen": 223739904 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753283851554664, + "loss": 1.58, + "theoretical_loss": 4.323448791886022, + "tokens_seen": 223805440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532678034102307, + "loss": 1.6923, + "theoretical_loss": 4.323294178579717, + "tokens_seen": 223870976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532517552657974, + "loss": 1.7731, + "theoretical_loss": 4.323139623197289, + "tokens_seen": 223936512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532357071213641, + "loss": 1.8041, + "theoretical_loss": 4.322985125700096, + "tokens_seen": 224002048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532196589769308, + "loss": 1.6624, + "theoretical_loss": 4.322830686049532, + "tokens_seen": 224067584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007532036108324975, + "loss": 1.8031, + "theoretical_loss": 4.322676304207027, + "tokens_seen": 224133120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531875626880642, + "loss": 1.6445, + "theoretical_loss": 4.32252198013405, + "tokens_seen": 224198656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531715145436309, + "loss": 1.6782, + "theoretical_loss": 4.32236771379211, + "tokens_seen": 224264192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531554663991976, + "loss": 1.7225, + "theoretical_loss": 4.322213505142744, + "tokens_seen": 224329728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531394182547644, + "loss": 1.775, + "theoretical_loss": 4.322059354147535, + "tokens_seen": 224395264 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753123370110331, + "loss": 1.7043, + "theoretical_loss": 4.321905260768097, + "tokens_seen": 224460800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007531073219658978, + "loss": 1.7811, + "theoretical_loss": 4.321751224966083, + "tokens_seen": 224526336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530912738214644, + "loss": 1.7081, + "theoretical_loss": 4.321597246703183, + "tokens_seen": 224591872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530752256770312, + "loss": 1.685, + "theoretical_loss": 4.321443325941123, + "tokens_seen": 224657408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530591775325978, + "loss": 1.6451, + "theoretical_loss": 4.3212894626416665, + "tokens_seen": 224722944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530431293881646, + "loss": 1.7142, + "theoretical_loss": 4.321135656766611, + "tokens_seen": 224788480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007530270812437312, + "loss": 1.755, + "theoretical_loss": 4.320981908277793, + "tokens_seen": 224854016 + }, + { + "epoch": 0.07, + "learning_rate": 0.000753011033099298, + "loss": 1.626, + "theoretical_loss": 4.320828217137086, + "tokens_seen": 224919552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529949849548646, + "loss": 1.6604, + "theoretical_loss": 4.320674583306396, + "tokens_seen": 224985088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529789368104314, + "loss": 1.6649, + "theoretical_loss": 4.32052100674767, + "tokens_seen": 225050624 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752962888665998, + "loss": 1.7843, + "theoretical_loss": 4.320367487422891, + "tokens_seen": 225116160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529468405215648, + "loss": 1.6409, + "theoretical_loss": 4.3202140252940735, + "tokens_seen": 225181696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529307923771314, + "loss": 1.6356, + "theoretical_loss": 4.3200606203232725, + "tokens_seen": 225247232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007529147442326982, + "loss": 1.7784, + "theoretical_loss": 4.319907272472579, + "tokens_seen": 225312768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528986960882648, + "loss": 1.6853, + "theoretical_loss": 4.319753981704119, + "tokens_seen": 225378304 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528826479438316, + "loss": 1.7475, + "theoretical_loss": 4.319600747980056, + "tokens_seen": 225443840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528665997993982, + "loss": 1.6991, + "theoretical_loss": 4.319447571262586, + "tokens_seen": 225509376 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752850551654965, + "loss": 1.6661, + "theoretical_loss": 4.3192944515139455, + "tokens_seen": 225574912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528345035105316, + "loss": 1.6993, + "theoretical_loss": 4.319141388696404, + "tokens_seen": 225640448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007528184553660984, + "loss": 1.7316, + "theoretical_loss": 4.318988382772269, + "tokens_seen": 225705984 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752802407221665, + "loss": 1.7177, + "theoretical_loss": 4.318835433703883, + "tokens_seen": 225771520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527863590772318, + "loss": 1.6656, + "theoretical_loss": 4.318682541453622, + "tokens_seen": 225837056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527703109327984, + "loss": 1.654, + "theoretical_loss": 4.318529705983903, + "tokens_seen": 225902592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527542627883652, + "loss": 1.7454, + "theoretical_loss": 4.318376927257174, + "tokens_seen": 225968128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527382146439318, + "loss": 1.6859, + "theoretical_loss": 4.3182242052359205, + "tokens_seen": 226033664 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527221664994986, + "loss": 1.7374, + "theoretical_loss": 4.318071539882665, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007527061183550652, + "loss": 1.6958, + "theoretical_loss": 4.317918931159963, + "tokens_seen": 226164736 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752690070210632, + "loss": 1.7292, + "theoretical_loss": 4.317766379030407, + "tokens_seen": 226230272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007526740220661986, + "loss": 1.641, + "theoretical_loss": 4.317613883456625, + "tokens_seen": 226295808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007526579739217654, + "loss": 1.7295, + "theoretical_loss": 4.317461444401281, + "tokens_seen": 226361344 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752641925777332, + "loss": 1.7554, + "theoretical_loss": 4.317309061827073, + "tokens_seen": 226426880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007526258776328988, + "loss": 1.7057, + "theoretical_loss": 4.317156735696736, + "tokens_seen": 226492416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007526098294884654, + "loss": 1.7868, + "theoretical_loss": 4.3170044659730396, + "tokens_seen": 226557952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525937813440322, + "loss": 1.7038, + "theoretical_loss": 4.316852252618788, + "tokens_seen": 226623488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525777331995988, + "loss": 1.5657, + "theoretical_loss": 4.316700095596822, + "tokens_seen": 226689024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525616850551656, + "loss": 1.7543, + "theoretical_loss": 4.3165479948700165, + "tokens_seen": 226754560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525456369107322, + "loss": 1.7003, + "theoretical_loss": 4.316395950401283, + "tokens_seen": 226820096 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752529588766299, + "loss": 1.6757, + "theoretical_loss": 4.316243962153566, + "tokens_seen": 226885632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007525135406218656, + "loss": 1.7257, + "theoretical_loss": 4.316092030089847, + "tokens_seen": 226951168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524974924774324, + "loss": 1.6205, + "theoretical_loss": 4.315940154173143, + "tokens_seen": 227016704 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752481444332999, + "loss": 1.735, + "theoretical_loss": 4.315788334366504, + "tokens_seen": 227082240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524653961885658, + "loss": 1.7005, + "theoretical_loss": 4.315636570633015, + "tokens_seen": 227147776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524493480441324, + "loss": 1.7894, + "theoretical_loss": 4.315484862935799, + "tokens_seen": 227213312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524332998996992, + "loss": 1.6059, + "theoretical_loss": 4.315333211238009, + "tokens_seen": 227278848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524172517552658, + "loss": 1.739, + "theoretical_loss": 4.315181615502838, + "tokens_seen": 227344384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007524012036108326, + "loss": 1.5948, + "theoretical_loss": 4.31503007569351, + "tokens_seen": 227409920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523851554663992, + "loss": 1.6074, + "theoretical_loss": 4.314878591773287, + "tokens_seen": 227475456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523691073219659, + "loss": 1.6405, + "theoretical_loss": 4.31472716370546, + "tokens_seen": 227540992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523530591775327, + "loss": 1.6388, + "theoretical_loss": 4.314575791453362, + "tokens_seen": 227606528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523370110330993, + "loss": 1.6625, + "theoretical_loss": 4.314424474980355, + "tokens_seen": 227672064 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752320962888666, + "loss": 1.5953, + "theoretical_loss": 4.31427321424984, + "tokens_seen": 227737600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007523049147442327, + "loss": 1.736, + "theoretical_loss": 4.314122009225248, + "tokens_seen": 227803136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522888665997995, + "loss": 1.7958, + "theoretical_loss": 4.313970859870048, + "tokens_seen": 227868672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522728184553661, + "loss": 1.7519, + "theoretical_loss": 4.313819766147741, + "tokens_seen": 227934208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522567703109329, + "loss": 1.7792, + "theoretical_loss": 4.313668728021866, + "tokens_seen": 227999744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522407221664995, + "loss": 1.7721, + "theoretical_loss": 4.313517745455991, + "tokens_seen": 228065280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522246740220661, + "loss": 1.635, + "theoretical_loss": 4.313366818413724, + "tokens_seen": 228130816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007522086258776329, + "loss": 1.7641, + "theoretical_loss": 4.313215946858702, + "tokens_seen": 228196352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521925777331995, + "loss": 1.6951, + "theoretical_loss": 4.313065130754601, + "tokens_seen": 228261888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521765295887663, + "loss": 1.6935, + "theoretical_loss": 4.312914370065128, + "tokens_seen": 228327424 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752160481444333, + "loss": 1.7565, + "theoretical_loss": 4.312763664754025, + "tokens_seen": 228392960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521444332998997, + "loss": 1.6678, + "theoretical_loss": 4.312613014785069, + "tokens_seen": 228458496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521283851554664, + "loss": 1.7712, + "theoretical_loss": 4.3124624201220705, + "tokens_seen": 228524032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007521123370110331, + "loss": 1.5884, + "theoretical_loss": 4.312311880728873, + "tokens_seen": 228589568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520962888665998, + "loss": 1.7449, + "theoretical_loss": 4.312161396569356, + "tokens_seen": 228655104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520802407221665, + "loss": 1.774, + "theoretical_loss": 4.312010967607431, + "tokens_seen": 228720640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520641925777332, + "loss": 1.7603, + "theoretical_loss": 4.311860593807045, + "tokens_seen": 228786176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520481444332999, + "loss": 1.7659, + "theoretical_loss": 4.311710275132178, + "tokens_seen": 228851712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520320962888666, + "loss": 1.7087, + "theoretical_loss": 4.3115600115468435, + "tokens_seen": 228917248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007520160481444333, + "loss": 1.7327, + "theoretical_loss": 4.311409803015089, + "tokens_seen": 228982784 + }, + { + "epoch": 0.07, + "learning_rate": 0.000752, + "loss": 1.7193, + "theoretical_loss": 4.3112596495009985, + "tokens_seen": 229048320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519839518555667, + "loss": 1.8119, + "theoretical_loss": 4.311109550968684, + "tokens_seen": 229113856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519679037111334, + "loss": 1.8058, + "theoretical_loss": 4.310959507382296, + "tokens_seen": 229179392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519518555667001, + "loss": 1.6538, + "theoretical_loss": 4.310809518706017, + "tokens_seen": 229244928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519358074222669, + "loss": 1.7477, + "theoretical_loss": 4.310659584904062, + "tokens_seen": 229310464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 92008, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.6449694633483887, + "objective/train/theoretical_loss": 4.310509705940682, + "objective/train/tokens_used": 249836000, + "theoretical_loss": 4.310509705940682, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519197592778335, + "loss": 1.645, + "theoretical_loss": 4.310509705940682, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007519037111334003, + "loss": 1.74, + "theoretical_loss": 4.310359881780159, + "tokens_seen": 229441536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518876629889669, + "loss": 1.761, + "theoretical_loss": 4.310210112386811, + "tokens_seen": 229507072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518716148445337, + "loss": 1.7839, + "theoretical_loss": 4.310060397724987, + "tokens_seen": 229572608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518555667001003, + "loss": 1.7724, + "theoretical_loss": 4.30991073775907, + "tokens_seen": 229638144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518395185556671, + "loss": 1.7281, + "theoretical_loss": 4.3097611324534775, + "tokens_seen": 229703680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518234704112337, + "loss": 1.7916, + "theoretical_loss": 4.309611581772659, + "tokens_seen": 229769216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007518074222668005, + "loss": 1.7421, + "theoretical_loss": 4.309462085681098, + "tokens_seen": 229834752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517913741223671, + "loss": 1.6439, + "theoretical_loss": 4.3093126441433105, + "tokens_seen": 229900288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517753259779339, + "loss": 1.6821, + "theoretical_loss": 4.3091632571238465, + "tokens_seen": 229965824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517592778335005, + "loss": 1.7777, + "theoretical_loss": 4.30901392458729, + "tokens_seen": 230031360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517432296890673, + "loss": 1.7295, + "theoretical_loss": 4.308864646498254, + "tokens_seen": 230096896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517271815446339, + "loss": 1.505, + "theoretical_loss": 4.30871542282139, + "tokens_seen": 230162432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007517111334002007, + "loss": 1.7193, + "theoretical_loss": 4.308566253521378, + "tokens_seen": 230227968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516950852557673, + "loss": 1.8099, + "theoretical_loss": 4.308417138562934, + "tokens_seen": 230293504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516790371113341, + "loss": 1.693, + "theoretical_loss": 4.308268077910806, + "tokens_seen": 230359040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516629889669007, + "loss": 1.7438, + "theoretical_loss": 4.308119071529774, + "tokens_seen": 230424576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516469408224675, + "loss": 1.7061, + "theoretical_loss": 4.3079701193846525, + "tokens_seen": 230490112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516308926780341, + "loss": 1.7257, + "theoretical_loss": 4.307821221440286, + "tokens_seen": 230555648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007516148445336009, + "loss": 1.7924, + "theoretical_loss": 4.3076723776615555, + "tokens_seen": 230621184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515987963891675, + "loss": 1.791, + "theoretical_loss": 4.307523588013371, + "tokens_seen": 230686720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515827482447343, + "loss": 1.733, + "theoretical_loss": 4.307374852460679, + "tokens_seen": 230752256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515667001003009, + "loss": 1.6646, + "theoretical_loss": 4.307226170968456, + "tokens_seen": 230817792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515506519558677, + "loss": 1.7016, + "theoretical_loss": 4.307077543501713, + "tokens_seen": 230883328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515346038114343, + "loss": 1.5857, + "theoretical_loss": 4.3069289700254885, + "tokens_seen": 230948864 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515185556670011, + "loss": 1.6563, + "theoretical_loss": 4.306780450504861, + "tokens_seen": 231014400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007515025075225677, + "loss": 1.6707, + "theoretical_loss": 4.3066319849049375, + "tokens_seen": 231079936 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514864593781345, + "loss": 1.7898, + "theoretical_loss": 4.306483573190857, + "tokens_seen": 231145472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514704112337011, + "loss": 1.932, + "theoretical_loss": 4.306335215327791, + "tokens_seen": 231211008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514543630892679, + "loss": 1.7709, + "theoretical_loss": 4.306186911280945, + "tokens_seen": 231276544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514383149448345, + "loss": 1.6175, + "theoretical_loss": 4.306038661015557, + "tokens_seen": 231342080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514222668004013, + "loss": 1.7677, + "theoretical_loss": 4.305890464496895, + "tokens_seen": 231407616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007514062186559679, + "loss": 1.8163, + "theoretical_loss": 4.305742321690261, + "tokens_seen": 231473152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513901705115347, + "loss": 1.7925, + "theoretical_loss": 4.305594232560988, + "tokens_seen": 231538688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513741223671013, + "loss": 1.7761, + "theoretical_loss": 4.305446197074443, + "tokens_seen": 231604224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513580742226681, + "loss": 1.6571, + "theoretical_loss": 4.305298215196023, + "tokens_seen": 231669760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513420260782347, + "loss": 1.7638, + "theoretical_loss": 4.305150286891159, + "tokens_seen": 231735296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513259779338015, + "loss": 1.7255, + "theoretical_loss": 4.3050024121253125, + "tokens_seen": 231800832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007513099297893681, + "loss": 1.7697, + "theoretical_loss": 4.304854590863979, + "tokens_seen": 231866368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512938816449349, + "loss": 1.6632, + "theoretical_loss": 4.304706823072683, + "tokens_seen": 231931904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512778335005015, + "loss": 1.8148, + "theoretical_loss": 4.304559108716984, + "tokens_seen": 231997440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512617853560683, + "loss": 1.5755, + "theoretical_loss": 4.304411447762472, + "tokens_seen": 232062976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512457372116349, + "loss": 1.7349, + "theoretical_loss": 4.304263840174768, + "tokens_seen": 232128512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512296890672017, + "loss": 1.8418, + "theoretical_loss": 4.304116285919527, + "tokens_seen": 232194048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007512136409227683, + "loss": 1.7876, + "theoretical_loss": 4.303968784962433, + "tokens_seen": 232259584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511975927783351, + "loss": 1.7659, + "theoretical_loss": 4.303821337269205, + "tokens_seen": 232325120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511815446339017, + "loss": 1.5599, + "theoretical_loss": 4.303673942805591, + "tokens_seen": 232390656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511654964894685, + "loss": 1.7451, + "theoretical_loss": 4.303526601537371, + "tokens_seen": 232456192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511494483450351, + "loss": 1.7257, + "theoretical_loss": 4.303379313430359, + "tokens_seen": 232521728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511334002006019, + "loss": 1.6267, + "theoretical_loss": 4.303232078450399, + "tokens_seen": 232587264 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511173520561685, + "loss": 1.5978, + "theoretical_loss": 4.303084896563364, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007511013039117353, + "loss": 1.7081, + "theoretical_loss": 4.302937767735164, + "tokens_seen": 232718336 + }, + { + "epoch": 0.07, + "learning_rate": 0.000751085255767302, + "loss": 1.7253, + "theoretical_loss": 4.302790691931736, + "tokens_seen": 232783872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510692076228687, + "loss": 1.7024, + "theoretical_loss": 4.30264366911905, + "tokens_seen": 232849408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510531594784353, + "loss": 1.8108, + "theoretical_loss": 4.3024966992631075, + "tokens_seen": 232914944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510371113340021, + "loss": 1.758, + "theoretical_loss": 4.302349782329942, + "tokens_seen": 232980480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510210631895687, + "loss": 1.7161, + "theoretical_loss": 4.302202918285616, + "tokens_seen": 233046016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007510050150451355, + "loss": 1.6465, + "theoretical_loss": 4.302056107096226, + "tokens_seen": 233111552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509889669007021, + "loss": 1.6963, + "theoretical_loss": 4.301909348727899, + "tokens_seen": 233177088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509729187562688, + "loss": 1.7785, + "theoretical_loss": 4.301762643146792, + "tokens_seen": 233242624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509568706118356, + "loss": 1.7259, + "theoretical_loss": 4.301615990319094, + "tokens_seen": 233308160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509408224674022, + "loss": 1.6942, + "theoretical_loss": 4.301469390211025, + "tokens_seen": 233373696 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750924774322969, + "loss": 1.6251, + "theoretical_loss": 4.301322842788839, + "tokens_seen": 233439232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007509087261785356, + "loss": 1.6583, + "theoretical_loss": 4.301176348018814, + "tokens_seen": 233504768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007508926780341024, + "loss": 1.7607, + "theoretical_loss": 4.301029905867265, + "tokens_seen": 233570304 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750876629889669, + "loss": 1.716, + "theoretical_loss": 4.300883516300539, + "tokens_seen": 233635840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007508605817452358, + "loss": 1.6477, + "theoretical_loss": 4.300737179285008, + "tokens_seen": 233701376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007508445336008024, + "loss": 1.6472, + "theoretical_loss": 4.30059089478708, + "tokens_seen": 233766912 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750828485456369, + "loss": 1.7246, + "theoretical_loss": 4.300444662773192, + "tokens_seen": 233832448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007508124373119358, + "loss": 1.7401, + "theoretical_loss": 4.30029848320981, + "tokens_seen": 233897984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507963891675024, + "loss": 1.7319, + "theoretical_loss": 4.300152356063435, + "tokens_seen": 233963520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507803410230692, + "loss": 1.7894, + "theoretical_loss": 4.300006281300598, + "tokens_seen": 234029056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507642928786359, + "loss": 1.5778, + "theoretical_loss": 4.299860258887856, + "tokens_seen": 234094592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507482447342026, + "loss": 1.705, + "theoretical_loss": 4.299714288791802, + "tokens_seen": 234160128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507321965897694, + "loss": 1.7241, + "theoretical_loss": 4.299568370979056, + "tokens_seen": 234225664 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750716148445336, + "loss": 1.6857, + "theoretical_loss": 4.299422505416272, + "tokens_seen": 234291200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007507001003009028, + "loss": 1.7266, + "theoretical_loss": 4.2992766920701335, + "tokens_seen": 234356736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506840521564694, + "loss": 1.589, + "theoretical_loss": 4.299130930907352, + "tokens_seen": 234422272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506680040120362, + "loss": 1.7423, + "theoretical_loss": 4.2989852218946725, + "tokens_seen": 234487808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506519558676028, + "loss": 1.6108, + "theoretical_loss": 4.298839564998869, + "tokens_seen": 234553344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506359077231696, + "loss": 1.7754, + "theoretical_loss": 4.298693960186747, + "tokens_seen": 234618880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007506198595787362, + "loss": 1.7338, + "theoretical_loss": 4.298548407425141, + "tokens_seen": 234684416 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750603811434303, + "loss": 1.8418, + "theoretical_loss": 4.298402906680918, + "tokens_seen": 234749952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505877632898696, + "loss": 1.7488, + "theoretical_loss": 4.298257457920973, + "tokens_seen": 234815488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505717151454364, + "loss": 1.7443, + "theoretical_loss": 4.298112061112232, + "tokens_seen": 234881024 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750555667001003, + "loss": 1.8159, + "theoretical_loss": 4.297966716221653, + "tokens_seen": 234946560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505396188565698, + "loss": 1.685, + "theoretical_loss": 4.297821423216222, + "tokens_seen": 235012096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505235707121364, + "loss": 1.7925, + "theoretical_loss": 4.297676182062955, + "tokens_seen": 235077632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007505075225677032, + "loss": 1.7185, + "theoretical_loss": 4.297530992728901, + "tokens_seen": 235143168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504914744232698, + "loss": 1.67, + "theoretical_loss": 4.297385855181136, + "tokens_seen": 235208704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504754262788366, + "loss": 1.728, + "theoretical_loss": 4.297240769386768, + "tokens_seen": 235274240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504593781344032, + "loss": 1.6595, + "theoretical_loss": 4.297095735312934, + "tokens_seen": 235339776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075044332998997, + "loss": 1.609, + "theoretical_loss": 4.296950752926801, + "tokens_seen": 235405312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504272818455366, + "loss": 1.7526, + "theoretical_loss": 4.296805822195568, + "tokens_seen": 235470848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007504112337011034, + "loss": 1.6949, + "theoretical_loss": 4.29666094308646, + "tokens_seen": 235536384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00075039518555667, + "loss": 1.7873, + "theoretical_loss": 4.296516115566736, + "tokens_seen": 235601920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503791374122368, + "loss": 1.7357, + "theoretical_loss": 4.296371339603684, + "tokens_seen": 235667456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503630892678034, + "loss": 1.7718, + "theoretical_loss": 4.296226615164618, + "tokens_seen": 235732992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503470411233702, + "loss": 1.6562, + "theoretical_loss": 4.296081942216887, + "tokens_seen": 235798528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503309929789368, + "loss": 1.7045, + "theoretical_loss": 4.295937320727866, + "tokens_seen": 235864064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 94470, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.7614660263061523, + "objective/train/theoretical_loss": 4.295792750664964, + "objective/train/tokens_used": 256389600, + "theoretical_loss": 4.295792750664964, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007503149448345036, + "loss": 1.7615, + "theoretical_loss": 4.295792750664964, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502988966900702, + "loss": 1.5895, + "theoretical_loss": 4.295648231995613, + "tokens_seen": 235995136 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750282848545637, + "loss": 1.6453, + "theoretical_loss": 4.295503764687281, + "tokens_seen": 236060672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502668004012036, + "loss": 1.7145, + "theoretical_loss": 4.295359348707462, + "tokens_seen": 236126208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502507522567704, + "loss": 1.8196, + "theoretical_loss": 4.2952149840236835, + "tokens_seen": 236191744 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750234704112337, + "loss": 1.6946, + "theoretical_loss": 4.295070670603496, + "tokens_seen": 236257280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502186559679038, + "loss": 1.7409, + "theoretical_loss": 4.294926408414486, + "tokens_seen": 236322816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007502026078234704, + "loss": 1.7793, + "theoretical_loss": 4.294782197424265, + "tokens_seen": 236388352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501865596790372, + "loss": 1.6357, + "theoretical_loss": 4.294638037600478, + "tokens_seen": 236453888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501705115346038, + "loss": 1.7133, + "theoretical_loss": 4.2944939289107955, + "tokens_seen": 236519424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501544633901706, + "loss": 1.7445, + "theoretical_loss": 4.29434987132292, + "tokens_seen": 236584960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501384152457372, + "loss": 1.6315, + "theoretical_loss": 4.294205864804581, + "tokens_seen": 236650496 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750122367101304, + "loss": 1.7542, + "theoretical_loss": 4.294061909323539, + "tokens_seen": 236716032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007501063189568706, + "loss": 1.7551, + "theoretical_loss": 4.293918004847585, + "tokens_seen": 236781568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500902708124374, + "loss": 1.7341, + "theoretical_loss": 4.293774151344537, + "tokens_seen": 236847104 + }, + { + "epoch": 0.07, + "learning_rate": 0.000750074222668004, + "loss": 1.7155, + "theoretical_loss": 4.2936303487822425, + "tokens_seen": 236912640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500581745235708, + "loss": 1.7439, + "theoretical_loss": 4.293486597128578, + "tokens_seen": 236978176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500421263791374, + "loss": 1.7712, + "theoretical_loss": 4.293342896351452, + "tokens_seen": 237043712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500260782347042, + "loss": 1.6608, + "theoretical_loss": 4.293199246418796, + "tokens_seen": 237109248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007500100300902708, + "loss": 1.7285, + "theoretical_loss": 4.293055647298578, + "tokens_seen": 237174784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007499939819458376, + "loss": 1.6871, + "theoretical_loss": 4.292912098958791, + "tokens_seen": 237240320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007499779338014042, + "loss": 1.7474, + "theoretical_loss": 4.292768601367455, + "tokens_seen": 237305856 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749961885656971, + "loss": 1.7256, + "theoretical_loss": 4.292625154492623, + "tokens_seen": 237371392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007499458375125376, + "loss": 1.6761, + "theoretical_loss": 4.292481758302375, + "tokens_seen": 237436928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007499297893681044, + "loss": 1.6787, + "theoretical_loss": 4.2923384127648205, + "tokens_seen": 237502464 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749913741223671, + "loss": 1.5678, + "theoretical_loss": 4.292195117848098, + "tokens_seen": 237568000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498976930792378, + "loss": 1.6483, + "theoretical_loss": 4.292051873520373, + "tokens_seen": 237633536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498816449348044, + "loss": 1.7666, + "theoretical_loss": 4.291908679749842, + "tokens_seen": 237699072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498655967903712, + "loss": 1.6182, + "theoretical_loss": 4.2917655365047285, + "tokens_seen": 237764608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498495486459378, + "loss": 1.6742, + "theoretical_loss": 4.291622443753287, + "tokens_seen": 237830144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498335005015046, + "loss": 1.6149, + "theoretical_loss": 4.291479401463799, + "tokens_seen": 237895680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007498174523570712, + "loss": 1.7008, + "theoretical_loss": 4.291336409604574, + "tokens_seen": 237961216 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749801404212638, + "loss": 1.6942, + "theoretical_loss": 4.291193468143952, + "tokens_seen": 238026752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497853560682046, + "loss": 1.66, + "theoretical_loss": 4.2910505770503, + "tokens_seen": 238092288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497693079237714, + "loss": 1.7244, + "theoretical_loss": 4.290907736292016, + "tokens_seen": 238157824 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749753259779338, + "loss": 1.7439, + "theoretical_loss": 4.290764945837522, + "tokens_seen": 238223360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497372116349048, + "loss": 1.6318, + "theoretical_loss": 4.290622205655273, + "tokens_seen": 238288896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497211634904714, + "loss": 1.6553, + "theoretical_loss": 4.290479515713751, + "tokens_seen": 238354432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007497051153460382, + "loss": 1.7517, + "theoretical_loss": 4.290336875981465, + "tokens_seen": 238419968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496890672016048, + "loss": 1.637, + "theoretical_loss": 4.290194286426955, + "tokens_seen": 238485504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496730190571716, + "loss": 1.6891, + "theoretical_loss": 4.290051747018785, + "tokens_seen": 238551040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496569709127382, + "loss": 1.6317, + "theoretical_loss": 4.289909257725553, + "tokens_seen": 238616576 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749640922768305, + "loss": 1.666, + "theoretical_loss": 4.289766818515882, + "tokens_seen": 238682112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496248746238716, + "loss": 1.6351, + "theoretical_loss": 4.2896244293584225, + "tokens_seen": 238747648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007496088264794384, + "loss": 1.7994, + "theoretical_loss": 4.2894820902218544, + "tokens_seen": 238813184 + }, + { + "epoch": 0.07, + "learning_rate": 0.000749592778335005, + "loss": 1.7, + "theoretical_loss": 4.289339801074888, + "tokens_seen": 238878720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495767301905718, + "loss": 1.7788, + "theoretical_loss": 4.289197561886258, + "tokens_seen": 238944256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495606820461385, + "loss": 1.7326, + "theoretical_loss": 4.289055372624729, + "tokens_seen": 239009792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495446339017051, + "loss": 1.6691, + "theoretical_loss": 4.288913233259092, + "tokens_seen": 239075328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495285857572719, + "loss": 1.6302, + "theoretical_loss": 4.28877114375817, + "tokens_seen": 239140864 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007495125376128385, + "loss": 1.8139, + "theoretical_loss": 4.288629104090811, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494964894684053, + "loss": 1.7045, + "theoretical_loss": 4.28848711422589, + "tokens_seen": 239271936 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494804413239719, + "loss": 1.7913, + "theoretical_loss": 4.288345174132313, + "tokens_seen": 239337472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494643931795387, + "loss": 1.7044, + "theoretical_loss": 4.288203283779012, + "tokens_seen": 239403008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494483450351053, + "loss": 1.6033, + "theoretical_loss": 4.288061443134947, + "tokens_seen": 239468544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494322968906721, + "loss": 1.7749, + "theoretical_loss": 4.287919652169107, + "tokens_seen": 239534080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494162487462387, + "loss": 1.7446, + "theoretical_loss": 4.287777910850507, + "tokens_seen": 239599616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007494002006018055, + "loss": 1.8335, + "theoretical_loss": 4.287636219148191, + "tokens_seen": 239665152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493841524573721, + "loss": 1.6768, + "theoretical_loss": 4.287494577031231, + "tokens_seen": 239730688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493681043129389, + "loss": 1.6988, + "theoretical_loss": 4.287352984468727, + "tokens_seen": 239796224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493520561685055, + "loss": 1.8031, + "theoretical_loss": 4.287211441429805, + "tokens_seen": 239861760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493360080240723, + "loss": 1.7529, + "theoretical_loss": 4.2870699478836185, + "tokens_seen": 239927296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493199598796389, + "loss": 1.6927, + "theoretical_loss": 4.286928503799352, + "tokens_seen": 239992832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007493039117352057, + "loss": 1.7997, + "theoretical_loss": 4.286787109146213, + "tokens_seen": 240058368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492878635907723, + "loss": 1.6076, + "theoretical_loss": 4.286645763893442, + "tokens_seen": 240123904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492718154463391, + "loss": 1.6384, + "theoretical_loss": 4.2865044680103015, + "tokens_seen": 240189440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492557673019057, + "loss": 1.5811, + "theoretical_loss": 4.286363221466084, + "tokens_seen": 240254976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492397191574725, + "loss": 1.7501, + "theoretical_loss": 4.286222024230111, + "tokens_seen": 240320512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492236710130391, + "loss": 1.5877, + "theoretical_loss": 4.286080876271729, + "tokens_seen": 240386048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007492076228686059, + "loss": 1.718, + "theoretical_loss": 4.285939777560312, + "tokens_seen": 240451584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491915747241725, + "loss": 1.7557, + "theoretical_loss": 4.285798728065264, + "tokens_seen": 240517120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491755265797393, + "loss": 1.7876, + "theoretical_loss": 4.285657727756012, + "tokens_seen": 240582656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491594784353059, + "loss": 1.6534, + "theoretical_loss": 4.285516776602016, + "tokens_seen": 240648192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491434302908727, + "loss": 1.6575, + "theoretical_loss": 4.2853758745727575, + "tokens_seen": 240713728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491273821464393, + "loss": 1.7133, + "theoretical_loss": 4.285235021637748, + "tokens_seen": 240779264 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007491113340020061, + "loss": 1.7635, + "theoretical_loss": 4.285094217766527, + "tokens_seen": 240844800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490952858575727, + "loss": 1.7549, + "theoretical_loss": 4.2849534629286605, + "tokens_seen": 240910336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490792377131395, + "loss": 1.7171, + "theoretical_loss": 4.284812757093741, + "tokens_seen": 240975872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490631895687061, + "loss": 1.6313, + "theoretical_loss": 4.284672100231388, + "tokens_seen": 241041408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490471414242729, + "loss": 1.6704, + "theoretical_loss": 4.284531492311248, + "tokens_seen": 241106944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490310932798395, + "loss": 1.6644, + "theoretical_loss": 4.284390933302997, + "tokens_seen": 241172480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007490150451354063, + "loss": 1.7358, + "theoretical_loss": 4.284250423176335, + "tokens_seen": 241238016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489989969909729, + "loss": 1.6679, + "theoretical_loss": 4.28410996190099, + "tokens_seen": 241303552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489829488465397, + "loss": 1.562, + "theoretical_loss": 4.283969549446717, + "tokens_seen": 241369088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489669007021063, + "loss": 1.6372, + "theoretical_loss": 4.2838291857833, + "tokens_seen": 241434624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489508525576731, + "loss": 1.7309, + "theoretical_loss": 4.283688870880545, + "tokens_seen": 241500160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489348044132397, + "loss": 1.7814, + "theoretical_loss": 4.28354860470829, + "tokens_seen": 241565696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489187562688065, + "loss": 1.6999, + "theoretical_loss": 4.283408387236397, + "tokens_seen": 241631232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007489027081243731, + "loss": 1.6034, + "theoretical_loss": 4.283268218434755, + "tokens_seen": 241696768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488866599799399, + "loss": 1.8609, + "theoretical_loss": 4.283128098273281, + "tokens_seen": 241762304 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488706118355065, + "loss": 1.8951, + "theoretical_loss": 4.282988026721917, + "tokens_seen": 241827840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488545636910733, + "loss": 1.6055, + "theoretical_loss": 4.282848003750634, + "tokens_seen": 241893376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488385155466399, + "loss": 1.5376, + "theoretical_loss": 4.282708029329427, + "tokens_seen": 241958912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488224674022067, + "loss": 1.7143, + "theoretical_loss": 4.2825681034283205, + "tokens_seen": 242024448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007488064192577733, + "loss": 1.6248, + "theoretical_loss": 4.2824282260173625, + "tokens_seen": 242089984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487903711133401, + "loss": 1.6945, + "theoretical_loss": 4.282288397066631, + "tokens_seen": 242155520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487743229689067, + "loss": 1.6584, + "theoretical_loss": 4.282148616546227, + "tokens_seen": 242221056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487582748244735, + "loss": 1.6498, + "theoretical_loss": 4.282008884426281, + "tokens_seen": 242286592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487422266800401, + "loss": 1.7651, + "theoretical_loss": 4.281869200676948, + "tokens_seen": 242352128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487261785356069, + "loss": 1.7393, + "theoretical_loss": 4.281729565268412, + "tokens_seen": 242417664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 97293, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.70721435546875, + "objective/train/theoretical_loss": 4.28158997817088, + "objective/train/tokens_used": 262943200, + "theoretical_loss": 4.28158997817088, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007487101303911735, + "loss": 1.7072, + "theoretical_loss": 4.28158997817088, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486940822467403, + "loss": 1.6776, + "theoretical_loss": 4.281450439354587, + "tokens_seen": 242548736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486780341023069, + "loss": 1.6901, + "theoretical_loss": 4.281310948789796, + "tokens_seen": 242614272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486619859578737, + "loss": 1.6518, + "theoretical_loss": 4.2811715064467935, + "tokens_seen": 242679808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486459378134403, + "loss": 1.6409, + "theoretical_loss": 4.281032112295894, + "tokens_seen": 242745344 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486298896690071, + "loss": 1.6907, + "theoretical_loss": 4.2808927663074385, + "tokens_seen": 242810880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007486138415245737, + "loss": 1.7182, + "theoretical_loss": 4.280753468451793, + "tokens_seen": 242876416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485977933801405, + "loss": 1.6676, + "theoretical_loss": 4.28061421869935, + "tokens_seen": 242941952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485817452357071, + "loss": 1.6912, + "theoretical_loss": 4.28047501702053, + "tokens_seen": 243007488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485656970912739, + "loss": 1.7135, + "theoretical_loss": 4.280335863385778, + "tokens_seen": 243073024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485496489468405, + "loss": 1.6867, + "theoretical_loss": 4.280196757765564, + "tokens_seen": 243138560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485336008024073, + "loss": 1.7557, + "theoretical_loss": 4.280057700130388, + "tokens_seen": 243204096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485175526579739, + "loss": 1.6174, + "theoretical_loss": 4.279918690450771, + "tokens_seen": 243269632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007485015045135407, + "loss": 1.5945, + "theoretical_loss": 4.279779728697263, + "tokens_seen": 243335168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484854563691073, + "loss": 1.7017, + "theoretical_loss": 4.279640814840441, + "tokens_seen": 243400704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484694082246741, + "loss": 1.8184, + "theoretical_loss": 4.279501948850905, + "tokens_seen": 243466240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484533600802407, + "loss": 1.6976, + "theoretical_loss": 4.279363130699284, + "tokens_seen": 243531776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484373119358075, + "loss": 1.7968, + "theoretical_loss": 4.279224360356231, + "tokens_seen": 243597312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484212637913741, + "loss": 1.671, + "theoretical_loss": 4.279085637792425, + "tokens_seen": 243662848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007484052156469409, + "loss": 1.7158, + "theoretical_loss": 4.278946962978573, + "tokens_seen": 243728384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483891675025075, + "loss": 1.6697, + "theoretical_loss": 4.278808335885403, + "tokens_seen": 243793920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483731193580743, + "loss": 1.8423, + "theoretical_loss": 4.278669756483674, + "tokens_seen": 243859456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483570712136409, + "loss": 1.7238, + "theoretical_loss": 4.278531224744167, + "tokens_seen": 243924992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483410230692077, + "loss": 1.6312, + "theoretical_loss": 4.278392740637694, + "tokens_seen": 243990528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483249749247745, + "loss": 1.6698, + "theoretical_loss": 4.278254304135085, + "tokens_seen": 244056064 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007483089267803411, + "loss": 1.6407, + "theoretical_loss": 4.2781159152072, + "tokens_seen": 244121600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482928786359077, + "loss": 1.6967, + "theoretical_loss": 4.277977573824928, + "tokens_seen": 244187136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482768304914745, + "loss": 1.6572, + "theoretical_loss": 4.277839279959177, + "tokens_seen": 244252672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482607823470411, + "loss": 1.581, + "theoretical_loss": 4.277701033580884, + "tokens_seen": 244318208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482447342026079, + "loss": 1.7374, + "theoretical_loss": 4.277562834661013, + "tokens_seen": 244383744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482286860581746, + "loss": 1.6872, + "theoretical_loss": 4.277424683170549, + "tokens_seen": 244449280 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007482126379137413, + "loss": 1.7246, + "theoretical_loss": 4.277286579080508, + "tokens_seen": 244514816 + }, + { + "epoch": 0.07, + "learning_rate": 0.000748196589769308, + "loss": 1.6254, + "theoretical_loss": 4.277148522361927, + "tokens_seen": 244580352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481805416248747, + "loss": 1.6898, + "theoretical_loss": 4.277010512985871, + "tokens_seen": 244645888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481644934804414, + "loss": 1.6823, + "theoretical_loss": 4.2768725509234296, + "tokens_seen": 244711424 + }, + { + "epoch": 0.07, + "learning_rate": 0.000748148445336008, + "loss": 1.6525, + "theoretical_loss": 4.276734636145717, + "tokens_seen": 244776960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481323971915748, + "loss": 1.6133, + "theoretical_loss": 4.276596768623875, + "tokens_seen": 244842496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481163490471414, + "loss": 1.6995, + "theoretical_loss": 4.276458948329068, + "tokens_seen": 244908032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007481003009027082, + "loss": 1.6637, + "theoretical_loss": 4.276321175232487, + "tokens_seen": 244973568 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480842527582748, + "loss": 1.7109, + "theoretical_loss": 4.276183449305348, + "tokens_seen": 245039104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480682046138416, + "loss": 1.6925, + "theoretical_loss": 4.276045770518895, + "tokens_seen": 245104640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480521564694082, + "loss": 1.632, + "theoretical_loss": 4.275908138844392, + "tokens_seen": 245170176 + }, + { + "epoch": 0.07, + "learning_rate": 0.000748036108324975, + "loss": 1.696, + "theoretical_loss": 4.275770554253132, + "tokens_seen": 245235712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480200601805416, + "loss": 1.636, + "theoretical_loss": 4.275633016716432, + "tokens_seen": 245301248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007480040120361084, + "loss": 1.6925, + "theoretical_loss": 4.275495526205635, + "tokens_seen": 245366784 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747987963891675, + "loss": 1.7086, + "theoretical_loss": 4.275358082692106, + "tokens_seen": 245432320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479719157472418, + "loss": 1.6197, + "theoretical_loss": 4.275220686147241, + "tokens_seen": 245497856 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479558676028084, + "loss": 1.7857, + "theoretical_loss": 4.275083336542455, + "tokens_seen": 245563392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479398194583752, + "loss": 1.7038, + "theoretical_loss": 4.274946033849189, + "tokens_seen": 245628928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479237713139418, + "loss": 1.6159, + "theoretical_loss": 4.2748087780389135, + "tokens_seen": 245694464 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007479077231695086, + "loss": 1.7605, + "theoretical_loss": 4.27467156908312, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007478916750250752, + "loss": 1.6676, + "theoretical_loss": 4.274534406953325, + "tokens_seen": 245825536 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747875626880642, + "loss": 1.6935, + "theoretical_loss": 4.274397291621073, + "tokens_seen": 245891072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007478595787362086, + "loss": 1.7408, + "theoretical_loss": 4.274260223057928, + "tokens_seen": 245956608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007478435305917754, + "loss": 1.747, + "theoretical_loss": 4.274123201235483, + "tokens_seen": 246022144 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747827482447342, + "loss": 1.7192, + "theoretical_loss": 4.273986226125356, + "tokens_seen": 246087680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007478114343029088, + "loss": 1.6998, + "theoretical_loss": 4.273849297699186, + "tokens_seen": 246153216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477953861584754, + "loss": 1.6497, + "theoretical_loss": 4.273712415928643, + "tokens_seen": 246218752 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477793380140422, + "loss": 1.6862, + "theoretical_loss": 4.273575580785415, + "tokens_seen": 246284288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477632898696088, + "loss": 1.7198, + "theoretical_loss": 4.27343879224122, + "tokens_seen": 246349824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477472417251756, + "loss": 1.7479, + "theoretical_loss": 4.273302050267796, + "tokens_seen": 246415360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007477311935807422, + "loss": 1.7629, + "theoretical_loss": 4.27316535483691, + "tokens_seen": 246480896 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747715145436309, + "loss": 1.6332, + "theoretical_loss": 4.273028705920351, + "tokens_seen": 246546432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476990972918756, + "loss": 1.7132, + "theoretical_loss": 4.272892103489934, + "tokens_seen": 246611968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476830491474424, + "loss": 1.8742, + "theoretical_loss": 4.272755547517495, + "tokens_seen": 246677504 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747667001003009, + "loss": 1.6886, + "theoretical_loss": 4.272619037974902, + "tokens_seen": 246743040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476509528585758, + "loss": 1.7306, + "theoretical_loss": 4.272482574834039, + "tokens_seen": 246808576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476349047141424, + "loss": 1.6748, + "theoretical_loss": 4.272346158066821, + "tokens_seen": 246874112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476188565697092, + "loss": 1.6958, + "theoretical_loss": 4.272209787645183, + "tokens_seen": 246939648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007476028084252758, + "loss": 1.7104, + "theoretical_loss": 4.272073463541087, + "tokens_seen": 247005184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007475867602808426, + "loss": 1.6468, + "theoretical_loss": 4.271937185726519, + "tokens_seen": 247070720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007475707121364092, + "loss": 1.6516, + "theoretical_loss": 4.2718009541734885, + "tokens_seen": 247136256 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747554663991976, + "loss": 1.6801, + "theoretical_loss": 4.27166476885403, + "tokens_seen": 247201792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007475386158475426, + "loss": 1.792, + "theoretical_loss": 4.271528629740201, + "tokens_seen": 247267328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007475225677031094, + "loss": 1.7957, + "theoretical_loss": 4.271392536804088, + "tokens_seen": 247332864 + }, + { + "epoch": 0.07, + "learning_rate": 0.000747506519558676, + "loss": 1.6654, + "theoretical_loss": 4.2712564900177945, + "tokens_seen": 247398400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0007474904714142428, + "loss": 1.826, + "theoretical_loss": 4.271120489353454, + "tokens_seen": 247463936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474744232698094, + "loss": 1.6521, + "theoretical_loss": 4.270984534783221, + "tokens_seen": 247529472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474583751253762, + "loss": 1.7131, + "theoretical_loss": 4.270848626279275, + "tokens_seen": 247595008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474423269809428, + "loss": 1.6221, + "theoretical_loss": 4.270712763813822, + "tokens_seen": 247660544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474262788365096, + "loss": 1.7582, + "theoretical_loss": 4.270576947359088, + "tokens_seen": 247726080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007474102306920762, + "loss": 1.6828, + "theoretical_loss": 4.270441176887327, + "tokens_seen": 247791616 + }, + { + "epoch": 0.08, + "learning_rate": 0.000747394182547643, + "loss": 1.695, + "theoretical_loss": 4.270305452370814, + "tokens_seen": 247857152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007473781344032096, + "loss": 1.5398, + "theoretical_loss": 4.27016977378185, + "tokens_seen": 247922688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007473620862587764, + "loss": 1.694, + "theoretical_loss": 4.270034141092759, + "tokens_seen": 247988224 + }, + { + "epoch": 0.08, + "learning_rate": 0.000747346038114343, + "loss": 1.654, + "theoretical_loss": 4.269898554275889, + "tokens_seen": 248053760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007473299899699098, + "loss": 1.703, + "theoretical_loss": 4.269763013303613, + "tokens_seen": 248119296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007473139418254764, + "loss": 1.7145, + "theoretical_loss": 4.269627518148326, + "tokens_seen": 248184832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472978936810432, + "loss": 1.6864, + "theoretical_loss": 4.26949206878245, + "tokens_seen": 248250368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472818455366098, + "loss": 1.6432, + "theoretical_loss": 4.269356665178426, + "tokens_seen": 248315904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472657973921766, + "loss": 1.7075, + "theoretical_loss": 4.269221307308725, + "tokens_seen": 248381440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472497492477432, + "loss": 1.5608, + "theoretical_loss": 4.269085995145837, + "tokens_seen": 248446976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074723370110331, + "loss": 1.6955, + "theoretical_loss": 4.268950728662278, + "tokens_seen": 248512512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472176529588766, + "loss": 1.6484, + "theoretical_loss": 4.268815507830587, + "tokens_seen": 248578048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007472016048144434, + "loss": 1.7139, + "theoretical_loss": 4.268680332623328, + "tokens_seen": 248643584 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074718555667001, + "loss": 1.6881, + "theoretical_loss": 4.268545203013085, + "tokens_seen": 248709120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007471695085255768, + "loss": 1.7474, + "theoretical_loss": 4.26841011897247, + "tokens_seen": 248774656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007471534603811434, + "loss": 1.6635, + "theoretical_loss": 4.268275080474118, + "tokens_seen": 248840192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007471374122367102, + "loss": 1.7351, + "theoretical_loss": 4.268140087490686, + "tokens_seen": 248905728 + }, + { + "epoch": 0.08, + "learning_rate": 0.000747121364092277, + "loss": 1.7056, + "theoretical_loss": 4.2680051399948535, + "tokens_seen": 248971264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 99585, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5767475366592407, + "objective/train/theoretical_loss": 4.267870237959327, + "objective/train/tokens_used": 269496800, + "theoretical_loss": 4.267870237959327, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007471053159478436, + "loss": 1.5767, + "theoretical_loss": 4.267870237959327, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470892678034103, + "loss": 1.8116, + "theoretical_loss": 4.267735381356835, + "tokens_seen": 249102336 + }, + { + "epoch": 0.08, + "learning_rate": 0.000747073219658977, + "loss": 1.8217, + "theoretical_loss": 4.26760057016013, + "tokens_seen": 249167872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470571715145438, + "loss": 1.7518, + "theoretical_loss": 4.2674658043419855, + "tokens_seen": 249233408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470411233701104, + "loss": 1.7357, + "theoretical_loss": 4.267331083875202, + "tokens_seen": 249298944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470250752256772, + "loss": 1.6793, + "theoretical_loss": 4.267196408732601, + "tokens_seen": 249364480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007470090270812438, + "loss": 1.7845, + "theoretical_loss": 4.267061778887029, + "tokens_seen": 249430016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007469929789368106, + "loss": 1.6544, + "theoretical_loss": 4.2669271943113545, + "tokens_seen": 249495552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007469769307923772, + "loss": 1.6735, + "theoretical_loss": 4.26679265497847, + "tokens_seen": 249561088 + }, + { + "epoch": 0.08, + "learning_rate": 0.000746960882647944, + "loss": 1.5407, + "theoretical_loss": 4.266658160861293, + "tokens_seen": 249626624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007469448345035106, + "loss": 1.7296, + "theoretical_loss": 4.266523711932761, + "tokens_seen": 249692160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007469287863590774, + "loss": 1.6292, + "theoretical_loss": 4.266389308165837, + "tokens_seen": 249757696 + }, + { + "epoch": 0.08, + "learning_rate": 0.000746912738214644, + "loss": 1.6179, + "theoretical_loss": 4.266254949533507, + "tokens_seen": 249823232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468966900702106, + "loss": 1.6641, + "theoretical_loss": 4.26612063600878, + "tokens_seen": 249888768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468806419257774, + "loss": 1.6639, + "theoretical_loss": 4.265986367564688, + "tokens_seen": 249954304 + }, + { + "epoch": 0.08, + "learning_rate": 0.000746864593781344, + "loss": 1.6032, + "theoretical_loss": 4.265852144174286, + "tokens_seen": 250019840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468485456369108, + "loss": 1.7491, + "theoretical_loss": 4.265717965810653, + "tokens_seen": 250085376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468324974924775, + "loss": 1.6103, + "theoretical_loss": 4.265583832446892, + "tokens_seen": 250150912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468164493480442, + "loss": 1.7388, + "theoretical_loss": 4.265449744056126, + "tokens_seen": 250216448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007468004012036109, + "loss": 1.6815, + "theoretical_loss": 4.265315700611504, + "tokens_seen": 250281984 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467843530591776, + "loss": 1.7446, + "theoretical_loss": 4.2651817020861955, + "tokens_seen": 250347520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467683049147443, + "loss": 1.6566, + "theoretical_loss": 4.265047748453396, + "tokens_seen": 250413056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467522567703109, + "loss": 1.6856, + "theoretical_loss": 4.264913839686323, + "tokens_seen": 250478592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467362086258777, + "loss": 1.7246, + "theoretical_loss": 4.264779975758215, + "tokens_seen": 250544128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467201604814443, + "loss": 1.7964, + "theoretical_loss": 4.2646461566423355, + "tokens_seen": 250609664 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007467041123370111, + "loss": 1.6955, + "theoretical_loss": 4.26451238231197, + "tokens_seen": 250675200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466880641925777, + "loss": 1.6534, + "theoretical_loss": 4.2643786527404295, + "tokens_seen": 250740736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466720160481445, + "loss": 1.6161, + "theoretical_loss": 4.264244967901043, + "tokens_seen": 250806272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466559679037111, + "loss": 1.679, + "theoretical_loss": 4.264111327767164, + "tokens_seen": 250871808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466399197592779, + "loss": 1.7451, + "theoretical_loss": 4.263977732312174, + "tokens_seen": 250937344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466238716148445, + "loss": 1.6456, + "theoretical_loss": 4.26384418150947, + "tokens_seen": 251002880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007466078234704113, + "loss": 1.6419, + "theoretical_loss": 4.263710675332476, + "tokens_seen": 251068416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465917753259779, + "loss": 1.6311, + "theoretical_loss": 4.263577213754638, + "tokens_seen": 251133952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465757271815447, + "loss": 1.5964, + "theoretical_loss": 4.2634437967494225, + "tokens_seen": 251199488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465596790371113, + "loss": 1.7977, + "theoretical_loss": 4.2633104242903235, + "tokens_seen": 251265024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465436308926781, + "loss": 1.7178, + "theoretical_loss": 4.263177096350852, + "tokens_seen": 251330560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465275827482447, + "loss": 1.6314, + "theoretical_loss": 4.263043812904547, + "tokens_seen": 251396096 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007465115346038115, + "loss": 1.7299, + "theoretical_loss": 4.262910573924965, + "tokens_seen": 251461632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464954864593781, + "loss": 1.575, + "theoretical_loss": 4.26277737938569, + "tokens_seen": 251527168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464794383149449, + "loss": 1.6518, + "theoretical_loss": 4.262644229260324, + "tokens_seen": 251592704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464633901705115, + "loss": 1.6766, + "theoretical_loss": 4.262511123522495, + "tokens_seen": 251658240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464473420260783, + "loss": 1.6524, + "theoretical_loss": 4.262378062145853, + "tokens_seen": 251723776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464312938816449, + "loss": 1.6439, + "theoretical_loss": 4.262245045104068, + "tokens_seen": 251789312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007464152457372117, + "loss": 1.7256, + "theoretical_loss": 4.262112072370837, + "tokens_seen": 251854848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463991975927783, + "loss": 1.7312, + "theoretical_loss": 4.261979143919873, + "tokens_seen": 251920384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463831494483451, + "loss": 1.6506, + "theoretical_loss": 4.261846259724917, + "tokens_seen": 251985920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463671013039117, + "loss": 1.7237, + "theoretical_loss": 4.261713419759731, + "tokens_seen": 252051456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463510531594785, + "loss": 1.7041, + "theoretical_loss": 4.2615806239981, + "tokens_seen": 252116992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463350050150451, + "loss": 1.7052, + "theoretical_loss": 4.261447872413827, + "tokens_seen": 252182528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463189568706119, + "loss": 1.699, + "theoretical_loss": 4.261315164980743, + "tokens_seen": 252248064 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007463029087261785, + "loss": 1.6671, + "theoretical_loss": 4.261182501672698, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462868605817453, + "loss": 1.722, + "theoretical_loss": 4.261049882463565, + "tokens_seen": 252379136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462708124373119, + "loss": 1.7046, + "theoretical_loss": 4.26091730732724, + "tokens_seen": 252444672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462547642928787, + "loss": 1.7357, + "theoretical_loss": 4.2607847762376405, + "tokens_seen": 252510208 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462387161484453, + "loss": 1.5987, + "theoretical_loss": 4.260652289168706, + "tokens_seen": 252575744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462226680040121, + "loss": 1.6433, + "theoretical_loss": 4.260519846094399, + "tokens_seen": 252641280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007462066198595787, + "loss": 1.659, + "theoretical_loss": 4.2603874469887035, + "tokens_seen": 252706816 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461905717151455, + "loss": 1.6698, + "theoretical_loss": 4.260255091825625, + "tokens_seen": 252772352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461745235707121, + "loss": 1.5611, + "theoretical_loss": 4.260122780579193, + "tokens_seen": 252837888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461584754262789, + "loss": 1.5983, + "theoretical_loss": 4.259990513223458, + "tokens_seen": 252903424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461424272818455, + "loss": 1.7221, + "theoretical_loss": 4.259858289732491, + "tokens_seen": 252968960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461263791374123, + "loss": 1.6704, + "theoretical_loss": 4.259726110080388, + "tokens_seen": 253034496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007461103309929789, + "loss": 1.7931, + "theoretical_loss": 4.259593974241264, + "tokens_seen": 253100032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460942828485457, + "loss": 1.664, + "theoretical_loss": 4.25946188218926, + "tokens_seen": 253165568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460782347041123, + "loss": 1.7322, + "theoretical_loss": 4.259329833898534, + "tokens_seen": 253231104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460621865596791, + "loss": 1.825, + "theoretical_loss": 4.25919782934327, + "tokens_seen": 253296640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460461384152457, + "loss": 1.7371, + "theoretical_loss": 4.259065868497672, + "tokens_seen": 253362176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460300902708125, + "loss": 1.6705, + "theoretical_loss": 4.258933951335965, + "tokens_seen": 253427712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007460140421263791, + "loss": 1.6526, + "theoretical_loss": 4.258802077832399, + "tokens_seen": 253493248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459979939819459, + "loss": 1.5827, + "theoretical_loss": 4.258670247961243, + "tokens_seen": 253558784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459819458375125, + "loss": 1.7689, + "theoretical_loss": 4.258538461696787, + "tokens_seen": 253624320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459658976930793, + "loss": 1.5772, + "theoretical_loss": 4.258406719013347, + "tokens_seen": 253689856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459498495486459, + "loss": 1.7136, + "theoretical_loss": 4.258275019885257, + "tokens_seen": 253755392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459338014042127, + "loss": 1.7662, + "theoretical_loss": 4.258143364286875, + "tokens_seen": 253820928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459177532597794, + "loss": 1.6651, + "theoretical_loss": 4.258011752192579, + "tokens_seen": 253886464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007459017051153461, + "loss": 1.6266, + "theoretical_loss": 4.257880183576768, + "tokens_seen": 253952000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458856569709128, + "loss": 1.5498, + "theoretical_loss": 4.257748658413867, + "tokens_seen": 254017536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458696088264795, + "loss": 1.6362, + "theoretical_loss": 4.257617176678317, + "tokens_seen": 254083072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458535606820462, + "loss": 1.6602, + "theoretical_loss": 4.2574857383445845, + "tokens_seen": 254148608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458375125376129, + "loss": 1.7236, + "theoretical_loss": 4.257354343387156, + "tokens_seen": 254214144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458214643931796, + "loss": 1.6552, + "theoretical_loss": 4.257222991780541, + "tokens_seen": 254279680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007458054162487463, + "loss": 1.6718, + "theoretical_loss": 4.257091683499268, + "tokens_seen": 254345216 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745789368104313, + "loss": 1.6494, + "theoretical_loss": 4.25696041851789, + "tokens_seen": 254410752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457733199598797, + "loss": 1.6609, + "theoretical_loss": 4.256829196810978, + "tokens_seen": 254476288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457572718154464, + "loss": 1.7628, + "theoretical_loss": 4.256698018353129, + "tokens_seen": 254541824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457412236710131, + "loss": 1.7087, + "theoretical_loss": 4.256566883118958, + "tokens_seen": 254607360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457251755265798, + "loss": 1.7376, + "theoretical_loss": 4.256435791083101, + "tokens_seen": 254672896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007457091273821465, + "loss": 1.7467, + "theoretical_loss": 4.256304742220219, + "tokens_seen": 254738432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456930792377133, + "loss": 1.6381, + "theoretical_loss": 4.25617373650499, + "tokens_seen": 254803968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456770310932799, + "loss": 1.7607, + "theoretical_loss": 4.256042773912117, + "tokens_seen": 254869504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456609829488467, + "loss": 1.697, + "theoretical_loss": 4.255911854416323, + "tokens_seen": 254935040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456449348044133, + "loss": 1.5103, + "theoretical_loss": 4.255780977992352, + "tokens_seen": 255000576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074562888665998, + "loss": 1.6685, + "theoretical_loss": 4.255650144614968, + "tokens_seen": 255066112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007456128385155467, + "loss": 1.623, + "theoretical_loss": 4.255519354258959, + "tokens_seen": 255131648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455967903711135, + "loss": 1.6435, + "theoretical_loss": 4.255388606899133, + "tokens_seen": 255197184 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455807422266801, + "loss": 1.5703, + "theoretical_loss": 4.255257902510319, + "tokens_seen": 255262720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455646940822469, + "loss": 1.7121, + "theoretical_loss": 4.255127241067368, + "tokens_seen": 255328256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455486459378135, + "loss": 1.5956, + "theoretical_loss": 4.25499662254515, + "tokens_seen": 255393792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455325977933803, + "loss": 1.58, + "theoretical_loss": 4.254866046918559, + "tokens_seen": 255459328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455165496489469, + "loss": 1.7176, + "theoretical_loss": 4.254735514162508, + "tokens_seen": 255524864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 101923, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.7840094566345215, + "objective/train/theoretical_loss": 4.254605024251933, + "objective/train/tokens_used": 276050400, + "theoretical_loss": 4.254605024251933, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007455005015045136, + "loss": 1.784, + "theoretical_loss": 4.254605024251933, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454844533600803, + "loss": 1.7116, + "theoretical_loss": 4.254474577161789, + "tokens_seen": 255655936 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745468405215647, + "loss": 1.6839, + "theoretical_loss": 4.254344172867053, + "tokens_seen": 255721472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454523570712137, + "loss": 1.6665, + "theoretical_loss": 4.254213811342723, + "tokens_seen": 255787008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454363089267804, + "loss": 1.7717, + "theoretical_loss": 4.254083492563819, + "tokens_seen": 255852544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454202607823471, + "loss": 1.6835, + "theoretical_loss": 4.253953216505381, + "tokens_seen": 255918080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007454042126379138, + "loss": 1.6512, + "theoretical_loss": 4.253822983142468, + "tokens_seen": 255983616 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453881644934805, + "loss": 1.695, + "theoretical_loss": 4.253692792450165, + "tokens_seen": 256049152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453721163490472, + "loss": 1.6956, + "theoretical_loss": 4.253562644403573, + "tokens_seen": 256114688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453560682046138, + "loss": 1.6822, + "theoretical_loss": 4.253432538977816, + "tokens_seen": 256180224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453400200601806, + "loss": 1.5904, + "theoretical_loss": 4.253302476148039, + "tokens_seen": 256245760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007453239719157472, + "loss": 1.7748, + "theoretical_loss": 4.253172455889407, + "tokens_seen": 256311296 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745307923771314, + "loss": 1.654, + "theoretical_loss": 4.2530424781771075, + "tokens_seen": 256376832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452918756268806, + "loss": 1.7685, + "theoretical_loss": 4.252912542986345, + "tokens_seen": 256442368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452758274824474, + "loss": 1.5897, + "theoretical_loss": 4.2527826502923505, + "tokens_seen": 256507904 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745259779338014, + "loss": 1.7146, + "theoretical_loss": 4.25265280007037, + "tokens_seen": 256573440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452437311935808, + "loss": 1.6872, + "theoretical_loss": 4.252522992295676, + "tokens_seen": 256638976 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452276830491474, + "loss": 1.7714, + "theoretical_loss": 4.252393226943555, + "tokens_seen": 256704512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007452116349047142, + "loss": 1.6481, + "theoretical_loss": 4.252263503989321, + "tokens_seen": 256770048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451955867602808, + "loss": 1.8716, + "theoretical_loss": 4.252133823408303, + "tokens_seen": 256835584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451795386158476, + "loss": 1.7463, + "theoretical_loss": 4.2520041851758545, + "tokens_seen": 256901120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451634904714142, + "loss": 1.6302, + "theoretical_loss": 4.251874589267347, + "tokens_seen": 256966656 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745147442326981, + "loss": 1.788, + "theoretical_loss": 4.251745035658176, + "tokens_seen": 257032192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451313941825476, + "loss": 1.5937, + "theoretical_loss": 4.251615524323752, + "tokens_seen": 257097728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007451153460381144, + "loss": 1.6688, + "theoretical_loss": 4.251486055239512, + "tokens_seen": 257163264 + }, + { + "epoch": 0.08, + "learning_rate": 0.000745099297893681, + "loss": 1.7729, + "theoretical_loss": 4.251356628380911, + "tokens_seen": 257228800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450832497492478, + "loss": 1.7324, + "theoretical_loss": 4.251227243723424, + "tokens_seen": 257294336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450672016048144, + "loss": 1.7021, + "theoretical_loss": 4.251097901242546, + "tokens_seen": 257359872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450511534603812, + "loss": 1.6532, + "theoretical_loss": 4.250968600913795, + "tokens_seen": 257425408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450351053159478, + "loss": 1.689, + "theoretical_loss": 4.250839342712707, + "tokens_seen": 257490944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450190571715146, + "loss": 1.6999, + "theoretical_loss": 4.250710126614838, + "tokens_seen": 257556480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007450030090270812, + "loss": 1.663, + "theoretical_loss": 4.250580952595769, + "tokens_seen": 257622016 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744986960882648, + "loss": 1.5993, + "theoretical_loss": 4.250451820631095, + "tokens_seen": 257687552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007449709127382146, + "loss": 1.7228, + "theoretical_loss": 4.2503227306964355, + "tokens_seen": 257753088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007449548645937814, + "loss": 1.7235, + "theoretical_loss": 4.25019368276743, + "tokens_seen": 257818624 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744938816449348, + "loss": 1.8186, + "theoretical_loss": 4.250064676819735, + "tokens_seen": 257884160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007449227683049148, + "loss": 1.7635, + "theoretical_loss": 4.249935712829033, + "tokens_seen": 257949696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007449067201604814, + "loss": 1.6662, + "theoretical_loss": 4.249806790771022, + "tokens_seen": 258015232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448906720160482, + "loss": 1.6486, + "theoretical_loss": 4.249677910621422, + "tokens_seen": 258080768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448746238716148, + "loss": 1.7737, + "theoretical_loss": 4.249549072355974, + "tokens_seen": 258146304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448585757271816, + "loss": 1.7493, + "theoretical_loss": 4.2494202759504365, + "tokens_seen": 258211840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448425275827482, + "loss": 1.6564, + "theoretical_loss": 4.249291521380591, + "tokens_seen": 258277376 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744826479438315, + "loss": 1.688, + "theoretical_loss": 4.249162808622237, + "tokens_seen": 258342912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007448104312938816, + "loss": 1.7598, + "theoretical_loss": 4.249034137651197, + "tokens_seen": 258408448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447943831494484, + "loss": 1.6927, + "theoretical_loss": 4.248905508443311, + "tokens_seen": 258473984 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744778335005015, + "loss": 1.6818, + "theoretical_loss": 4.248776920974439, + "tokens_seen": 258539520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447622868605818, + "loss": 1.6458, + "theoretical_loss": 4.248648375220464, + "tokens_seen": 258605056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447462387161484, + "loss": 1.7227, + "theoretical_loss": 4.248519871157285, + "tokens_seen": 258670592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447301905717152, + "loss": 1.7057, + "theoretical_loss": 4.248391408760824, + "tokens_seen": 258736128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007447141424272819, + "loss": 1.63, + "theoretical_loss": 4.248262988007022, + "tokens_seen": 258801664 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446980942828486, + "loss": 1.7302, + "theoretical_loss": 4.248134608871841, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446820461384153, + "loss": 1.6269, + "theoretical_loss": 4.24800627133126, + "tokens_seen": 258932736 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744665997993982, + "loss": 1.6952, + "theoretical_loss": 4.247877975361281, + "tokens_seen": 258998272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446499498495487, + "loss": 1.6942, + "theoretical_loss": 4.247749720937925, + "tokens_seen": 259063808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446339017051154, + "loss": 1.6828, + "theoretical_loss": 4.247621508037232, + "tokens_seen": 259129344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446178535606821, + "loss": 1.75, + "theoretical_loss": 4.247493336635264, + "tokens_seen": 259194880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007446018054162488, + "loss": 1.6661, + "theoretical_loss": 4.2473652067081, + "tokens_seen": 259260416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445857572718155, + "loss": 1.6707, + "theoretical_loss": 4.247237118231841, + "tokens_seen": 259325952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445697091273822, + "loss": 1.6324, + "theoretical_loss": 4.247109071182607, + "tokens_seen": 259391488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445536609829489, + "loss": 1.6038, + "theoretical_loss": 4.246981065536538, + "tokens_seen": 259457024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445376128385156, + "loss": 1.5675, + "theoretical_loss": 4.246853101269793, + "tokens_seen": 259522560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007445215646940823, + "loss": 1.7639, + "theoretical_loss": 4.2467251783585525, + "tokens_seen": 259588096 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744505516549649, + "loss": 1.7606, + "theoretical_loss": 4.246597296779014, + "tokens_seen": 259653632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444894684052157, + "loss": 1.7239, + "theoretical_loss": 4.246469456507398, + "tokens_seen": 259719168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444734202607824, + "loss": 1.725, + "theoretical_loss": 4.246341657519943, + "tokens_seen": 259784704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444573721163491, + "loss": 1.7266, + "theoretical_loss": 4.246213899792905, + "tokens_seen": 259850240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444413239719158, + "loss": 1.6228, + "theoretical_loss": 4.246086183302564, + "tokens_seen": 259915776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444252758274825, + "loss": 1.5962, + "theoretical_loss": 4.2459585080252165, + "tokens_seen": 259981312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007444092276830492, + "loss": 1.6643, + "theoretical_loss": 4.245830873937179, + "tokens_seen": 260046848 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744393179538616, + "loss": 1.768, + "theoretical_loss": 4.24570328101479, + "tokens_seen": 260112384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007443771313941826, + "loss": 1.73, + "theoretical_loss": 4.245575729234403, + "tokens_seen": 260177920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007443610832497493, + "loss": 1.5881, + "theoretical_loss": 4.245448218572396, + "tokens_seen": 260243456 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744345035105316, + "loss": 1.7745, + "theoretical_loss": 4.245320749005163, + "tokens_seen": 260308992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007443289869608828, + "loss": 1.6264, + "theoretical_loss": 4.245193320509118, + "tokens_seen": 260374528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007443129388164494, + "loss": 1.6443, + "theoretical_loss": 4.245065933060697, + "tokens_seen": 260440064 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442968906720162, + "loss": 1.6297, + "theoretical_loss": 4.244938586636351, + "tokens_seen": 260505600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442808425275828, + "loss": 1.6536, + "theoretical_loss": 4.244811281212556, + "tokens_seen": 260571136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442647943831496, + "loss": 1.7296, + "theoretical_loss": 4.244684016765802, + "tokens_seen": 260636672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442487462387162, + "loss": 1.647, + "theoretical_loss": 4.2445567932726025, + "tokens_seen": 260702208 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744232698094283, + "loss": 1.7275, + "theoretical_loss": 4.244429610709488, + "tokens_seen": 260767744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442166499498496, + "loss": 1.5733, + "theoretical_loss": 4.244302469053007, + "tokens_seen": 260833280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007442006018054164, + "loss": 1.6508, + "theoretical_loss": 4.244175368279733, + "tokens_seen": 260898816 + }, + { + "epoch": 0.08, + "learning_rate": 0.000744184553660983, + "loss": 1.6887, + "theoretical_loss": 4.244048308366253, + "tokens_seen": 260964352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441685055165498, + "loss": 1.5841, + "theoretical_loss": 4.243921289289175, + "tokens_seen": 261029888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441524573721164, + "loss": 1.6457, + "theoretical_loss": 4.243794311025128, + "tokens_seen": 261095424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441364092276832, + "loss": 1.7269, + "theoretical_loss": 4.243667373550757, + "tokens_seen": 261160960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441203610832498, + "loss": 1.6444, + "theoretical_loss": 4.24354047684273, + "tokens_seen": 261226496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007441043129388166, + "loss": 1.6782, + "theoretical_loss": 4.243413620877732, + "tokens_seen": 261292032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440882647943832, + "loss": 1.5928, + "theoretical_loss": 4.243286805632467, + "tokens_seen": 261357568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440722166499499, + "loss": 1.6889, + "theoretical_loss": 4.243160031083658, + "tokens_seen": 261423104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440561685055166, + "loss": 1.5918, + "theoretical_loss": 4.24303329720805, + "tokens_seen": 261488640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440401203610833, + "loss": 1.6905, + "theoretical_loss": 4.242906603982402, + "tokens_seen": 261554176 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074402407221665, + "loss": 1.7469, + "theoretical_loss": 4.242779951383497, + "tokens_seen": 261619712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007440080240722167, + "loss": 1.6461, + "theoretical_loss": 4.242653339388135, + "tokens_seen": 261685248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439919759277834, + "loss": 1.7377, + "theoretical_loss": 4.242526767973134, + "tokens_seen": 261750784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439759277833501, + "loss": 1.6465, + "theoretical_loss": 4.242400237115333, + "tokens_seen": 261816320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439598796389168, + "loss": 1.5911, + "theoretical_loss": 4.242273746791589, + "tokens_seen": 261881856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439438314944835, + "loss": 1.765, + "theoretical_loss": 4.242147296978778, + "tokens_seen": 261947392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439277833500501, + "loss": 1.7125, + "theoretical_loss": 4.242020887653795, + "tokens_seen": 262012928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007439117352056169, + "loss": 1.6799, + "theoretical_loss": 4.241894518793555, + "tokens_seen": 262078464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 104233, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.6709940433502197, + "objective/train/theoretical_loss": 4.24176819037499, + "objective/train/tokens_used": 282604000, + "theoretical_loss": 4.24176819037499, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438956870611835, + "loss": 1.671, + "theoretical_loss": 4.24176819037499, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438796389167503, + "loss": 1.7132, + "theoretical_loss": 4.2416419023750525, + "tokens_seen": 262209536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438635907723169, + "loss": 1.6513, + "theoretical_loss": 4.241515654770713, + "tokens_seen": 262275072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438475426278837, + "loss": 1.6345, + "theoretical_loss": 4.241389447538961, + "tokens_seen": 262340608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438314944834503, + "loss": 1.6632, + "theoretical_loss": 4.241263280656806, + "tokens_seen": 262406144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007438154463390171, + "loss": 1.6645, + "theoretical_loss": 4.241137154101275, + "tokens_seen": 262471680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437993981945837, + "loss": 1.6946, + "theoretical_loss": 4.2410110678494135, + "tokens_seen": 262537216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437833500501505, + "loss": 1.6594, + "theoretical_loss": 4.240885021878286, + "tokens_seen": 262602752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437673019057171, + "loss": 1.6579, + "theoretical_loss": 4.240759016164978, + "tokens_seen": 262668288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437512537612839, + "loss": 1.682, + "theoretical_loss": 4.240633050686592, + "tokens_seen": 262733824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437352056168505, + "loss": 1.6489, + "theoretical_loss": 4.2405071254202475, + "tokens_seen": 262799360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437191574724173, + "loss": 1.7881, + "theoretical_loss": 4.240381240343085, + "tokens_seen": 262864896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007437031093279839, + "loss": 1.7095, + "theoretical_loss": 4.240255395432264, + "tokens_seen": 262930432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436870611835507, + "loss": 1.6394, + "theoretical_loss": 4.240129590664961, + "tokens_seen": 262995968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436710130391173, + "loss": 1.6244, + "theoretical_loss": 4.240003826018373, + "tokens_seen": 263061504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436549648946841, + "loss": 1.7085, + "theoretical_loss": 4.239878101469713, + "tokens_seen": 263127040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436389167502507, + "loss": 1.722, + "theoretical_loss": 4.239752416996215, + "tokens_seen": 263192576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436228686058175, + "loss": 1.7182, + "theoretical_loss": 4.239626772575131, + "tokens_seen": 263258112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007436068204613841, + "loss": 1.5903, + "theoretical_loss": 4.239501168183732, + "tokens_seen": 263323648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435907723169509, + "loss": 1.5815, + "theoretical_loss": 4.239375603799305, + "tokens_seen": 263389184 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435747241725175, + "loss": 1.6607, + "theoretical_loss": 4.2392500793991585, + "tokens_seen": 263454720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435586760280843, + "loss": 1.5409, + "theoretical_loss": 4.239124594960619, + "tokens_seen": 263520256 + }, + { + "epoch": 0.08, + "learning_rate": 0.000743542627883651, + "loss": 1.6256, + "theoretical_loss": 4.23899915046103, + "tokens_seen": 263585792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435265797392177, + "loss": 1.6867, + "theoretical_loss": 4.238873745877756, + "tokens_seen": 263651328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007435105315947844, + "loss": 1.6541, + "theoretical_loss": 4.238748381188176, + "tokens_seen": 263716864 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434944834503511, + "loss": 1.8033, + "theoretical_loss": 4.238623056369692, + "tokens_seen": 263782400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434784353059178, + "loss": 1.6045, + "theoretical_loss": 4.238497771399721, + "tokens_seen": 263847936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434623871614845, + "loss": 1.7191, + "theoretical_loss": 4.2383725262556995, + "tokens_seen": 263913472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434463390170512, + "loss": 1.6445, + "theoretical_loss": 4.238247320915083, + "tokens_seen": 263979008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434302908726179, + "loss": 1.5632, + "theoretical_loss": 4.238122155355345, + "tokens_seen": 264044544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007434142427281846, + "loss": 1.702, + "theoretical_loss": 4.237997029553977, + "tokens_seen": 264110080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433981945837513, + "loss": 1.6446, + "theoretical_loss": 4.237871943488488, + "tokens_seen": 264175616 + }, + { + "epoch": 0.08, + "learning_rate": 0.000743382146439318, + "loss": 1.7052, + "theoretical_loss": 4.237746897136408, + "tokens_seen": 264241152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433660982948847, + "loss": 1.6234, + "theoretical_loss": 4.237621890475284, + "tokens_seen": 264306688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433500501504514, + "loss": 1.6944, + "theoretical_loss": 4.237496923482678, + "tokens_seen": 264372224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433340020060181, + "loss": 1.6701, + "theoretical_loss": 4.237371996136175, + "tokens_seen": 264437760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433179538615848, + "loss": 1.6702, + "theoretical_loss": 4.237247108413376, + "tokens_seen": 264503296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007433019057171515, + "loss": 1.5667, + "theoretical_loss": 4.237122260291901, + "tokens_seen": 264568832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432858575727182, + "loss": 1.6129, + "theoretical_loss": 4.236997451749387, + "tokens_seen": 264634368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432698094282849, + "loss": 1.7445, + "theoretical_loss": 4.236872682763489, + "tokens_seen": 264699904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432537612838516, + "loss": 1.6724, + "theoretical_loss": 4.236747953311884, + "tokens_seen": 264765440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432377131394183, + "loss": 1.6016, + "theoretical_loss": 4.236623263372261, + "tokens_seen": 264830976 + }, + { + "epoch": 0.08, + "learning_rate": 0.000743221664994985, + "loss": 1.6213, + "theoretical_loss": 4.236498612922331, + "tokens_seen": 264896512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007432056168505517, + "loss": 1.6128, + "theoretical_loss": 4.236374001939822, + "tokens_seen": 264962048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431895687061184, + "loss": 1.6469, + "theoretical_loss": 4.2362494304024825, + "tokens_seen": 265027584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431735205616851, + "loss": 1.7452, + "theoretical_loss": 4.236124898288074, + "tokens_seen": 265093120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431574724172518, + "loss": 1.6845, + "theoretical_loss": 4.236000405574382, + "tokens_seen": 265158656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431414242728185, + "loss": 1.7285, + "theoretical_loss": 4.235875952239204, + "tokens_seen": 265224192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431253761283852, + "loss": 1.6745, + "theoretical_loss": 4.235751538260359, + "tokens_seen": 265289728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007431093279839519, + "loss": 1.6798, + "theoretical_loss": 4.2356271636156855, + "tokens_seen": 265355264 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430932798395186, + "loss": 1.6917, + "theoretical_loss": 4.235502828283035, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430772316950853, + "loss": 1.7893, + "theoretical_loss": 4.235378532240283, + "tokens_seen": 265486336 + }, + { + "epoch": 0.08, + "learning_rate": 0.000743061183550652, + "loss": 1.7136, + "theoretical_loss": 4.235254275465317, + "tokens_seen": 265551872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430451354062187, + "loss": 1.6035, + "theoretical_loss": 4.235130057936045, + "tokens_seen": 265617408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430290872617854, + "loss": 1.5738, + "theoretical_loss": 4.235005879630395, + "tokens_seen": 265682944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007430130391173521, + "loss": 1.6699, + "theoretical_loss": 4.23488174052631, + "tokens_seen": 265748480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429969909729188, + "loss": 1.6366, + "theoretical_loss": 4.234757640601751, + "tokens_seen": 265814016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429809428284855, + "loss": 1.5538, + "theoretical_loss": 4.234633579834698, + "tokens_seen": 265879552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429648946840523, + "loss": 1.6776, + "theoretical_loss": 4.234509558203149, + "tokens_seen": 265945088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429488465396189, + "loss": 1.6846, + "theoretical_loss": 4.234385575685116, + "tokens_seen": 266010624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429327983951857, + "loss": 1.7382, + "theoretical_loss": 4.234261632258636, + "tokens_seen": 266076160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007429167502507523, + "loss": 1.7313, + "theoretical_loss": 4.234137727901757, + "tokens_seen": 266141696 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742900702106319, + "loss": 1.5754, + "theoretical_loss": 4.234013862592548, + "tokens_seen": 266207232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428846539618857, + "loss": 1.7733, + "theoretical_loss": 4.233890036309095, + "tokens_seen": 266272768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428686058174525, + "loss": 1.7302, + "theoretical_loss": 4.233766249029501, + "tokens_seen": 266338304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428525576730191, + "loss": 1.6632, + "theoretical_loss": 4.233642500731889, + "tokens_seen": 266403840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428365095285859, + "loss": 1.6486, + "theoretical_loss": 4.233518791394396, + "tokens_seen": 266469376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428204613841525, + "loss": 1.5989, + "theoretical_loss": 4.233395120995181, + "tokens_seen": 266534912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007428044132397193, + "loss": 1.6482, + "theoretical_loss": 4.233271489512416, + "tokens_seen": 266600448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427883650952859, + "loss": 1.6986, + "theoretical_loss": 4.233147896924294, + "tokens_seen": 266665984 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427723169508527, + "loss": 1.7408, + "theoretical_loss": 4.233024343209023, + "tokens_seen": 266731520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427562688064193, + "loss": 1.5948, + "theoretical_loss": 4.232900828344833, + "tokens_seen": 266797056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427402206619861, + "loss": 1.7545, + "theoretical_loss": 4.232777352309966, + "tokens_seen": 266862592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427241725175527, + "loss": 1.7316, + "theoretical_loss": 4.232653915082684, + "tokens_seen": 266928128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007427081243731195, + "loss": 1.6597, + "theoretical_loss": 4.232530516641267, + "tokens_seen": 266993664 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426920762286861, + "loss": 1.6585, + "theoretical_loss": 4.232407156964014, + "tokens_seen": 267059200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426760280842528, + "loss": 1.6008, + "theoretical_loss": 4.232283836029237, + "tokens_seen": 267124736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426599799398195, + "loss": 1.6104, + "theoretical_loss": 4.232160553815269, + "tokens_seen": 267190272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426439317953862, + "loss": 1.6741, + "theoretical_loss": 4.232037310300458, + "tokens_seen": 267255808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426278836509529, + "loss": 1.6344, + "theoretical_loss": 4.231914105463173, + "tokens_seen": 267321344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007426118355065196, + "loss": 1.7326, + "theoretical_loss": 4.231790939281797, + "tokens_seen": 267386880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007425957873620863, + "loss": 1.7199, + "theoretical_loss": 4.231667811734732, + "tokens_seen": 267452416 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742579739217653, + "loss": 1.602, + "theoretical_loss": 4.231544722800395, + "tokens_seen": 267517952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007425636910732197, + "loss": 1.5649, + "theoretical_loss": 4.231421672457225, + "tokens_seen": 267583488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007425476429287864, + "loss": 1.6479, + "theoretical_loss": 4.231298660683675, + "tokens_seen": 267649024 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742531594784353, + "loss": 1.772, + "theoretical_loss": 4.231175687458214, + "tokens_seen": 267714560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007425155466399198, + "loss": 1.696, + "theoretical_loss": 4.231052752759332, + "tokens_seen": 267780096 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424994984954864, + "loss": 1.6956, + "theoretical_loss": 4.230929856565535, + "tokens_seen": 267845632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424834503510532, + "loss": 1.7029, + "theoretical_loss": 4.230806998855343, + "tokens_seen": 267911168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424674022066198, + "loss": 1.7181, + "theoretical_loss": 4.230684179607297, + "tokens_seen": 267976704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424513540621866, + "loss": 1.7139, + "theoretical_loss": 4.2305613987999555, + "tokens_seen": 268042240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424353059177532, + "loss": 1.7342, + "theoretical_loss": 4.230438656411891, + "tokens_seen": 268107776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074241925777332, + "loss": 1.691, + "theoretical_loss": 4.230315952421697, + "tokens_seen": 268173312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007424032096288866, + "loss": 1.6905, + "theoretical_loss": 4.230193286807981, + "tokens_seen": 268238848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423871614844534, + "loss": 1.6338, + "theoretical_loss": 4.230070659549368, + "tokens_seen": 268304384 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074237111334002, + "loss": 1.719, + "theoretical_loss": 4.229948070624502, + "tokens_seen": 268369920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423550651955868, + "loss": 1.6452, + "theoretical_loss": 4.229825520012042, + "tokens_seen": 268435456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423390170511535, + "loss": 1.6339, + "theoretical_loss": 4.2297030076906665, + "tokens_seen": 268500992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423229689067202, + "loss": 1.6726, + "theoretical_loss": 4.229580533639068, + "tokens_seen": 268566528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007423069207622869, + "loss": 1.6218, + "theoretical_loss": 4.229458097835959, + "tokens_seen": 268632064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 106547, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.7015962600708008, + "objective/train/theoretical_loss": 4.2293357002600676, + "objective/train/tokens_used": 289157600, + "theoretical_loss": 4.2293357002600676, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422908726178536, + "loss": 1.7016, + "theoretical_loss": 4.2293357002600676, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422748244734203, + "loss": 1.7635, + "theoretical_loss": 4.229213340890139, + "tokens_seen": 268763136 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742258776328987, + "loss": 1.669, + "theoretical_loss": 4.229091019704935, + "tokens_seen": 268828672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422427281845537, + "loss": 1.5246, + "theoretical_loss": 4.2289687366832345, + "tokens_seen": 268894208 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422266800401204, + "loss": 1.6254, + "theoretical_loss": 4.228846491803835, + "tokens_seen": 268959744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007422106318956871, + "loss": 1.6714, + "theoretical_loss": 4.228724285045549, + "tokens_seen": 269025280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421945837512538, + "loss": 1.5393, + "theoretical_loss": 4.228602116387206, + "tokens_seen": 269090816 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421785356068205, + "loss": 1.6996, + "theoretical_loss": 4.228479985807653, + "tokens_seen": 269156352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421624874623872, + "loss": 1.8339, + "theoretical_loss": 4.228357893285755, + "tokens_seen": 269221888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421464393179539, + "loss": 1.7352, + "theoretical_loss": 4.2282358388003916, + "tokens_seen": 269287424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421303911735206, + "loss": 1.5974, + "theoretical_loss": 4.228113822330461, + "tokens_seen": 269352960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007421143430290873, + "loss": 1.7363, + "theoretical_loss": 4.227991843854878, + "tokens_seen": 269418496 + }, + { + "epoch": 0.08, + "learning_rate": 0.000742098294884654, + "loss": 1.7108, + "theoretical_loss": 4.227869903352572, + "tokens_seen": 269484032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420822467402207, + "loss": 1.6682, + "theoretical_loss": 4.227748000802493, + "tokens_seen": 269549568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420661985957874, + "loss": 1.6003, + "theoretical_loss": 4.227626136183606, + "tokens_seen": 269615104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420501504513541, + "loss": 1.5697, + "theoretical_loss": 4.227504309474892, + "tokens_seen": 269680640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420341023069208, + "loss": 1.7448, + "theoretical_loss": 4.227382520655348, + "tokens_seen": 269746176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420180541624875, + "loss": 1.7072, + "theoretical_loss": 4.227260769703991, + "tokens_seen": 269811712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007420020060180542, + "loss": 1.6838, + "theoretical_loss": 4.227139056599851, + "tokens_seen": 269877248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419859578736209, + "loss": 1.6778, + "theoretical_loss": 4.2270173813219785, + "tokens_seen": 269942784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419699097291876, + "loss": 1.6301, + "theoretical_loss": 4.226895743849437, + "tokens_seen": 270008320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419538615847543, + "loss": 1.6792, + "theoretical_loss": 4.226774144161309, + "tokens_seen": 270073856 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741937813440321, + "loss": 1.6911, + "theoretical_loss": 4.226652582236694, + "tokens_seen": 270139392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419217652958877, + "loss": 1.5549, + "theoretical_loss": 4.226531058054706, + "tokens_seen": 270204928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007419057171514544, + "loss": 1.5975, + "theoretical_loss": 4.226409571594477, + "tokens_seen": 270270464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418896690070211, + "loss": 1.6987, + "theoretical_loss": 4.226288122835156, + "tokens_seen": 270336000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418736208625878, + "loss": 1.5463, + "theoretical_loss": 4.226166711755906, + "tokens_seen": 270401536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418575727181545, + "loss": 1.7582, + "theoretical_loss": 4.2260453383359105, + "tokens_seen": 270467072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418415245737212, + "loss": 1.7113, + "theoretical_loss": 4.2259240025543665, + "tokens_seen": 270532608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418254764292879, + "loss": 1.6721, + "theoretical_loss": 4.225802704390488, + "tokens_seen": 270598144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007418094282848546, + "loss": 1.6108, + "theoretical_loss": 4.2256814438235075, + "tokens_seen": 270663680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417933801404213, + "loss": 1.6607, + "theoretical_loss": 4.225560220832672, + "tokens_seen": 270729216 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741777331995988, + "loss": 1.681, + "theoretical_loss": 4.225439035397246, + "tokens_seen": 270794752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417612838515547, + "loss": 1.7115, + "theoretical_loss": 4.225317887496508, + "tokens_seen": 270860288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417452357071214, + "loss": 1.6303, + "theoretical_loss": 4.2251967771097565, + "tokens_seen": 270925824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417291875626881, + "loss": 1.6108, + "theoretical_loss": 4.225075704216305, + "tokens_seen": 270991360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007417131394182548, + "loss": 1.746, + "theoretical_loss": 4.224954668795483, + "tokens_seen": 271056896 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416970912738215, + "loss": 1.6751, + "theoretical_loss": 4.224833670826636, + "tokens_seen": 271122432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416810431293882, + "loss": 1.5968, + "theoretical_loss": 4.224712710289127, + "tokens_seen": 271187968 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741664994984955, + "loss": 1.6471, + "theoretical_loss": 4.2245917871623355, + "tokens_seen": 271253504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416489468405216, + "loss": 1.4978, + "theoretical_loss": 4.224470901425655, + "tokens_seen": 271319040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416328986960883, + "loss": 1.674, + "theoretical_loss": 4.224350053058499, + "tokens_seen": 271384576 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741616850551655, + "loss": 1.6004, + "theoretical_loss": 4.224229242040295, + "tokens_seen": 271450112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007416008024072218, + "loss": 1.6213, + "theoretical_loss": 4.224108468350485, + "tokens_seen": 271515648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415847542627884, + "loss": 1.7061, + "theoretical_loss": 4.223987731968532, + "tokens_seen": 271581184 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415687061183552, + "loss": 1.6753, + "theoretical_loss": 4.223867032873912, + "tokens_seen": 271646720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415526579739218, + "loss": 1.8097, + "theoretical_loss": 4.223746371046116, + "tokens_seen": 271712256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415366098294886, + "loss": 1.5684, + "theoretical_loss": 4.223625746464656, + "tokens_seen": 271777792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007415205616850552, + "loss": 1.6576, + "theoretical_loss": 4.223505159109054, + "tokens_seen": 271843328 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741504513540622, + "loss": 1.7029, + "theoretical_loss": 4.223384608958855, + "tokens_seen": 271908864 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414884653961886, + "loss": 1.7804, + "theoretical_loss": 4.223264095993615, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414724172517554, + "loss": 1.6577, + "theoretical_loss": 4.223143620192907, + "tokens_seen": 272039936 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741456369107322, + "loss": 1.5331, + "theoretical_loss": 4.223023181536323, + "tokens_seen": 272105472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414403209628888, + "loss": 1.6128, + "theoretical_loss": 4.2229027800034675, + "tokens_seen": 272171008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414242728184554, + "loss": 1.5997, + "theoretical_loss": 4.222782415573962, + "tokens_seen": 272236544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007414082246740222, + "loss": 1.703, + "theoretical_loss": 4.222662088227447, + "tokens_seen": 272302080 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413921765295888, + "loss": 1.5279, + "theoretical_loss": 4.222541797943576, + "tokens_seen": 272367616 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413761283851556, + "loss": 1.7537, + "theoretical_loss": 4.222421544702018, + "tokens_seen": 272433152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413600802407222, + "loss": 1.7776, + "theoretical_loss": 4.2223013284824615, + "tokens_seen": 272498688 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741344032096289, + "loss": 1.7529, + "theoretical_loss": 4.222181149264608, + "tokens_seen": 272564224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413279839518556, + "loss": 1.6538, + "theoretical_loss": 4.222061007028175, + "tokens_seen": 272629760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007413119358074224, + "loss": 1.647, + "theoretical_loss": 4.221940901752899, + "tokens_seen": 272695296 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741295887662989, + "loss": 1.6851, + "theoretical_loss": 4.221820833418529, + "tokens_seen": 272760832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412798395185557, + "loss": 1.6389, + "theoretical_loss": 4.221700802004832, + "tokens_seen": 272826368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412637913741224, + "loss": 1.7206, + "theoretical_loss": 4.2215808074915895, + "tokens_seen": 272891904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412477432296891, + "loss": 1.5648, + "theoretical_loss": 4.221460849858601, + "tokens_seen": 272957440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412316950852558, + "loss": 1.6165, + "theoretical_loss": 4.221340929085679, + "tokens_seen": 273022976 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007412156469408225, + "loss": 1.582, + "theoretical_loss": 4.221221045152655, + "tokens_seen": 273088512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411995987963892, + "loss": 1.7083, + "theoretical_loss": 4.221101198039375, + "tokens_seen": 273154048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411835506519559, + "loss": 1.6301, + "theoretical_loss": 4.2209813877257, + "tokens_seen": 273219584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411675025075226, + "loss": 1.6637, + "theoretical_loss": 4.220861614191508, + "tokens_seen": 273285120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411514543630893, + "loss": 1.674, + "theoretical_loss": 4.220741877416691, + "tokens_seen": 273350656 + }, + { + "epoch": 0.08, + "learning_rate": 0.000741135406218656, + "loss": 1.7409, + "theoretical_loss": 4.22062217738116, + "tokens_seen": 273416192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411193580742227, + "loss": 1.7326, + "theoretical_loss": 4.22050251406484, + "tokens_seen": 273481728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007411033099297894, + "loss": 1.6014, + "theoretical_loss": 4.220382887447671, + "tokens_seen": 273547264 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410872617853561, + "loss": 1.676, + "theoretical_loss": 4.22026329750961, + "tokens_seen": 273612800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410712136409228, + "loss": 1.6044, + "theoretical_loss": 4.220143744230629, + "tokens_seen": 273678336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410551654964895, + "loss": 1.7051, + "theoretical_loss": 4.220024227590717, + "tokens_seen": 273743872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410391173520562, + "loss": 1.628, + "theoretical_loss": 4.219904747569877, + "tokens_seen": 273809408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410230692076229, + "loss": 1.72, + "theoretical_loss": 4.219785304148129, + "tokens_seen": 273874944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007410070210631896, + "loss": 1.7064, + "theoretical_loss": 4.219665897305506, + "tokens_seen": 273940480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409909729187563, + "loss": 1.6476, + "theoretical_loss": 4.219546527022062, + "tokens_seen": 274006016 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740974924774323, + "loss": 1.6121, + "theoretical_loss": 4.219427193277861, + "tokens_seen": 274071552 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409588766298897, + "loss": 1.8465, + "theoretical_loss": 4.219307896052987, + "tokens_seen": 274137088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409428284854564, + "loss": 1.7803, + "theoretical_loss": 4.219188635327535, + "tokens_seen": 274202624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409267803410231, + "loss": 1.6073, + "theoretical_loss": 4.219069411081622, + "tokens_seen": 274268160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007409107321965898, + "loss": 1.7041, + "theoretical_loss": 4.218950223295374, + "tokens_seen": 274333696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408946840521565, + "loss": 1.6319, + "theoretical_loss": 4.218831071948936, + "tokens_seen": 274399232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408786359077232, + "loss": 1.6451, + "theoretical_loss": 4.218711957022469, + "tokens_seen": 274464768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408625877632899, + "loss": 1.7653, + "theoretical_loss": 4.218592878496148, + "tokens_seen": 274530304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408465396188566, + "loss": 1.6506, + "theoretical_loss": 4.218473836350164, + "tokens_seen": 274595840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007408304914744233, + "loss": 1.6903, + "theoretical_loss": 4.2183548305647225, + "tokens_seen": 274661376 + }, + { + "epoch": 0.08, + "learning_rate": 0.00074081444332999, + "loss": 1.7129, + "theoretical_loss": 4.218235861120047, + "tokens_seen": 274726912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407983951855567, + "loss": 1.6008, + "theoretical_loss": 4.218116927996375, + "tokens_seen": 274792448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407823470411234, + "loss": 1.6658, + "theoretical_loss": 4.217998031173959, + "tokens_seen": 274857984 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407662988966901, + "loss": 1.5342, + "theoretical_loss": 4.217879170633067, + "tokens_seen": 274923520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407502507522568, + "loss": 1.6918, + "theoretical_loss": 4.217760346353984, + "tokens_seen": 274989056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407342026078235, + "loss": 1.6561, + "theoretical_loss": 4.217641558317008, + "tokens_seen": 275054592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407181544633902, + "loss": 1.6212, + "theoretical_loss": 4.217522806502455, + "tokens_seen": 275120128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007407021063189569, + "loss": 1.5607, + "theoretical_loss": 4.217404090890653, + "tokens_seen": 275185664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 108722, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.6769760847091675, + "objective/train/theoretical_loss": 4.217285411461949, + "objective/train/tokens_used": 295711200, + "theoretical_loss": 4.217285411461949, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406860581745236, + "loss": 1.677, + "theoretical_loss": 4.217285411461949, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406700100300903, + "loss": 1.6009, + "theoretical_loss": 4.217166768196704, + "tokens_seen": 275316736 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740653961885657, + "loss": 1.7233, + "theoretical_loss": 4.217048161075294, + "tokens_seen": 275382272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406379137412237, + "loss": 1.6772, + "theoretical_loss": 4.2169295900781085, + "tokens_seen": 275447808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406218655967904, + "loss": 1.6218, + "theoretical_loss": 4.216811055185556, + "tokens_seen": 275513344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007406058174523571, + "loss": 1.6164, + "theoretical_loss": 4.216692556378058, + "tokens_seen": 275578880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405897693079238, + "loss": 1.5954, + "theoretical_loss": 4.216574093636053, + "tokens_seen": 275644416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405737211634905, + "loss": 1.5856, + "theoretical_loss": 4.216455666939991, + "tokens_seen": 275709952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405576730190572, + "loss": 1.7238, + "theoretical_loss": 4.216337276270341, + "tokens_seen": 275775488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405416248746239, + "loss": 1.7341, + "theoretical_loss": 4.216218921607586, + "tokens_seen": 275841024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405255767301906, + "loss": 1.6413, + "theoretical_loss": 4.216100602932224, + "tokens_seen": 275906560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007405095285857573, + "loss": 1.6513, + "theoretical_loss": 4.21598232022477, + "tokens_seen": 275972096 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740493480441324, + "loss": 1.6008, + "theoretical_loss": 4.215864073465749, + "tokens_seen": 276037632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404774322968907, + "loss": 1.6939, + "theoretical_loss": 4.215745862635708, + "tokens_seen": 276103168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404613841524574, + "loss": 1.639, + "theoretical_loss": 4.2156276877152035, + "tokens_seen": 276168704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404453360080241, + "loss": 1.6366, + "theoretical_loss": 4.21550954868481, + "tokens_seen": 276234240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404292878635908, + "loss": 1.5483, + "theoretical_loss": 4.2153914455251185, + "tokens_seen": 276299776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007404132397191575, + "loss": 1.648, + "theoretical_loss": 4.215273378216731, + "tokens_seen": 276365312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403971915747242, + "loss": 1.686, + "theoretical_loss": 4.215155346740268, + "tokens_seen": 276430848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403811434302909, + "loss": 1.616, + "theoretical_loss": 4.215037351076363, + "tokens_seen": 276496384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403650952858576, + "loss": 1.6878, + "theoretical_loss": 4.214919391205665, + "tokens_seen": 276561920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403490471414243, + "loss": 1.6904, + "theoretical_loss": 4.21480146710884, + "tokens_seen": 276627456 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740332998996991, + "loss": 1.6566, + "theoretical_loss": 4.214683578766566, + "tokens_seen": 276692992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403169508525577, + "loss": 1.5896, + "theoretical_loss": 4.214565726159539, + "tokens_seen": 276758528 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007403009027081244, + "loss": 1.6907, + "theoretical_loss": 4.214447909268467, + "tokens_seen": 276824064 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402848545636911, + "loss": 1.6596, + "theoretical_loss": 4.214330128074074, + "tokens_seen": 276889600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402688064192578, + "loss": 1.7149, + "theoretical_loss": 4.214212382557101, + "tokens_seen": 276955136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402527582748245, + "loss": 1.6435, + "theoretical_loss": 4.214094672698302, + "tokens_seen": 277020672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402367101303912, + "loss": 1.5875, + "theoretical_loss": 4.213976998478445, + "tokens_seen": 277086208 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402206619859579, + "loss": 1.6132, + "theoretical_loss": 4.213859359878316, + "tokens_seen": 277151744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007402046138415247, + "loss": 1.6561, + "theoretical_loss": 4.213741756878713, + "tokens_seen": 277217280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401885656970913, + "loss": 1.5125, + "theoretical_loss": 4.213624189460449, + "tokens_seen": 277282816 + }, + { + "epoch": 0.08, + "learning_rate": 0.000740172517552658, + "loss": 1.742, + "theoretical_loss": 4.213506657604356, + "tokens_seen": 277348352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401564694082247, + "loss": 1.6136, + "theoretical_loss": 4.213389161291273, + "tokens_seen": 277413888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401404212637915, + "loss": 1.6431, + "theoretical_loss": 4.213271700502062, + "tokens_seen": 277479424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401243731193581, + "loss": 1.6536, + "theoretical_loss": 4.213154275217597, + "tokens_seen": 277544960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007401083249749249, + "loss": 1.5936, + "theoretical_loss": 4.213036885418763, + "tokens_seen": 277610496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400922768304915, + "loss": 1.6004, + "theoretical_loss": 4.212919531086466, + "tokens_seen": 277676032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400762286860583, + "loss": 1.7002, + "theoretical_loss": 4.2128022122016215, + "tokens_seen": 277741568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400601805416249, + "loss": 1.7299, + "theoretical_loss": 4.212684928745163, + "tokens_seen": 277807104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400441323971917, + "loss": 1.7295, + "theoretical_loss": 4.212567680698038, + "tokens_seen": 277872640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400280842527583, + "loss": 1.5592, + "theoretical_loss": 4.212450468041208, + "tokens_seen": 277938176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007400120361083251, + "loss": 1.609, + "theoretical_loss": 4.21233329075565, + "tokens_seen": 278003712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399959879638917, + "loss": 1.6057, + "theoretical_loss": 4.212216148822355, + "tokens_seen": 278069248 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399799398194585, + "loss": 1.5876, + "theoretical_loss": 4.212099042222331, + "tokens_seen": 278134784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399638916750251, + "loss": 1.5245, + "theoretical_loss": 4.211981970936595, + "tokens_seen": 278200320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399478435305919, + "loss": 1.5892, + "theoretical_loss": 4.2118649349461865, + "tokens_seen": 278265856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399317953861585, + "loss": 1.5567, + "theoretical_loss": 4.211747934232153, + "tokens_seen": 278331392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007399157472417253, + "loss": 1.584, + "theoretical_loss": 4.211630968775561, + "tokens_seen": 278396928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398996990972919, + "loss": 1.6937, + "theoretical_loss": 4.211514038557488, + "tokens_seen": 278462464 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398836509528586, + "loss": 1.6164, + "theoretical_loss": 4.211397143559029, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398676028084253, + "loss": 1.5245, + "theoretical_loss": 4.211280283761294, + "tokens_seen": 278593536 + }, + { + "epoch": 0.08, + "learning_rate": 0.000739851554663992, + "loss": 1.66, + "theoretical_loss": 4.211163459145403, + "tokens_seen": 278659072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398355065195587, + "loss": 1.6867, + "theoretical_loss": 4.211046669692496, + "tokens_seen": 278724608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398194583751254, + "loss": 1.5391, + "theoretical_loss": 4.2109299153837245, + "tokens_seen": 278790144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007398034102306921, + "loss": 1.6044, + "theoretical_loss": 4.210813196200255, + "tokens_seen": 278855680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397873620862588, + "loss": 1.7526, + "theoretical_loss": 4.21069651212327, + "tokens_seen": 278921216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397713139418255, + "loss": 1.6291, + "theoretical_loss": 4.2105798631339635, + "tokens_seen": 278986752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397552657973922, + "loss": 1.6543, + "theoretical_loss": 4.2104632492135465, + "tokens_seen": 279052288 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397392176529589, + "loss": 1.6405, + "theoretical_loss": 4.210346670343244, + "tokens_seen": 279117824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397231695085256, + "loss": 1.7012, + "theoretical_loss": 4.210230126504296, + "tokens_seen": 279183360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007397071213640923, + "loss": 1.6207, + "theoretical_loss": 4.210113617677955, + "tokens_seen": 279248896 + }, + { + "epoch": 0.08, + "learning_rate": 0.000739691073219659, + "loss": 1.6666, + "theoretical_loss": 4.20999714384549, + "tokens_seen": 279314432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396750250752257, + "loss": 1.6338, + "theoretical_loss": 4.209880704988181, + "tokens_seen": 279379968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396589769307924, + "loss": 1.6907, + "theoretical_loss": 4.209764301087327, + "tokens_seen": 279445504 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396429287863591, + "loss": 1.7038, + "theoretical_loss": 4.20964793212424, + "tokens_seen": 279511040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396268806419258, + "loss": 1.6824, + "theoretical_loss": 4.209531598080245, + "tokens_seen": 279576576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007396108324974925, + "loss": 1.6282, + "theoretical_loss": 4.20941529893668, + "tokens_seen": 279642112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395947843530592, + "loss": 1.545, + "theoretical_loss": 4.209299034674902, + "tokens_seen": 279707648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395787362086259, + "loss": 1.586, + "theoretical_loss": 4.209182805276278, + "tokens_seen": 279773184 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395626880641926, + "loss": 1.7454, + "theoretical_loss": 4.209066610722192, + "tokens_seen": 279838720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395466399197593, + "loss": 1.5542, + "theoretical_loss": 4.208950450994041, + "tokens_seen": 279904256 + }, + { + "epoch": 0.08, + "learning_rate": 0.000739530591775326, + "loss": 1.6669, + "theoretical_loss": 4.2088343260732355, + "tokens_seen": 279969792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007395145436308927, + "loss": 1.605, + "theoretical_loss": 4.208718235941204, + "tokens_seen": 280035328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394984954864594, + "loss": 1.7347, + "theoretical_loss": 4.2086021805793825, + "tokens_seen": 280100864 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394824473420261, + "loss": 1.719, + "theoretical_loss": 4.208486159969229, + "tokens_seen": 280166400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394663991975928, + "loss": 1.701, + "theoretical_loss": 4.20837017409221, + "tokens_seen": 280231936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394503510531595, + "loss": 1.6611, + "theoretical_loss": 4.20825422292981, + "tokens_seen": 280297472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394343029087262, + "loss": 1.5767, + "theoretical_loss": 4.208138306463524, + "tokens_seen": 280363008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394182547642929, + "loss": 1.68, + "theoretical_loss": 4.208022424674864, + "tokens_seen": 280428544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0007394022066198596, + "loss": 1.6801, + "theoretical_loss": 4.207906577545356, + "tokens_seen": 280494080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393861584754263, + "loss": 1.5856, + "theoretical_loss": 4.207790765056538, + "tokens_seen": 280559616 + }, + { + "epoch": 0.09, + "learning_rate": 0.000739370110330993, + "loss": 1.6837, + "theoretical_loss": 4.207674987189965, + "tokens_seen": 280625152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393540621865597, + "loss": 1.8647, + "theoretical_loss": 4.2075592439272045, + "tokens_seen": 280690688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393380140421264, + "loss": 1.6042, + "theoretical_loss": 4.207443535249838, + "tokens_seen": 280756224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393219658976931, + "loss": 1.5646, + "theoretical_loss": 4.207327861139461, + "tokens_seen": 280821760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007393059177532598, + "loss": 1.7132, + "theoretical_loss": 4.207212221577684, + "tokens_seen": 280887296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392898696088265, + "loss": 1.731, + "theoretical_loss": 4.207096616546132, + "tokens_seen": 280952832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392738214643932, + "loss": 1.6299, + "theoretical_loss": 4.206981046026442, + "tokens_seen": 281018368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392577733199599, + "loss": 1.5683, + "theoretical_loss": 4.2068655100002665, + "tokens_seen": 281083904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392417251755266, + "loss": 1.5759, + "theoretical_loss": 4.206750008449272, + "tokens_seen": 281149440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007392256770310933, + "loss": 1.6443, + "theoretical_loss": 4.206634541355138, + "tokens_seen": 281214976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00073920962888666, + "loss": 1.7292, + "theoretical_loss": 4.20651910869956, + "tokens_seen": 281280512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391935807422267, + "loss": 1.6339, + "theoretical_loss": 4.2064037104642455, + "tokens_seen": 281346048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391775325977934, + "loss": 1.5848, + "theoretical_loss": 4.206288346630917, + "tokens_seen": 281411584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391614844533601, + "loss": 1.6348, + "theoretical_loss": 4.206173017181309, + "tokens_seen": 281477120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391454363089268, + "loss": 1.7069, + "theoretical_loss": 4.206057722097174, + "tokens_seen": 281542656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391293881644935, + "loss": 1.5232, + "theoretical_loss": 4.205942461360276, + "tokens_seen": 281608192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007391133400200602, + "loss": 1.5416, + "theoretical_loss": 4.205827234952391, + "tokens_seen": 281673728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390972918756269, + "loss": 1.5954, + "theoretical_loss": 4.205712042855312, + "tokens_seen": 281739264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 111427, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5511796474456787, + "objective/train/theoretical_loss": 4.205596885050846, + "objective/train/tokens_used": 302264800, + "theoretical_loss": 4.205596885050846, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390812437311936, + "loss": 1.5512, + "theoretical_loss": 4.205596885050846, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390651955867603, + "loss": 1.6241, + "theoretical_loss": 4.205481761520811, + "tokens_seen": 281870336 + }, + { + "epoch": 0.09, + "learning_rate": 0.000739049147442327, + "loss": 1.6993, + "theoretical_loss": 4.20536667224704, + "tokens_seen": 281935872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390330992978937, + "loss": 1.6109, + "theoretical_loss": 4.205251617211382, + "tokens_seen": 282001408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390170511534604, + "loss": 1.693, + "theoretical_loss": 4.205136596395698, + "tokens_seen": 282066944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007390010030090271, + "loss": 1.6325, + "theoretical_loss": 4.205021609781861, + "tokens_seen": 282132480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389849548645938, + "loss": 1.5679, + "theoretical_loss": 4.204906657351763, + "tokens_seen": 282198016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389689067201605, + "loss": 1.5551, + "theoretical_loss": 4.2047917390873035, + "tokens_seen": 282263552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389528585757272, + "loss": 1.5724, + "theoretical_loss": 4.204676854970401, + "tokens_seen": 282329088 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738936810431294, + "loss": 1.7219, + "theoretical_loss": 4.204562004982986, + "tokens_seen": 282394624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389207622868606, + "loss": 1.6348, + "theoretical_loss": 4.204447189107, + "tokens_seen": 282460160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007389047141424273, + "loss": 1.6085, + "theoretical_loss": 4.204332407324403, + "tokens_seen": 282525696 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738888665997994, + "loss": 1.6378, + "theoretical_loss": 4.204217659617164, + "tokens_seen": 282591232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388726178535607, + "loss": 1.5899, + "theoretical_loss": 4.2041029459672705, + "tokens_seen": 282656768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388565697091274, + "loss": 1.5708, + "theoretical_loss": 4.2039882663567205, + "tokens_seen": 282722304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388405215646942, + "loss": 1.6657, + "theoretical_loss": 4.203873620767526, + "tokens_seen": 282787840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388244734202608, + "loss": 1.6038, + "theoretical_loss": 4.203759009181714, + "tokens_seen": 282853376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007388084252758276, + "loss": 1.813, + "theoretical_loss": 4.203644431581323, + "tokens_seen": 282918912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387923771313942, + "loss": 1.6019, + "theoretical_loss": 4.203529887948408, + "tokens_seen": 282984448 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738776328986961, + "loss": 1.6598, + "theoretical_loss": 4.203415378265034, + "tokens_seen": 283049984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387602808425276, + "loss": 1.5217, + "theoretical_loss": 4.203300902513284, + "tokens_seen": 283115520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387442326980944, + "loss": 1.6503, + "theoretical_loss": 4.203186460675251, + "tokens_seen": 283181056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387281845536611, + "loss": 1.5696, + "theoretical_loss": 4.203072052733043, + "tokens_seen": 283246592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007387121364092278, + "loss": 1.683, + "theoretical_loss": 4.202957678668782, + "tokens_seen": 283312128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386960882647945, + "loss": 1.6325, + "theoretical_loss": 4.202843338464602, + "tokens_seen": 283377664 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386800401203612, + "loss": 1.7068, + "theoretical_loss": 4.202729032102653, + "tokens_seen": 283443200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386639919759279, + "loss": 1.6819, + "theoretical_loss": 4.202614759565096, + "tokens_seen": 283508736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386479438314946, + "loss": 1.6655, + "theoretical_loss": 4.202500520834106, + "tokens_seen": 283574272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007386318956870613, + "loss": 1.6321, + "theoretical_loss": 4.202386315891874, + "tokens_seen": 283639808 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738615847542628, + "loss": 1.6632, + "theoretical_loss": 4.202272144720602, + "tokens_seen": 283705344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385997993981946, + "loss": 1.6903, + "theoretical_loss": 4.202158007302506, + "tokens_seen": 283770880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385837512537614, + "loss": 1.6503, + "theoretical_loss": 4.202043903619814, + "tokens_seen": 283836416 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738567703109328, + "loss": 1.6348, + "theoretical_loss": 4.201929833654772, + "tokens_seen": 283901952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385516549648948, + "loss": 1.6455, + "theoretical_loss": 4.201815797389634, + "tokens_seen": 283967488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385356068204614, + "loss": 1.6176, + "theoretical_loss": 4.201701794806672, + "tokens_seen": 284033024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385195586760282, + "loss": 1.656, + "theoretical_loss": 4.201587825888167, + "tokens_seen": 284098560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007385035105315948, + "loss": 1.6294, + "theoretical_loss": 4.201473890616418, + "tokens_seen": 284164096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384874623871615, + "loss": 1.5932, + "theoretical_loss": 4.201359988973733, + "tokens_seen": 284229632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384714142427282, + "loss": 1.6699, + "theoretical_loss": 4.201246120942438, + "tokens_seen": 284295168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384553660982949, + "loss": 1.6361, + "theoretical_loss": 4.201132286504867, + "tokens_seen": 284360704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384393179538616, + "loss": 1.6764, + "theoretical_loss": 4.201018485643373, + "tokens_seen": 284426240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007384232698094283, + "loss": 1.7661, + "theoretical_loss": 4.200904718340317, + "tokens_seen": 284491776 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738407221664995, + "loss": 1.6275, + "theoretical_loss": 4.200790984578078, + "tokens_seen": 284557312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383911735205617, + "loss": 1.6574, + "theoretical_loss": 4.200677284339045, + "tokens_seen": 284622848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383751253761284, + "loss": 1.6439, + "theoretical_loss": 4.2005636176056225, + "tokens_seen": 284688384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383590772316951, + "loss": 1.6343, + "theoretical_loss": 4.200449984360226, + "tokens_seen": 284753920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383430290872618, + "loss": 1.6418, + "theoretical_loss": 4.200336384585286, + "tokens_seen": 284819456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383269809428285, + "loss": 1.6593, + "theoretical_loss": 4.200222818263246, + "tokens_seen": 284884992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007383109327983952, + "loss": 1.6162, + "theoretical_loss": 4.200109285376563, + "tokens_seen": 284950528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382948846539619, + "loss": 1.5544, + "theoretical_loss": 4.1999957859077055, + "tokens_seen": 285016064 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382788365095286, + "loss": 1.6838, + "theoretical_loss": 4.1998823198391575, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382627883650953, + "loss": 1.5549, + "theoretical_loss": 4.199768887153415, + "tokens_seen": 285147136 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738246740220662, + "loss": 1.7648, + "theoretical_loss": 4.199655487832987, + "tokens_seen": 285212672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382306920762287, + "loss": 1.6146, + "theoretical_loss": 4.199542121860397, + "tokens_seen": 285278208 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007382146439317954, + "loss": 1.5645, + "theoretical_loss": 4.199428789218181, + "tokens_seen": 285343744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381985957873621, + "loss": 1.6011, + "theoretical_loss": 4.199315489888885, + "tokens_seen": 285409280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381825476429288, + "loss": 1.6522, + "theoretical_loss": 4.199202223855075, + "tokens_seen": 285474816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381664994984955, + "loss": 1.6987, + "theoretical_loss": 4.199088991099323, + "tokens_seen": 285540352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381504513540622, + "loss": 1.7089, + "theoretical_loss": 4.19897579160422, + "tokens_seen": 285605888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381344032096289, + "loss": 1.5844, + "theoretical_loss": 4.1988626253523655, + "tokens_seen": 285671424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381183550651956, + "loss": 1.5387, + "theoretical_loss": 4.198749492326374, + "tokens_seen": 285736960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007381023069207623, + "loss": 1.553, + "theoretical_loss": 4.198636392508874, + "tokens_seen": 285802496 + }, + { + "epoch": 0.09, + "learning_rate": 0.000738086258776329, + "loss": 1.6633, + "theoretical_loss": 4.198523325882505, + "tokens_seen": 285868032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380702106318957, + "loss": 1.6429, + "theoretical_loss": 4.198410292429922, + "tokens_seen": 285933568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380541624874624, + "loss": 1.6591, + "theoretical_loss": 4.19829729213379, + "tokens_seen": 285999104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380381143430291, + "loss": 1.6118, + "theoretical_loss": 4.198184324976791, + "tokens_seen": 286064640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380220661985958, + "loss": 1.6294, + "theoretical_loss": 4.198071390941616, + "tokens_seen": 286130176 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007380060180541625, + "loss": 1.6333, + "theoretical_loss": 4.197958490010971, + "tokens_seen": 286195712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379899699097292, + "loss": 1.6301, + "theoretical_loss": 4.197845622167575, + "tokens_seen": 286261248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379739217652959, + "loss": 1.6499, + "theoretical_loss": 4.197732787394161, + "tokens_seen": 286326784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379578736208626, + "loss": 1.69, + "theoretical_loss": 4.197619985673471, + "tokens_seen": 286392320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379418254764293, + "loss": 1.6694, + "theoretical_loss": 4.197507216988265, + "tokens_seen": 286457856 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737925777331996, + "loss": 1.621, + "theoretical_loss": 4.197394481321312, + "tokens_seen": 286523392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007379097291875627, + "loss": 1.6053, + "theoretical_loss": 4.1972817786553955, + "tokens_seen": 286588928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378936810431294, + "loss": 1.7107, + "theoretical_loss": 4.197169108973313, + "tokens_seen": 286654464 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378776328986961, + "loss": 1.6213, + "theoretical_loss": 4.197056472257874, + "tokens_seen": 286720000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378615847542628, + "loss": 1.6317, + "theoretical_loss": 4.196943868491899, + "tokens_seen": 286785536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378455366098295, + "loss": 1.5911, + "theoretical_loss": 4.196831297658225, + "tokens_seen": 286851072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378294884653962, + "loss": 1.6587, + "theoretical_loss": 4.1967187597396975, + "tokens_seen": 286916608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007378134403209629, + "loss": 1.5819, + "theoretical_loss": 4.196606254719179, + "tokens_seen": 286982144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377973921765296, + "loss": 1.6383, + "theoretical_loss": 4.196493782579544, + "tokens_seen": 287047680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377813440320963, + "loss": 1.5129, + "theoretical_loss": 4.196381343303676, + "tokens_seen": 287113216 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737765295887663, + "loss": 1.6588, + "theoretical_loss": 4.1962689368744766, + "tokens_seen": 287178752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377492477432297, + "loss": 1.73, + "theoretical_loss": 4.196156563274858, + "tokens_seen": 287244288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377331995987964, + "loss": 1.7149, + "theoretical_loss": 4.1960442224877434, + "tokens_seen": 287309824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377171514543631, + "loss": 1.6242, + "theoretical_loss": 4.195931914496072, + "tokens_seen": 287375360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007377011033099298, + "loss": 1.6117, + "theoretical_loss": 4.195819639282792, + "tokens_seen": 287440896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376850551654965, + "loss": 1.6406, + "theoretical_loss": 4.195707396830869, + "tokens_seen": 287506432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376690070210632, + "loss": 1.6764, + "theoretical_loss": 4.195595187123277, + "tokens_seen": 287571968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376529588766299, + "loss": 1.5057, + "theoretical_loss": 4.195483010143006, + "tokens_seen": 287637504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376369107321966, + "loss": 1.5263, + "theoretical_loss": 4.1953708658730555, + "tokens_seen": 287703040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007376208625877633, + "loss": 1.5509, + "theoretical_loss": 4.195258754296441, + "tokens_seen": 287768576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00073760481444333, + "loss": 1.5322, + "theoretical_loss": 4.195146675396189, + "tokens_seen": 287834112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375887662988967, + "loss": 1.5192, + "theoretical_loss": 4.195034629155338, + "tokens_seen": 287899648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375727181544634, + "loss": 1.6154, + "theoretical_loss": 4.194922615556941, + "tokens_seen": 287965184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375566700100301, + "loss": 1.6231, + "theoretical_loss": 4.194810634584061, + "tokens_seen": 288030720 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375406218655968, + "loss": 1.6379, + "theoretical_loss": 4.194698686219778, + "tokens_seen": 288096256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375245737211636, + "loss": 1.6091, + "theoretical_loss": 4.194586770447179, + "tokens_seen": 288161792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007375085255767302, + "loss": 1.5601, + "theoretical_loss": 4.194474887249369, + "tokens_seen": 288227328 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737492477432297, + "loss": 1.5731, + "theoretical_loss": 4.19436303660946, + "tokens_seen": 288292864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 113697, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.635483741760254, + "objective/train/theoretical_loss": 4.194251218510583, + "objective/train/tokens_used": 308818400, + "theoretical_loss": 4.194251218510583, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007374764292878637, + "loss": 1.6355, + "theoretical_loss": 4.194251218510583, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007374603811434304, + "loss": 1.6591, + "theoretical_loss": 4.194139432935877, + "tokens_seen": 288423936 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737444332998997, + "loss": 1.6107, + "theoretical_loss": 4.194027679868495, + "tokens_seen": 288489472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007374282848545638, + "loss": 1.6476, + "theoretical_loss": 4.193915959291601, + "tokens_seen": 288555008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007374122367101305, + "loss": 1.6513, + "theoretical_loss": 4.193804271188376, + "tokens_seen": 288620544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373961885656972, + "loss": 1.5988, + "theoretical_loss": 4.193692615542007, + "tokens_seen": 288686080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373801404212639, + "loss": 1.6891, + "theoretical_loss": 4.193580992335699, + "tokens_seen": 288751616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373640922768306, + "loss": 1.6256, + "theoretical_loss": 4.193469401552667, + "tokens_seen": 288817152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373480441323973, + "loss": 1.5912, + "theoretical_loss": 4.193357843176139, + "tokens_seen": 288882688 + }, + { + "epoch": 0.09, + "learning_rate": 0.000737331995987964, + "loss": 1.6378, + "theoretical_loss": 4.193246317189356, + "tokens_seen": 288948224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007373159478435307, + "loss": 1.5431, + "theoretical_loss": 4.19313482357557, + "tokens_seen": 289013760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372998996990974, + "loss": 1.5822, + "theoretical_loss": 4.193023362318047, + "tokens_seen": 289079296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372838515546641, + "loss": 1.7027, + "theoretical_loss": 4.1929119334000635, + "tokens_seen": 289144832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372678034102308, + "loss": 1.635, + "theoretical_loss": 4.192800536804912, + "tokens_seen": 289210368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372517552657975, + "loss": 1.6755, + "theoretical_loss": 4.192689172515893, + "tokens_seen": 289275904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372357071213642, + "loss": 1.6572, + "theoretical_loss": 4.192577840516323, + "tokens_seen": 289341440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372196589769309, + "loss": 1.6666, + "theoretical_loss": 4.192466540789528, + "tokens_seen": 289406976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007372036108324975, + "loss": 1.6041, + "theoretical_loss": 4.19235527331885, + "tokens_seen": 289472512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371875626880643, + "loss": 1.5248, + "theoretical_loss": 4.19224403808764, + "tokens_seen": 289538048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371715145436309, + "loss": 1.7546, + "theoretical_loss": 4.192132835079262, + "tokens_seen": 289603584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371554663991977, + "loss": 1.5898, + "theoretical_loss": 4.192021664277094, + "tokens_seen": 289669120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371394182547643, + "loss": 1.6041, + "theoretical_loss": 4.191910525664524, + "tokens_seen": 289734656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371233701103311, + "loss": 1.7532, + "theoretical_loss": 4.191799419224955, + "tokens_seen": 289800192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007371073219658977, + "loss": 1.5373, + "theoretical_loss": 4.1916883449418, + "tokens_seen": 289865728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370912738214645, + "loss": 1.5238, + "theoretical_loss": 4.1915773027984855, + "tokens_seen": 289931264 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370752256770311, + "loss": 1.6475, + "theoretical_loss": 4.19146629277845, + "tokens_seen": 289996800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370591775325978, + "loss": 1.6677, + "theoretical_loss": 4.1913553148651435, + "tokens_seen": 290062336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370431293881645, + "loss": 1.6085, + "theoretical_loss": 4.191244369042029, + "tokens_seen": 290127872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370270812437312, + "loss": 1.616, + "theoretical_loss": 4.191133455292583, + "tokens_seen": 290193408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007370110330992979, + "loss": 1.652, + "theoretical_loss": 4.191022573600293, + "tokens_seen": 290258944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369949849548646, + "loss": 1.5688, + "theoretical_loss": 4.190911723948657, + "tokens_seen": 290324480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369789368104313, + "loss": 1.5422, + "theoretical_loss": 4.190800906321188, + "tokens_seen": 290390016 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736962888665998, + "loss": 1.7319, + "theoretical_loss": 4.190690120701412, + "tokens_seen": 290455552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369468405215647, + "loss": 1.5619, + "theoretical_loss": 4.190579367072861, + "tokens_seen": 290521088 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369307923771314, + "loss": 1.674, + "theoretical_loss": 4.190468645419086, + "tokens_seen": 290586624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007369147442326981, + "loss": 1.6522, + "theoretical_loss": 4.190357955723649, + "tokens_seen": 290652160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368986960882648, + "loss": 1.6744, + "theoretical_loss": 4.190247297970121, + "tokens_seen": 290717696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368826479438315, + "loss": 1.6186, + "theoretical_loss": 4.190136672142088, + "tokens_seen": 290783232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368665997993982, + "loss": 1.6333, + "theoretical_loss": 4.190026078223145, + "tokens_seen": 290848768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368505516549649, + "loss": 1.6789, + "theoretical_loss": 4.189915516196904, + "tokens_seen": 290914304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368345035105316, + "loss": 1.7059, + "theoretical_loss": 4.189804986046984, + "tokens_seen": 290979840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007368184553660983, + "loss": 1.6899, + "theoretical_loss": 4.18969448775702, + "tokens_seen": 291045376 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736802407221665, + "loss": 1.7178, + "theoretical_loss": 4.189584021310657, + "tokens_seen": 291110912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367863590772317, + "loss": 1.5942, + "theoretical_loss": 4.189473586691554, + "tokens_seen": 291176448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367703109327984, + "loss": 1.6982, + "theoretical_loss": 4.189363183883377, + "tokens_seen": 291241984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367542627883651, + "loss": 1.6072, + "theoretical_loss": 4.189252812869811, + "tokens_seen": 291307520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367382146439318, + "loss": 1.6422, + "theoretical_loss": 4.18914247363455, + "tokens_seen": 291373056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367221664994985, + "loss": 1.5706, + "theoretical_loss": 4.189032166161297, + "tokens_seen": 291438592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007367061183550652, + "loss": 1.6603, + "theoretical_loss": 4.188921890433772, + "tokens_seen": 291504128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366900702106319, + "loss": 1.6234, + "theoretical_loss": 4.1888116464357035, + "tokens_seen": 291569664 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366740220661986, + "loss": 1.654, + "theoretical_loss": 4.188701434150834, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366579739217653, + "loss": 1.6638, + "theoretical_loss": 4.188591253562918, + "tokens_seen": 291700736 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736641925777332, + "loss": 1.6495, + "theoretical_loss": 4.18848110465572, + "tokens_seen": 291766272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366258776328987, + "loss": 1.5695, + "theoretical_loss": 4.188370987413018, + "tokens_seen": 291831808 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007366098294884654, + "loss": 1.5701, + "theoretical_loss": 4.188260901818602, + "tokens_seen": 291897344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365937813440321, + "loss": 1.5787, + "theoretical_loss": 4.1881508478562735, + "tokens_seen": 291962880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365777331995988, + "loss": 1.5897, + "theoretical_loss": 4.188040825509845, + "tokens_seen": 292028416 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365616850551655, + "loss": 1.6281, + "theoretical_loss": 4.1879308347631445, + "tokens_seen": 292093952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365456369107322, + "loss": 1.5942, + "theoretical_loss": 4.187820875600006, + "tokens_seen": 292159488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365295887662989, + "loss": 1.6395, + "theoretical_loss": 4.187710948004281, + "tokens_seen": 292225024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007365135406218656, + "loss": 1.706, + "theoretical_loss": 4.187601051959831, + "tokens_seen": 292290560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364974924774323, + "loss": 1.7282, + "theoretical_loss": 4.187491187450527, + "tokens_seen": 292356096 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736481444332999, + "loss": 1.6488, + "theoretical_loss": 4.187381354460255, + "tokens_seen": 292421632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364653961885657, + "loss": 1.7034, + "theoretical_loss": 4.187271552972911, + "tokens_seen": 292487168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364493480441324, + "loss": 1.6495, + "theoretical_loss": 4.187161782972405, + "tokens_seen": 292552704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364332998996991, + "loss": 1.7075, + "theoretical_loss": 4.187052044442655, + "tokens_seen": 292618240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364172517552658, + "loss": 1.6143, + "theoretical_loss": 4.1869423373675945, + "tokens_seen": 292683776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007364012036108325, + "loss": 1.6848, + "theoretical_loss": 4.1868326617311675, + "tokens_seen": 292749312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363851554663992, + "loss": 1.6273, + "theoretical_loss": 4.18672301751733, + "tokens_seen": 292814848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363691073219659, + "loss": 1.6245, + "theoretical_loss": 4.186613404710048, + "tokens_seen": 292880384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363530591775326, + "loss": 1.6515, + "theoretical_loss": 4.186503823293303, + "tokens_seen": 292945920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363370110330993, + "loss": 1.6875, + "theoretical_loss": 4.186394273251084, + "tokens_seen": 293011456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363209628886661, + "loss": 1.7158, + "theoretical_loss": 4.186284754567395, + "tokens_seen": 293076992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007363049147442327, + "loss": 1.6544, + "theoretical_loss": 4.1861752672262496, + "tokens_seen": 293142528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362888665997995, + "loss": 1.6383, + "theoretical_loss": 4.186065811211675, + "tokens_seen": 293208064 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362728184553661, + "loss": 1.6027, + "theoretical_loss": 4.185956386507708, + "tokens_seen": 293273600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362567703109329, + "loss": 1.5945, + "theoretical_loss": 4.185846993098398, + "tokens_seen": 293339136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362407221664995, + "loss": 1.6038, + "theoretical_loss": 4.185737630967808, + "tokens_seen": 293404672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007362246740220663, + "loss": 1.6761, + "theoretical_loss": 4.18562830010001, + "tokens_seen": 293470208 + }, + { + "epoch": 0.09, + "learning_rate": 0.000736208625877633, + "loss": 1.707, + "theoretical_loss": 4.185519000479089, + "tokens_seen": 293535744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361925777331997, + "loss": 1.5523, + "theoretical_loss": 4.18540973208914, + "tokens_seen": 293601280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361765295887663, + "loss": 1.6931, + "theoretical_loss": 4.185300494914273, + "tokens_seen": 293666816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361604814443331, + "loss": 1.555, + "theoretical_loss": 4.185191288938606, + "tokens_seen": 293732352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361444332998997, + "loss": 1.6381, + "theoretical_loss": 4.185082114146271, + "tokens_seen": 293797888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361283851554665, + "loss": 1.7155, + "theoretical_loss": 4.184972970521409, + "tokens_seen": 293863424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007361123370110332, + "loss": 1.5649, + "theoretical_loss": 4.184863858048177, + "tokens_seen": 293928960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360962888665999, + "loss": 1.6585, + "theoretical_loss": 4.18475477671074, + "tokens_seen": 293994496 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360802407221666, + "loss": 1.6703, + "theoretical_loss": 4.184645726493275, + "tokens_seen": 294060032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360641925777333, + "loss": 1.5013, + "theoretical_loss": 4.184536707379972, + "tokens_seen": 294125568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360481444333, + "loss": 1.6214, + "theoretical_loss": 4.184427719355031, + "tokens_seen": 294191104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360320962888667, + "loss": 1.6508, + "theoretical_loss": 4.184318762402664, + "tokens_seen": 294256640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360160481444334, + "loss": 1.576, + "theoretical_loss": 4.184209836507096, + "tokens_seen": 294322176 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007360000000000001, + "loss": 1.6761, + "theoretical_loss": 4.18410094165256, + "tokens_seen": 294387712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359839518555668, + "loss": 1.5281, + "theoretical_loss": 4.183992077823305, + "tokens_seen": 294453248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359679037111335, + "loss": 1.5962, + "theoretical_loss": 4.183883245003588, + "tokens_seen": 294518784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359518555667002, + "loss": 1.584, + "theoretical_loss": 4.183774443177679, + "tokens_seen": 294584320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359358074222669, + "loss": 1.6362, + "theoretical_loss": 4.183665672329861, + "tokens_seen": 294649856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359197592778336, + "loss": 1.5156, + "theoretical_loss": 4.183556932444424, + "tokens_seen": 294715392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007359037111334003, + "loss": 1.5991, + "theoretical_loss": 4.183448223505673, + "tokens_seen": 294780928 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735887662988967, + "loss": 1.7028, + "theoretical_loss": 4.183339545497924, + "tokens_seen": 294846464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 116257, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.6783287525177002, + "objective/train/theoretical_loss": 4.1832308984055055, + "objective/train/tokens_used": 315372000, + "theoretical_loss": 4.1832308984055055, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358716148445337, + "loss": 1.6783, + "theoretical_loss": 4.1832308984055055, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358555667001004, + "loss": 1.5823, + "theoretical_loss": 4.183122282212754, + "tokens_seen": 294977536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358395185556671, + "loss": 1.695, + "theoretical_loss": 4.183013696904018, + "tokens_seen": 295043072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358234704112338, + "loss": 1.5844, + "theoretical_loss": 4.182905142463662, + "tokens_seen": 295108608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007358074222668004, + "loss": 1.4783, + "theoretical_loss": 4.182796618876057, + "tokens_seen": 295174144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357913741223672, + "loss": 1.7181, + "theoretical_loss": 4.182688126125587, + "tokens_seen": 295239680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357753259779338, + "loss": 1.652, + "theoretical_loss": 4.182579664196648, + "tokens_seen": 295305216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357592778335006, + "loss": 1.6404, + "theoretical_loss": 4.182471233073647, + "tokens_seen": 295370752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357432296890672, + "loss": 1.5395, + "theoretical_loss": 4.182362832741001, + "tokens_seen": 295436288 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735727181544634, + "loss": 1.6276, + "theoretical_loss": 4.182254463183139, + "tokens_seen": 295501824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007357111334002006, + "loss": 1.6724, + "theoretical_loss": 4.182146124384504, + "tokens_seen": 295567360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356950852557674, + "loss": 1.6746, + "theoretical_loss": 4.182037816329547, + "tokens_seen": 295632896 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735679037111334, + "loss": 1.6133, + "theoretical_loss": 4.181929539002731, + "tokens_seen": 295698432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356629889669007, + "loss": 1.7129, + "theoretical_loss": 4.1818212923885305, + "tokens_seen": 295763968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356469408224674, + "loss": 1.6663, + "theoretical_loss": 4.181713076471432, + "tokens_seen": 295829504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356308926780341, + "loss": 1.5642, + "theoretical_loss": 4.181604891235933, + "tokens_seen": 295895040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007356148445336008, + "loss": 1.5794, + "theoretical_loss": 4.181496736666542, + "tokens_seen": 295960576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355987963891675, + "loss": 1.6991, + "theoretical_loss": 4.181388612747778, + "tokens_seen": 296026112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355827482447342, + "loss": 1.6103, + "theoretical_loss": 4.181280519464172, + "tokens_seen": 296091648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355667001003009, + "loss": 1.5653, + "theoretical_loss": 4.181172456800267, + "tokens_seen": 296157184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355506519558676, + "loss": 1.6646, + "theoretical_loss": 4.181064424740617, + "tokens_seen": 296222720 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355346038114343, + "loss": 1.5364, + "theoretical_loss": 4.180956423269784, + "tokens_seen": 296288256 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735518555667001, + "loss": 1.565, + "theoretical_loss": 4.180848452372347, + "tokens_seen": 296353792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007355025075225677, + "loss": 1.6551, + "theoretical_loss": 4.180740512032891, + "tokens_seen": 296419328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354864593781344, + "loss": 1.6264, + "theoretical_loss": 4.180632602236014, + "tokens_seen": 296484864 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354704112337011, + "loss": 1.5132, + "theoretical_loss": 4.180524722966327, + "tokens_seen": 296550400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354543630892678, + "loss": 1.6651, + "theoretical_loss": 4.180416874208449, + "tokens_seen": 296615936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354383149448345, + "loss": 1.6225, + "theoretical_loss": 4.180309055947012, + "tokens_seen": 296681472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354222668004012, + "loss": 1.6646, + "theoretical_loss": 4.18020126816666, + "tokens_seen": 296747008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007354062186559679, + "loss": 1.6745, + "theoretical_loss": 4.180093510852045, + "tokens_seen": 296812544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353901705115346, + "loss": 1.7138, + "theoretical_loss": 4.179985783987834, + "tokens_seen": 296878080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353741223671013, + "loss": 1.6246, + "theoretical_loss": 4.179878087558702, + "tokens_seen": 296943616 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735358074222668, + "loss": 1.5611, + "theoretical_loss": 4.179770421549336, + "tokens_seen": 297009152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353420260782347, + "loss": 1.6443, + "theoretical_loss": 4.179662785944435, + "tokens_seen": 297074688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353259779338014, + "loss": 1.4962, + "theoretical_loss": 4.179555180728708, + "tokens_seen": 297140224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007353099297893681, + "loss": 1.6492, + "theoretical_loss": 4.179447605886876, + "tokens_seen": 297205760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352938816449348, + "loss": 1.6408, + "theoretical_loss": 4.1793400614036695, + "tokens_seen": 297271296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352778335005015, + "loss": 1.6694, + "theoretical_loss": 4.179232547263831, + "tokens_seen": 297336832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352617853560682, + "loss": 1.6532, + "theoretical_loss": 4.1791250634521155, + "tokens_seen": 297402368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352457372116349, + "loss": 1.7528, + "theoretical_loss": 4.179017609953287, + "tokens_seen": 297467904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352296890672016, + "loss": 1.6727, + "theoretical_loss": 4.17891018675212, + "tokens_seen": 297533440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007352136409227683, + "loss": 1.7006, + "theoretical_loss": 4.178802793833402, + "tokens_seen": 297598976 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735197592778335, + "loss": 1.4933, + "theoretical_loss": 4.17869543118193, + "tokens_seen": 297664512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351815446339017, + "loss": 1.6455, + "theoretical_loss": 4.178588098782513, + "tokens_seen": 297730048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351654964894684, + "loss": 1.5437, + "theoretical_loss": 4.178480796619971, + "tokens_seen": 297795584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351494483450351, + "loss": 1.7077, + "theoretical_loss": 4.178373524679134, + "tokens_seen": 297861120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351334002006018, + "loss": 1.6845, + "theoretical_loss": 4.1782662829448425, + "tokens_seen": 297926656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351173520561686, + "loss": 1.6499, + "theoretical_loss": 4.17815907140195, + "tokens_seen": 297992192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007351013039117352, + "loss": 1.5718, + "theoretical_loss": 4.178051890035319, + "tokens_seen": 298057728 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735085255767302, + "loss": 1.6246, + "theoretical_loss": 4.177944738829825, + "tokens_seen": 298123264 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007350692076228686, + "loss": 1.5704, + "theoretical_loss": 4.177837617770352, + "tokens_seen": 298188800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007350531594784354, + "loss": 1.7485, + "theoretical_loss": 4.1777305268417955, + "tokens_seen": 298254336 + }, + { + "epoch": 0.09, + "learning_rate": 0.000735037111334002, + "loss": 1.5856, + "theoretical_loss": 4.177623466029064, + "tokens_seen": 298319872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007350210631895688, + "loss": 1.5798, + "theoretical_loss": 4.177516435317074, + "tokens_seen": 298385408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007350050150451354, + "loss": 1.6708, + "theoretical_loss": 4.177409434690754, + "tokens_seen": 298450944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349889669007022, + "loss": 1.6167, + "theoretical_loss": 4.177302464135044, + "tokens_seen": 298516480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349729187562688, + "loss": 1.5887, + "theoretical_loss": 4.177195523634895, + "tokens_seen": 298582016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349568706118356, + "loss": 1.6746, + "theoretical_loss": 4.1770886131752665, + "tokens_seen": 298647552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349408224674022, + "loss": 1.6054, + "theoretical_loss": 4.176981732741131, + "tokens_seen": 298713088 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734924774322969, + "loss": 1.6931, + "theoretical_loss": 4.176874882317472, + "tokens_seen": 298778624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007349087261785356, + "loss": 1.5524, + "theoretical_loss": 4.176768061889283, + "tokens_seen": 298844160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348926780341024, + "loss": 1.6189, + "theoretical_loss": 4.176661271441568, + "tokens_seen": 298909696 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734876629889669, + "loss": 1.635, + "theoretical_loss": 4.176554510959342, + "tokens_seen": 298975232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348605817452358, + "loss": 1.7508, + "theoretical_loss": 4.1764477804276305, + "tokens_seen": 299040768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348445336008024, + "loss": 1.5193, + "theoretical_loss": 4.176341079831471, + "tokens_seen": 299106304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348284854563692, + "loss": 1.7028, + "theoretical_loss": 4.176234409155912, + "tokens_seen": 299171840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007348124373119358, + "loss": 1.6708, + "theoretical_loss": 4.176127768386008, + "tokens_seen": 299237376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347963891675026, + "loss": 1.5564, + "theoretical_loss": 4.176021157506832, + "tokens_seen": 299302912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347803410230692, + "loss": 1.6287, + "theoretical_loss": 4.1759145765034615, + "tokens_seen": 299368448 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734764292878636, + "loss": 1.6559, + "theoretical_loss": 4.175808025360988, + "tokens_seen": 299433984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347482447342027, + "loss": 1.6383, + "theoretical_loss": 4.175701504064511, + "tokens_seen": 299499520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347321965897694, + "loss": 1.595, + "theoretical_loss": 4.175595012599143, + "tokens_seen": 299565056 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734716148445336, + "loss": 1.6399, + "theoretical_loss": 4.175488550950007, + "tokens_seen": 299630592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007347001003009028, + "loss": 1.628, + "theoretical_loss": 4.175382119102235, + "tokens_seen": 299696128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346840521564695, + "loss": 1.731, + "theoretical_loss": 4.175275717040972, + "tokens_seen": 299761664 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346680040120362, + "loss": 1.5828, + "theoretical_loss": 4.175169344751371, + "tokens_seen": 299827200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346519558676029, + "loss": 1.6928, + "theoretical_loss": 4.175063002218598, + "tokens_seen": 299892736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346359077231696, + "loss": 1.6022, + "theoretical_loss": 4.174956689427829, + "tokens_seen": 299958272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007346198595787363, + "loss": 1.6119, + "theoretical_loss": 4.17485040636425, + "tokens_seen": 300023808 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734603811434303, + "loss": 1.6451, + "theoretical_loss": 4.174744153013057, + "tokens_seen": 300089344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345877632898697, + "loss": 1.5967, + "theoretical_loss": 4.174637929359458, + "tokens_seen": 300154880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345717151454364, + "loss": 1.6597, + "theoretical_loss": 4.174531735388673, + "tokens_seen": 300220416 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345556670010031, + "loss": 1.5078, + "theoretical_loss": 4.174425571085927, + "tokens_seen": 300285952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345396188565698, + "loss": 1.6415, + "theoretical_loss": 4.174319436436462, + "tokens_seen": 300351488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345235707121365, + "loss": 1.5989, + "theoretical_loss": 4.174213331425527, + "tokens_seen": 300417024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007345075225677032, + "loss": 1.5678, + "theoretical_loss": 4.174107256038383, + "tokens_seen": 300482560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344914744232699, + "loss": 1.5294, + "theoretical_loss": 4.1740012102603, + "tokens_seen": 300548096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344754262788366, + "loss": 1.5301, + "theoretical_loss": 4.17389519407656, + "tokens_seen": 300613632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344593781344033, + "loss": 1.6422, + "theoretical_loss": 4.173789207472455, + "tokens_seen": 300679168 + }, + { + "epoch": 0.09, + "learning_rate": 0.00073444332998997, + "loss": 1.7038, + "theoretical_loss": 4.173683250433287, + "tokens_seen": 300744704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344272818455367, + "loss": 1.7171, + "theoretical_loss": 4.17357732294437, + "tokens_seen": 300810240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007344112337011033, + "loss": 1.6093, + "theoretical_loss": 4.1734714249910265, + "tokens_seen": 300875776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343951855566701, + "loss": 1.4467, + "theoretical_loss": 4.173365556558591, + "tokens_seen": 300941312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343791374122367, + "loss": 1.6797, + "theoretical_loss": 4.1732597176324076, + "tokens_seen": 301006848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343630892678035, + "loss": 1.662, + "theoretical_loss": 4.173153908197831, + "tokens_seen": 301072384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343470411233701, + "loss": 1.5466, + "theoretical_loss": 4.173048128240227, + "tokens_seen": 301137920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343309929789369, + "loss": 1.6066, + "theoretical_loss": 4.172942377744972, + "tokens_seen": 301203456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007343149448345035, + "loss": 1.5853, + "theoretical_loss": 4.17283665669745, + "tokens_seen": 301268992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342988966900703, + "loss": 1.6151, + "theoretical_loss": 4.172730965083061, + "tokens_seen": 301334528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342828485456369, + "loss": 1.7252, + "theoretical_loss": 4.1726253028872105, + "tokens_seen": 301400064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 118641, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.509371042251587, + "objective/train/theoretical_loss": 4.172519670095315, + "objective/train/tokens_used": 321925600, + "theoretical_loss": 4.172519670095315, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342668004012036, + "loss": 1.5094, + "theoretical_loss": 4.172519670095315, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342507522567703, + "loss": 1.7312, + "theoretical_loss": 4.172414066692804, + "tokens_seen": 301531136 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734234704112337, + "loss": 1.6671, + "theoretical_loss": 4.172308492665114, + "tokens_seen": 301596672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342186559679037, + "loss": 1.6761, + "theoretical_loss": 4.172202947997695, + "tokens_seen": 301662208 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007342026078234704, + "loss": 1.5495, + "theoretical_loss": 4.172097432676006, + "tokens_seen": 301727744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341865596790371, + "loss": 1.5774, + "theoretical_loss": 4.171991946685516, + "tokens_seen": 301793280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341705115346038, + "loss": 1.6569, + "theoretical_loss": 4.171886490011705, + "tokens_seen": 301858816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341544633901705, + "loss": 1.6768, + "theoretical_loss": 4.171781062640063, + "tokens_seen": 301924352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341384152457372, + "loss": 1.747, + "theoretical_loss": 4.17167566455609, + "tokens_seen": 301989888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341223671013039, + "loss": 1.6123, + "theoretical_loss": 4.171570295745297, + "tokens_seen": 302055424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007341063189568706, + "loss": 1.6138, + "theoretical_loss": 4.171464956193205, + "tokens_seen": 302120960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340902708124373, + "loss": 1.7125, + "theoretical_loss": 4.171359645885346, + "tokens_seen": 302186496 + }, + { + "epoch": 0.09, + "learning_rate": 0.000734074222668004, + "loss": 1.6021, + "theoretical_loss": 4.17125436480726, + "tokens_seen": 302252032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340581745235707, + "loss": 1.5513, + "theoretical_loss": 4.171149112944501, + "tokens_seen": 302317568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340421263791374, + "loss": 1.6594, + "theoretical_loss": 4.171043890282629, + "tokens_seen": 302383104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340260782347041, + "loss": 1.6129, + "theoretical_loss": 4.170938696807218, + "tokens_seen": 302448640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007340100300902708, + "loss": 1.534, + "theoretical_loss": 4.17083353250385, + "tokens_seen": 302514176 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339939819458375, + "loss": 1.6151, + "theoretical_loss": 4.170728397358118, + "tokens_seen": 302579712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339779338014042, + "loss": 1.6966, + "theoretical_loss": 4.170623291355626, + "tokens_seen": 302645248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339618856569709, + "loss": 1.5945, + "theoretical_loss": 4.1705182144819855, + "tokens_seen": 302710784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339458375125376, + "loss": 1.5896, + "theoretical_loss": 4.170413166722822, + "tokens_seen": 302776320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339297893681043, + "loss": 1.717, + "theoretical_loss": 4.170308148063769, + "tokens_seen": 302841856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007339137412236711, + "loss": 1.6862, + "theoretical_loss": 4.1702031584904695, + "tokens_seen": 302907392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338976930792377, + "loss": 1.5662, + "theoretical_loss": 4.170098197988578, + "tokens_seen": 302972928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338816449348045, + "loss": 1.6809, + "theoretical_loss": 4.169993266543759, + "tokens_seen": 303038464 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338655967903711, + "loss": 1.6761, + "theoretical_loss": 4.169888364141689, + "tokens_seen": 303104000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338495486459379, + "loss": 1.6053, + "theoretical_loss": 4.169783490768049, + "tokens_seen": 303169536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338335005015045, + "loss": 1.7181, + "theoretical_loss": 4.169678646408536, + "tokens_seen": 303235072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338174523570713, + "loss": 1.5548, + "theoretical_loss": 4.169573831048854, + "tokens_seen": 303300608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007338014042126379, + "loss": 1.597, + "theoretical_loss": 4.16946904467472, + "tokens_seen": 303366144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337853560682047, + "loss": 1.6502, + "theoretical_loss": 4.169364287271858, + "tokens_seen": 303431680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337693079237713, + "loss": 1.6468, + "theoretical_loss": 4.169259558826003, + "tokens_seen": 303497216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337532597793381, + "loss": 1.756, + "theoretical_loss": 4.169154859322901, + "tokens_seen": 303562752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337372116349047, + "loss": 1.6479, + "theoretical_loss": 4.169050188748307, + "tokens_seen": 303628288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337211634904715, + "loss": 1.6078, + "theoretical_loss": 4.168945547087987, + "tokens_seen": 303693824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007337051153460381, + "loss": 1.6399, + "theoretical_loss": 4.168840934327717, + "tokens_seen": 303759360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336890672016049, + "loss": 1.7228, + "theoretical_loss": 4.1687363504532815, + "tokens_seen": 303824896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336730190571715, + "loss": 1.5608, + "theoretical_loss": 4.168631795450478, + "tokens_seen": 303890432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336569709127383, + "loss": 1.7236, + "theoretical_loss": 4.168527269305112, + "tokens_seen": 303955968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336409227683049, + "loss": 1.6453, + "theoretical_loss": 4.168422772002998, + "tokens_seen": 304021504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336248746238717, + "loss": 1.6206, + "theoretical_loss": 4.168318303529963, + "tokens_seen": 304087040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007336088264794383, + "loss": 1.5473, + "theoretical_loss": 4.168213863871843, + "tokens_seen": 304152576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335927783350051, + "loss": 1.6138, + "theoretical_loss": 4.168109453014484, + "tokens_seen": 304218112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335767301905717, + "loss": 1.6361, + "theoretical_loss": 4.1680050709437415, + "tokens_seen": 304283648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335606820461385, + "loss": 1.6445, + "theoretical_loss": 4.167900717645482, + "tokens_seen": 304349184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335446339017051, + "loss": 1.5995, + "theoretical_loss": 4.16779639310558, + "tokens_seen": 304414720 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335285857572719, + "loss": 1.6864, + "theoretical_loss": 4.167692097309923, + "tokens_seen": 304480256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007335125376128385, + "loss": 1.5532, + "theoretical_loss": 4.167587830244408, + "tokens_seen": 304545792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334964894684053, + "loss": 1.6382, + "theoretical_loss": 4.167483591894937, + "tokens_seen": 304611328 + }, + { + "epoch": 0.09, + "learning_rate": 0.000733480441323972, + "loss": 1.6111, + "theoretical_loss": 4.167379382247429, + "tokens_seen": 304676864 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334643931795387, + "loss": 1.6875, + "theoretical_loss": 4.167275201287808, + "tokens_seen": 304742400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334483450351053, + "loss": 1.5534, + "theoretical_loss": 4.16717104900201, + "tokens_seen": 304807936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334322968906721, + "loss": 1.5838, + "theoretical_loss": 4.167066925375981, + "tokens_seen": 304873472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334162487462387, + "loss": 1.574, + "theoretical_loss": 4.1669628303956765, + "tokens_seen": 304939008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007334002006018055, + "loss": 1.5887, + "theoretical_loss": 4.166858764047062, + "tokens_seen": 305004544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333841524573721, + "loss": 1.5975, + "theoretical_loss": 4.166754726316111, + "tokens_seen": 305070080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333681043129389, + "loss": 1.5917, + "theoretical_loss": 4.16665071718881, + "tokens_seen": 305135616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333520561685056, + "loss": 1.5893, + "theoretical_loss": 4.1665467366511555, + "tokens_seen": 305201152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333360080240723, + "loss": 1.6601, + "theoretical_loss": 4.166442784689149, + "tokens_seen": 305266688 + }, + { + "epoch": 0.09, + "learning_rate": 0.000733319959879639, + "loss": 1.668, + "theoretical_loss": 4.166338861288808, + "tokens_seen": 305332224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007333039117352057, + "loss": 1.5953, + "theoretical_loss": 4.166234966436155, + "tokens_seen": 305397760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332878635907724, + "loss": 1.6166, + "theoretical_loss": 4.166131100117226, + "tokens_seen": 305463296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332718154463391, + "loss": 1.6878, + "theoretical_loss": 4.166027262318064, + "tokens_seen": 305528832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332557673019058, + "loss": 1.5142, + "theoretical_loss": 4.165923453024724, + "tokens_seen": 305594368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332397191574725, + "loss": 1.6742, + "theoretical_loss": 4.165819672223269, + "tokens_seen": 305659904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332236710130392, + "loss": 1.6469, + "theoretical_loss": 4.165715919899773, + "tokens_seen": 305725440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007332076228686059, + "loss": 1.5844, + "theoretical_loss": 4.165612196040319, + "tokens_seen": 305790976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331915747241726, + "loss": 1.77, + "theoretical_loss": 4.1655085006310015, + "tokens_seen": 305856512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331755265797393, + "loss": 1.5801, + "theoretical_loss": 4.165404833657923, + "tokens_seen": 305922048 + }, + { + "epoch": 0.09, + "learning_rate": 0.000733159478435306, + "loss": 1.6918, + "theoretical_loss": 4.165301195107195, + "tokens_seen": 305987584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331434302908727, + "loss": 1.5664, + "theoretical_loss": 4.1651975849649405, + "tokens_seen": 306053120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331273821464394, + "loss": 1.6055, + "theoretical_loss": 4.165094003217293, + "tokens_seen": 306118656 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007331113340020061, + "loss": 1.704, + "theoretical_loss": 4.164990449850393, + "tokens_seen": 306184192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330952858575728, + "loss": 1.7277, + "theoretical_loss": 4.164886924850393, + "tokens_seen": 306249728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330792377131395, + "loss": 1.5589, + "theoretical_loss": 4.164783428203453, + "tokens_seen": 306315264 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330631895687062, + "loss": 1.6528, + "theoretical_loss": 4.1646799598957465, + "tokens_seen": 306380800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330471414242729, + "loss": 1.6215, + "theoretical_loss": 4.164576519913453, + "tokens_seen": 306446336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330310932798396, + "loss": 1.6438, + "theoretical_loss": 4.164473108242763, + "tokens_seen": 306511872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007330150451354062, + "loss": 1.6968, + "theoretical_loss": 4.164369724869878, + "tokens_seen": 306577408 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732998996990973, + "loss": 1.6686, + "theoretical_loss": 4.164266369781005, + "tokens_seen": 306642944 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329829488465396, + "loss": 1.5945, + "theoretical_loss": 4.164163042962366, + "tokens_seen": 306708480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329669007021064, + "loss": 1.6439, + "theoretical_loss": 4.1640597444001894, + "tokens_seen": 306774016 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732950852557673, + "loss": 1.6308, + "theoretical_loss": 4.163956474080715, + "tokens_seen": 306839552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329348044132398, + "loss": 1.6596, + "theoretical_loss": 4.16385323199019, + "tokens_seen": 306905088 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329187562688064, + "loss": 1.627, + "theoretical_loss": 4.1637500181148726, + "tokens_seen": 306970624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007329027081243732, + "loss": 1.6526, + "theoretical_loss": 4.163646832441031, + "tokens_seen": 307036160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328866599799398, + "loss": 1.6554, + "theoretical_loss": 4.163543674954942, + "tokens_seen": 307101696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328706118355065, + "loss": 1.6339, + "theoretical_loss": 4.163440545642894, + "tokens_seen": 307167232 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328545636910732, + "loss": 1.6537, + "theoretical_loss": 4.163337444491183, + "tokens_seen": 307232768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328385155466399, + "loss": 1.5867, + "theoretical_loss": 4.163234371486114, + "tokens_seen": 307298304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328224674022066, + "loss": 1.5403, + "theoretical_loss": 4.163131326614003, + "tokens_seen": 307363840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007328064192577733, + "loss": 1.6822, + "theoretical_loss": 4.163028309861176, + "tokens_seen": 307429376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00073279037111334, + "loss": 1.6459, + "theoretical_loss": 4.162925321213968, + "tokens_seen": 307494912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327743229689067, + "loss": 1.585, + "theoretical_loss": 4.162822360658723, + "tokens_seen": 307560448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327582748244734, + "loss": 1.6779, + "theoretical_loss": 4.162719428181794, + "tokens_seen": 307625984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327422266800401, + "loss": 1.6442, + "theoretical_loss": 4.162616523769547, + "tokens_seen": 307691520 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327261785356068, + "loss": 1.5705, + "theoretical_loss": 4.1625136474083515, + "tokens_seen": 307757056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007327101303911736, + "loss": 1.6629, + "theoretical_loss": 4.1624107990845935, + "tokens_seen": 307822592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007326940822467402, + "loss": 1.607, + "theoretical_loss": 4.162307978784662, + "tokens_seen": 307888128 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732678034102307, + "loss": 1.6092, + "theoretical_loss": 4.162205186494962, + "tokens_seen": 307953664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 120992, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5098984241485596, + "objective/train/theoretical_loss": 4.162102422201901, + "objective/train/tokens_used": 328479200, + "theoretical_loss": 4.162102422201901, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007326619859578736, + "loss": 1.5099, + "theoretical_loss": 4.162102422201901, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007326459378134404, + "loss": 1.737, + "theoretical_loss": 4.161999685891901, + "tokens_seen": 308084736 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732629889669007, + "loss": 1.5714, + "theoretical_loss": 4.161896977551393, + "tokens_seen": 308150272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007326138415245738, + "loss": 1.6119, + "theoretical_loss": 4.161794297166815, + "tokens_seen": 308215808 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325977933801404, + "loss": 1.6664, + "theoretical_loss": 4.161691644724616, + "tokens_seen": 308281344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325817452357072, + "loss": 1.4754, + "theoretical_loss": 4.161589020211256, + "tokens_seen": 308346880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325656970912738, + "loss": 1.6394, + "theoretical_loss": 4.1614864236132, + "tokens_seen": 308412416 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325496489468406, + "loss": 1.6824, + "theoretical_loss": 4.161383854916926, + "tokens_seen": 308477952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325336008024072, + "loss": 1.6069, + "theoretical_loss": 4.161281314108923, + "tokens_seen": 308543488 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732517552657974, + "loss": 1.56, + "theoretical_loss": 4.1611788011756845, + "tokens_seen": 308609024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007325015045135406, + "loss": 1.5524, + "theoretical_loss": 4.161076316103717, + "tokens_seen": 308674560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324854563691074, + "loss": 1.6594, + "theoretical_loss": 4.160973858879535, + "tokens_seen": 308740096 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732469408224674, + "loss": 1.5346, + "theoretical_loss": 4.160871429489663, + "tokens_seen": 308805632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324533600802408, + "loss": 1.5763, + "theoretical_loss": 4.160769027920635, + "tokens_seen": 308871168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324373119358074, + "loss": 1.7231, + "theoretical_loss": 4.160666654158993, + "tokens_seen": 308936704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324212637913742, + "loss": 1.6798, + "theoretical_loss": 4.160564308191289, + "tokens_seen": 309002240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007324052156469408, + "loss": 1.5925, + "theoretical_loss": 4.1604619900040865, + "tokens_seen": 309067776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007323891675025076, + "loss": 1.6104, + "theoretical_loss": 4.160359699583955, + "tokens_seen": 309133312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007323731193580742, + "loss": 1.5745, + "theoretical_loss": 4.160257436917476, + "tokens_seen": 309198848 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732357071213641, + "loss": 1.5826, + "theoretical_loss": 4.160155201991238, + "tokens_seen": 309264384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007323410230692076, + "loss": 1.56, + "theoretical_loss": 4.160052994791841, + "tokens_seen": 309329920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007323249749247744, + "loss": 1.6606, + "theoretical_loss": 4.159950815305893, + "tokens_seen": 309395456 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732308926780341, + "loss": 1.708, + "theoretical_loss": 4.1598486635200125, + "tokens_seen": 309460992 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322928786359078, + "loss": 1.5914, + "theoretical_loss": 4.159746539420825, + "tokens_seen": 309526528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322768304914744, + "loss": 1.5279, + "theoretical_loss": 4.159644442994967, + "tokens_seen": 309592064 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322607823470412, + "loss": 1.7254, + "theoretical_loss": 4.159542374229084, + "tokens_seen": 309657600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322447342026078, + "loss": 1.5446, + "theoretical_loss": 4.159440333109832, + "tokens_seen": 309723136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322286860581746, + "loss": 1.6927, + "theoretical_loss": 4.159338319623874, + "tokens_seen": 309788672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007322126379137412, + "loss": 1.6384, + "theoretical_loss": 4.1592363337578835, + "tokens_seen": 309854208 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732196589769308, + "loss": 1.586, + "theoretical_loss": 4.159134375498542, + "tokens_seen": 309919744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321805416248746, + "loss": 1.4855, + "theoretical_loss": 4.159032444832543, + "tokens_seen": 309985280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321644934804414, + "loss": 1.616, + "theoretical_loss": 4.158930541746587, + "tokens_seen": 310050816 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732148445336008, + "loss": 1.6506, + "theoretical_loss": 4.158828666227383, + "tokens_seen": 310116352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321323971915748, + "loss": 1.6238, + "theoretical_loss": 4.158726818261652, + "tokens_seen": 310181888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321163490471414, + "loss": 1.64, + "theoretical_loss": 4.158624997836123, + "tokens_seen": 310247424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007321003009027082, + "loss": 1.7264, + "theoretical_loss": 4.158523204937531, + "tokens_seen": 310312960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320842527582748, + "loss": 1.6462, + "theoretical_loss": 4.158421439552626, + "tokens_seen": 310378496 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320682046138416, + "loss": 1.6258, + "theoretical_loss": 4.158319701668162, + "tokens_seen": 310444032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320521564694082, + "loss": 1.6203, + "theoretical_loss": 4.158217991270906, + "tokens_seen": 310509568 + }, + { + "epoch": 0.09, + "learning_rate": 0.000732036108324975, + "loss": 1.7848, + "theoretical_loss": 4.158116308347632, + "tokens_seen": 310575104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320200601805416, + "loss": 1.7354, + "theoretical_loss": 4.1580146528851225, + "tokens_seen": 310640640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007320040120361084, + "loss": 1.5643, + "theoretical_loss": 4.157913024870172, + "tokens_seen": 310706176 + }, + { + "epoch": 0.09, + "learning_rate": 0.000731987963891675, + "loss": 1.6105, + "theoretical_loss": 4.157811424289582, + "tokens_seen": 310771712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319719157472418, + "loss": 1.6735, + "theoretical_loss": 4.1577098511301624, + "tokens_seen": 310837248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319558676028085, + "loss": 1.6173, + "theoretical_loss": 4.157608305378735, + "tokens_seen": 310902784 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319398194583752, + "loss": 1.7382, + "theoretical_loss": 4.157506787022128, + "tokens_seen": 310968320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319237713139419, + "loss": 1.6054, + "theoretical_loss": 4.15740529604718, + "tokens_seen": 311033856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007319077231695086, + "loss": 1.5183, + "theoretical_loss": 4.15730383244074, + "tokens_seen": 311099392 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318916750250753, + "loss": 1.5937, + "theoretical_loss": 4.157202396189662, + "tokens_seen": 311164928 + }, + { + "epoch": 0.09, + "learning_rate": 0.000731875626880642, + "loss": 1.5996, + "theoretical_loss": 4.157100987280814, + "tokens_seen": 311230464 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318595787362087, + "loss": 1.6759, + "theoretical_loss": 4.15699960570107, + "tokens_seen": 311296000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318435305917754, + "loss": 1.7264, + "theoretical_loss": 4.156898251437312, + "tokens_seen": 311361536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318274824473421, + "loss": 1.6315, + "theoretical_loss": 4.156796924476435, + "tokens_seen": 311427072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007318114343029088, + "loss": 1.7101, + "theoretical_loss": 4.156695624805341, + "tokens_seen": 311492608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317953861584755, + "loss": 1.6406, + "theoretical_loss": 4.15659435241094, + "tokens_seen": 311558144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317793380140422, + "loss": 1.5463, + "theoretical_loss": 4.15649310728015, + "tokens_seen": 311623680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317632898696089, + "loss": 1.7026, + "theoretical_loss": 4.156391889399904, + "tokens_seen": 311689216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317472417251756, + "loss": 1.5756, + "theoretical_loss": 4.156290698757139, + "tokens_seen": 311754752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007317311935807423, + "loss": 1.5774, + "theoretical_loss": 4.156189535338799, + "tokens_seen": 311820288 + }, + { + "epoch": 0.09, + "learning_rate": 0.000731715145436309, + "loss": 1.5761, + "theoretical_loss": 4.156088399131844, + "tokens_seen": 311885824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316990972918757, + "loss": 1.4766, + "theoretical_loss": 4.155987290123236, + "tokens_seen": 311951360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316830491474424, + "loss": 1.5592, + "theoretical_loss": 4.15588620829995, + "tokens_seen": 312016896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316670010030091, + "loss": 1.6467, + "theoretical_loss": 4.15578515364897, + "tokens_seen": 312082432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316509528585758, + "loss": 1.6672, + "theoretical_loss": 4.155684126157286, + "tokens_seen": 312147968 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316349047141425, + "loss": 1.6334, + "theoretical_loss": 4.1555831258119005, + "tokens_seen": 312213504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316188565697092, + "loss": 1.6194, + "theoretical_loss": 4.155482152599823, + "tokens_seen": 312279040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007316028084252759, + "loss": 1.6357, + "theoretical_loss": 4.155381206508071, + "tokens_seen": 312344576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315867602808425, + "loss": 1.7414, + "theoretical_loss": 4.155280287523674, + "tokens_seen": 312410112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315707121364093, + "loss": 1.6323, + "theoretical_loss": 4.155179395633667, + "tokens_seen": 312475648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315546639919759, + "loss": 1.6686, + "theoretical_loss": 4.1550785308250955, + "tokens_seen": 312541184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315386158475427, + "loss": 1.5955, + "theoretical_loss": 4.154977693085016, + "tokens_seen": 312606720 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315225677031093, + "loss": 1.633, + "theoretical_loss": 4.154876882400489, + "tokens_seen": 312672256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007315065195586761, + "loss": 1.5965, + "theoretical_loss": 4.154776098758589, + "tokens_seen": 312737792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314904714142427, + "loss": 1.5974, + "theoretical_loss": 4.154675342146397, + "tokens_seen": 312803328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314744232698095, + "loss": 1.7647, + "theoretical_loss": 4.154574612551001, + "tokens_seen": 312868864 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314583751253761, + "loss": 1.5976, + "theoretical_loss": 4.154473909959503, + "tokens_seen": 312934400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314423269809429, + "loss": 1.7073, + "theoretical_loss": 4.154373234359007, + "tokens_seen": 312999936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314262788365095, + "loss": 1.7557, + "theoretical_loss": 4.154272585736632, + "tokens_seen": 313065472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007314102306920763, + "loss": 1.6123, + "theoretical_loss": 4.154171964079504, + "tokens_seen": 313131008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313941825476429, + "loss": 1.6845, + "theoretical_loss": 4.154071369374755, + "tokens_seen": 313196544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313781344032097, + "loss": 1.5383, + "theoretical_loss": 4.15397080160953, + "tokens_seen": 313262080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313620862587763, + "loss": 1.5845, + "theoretical_loss": 4.153870260770979, + "tokens_seen": 313327616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313460381143431, + "loss": 1.6715, + "theoretical_loss": 4.153769746846265, + "tokens_seen": 313393152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0007313299899699097, + "loss": 1.8037, + "theoretical_loss": 4.153669259822556, + "tokens_seen": 313458688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007313139418254765, + "loss": 1.716, + "theoretical_loss": 4.153568799687031, + "tokens_seen": 313524224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312978936810431, + "loss": 1.6919, + "theoretical_loss": 4.153468366426877, + "tokens_seen": 313589760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312818455366099, + "loss": 1.587, + "theoretical_loss": 4.153367960029288, + "tokens_seen": 313655296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312657973921765, + "loss": 1.5868, + "theoretical_loss": 4.153267580481472, + "tokens_seen": 313720832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312497492477433, + "loss": 1.57, + "theoretical_loss": 4.15316722777064, + "tokens_seen": 313786368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312337011033099, + "loss": 1.6001, + "theoretical_loss": 4.153066901884016, + "tokens_seen": 313851904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312176529588767, + "loss": 1.7275, + "theoretical_loss": 4.15296660280883, + "tokens_seen": 313917440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007312016048144433, + "loss": 1.6171, + "theoretical_loss": 4.152866330532321, + "tokens_seen": 313982976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311855566700101, + "loss": 1.6996, + "theoretical_loss": 4.152766085041737, + "tokens_seen": 314048512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311695085255767, + "loss": 1.6482, + "theoretical_loss": 4.152665866324337, + "tokens_seen": 314114048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311534603811435, + "loss": 1.5343, + "theoretical_loss": 4.152565674367387, + "tokens_seen": 314179584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311374122367101, + "loss": 1.6345, + "theoretical_loss": 4.15246550915816, + "tokens_seen": 314245120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311213640922769, + "loss": 1.635, + "theoretical_loss": 4.15236537068394, + "tokens_seen": 314310656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007311053159478435, + "loss": 1.6151, + "theoretical_loss": 4.152265258932019, + "tokens_seen": 314376192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310892678034103, + "loss": 1.5423, + "theoretical_loss": 4.152165173889697, + "tokens_seen": 314441728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310732196589769, + "loss": 1.5504, + "theoretical_loss": 4.152065115544284, + "tokens_seen": 314507264 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 123650, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5619208812713623, + "objective/train/theoretical_loss": 4.151965083883098, + "objective/train/tokens_used": 335032800, + "theoretical_loss": 4.151965083883098, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310571715145437, + "loss": 1.5619, + "theoretical_loss": 4.151965083883098, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310411233701103, + "loss": 1.5635, + "theoretical_loss": 4.151865078893466, + "tokens_seen": 314638336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310250752256771, + "loss": 1.6122, + "theoretical_loss": 4.151765100562723, + "tokens_seen": 314703872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007310090270812437, + "loss": 1.6883, + "theoretical_loss": 4.151665148878212, + "tokens_seen": 314769408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309929789368105, + "loss": 1.6062, + "theoretical_loss": 4.151565223827288, + "tokens_seen": 314834944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309769307923771, + "loss": 1.5542, + "theoretical_loss": 4.15146532539731, + "tokens_seen": 314900480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309608826479439, + "loss": 1.6221, + "theoretical_loss": 4.151365453575648, + "tokens_seen": 314966016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309448345035105, + "loss": 1.5982, + "theoretical_loss": 4.151265608349682, + "tokens_seen": 315031552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309287863590773, + "loss": 1.5673, + "theoretical_loss": 4.151165789706798, + "tokens_seen": 315097088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007309127382146439, + "loss": 1.6323, + "theoretical_loss": 4.151065997634393, + "tokens_seen": 315162624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308966900702107, + "loss": 1.5159, + "theoretical_loss": 4.1509662321198695, + "tokens_seen": 315228160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308806419257773, + "loss": 1.5806, + "theoretical_loss": 4.150866493150642, + "tokens_seen": 315293696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308645937813441, + "loss": 1.5716, + "theoretical_loss": 4.150766780714131, + "tokens_seen": 315359232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308485456369107, + "loss": 1.7149, + "theoretical_loss": 4.150667094797766, + "tokens_seen": 315424768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308324974924775, + "loss": 1.6208, + "theoretical_loss": 4.150567435388988, + "tokens_seen": 315490304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308164493480441, + "loss": 1.6424, + "theoretical_loss": 4.150467802475241, + "tokens_seen": 315555840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007308004012036109, + "loss": 1.5372, + "theoretical_loss": 4.150368196043983, + "tokens_seen": 315621376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307843530591775, + "loss": 1.5855, + "theoretical_loss": 4.150268616082678, + "tokens_seen": 315686912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307683049147443, + "loss": 1.6299, + "theoretical_loss": 4.150169062578798, + "tokens_seen": 315752448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307522567703109, + "loss": 1.6293, + "theoretical_loss": 4.150069535519825, + "tokens_seen": 315817984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307362086258777, + "loss": 1.6443, + "theoretical_loss": 4.149970034893249, + "tokens_seen": 315883520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307201604814443, + "loss": 1.5875, + "theoretical_loss": 4.149870560686567, + "tokens_seen": 315949056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007307041123370111, + "loss": 1.6472, + "theoretical_loss": 4.149771112887288, + "tokens_seen": 316014592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306880641925777, + "loss": 1.5727, + "theoretical_loss": 4.149671691482926, + "tokens_seen": 316080128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306720160481445, + "loss": 1.7287, + "theoretical_loss": 4.149572296461005, + "tokens_seen": 316145664 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306559679037111, + "loss": 1.5935, + "theoretical_loss": 4.149472927809057, + "tokens_seen": 316211200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306399197592779, + "loss": 1.6763, + "theoretical_loss": 4.149373585514625, + "tokens_seen": 316276736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306238716148446, + "loss": 1.6598, + "theoretical_loss": 4.149274269565256, + "tokens_seen": 316342272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007306078234704113, + "loss": 1.676, + "theoretical_loss": 4.149174979948507, + "tokens_seen": 316407808 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730591775325978, + "loss": 1.5757, + "theoretical_loss": 4.1490757166519465, + "tokens_seen": 316473344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305757271815447, + "loss": 1.5618, + "theoretical_loss": 4.148976479663148, + "tokens_seen": 316538880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305596790371114, + "loss": 1.5078, + "theoretical_loss": 4.148877268969694, + "tokens_seen": 316604416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305436308926781, + "loss": 1.6376, + "theoretical_loss": 4.1487780845591775, + "tokens_seen": 316669952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305275827482448, + "loss": 1.6326, + "theoretical_loss": 4.148678926419197, + "tokens_seen": 316735488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007305115346038115, + "loss": 1.6067, + "theoretical_loss": 4.148579794537362, + "tokens_seen": 316801024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304954864593782, + "loss": 1.578, + "theoretical_loss": 4.148480688901288, + "tokens_seen": 316866560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304794383149449, + "loss": 1.694, + "theoretical_loss": 4.148381609498602, + "tokens_seen": 316932096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304633901705116, + "loss": 1.5567, + "theoretical_loss": 4.148282556316936, + "tokens_seen": 316997632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304473420260783, + "loss": 1.6117, + "theoretical_loss": 4.148183529343932, + "tokens_seen": 317063168 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730431293881645, + "loss": 1.5606, + "theoretical_loss": 4.148084528567241, + "tokens_seen": 317128704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007304152457372117, + "loss": 1.6511, + "theoretical_loss": 4.147985553974522, + "tokens_seen": 317194240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303991975927784, + "loss": 1.5511, + "theoretical_loss": 4.14788660555344, + "tokens_seen": 317259776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303831494483451, + "loss": 1.5398, + "theoretical_loss": 4.147787683291672, + "tokens_seen": 317325312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303671013039118, + "loss": 1.7024, + "theoretical_loss": 4.147688787176902, + "tokens_seen": 317390848 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303510531594785, + "loss": 1.5716, + "theoretical_loss": 4.147589917196821, + "tokens_seen": 317456384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303350050150452, + "loss": 1.5588, + "theoretical_loss": 4.14749107333913, + "tokens_seen": 317521920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303189568706119, + "loss": 1.6169, + "theoretical_loss": 4.147392255591538, + "tokens_seen": 317587456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007303029087261786, + "loss": 1.7814, + "theoretical_loss": 4.147293463941762, + "tokens_seen": 317652992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302868605817453, + "loss": 1.4771, + "theoretical_loss": 4.147194698377526, + "tokens_seen": 317718528 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730270812437312, + "loss": 1.6387, + "theoretical_loss": 4.147095958886566, + "tokens_seen": 317784064 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302547642928787, + "loss": 1.6541, + "theoretical_loss": 4.146997245456623, + "tokens_seen": 317849600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302387161484454, + "loss": 1.6418, + "theoretical_loss": 4.146898558075446, + "tokens_seen": 317915136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302226680040121, + "loss": 1.5862, + "theoretical_loss": 4.146799896730794, + "tokens_seen": 317980672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007302066198595788, + "loss": 1.5395, + "theoretical_loss": 4.146701261410435, + "tokens_seen": 318046208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301905717151454, + "loss": 1.5877, + "theoretical_loss": 4.1466026521021435, + "tokens_seen": 318111744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301745235707122, + "loss": 1.7137, + "theoretical_loss": 4.146504068793703, + "tokens_seen": 318177280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301584754262788, + "loss": 1.6126, + "theoretical_loss": 4.146405511472905, + "tokens_seen": 318242816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301424272818456, + "loss": 1.6433, + "theoretical_loss": 4.146306980127548, + "tokens_seen": 318308352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007301263791374122, + "loss": 1.6642, + "theoretical_loss": 4.146208474745442, + "tokens_seen": 318373888 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730110330992979, + "loss": 1.6511, + "theoretical_loss": 4.1461099953144025, + "tokens_seen": 318439424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300942828485456, + "loss": 1.5287, + "theoretical_loss": 4.146011541822253, + "tokens_seen": 318504960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300782347041124, + "loss": 1.538, + "theoretical_loss": 4.145913114256829, + "tokens_seen": 318570496 + }, + { + "epoch": 0.1, + "learning_rate": 0.000730062186559679, + "loss": 1.5778, + "theoretical_loss": 4.145814712605969, + "tokens_seen": 318636032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300461384152458, + "loss": 1.5975, + "theoretical_loss": 4.1457163368575225, + "tokens_seen": 318701568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300300902708124, + "loss": 1.552, + "theoretical_loss": 4.145617986999348, + "tokens_seen": 318767104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007300140421263792, + "loss": 1.6324, + "theoretical_loss": 4.145519663019309, + "tokens_seen": 318832640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299979939819458, + "loss": 1.5314, + "theoretical_loss": 4.145421364905282, + "tokens_seen": 318898176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299819458375126, + "loss": 1.5986, + "theoretical_loss": 4.145323092645146, + "tokens_seen": 318963712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299658976930792, + "loss": 1.6539, + "theoretical_loss": 4.145224846226793, + "tokens_seen": 319029248 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729949849548646, + "loss": 1.6846, + "theoretical_loss": 4.14512662563812, + "tokens_seen": 319094784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299338014042126, + "loss": 1.5786, + "theoretical_loss": 4.145028430867035, + "tokens_seen": 319160320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007299177532597794, + "loss": 1.6125, + "theoretical_loss": 4.144930261901451, + "tokens_seen": 319225856 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729901705115346, + "loss": 1.5453, + "theoretical_loss": 4.144832118729291, + "tokens_seen": 319291392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298856569709128, + "loss": 1.6036, + "theoretical_loss": 4.144734001338486, + "tokens_seen": 319356928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298696088264794, + "loss": 1.7081, + "theoretical_loss": 4.144635909716976, + "tokens_seen": 319422464 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298535606820462, + "loss": 1.6158, + "theoretical_loss": 4.144537843852705, + "tokens_seen": 319488000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298375125376128, + "loss": 1.4767, + "theoretical_loss": 4.144439803733631, + "tokens_seen": 319553536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298214643931796, + "loss": 1.5602, + "theoretical_loss": 4.144341789347715, + "tokens_seen": 319619072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007298054162487462, + "loss": 1.6508, + "theoretical_loss": 4.144243800682931, + "tokens_seen": 319684608 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729789368104313, + "loss": 1.5997, + "theoretical_loss": 4.1441458377272555, + "tokens_seen": 319750144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007297733199598796, + "loss": 1.6316, + "theoretical_loss": 4.144047900468678, + "tokens_seen": 319815680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007297572718154464, + "loss": 1.6584, + "theoretical_loss": 4.143949988895193, + "tokens_seen": 319881216 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729741223671013, + "loss": 1.6522, + "theoretical_loss": 4.1438521029948046, + "tokens_seen": 319946752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007297251755265798, + "loss": 1.5663, + "theoretical_loss": 4.1437542427555245, + "tokens_seen": 320012288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007297091273821464, + "loss": 1.5859, + "theoretical_loss": 4.143656408165373, + "tokens_seen": 320077824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296930792377132, + "loss": 1.6449, + "theoretical_loss": 4.143558599212376, + "tokens_seen": 320143360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296770310932798, + "loss": 1.6123, + "theoretical_loss": 4.143460815884571, + "tokens_seen": 320208896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296609829488466, + "loss": 1.6351, + "theoretical_loss": 4.143363058170001, + "tokens_seen": 320274432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296449348044132, + "loss": 1.6385, + "theoretical_loss": 4.143265326056719, + "tokens_seen": 320339968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00072962888665998, + "loss": 1.6856, + "theoretical_loss": 4.1431676195327825, + "tokens_seen": 320405504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007296128385155466, + "loss": 1.7328, + "theoretical_loss": 4.143069938586263, + "tokens_seen": 320471040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295967903711134, + "loss": 1.7697, + "theoretical_loss": 4.142972283205233, + "tokens_seen": 320536576 + }, + { + "epoch": 0.1, + "learning_rate": 0.00072958074222668, + "loss": 1.5611, + "theoretical_loss": 4.142874653377778, + "tokens_seen": 320602112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295646940822468, + "loss": 1.6218, + "theoretical_loss": 4.14277704909199, + "tokens_seen": 320667648 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295486459378134, + "loss": 1.5247, + "theoretical_loss": 4.142679470335969, + "tokens_seen": 320733184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295325977933802, + "loss": 1.5723, + "theoretical_loss": 4.142581917097821, + "tokens_seen": 320798720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295165496489468, + "loss": 1.6569, + "theoretical_loss": 4.142484389365664, + "tokens_seen": 320864256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007295005015045136, + "loss": 1.5901, + "theoretical_loss": 4.14238688712762, + "tokens_seen": 320929792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007294844533600802, + "loss": 1.6504, + "theoretical_loss": 4.142289410371823, + "tokens_seen": 320995328 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729468405215647, + "loss": 1.6673, + "theoretical_loss": 4.14219195908641, + "tokens_seen": 321060864 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 125851, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5615171194076538, + "objective/train/theoretical_loss": 4.14209453325953, + "objective/train/tokens_used": 341586400, + "theoretical_loss": 4.14209453325953, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007294523570712136, + "loss": 1.5615, + "theoretical_loss": 4.14209453325953, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007294363089267804, + "loss": 1.6042, + "theoretical_loss": 4.1419971328793395, + "tokens_seen": 321191936 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729420260782347, + "loss": 1.6517, + "theoretical_loss": 4.141899757934, + "tokens_seen": 321257472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007294042126379138, + "loss": 1.5173, + "theoretical_loss": 4.141802408411683, + "tokens_seen": 321323008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293881644934804, + "loss": 1.595, + "theoretical_loss": 4.14170508430057, + "tokens_seen": 321388544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293721163490472, + "loss": 1.6864, + "theoretical_loss": 4.141607785588846, + "tokens_seen": 321454080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293560682046138, + "loss": 1.6457, + "theoretical_loss": 4.141510512264707, + "tokens_seen": 321519616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293400200601806, + "loss": 1.6577, + "theoretical_loss": 4.141413264316356, + "tokens_seen": 321585152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007293239719157472, + "loss": 1.7241, + "theoretical_loss": 4.141316041732004, + "tokens_seen": 321650688 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729307923771314, + "loss": 1.657, + "theoretical_loss": 4.141218844499869, + "tokens_seen": 321716224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292918756268806, + "loss": 1.6004, + "theoretical_loss": 4.141121672608178, + "tokens_seen": 321781760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292758274824474, + "loss": 1.6061, + "theoretical_loss": 4.141024526045166, + "tokens_seen": 321847296 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729259779338014, + "loss": 1.725, + "theoretical_loss": 4.140927404799075, + "tokens_seen": 321912832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292437311935808, + "loss": 1.6464, + "theoretical_loss": 4.140830308858154, + "tokens_seen": 321978368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292276830491475, + "loss": 1.5866, + "theoretical_loss": 4.140733238210664, + "tokens_seen": 322043904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007292116349047142, + "loss": 1.5196, + "theoretical_loss": 4.140636192844867, + "tokens_seen": 322109440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291955867602809, + "loss": 1.6769, + "theoretical_loss": 4.1405391727490395, + "tokens_seen": 322174976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291795386158476, + "loss": 1.6522, + "theoretical_loss": 4.140442177911462, + "tokens_seen": 322240512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291634904714143, + "loss": 1.5604, + "theoretical_loss": 4.140345208320425, + "tokens_seen": 322306048 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729147442326981, + "loss": 1.6293, + "theoretical_loss": 4.1402482639642235, + "tokens_seen": 322371584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291313941825477, + "loss": 1.5509, + "theoretical_loss": 4.140151344831164, + "tokens_seen": 322437120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007291153460381144, + "loss": 1.6253, + "theoretical_loss": 4.14005445090956, + "tokens_seen": 322502656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290992978936812, + "loss": 1.619, + "theoretical_loss": 4.13995758218773, + "tokens_seen": 322568192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290832497492478, + "loss": 1.6907, + "theoretical_loss": 4.1398607386540025, + "tokens_seen": 322633728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290672016048146, + "loss": 1.6889, + "theoretical_loss": 4.139763920296716, + "tokens_seen": 322699264 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290511534603812, + "loss": 1.6077, + "theoretical_loss": 4.139667127104211, + "tokens_seen": 322764800 + }, + { + "epoch": 0.1, + "learning_rate": 0.000729035105315948, + "loss": 1.5675, + "theoretical_loss": 4.139570359064842, + "tokens_seen": 322830336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290190571715146, + "loss": 1.5447, + "theoretical_loss": 4.139473616166967, + "tokens_seen": 322895872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007290030090270814, + "loss": 1.4649, + "theoretical_loss": 4.139376898398953, + "tokens_seen": 322961408 + }, + { + "epoch": 0.1, + "learning_rate": 0.000728986960882648, + "loss": 1.6747, + "theoretical_loss": 4.139280205749175, + "tokens_seen": 323026944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289709127382148, + "loss": 1.4629, + "theoretical_loss": 4.139183538206017, + "tokens_seen": 323092480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289548645937814, + "loss": 1.7784, + "theoretical_loss": 4.139086895757868, + "tokens_seen": 323158016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289388164493481, + "loss": 1.6418, + "theoretical_loss": 4.138990278393125, + "tokens_seen": 323223552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289227683049148, + "loss": 1.5323, + "theoretical_loss": 4.138893686100196, + "tokens_seen": 323289088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007289067201604815, + "loss": 1.5261, + "theoretical_loss": 4.138797118867493, + "tokens_seen": 323354624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288906720160482, + "loss": 1.6058, + "theoretical_loss": 4.138700576683437, + "tokens_seen": 323420160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288746238716149, + "loss": 1.5333, + "theoretical_loss": 4.138604059536458, + "tokens_seen": 323485696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288585757271816, + "loss": 1.5597, + "theoretical_loss": 4.138507567414993, + "tokens_seen": 323551232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288425275827483, + "loss": 1.5813, + "theoretical_loss": 4.138411100307485, + "tokens_seen": 323616768 + }, + { + "epoch": 0.1, + "learning_rate": 0.000728826479438315, + "loss": 1.6585, + "theoretical_loss": 4.138314658202385, + "tokens_seen": 323682304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007288104312938817, + "loss": 1.6357, + "theoretical_loss": 4.138218241088156, + "tokens_seen": 323747840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287943831494483, + "loss": 1.5418, + "theoretical_loss": 4.138121848953261, + "tokens_seen": 323813376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287783350050151, + "loss": 1.6357, + "theoretical_loss": 4.138025481786178, + "tokens_seen": 323878912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287622868605817, + "loss": 1.599, + "theoretical_loss": 4.137929139575387, + "tokens_seen": 323944448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287462387161485, + "loss": 1.6741, + "theoretical_loss": 4.137832822309382, + "tokens_seen": 324009984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287301905717151, + "loss": 1.6565, + "theoretical_loss": 4.1377365299766575, + "tokens_seen": 324075520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007287141424272819, + "loss": 1.5809, + "theoretical_loss": 4.13764026256572, + "tokens_seen": 324141056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286980942828485, + "loss": 1.6851, + "theoretical_loss": 4.137544020065082, + "tokens_seen": 324206592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286820461384153, + "loss": 1.5922, + "theoretical_loss": 4.1374478024632655, + "tokens_seen": 324272128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286659979939819, + "loss": 1.7437, + "theoretical_loss": 4.137351609748798, + "tokens_seen": 324337664 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286499498495487, + "loss": 1.6601, + "theoretical_loss": 4.137255441910215, + "tokens_seen": 324403200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286339017051153, + "loss": 1.6656, + "theoretical_loss": 4.13715929893606, + "tokens_seen": 324468736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286178535606821, + "loss": 1.6646, + "theoretical_loss": 4.137063180814886, + "tokens_seen": 324534272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007286018054162487, + "loss": 1.5173, + "theoretical_loss": 4.136967087535249, + "tokens_seen": 324599808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285857572718155, + "loss": 1.4389, + "theoretical_loss": 4.136871019085717, + "tokens_seen": 324665344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285697091273821, + "loss": 1.6198, + "theoretical_loss": 4.136774975454863, + "tokens_seen": 324730880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285536609829489, + "loss": 1.5379, + "theoretical_loss": 4.136678956631269, + "tokens_seen": 324796416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285376128385155, + "loss": 1.5541, + "theoretical_loss": 4.136582962603523, + "tokens_seen": 324861952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285215646940823, + "loss": 1.6251, + "theoretical_loss": 4.136486993360222, + "tokens_seen": 324927488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007285055165496489, + "loss": 1.564, + "theoretical_loss": 4.136391048889971, + "tokens_seen": 324993024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284894684052157, + "loss": 1.5952, + "theoretical_loss": 4.136295129181381, + "tokens_seen": 325058560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284734202607823, + "loss": 1.6159, + "theoretical_loss": 4.136199234223069, + "tokens_seen": 325124096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284573721163491, + "loss": 1.5685, + "theoretical_loss": 4.136103364003665, + "tokens_seen": 325189632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284413239719157, + "loss": 1.6032, + "theoretical_loss": 4.1360075185118, + "tokens_seen": 325255168 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284252758274825, + "loss": 1.7056, + "theoretical_loss": 4.135911697736118, + "tokens_seen": 325320704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007284092276830491, + "loss": 1.5234, + "theoretical_loss": 4.135815901665267, + "tokens_seen": 325386240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283931795386159, + "loss": 1.65, + "theoretical_loss": 4.135720130287904, + "tokens_seen": 325451776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283771313941825, + "loss": 1.6589, + "theoretical_loss": 4.135624383592694, + "tokens_seen": 325517312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283610832497493, + "loss": 1.572, + "theoretical_loss": 4.135528661568307, + "tokens_seen": 325582848 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283450351053159, + "loss": 1.5901, + "theoretical_loss": 4.135432964203422, + "tokens_seen": 325648384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283289869608827, + "loss": 1.7418, + "theoretical_loss": 4.135337291486727, + "tokens_seen": 325713920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007283129388164493, + "loss": 1.6472, + "theoretical_loss": 4.135241643406915, + "tokens_seen": 325779456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282968906720161, + "loss": 1.629, + "theoretical_loss": 4.135146019952688, + "tokens_seen": 325844992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282808425275827, + "loss": 1.5419, + "theoretical_loss": 4.135050421112755, + "tokens_seen": 325910528 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282647943831495, + "loss": 1.646, + "theoretical_loss": 4.134954846875832, + "tokens_seen": 325976064 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282487462387161, + "loss": 1.6814, + "theoretical_loss": 4.134859297230644, + "tokens_seen": 326041600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282326980942829, + "loss": 1.6225, + "theoretical_loss": 4.13476377216592, + "tokens_seen": 326107136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282166499498495, + "loss": 1.651, + "theoretical_loss": 4.134668271670401, + "tokens_seen": 326172672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007282006018054163, + "loss": 1.6105, + "theoretical_loss": 4.134572795732832, + "tokens_seen": 326238208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281845536609829, + "loss": 1.6282, + "theoretical_loss": 4.134477344341967, + "tokens_seen": 326303744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281685055165497, + "loss": 1.6764, + "theoretical_loss": 4.134381917486565, + "tokens_seen": 326369280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281524573721163, + "loss": 1.5892, + "theoretical_loss": 4.134286515155398, + "tokens_seen": 326434816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281364092276831, + "loss": 1.5875, + "theoretical_loss": 4.134191137337238, + "tokens_seen": 326500352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281203610832497, + "loss": 1.6182, + "theoretical_loss": 4.134095784020871, + "tokens_seen": 326565888 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007281043129388165, + "loss": 1.5375, + "theoretical_loss": 4.1340004551950855, + "tokens_seen": 326631424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280882647943831, + "loss": 1.5434, + "theoretical_loss": 4.133905150848681, + "tokens_seen": 326696960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280722166499499, + "loss": 1.5871, + "theoretical_loss": 4.133809870970461, + "tokens_seen": 326762496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280561685055165, + "loss": 1.6211, + "theoretical_loss": 4.1337146155492395, + "tokens_seen": 326828032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280401203610833, + "loss": 1.4744, + "theoretical_loss": 4.133619384573836, + "tokens_seen": 326893568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280240722166499, + "loss": 1.5541, + "theoretical_loss": 4.1335241780330785, + "tokens_seen": 326959104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007280080240722167, + "loss": 1.5424, + "theoretical_loss": 4.1334289959158, + "tokens_seen": 327024640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279919759277833, + "loss": 1.5648, + "theoretical_loss": 4.133333838210844, + "tokens_seen": 327090176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279759277833501, + "loss": 1.7727, + "theoretical_loss": 4.133238704907059, + "tokens_seen": 327155712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279598796389167, + "loss": 1.6525, + "theoretical_loss": 4.133143595993303, + "tokens_seen": 327221248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279438314944835, + "loss": 1.6713, + "theoretical_loss": 4.133048511458438, + "tokens_seen": 327286784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279277833500503, + "loss": 1.705, + "theoretical_loss": 4.132953451291337, + "tokens_seen": 327352320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007279117352056169, + "loss": 1.5638, + "theoretical_loss": 4.132858415480877, + "tokens_seen": 327417856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278956870611837, + "loss": 1.6074, + "theoretical_loss": 4.132763404015946, + "tokens_seen": 327483392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278796389167503, + "loss": 1.5912, + "theoretical_loss": 4.132668416885435, + "tokens_seen": 327548928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278635907723171, + "loss": 1.5823, + "theoretical_loss": 4.132573454078246, + "tokens_seen": 327614464 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 128204, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5737180709838867, + "objective/train/theoretical_loss": 4.132478515583286, + "objective/train/tokens_used": 348140000, + "theoretical_loss": 4.132478515583286, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278475426278837, + "loss": 1.5737, + "theoretical_loss": 4.132478515583286, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278314944834505, + "loss": 1.6624, + "theoretical_loss": 4.132383601389471, + "tokens_seen": 327745536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007278154463390171, + "loss": 1.7106, + "theoretical_loss": 4.132288711485721, + "tokens_seen": 327811072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277993981945839, + "loss": 1.7156, + "theoretical_loss": 4.132193845860969, + "tokens_seen": 327876608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277833500501505, + "loss": 1.5011, + "theoretical_loss": 4.132099004504149, + "tokens_seen": 327942144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277673019057173, + "loss": 1.6679, + "theoretical_loss": 4.132004187404206, + "tokens_seen": 328007680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277512537612839, + "loss": 1.5978, + "theoretical_loss": 4.131909394550092, + "tokens_seen": 328073216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277352056168507, + "loss": 1.6576, + "theoretical_loss": 4.131814625930764, + "tokens_seen": 328138752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277191574724173, + "loss": 1.5793, + "theoretical_loss": 4.13171988153519, + "tokens_seen": 328204288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007277031093279841, + "loss": 1.6284, + "theoretical_loss": 4.131625161352342, + "tokens_seen": 328269824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276870611835507, + "loss": 1.5509, + "theoretical_loss": 4.131530465371199, + "tokens_seen": 328335360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276710130391175, + "loss": 1.5893, + "theoretical_loss": 4.131435793580749, + "tokens_seen": 328400896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276549648946841, + "loss": 1.528, + "theoretical_loss": 4.131341145969989, + "tokens_seen": 328466432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276389167502509, + "loss": 1.6218, + "theoretical_loss": 4.131246522527918, + "tokens_seen": 328531968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276228686058175, + "loss": 1.6846, + "theoretical_loss": 4.1311519232435465, + "tokens_seen": 328597504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007276068204613843, + "loss": 1.599, + "theoretical_loss": 4.131057348105889, + "tokens_seen": 328663040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275907723169509, + "loss": 1.57, + "theoretical_loss": 4.1309627971039715, + "tokens_seen": 328728576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275747241725177, + "loss": 1.5081, + "theoretical_loss": 4.130868270226824, + "tokens_seen": 328794112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275586760280843, + "loss": 1.6188, + "theoretical_loss": 4.130773767463483, + "tokens_seen": 328859648 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727542627883651, + "loss": 1.5607, + "theoretical_loss": 4.130679288802993, + "tokens_seen": 328925184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275265797392177, + "loss": 1.6072, + "theoretical_loss": 4.1305848342344085, + "tokens_seen": 328990720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007275105315947844, + "loss": 1.6061, + "theoretical_loss": 4.130490403746787, + "tokens_seen": 329056256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274944834503511, + "loss": 1.6438, + "theoretical_loss": 4.130395997329196, + "tokens_seen": 329121792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274784353059178, + "loss": 1.6562, + "theoretical_loss": 4.1303016149707075, + "tokens_seen": 329187328 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274623871614845, + "loss": 1.6846, + "theoretical_loss": 4.130207256660404, + "tokens_seen": 329252864 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274463390170512, + "loss": 1.575, + "theoretical_loss": 4.130112922387372, + "tokens_seen": 329318400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274302908726179, + "loss": 1.5378, + "theoretical_loss": 4.130018612140707, + "tokens_seen": 329383936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007274142427281846, + "loss": 1.6047, + "theoretical_loss": 4.12992432590951, + "tokens_seen": 329449472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273981945837512, + "loss": 1.5478, + "theoretical_loss": 4.129830063682892, + "tokens_seen": 329515008 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727382146439318, + "loss": 1.5945, + "theoretical_loss": 4.129735825449968, + "tokens_seen": 329580544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273660982948846, + "loss": 1.5524, + "theoretical_loss": 4.129641611199862, + "tokens_seen": 329646080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273500501504514, + "loss": 1.4979, + "theoretical_loss": 4.1295474209217025, + "tokens_seen": 329711616 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727334002006018, + "loss": 1.6272, + "theoretical_loss": 4.12945325460463, + "tokens_seen": 329777152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273179538615848, + "loss": 1.6828, + "theoretical_loss": 4.129359112237787, + "tokens_seen": 329842688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007273019057171514, + "loss": 1.5937, + "theoretical_loss": 4.1292649938103265, + "tokens_seen": 329908224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272858575727182, + "loss": 1.6014, + "theoretical_loss": 4.129170899311406, + "tokens_seen": 329973760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272698094282848, + "loss": 1.6816, + "theoretical_loss": 4.129076828730192, + "tokens_seen": 330039296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272537612838516, + "loss": 1.5572, + "theoretical_loss": 4.128982782055858, + "tokens_seen": 330104832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272377131394182, + "loss": 1.5982, + "theoretical_loss": 4.128888759277583, + "tokens_seen": 330170368 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727221664994985, + "loss": 1.5945, + "theoretical_loss": 4.1287947603845545, + "tokens_seen": 330235904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007272056168505516, + "loss": 1.5861, + "theoretical_loss": 4.128700785365965, + "tokens_seen": 330301440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271895687061184, + "loss": 1.5119, + "theoretical_loss": 4.128606834211018, + "tokens_seen": 330366976 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727173520561685, + "loss": 1.6059, + "theoretical_loss": 4.128512906908919, + "tokens_seen": 330432512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271574724172518, + "loss": 1.6739, + "theoretical_loss": 4.128419003448886, + "tokens_seen": 330498048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271414242728184, + "loss": 1.5753, + "theoretical_loss": 4.128325123820138, + "tokens_seen": 330563584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271253761283852, + "loss": 1.6488, + "theoretical_loss": 4.1282312680119055, + "tokens_seen": 330629120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007271093279839518, + "loss": 1.6453, + "theoretical_loss": 4.128137436013425, + "tokens_seen": 330694656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007270932798395186, + "loss": 1.5761, + "theoretical_loss": 4.128043627813939, + "tokens_seen": 330760192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007270772316950852, + "loss": 1.5743, + "theoretical_loss": 4.127949843402699, + "tokens_seen": 330825728 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727061183550652, + "loss": 1.6475, + "theoretical_loss": 4.12785608276896, + "tokens_seen": 330891264 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007270451354062186, + "loss": 1.584, + "theoretical_loss": 4.127762345901987, + "tokens_seen": 330956800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007270290872617854, + "loss": 1.5983, + "theoretical_loss": 4.1276686327910515, + "tokens_seen": 331022336 + }, + { + "epoch": 0.1, + "learning_rate": 0.000727013039117352, + "loss": 1.6357, + "theoretical_loss": 4.127574943425429, + "tokens_seen": 331087872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269969909729188, + "loss": 1.7035, + "theoretical_loss": 4.127481277794408, + "tokens_seen": 331153408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269809428284854, + "loss": 1.6127, + "theoretical_loss": 4.1273876358872785, + "tokens_seen": 331218944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269648946840522, + "loss": 1.4817, + "theoretical_loss": 4.127294017693339, + "tokens_seen": 331284480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269488465396188, + "loss": 1.5948, + "theoretical_loss": 4.127200423201897, + "tokens_seen": 331350016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269327983951856, + "loss": 1.5819, + "theoretical_loss": 4.127106852402264, + "tokens_seen": 331415552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007269167502507522, + "loss": 1.5774, + "theoretical_loss": 4.127013305283759, + "tokens_seen": 331481088 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726900702106319, + "loss": 1.6623, + "theoretical_loss": 4.12691978183571, + "tokens_seen": 331546624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268846539618856, + "loss": 1.5851, + "theoretical_loss": 4.126826282047449, + "tokens_seen": 331612160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268686058174524, + "loss": 1.5979, + "theoretical_loss": 4.1267328059083175, + "tokens_seen": 331677696 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726852557673019, + "loss": 1.7116, + "theoretical_loss": 4.126639353407663, + "tokens_seen": 331743232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268365095285858, + "loss": 1.5787, + "theoretical_loss": 4.12654592453484, + "tokens_seen": 331808768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268204613841524, + "loss": 1.6028, + "theoretical_loss": 4.126452519279208, + "tokens_seen": 331874304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007268044132397192, + "loss": 1.582, + "theoretical_loss": 4.126359137630136, + "tokens_seen": 331939840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267883650952858, + "loss": 1.6223, + "theoretical_loss": 4.126265779576999, + "tokens_seen": 332005376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267723169508526, + "loss": 1.6756, + "theoretical_loss": 4.126172445109178, + "tokens_seen": 332070912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267562688064192, + "loss": 1.5595, + "theoretical_loss": 4.126079134216063, + "tokens_seen": 332136448 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726740220661986, + "loss": 1.634, + "theoretical_loss": 4.125985846887049, + "tokens_seen": 332201984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267241725175528, + "loss": 1.5538, + "theoretical_loss": 4.125892583111538, + "tokens_seen": 332267520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007267081243731194, + "loss": 1.6569, + "theoretical_loss": 4.125799342878939, + "tokens_seen": 332333056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266920762286862, + "loss": 1.604, + "theoretical_loss": 4.1257061261786685, + "tokens_seen": 332398592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266760280842528, + "loss": 1.7223, + "theoretical_loss": 4.12561293300015, + "tokens_seen": 332464128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266599799398196, + "loss": 1.4625, + "theoretical_loss": 4.125519763332813, + "tokens_seen": 332529664 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266439317953862, + "loss": 1.5883, + "theoretical_loss": 4.1254266171660925, + "tokens_seen": 332595200 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726627883650953, + "loss": 1.5598, + "theoretical_loss": 4.125333494489434, + "tokens_seen": 332660736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007266118355065196, + "loss": 1.4813, + "theoretical_loss": 4.125240395292286, + "tokens_seen": 332726272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265957873620864, + "loss": 1.6135, + "theoretical_loss": 4.125147319564107, + "tokens_seen": 332791808 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726579739217653, + "loss": 1.7149, + "theoretical_loss": 4.12505426729436, + "tokens_seen": 332857344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265636910732198, + "loss": 1.6416, + "theoretical_loss": 4.124961238472517, + "tokens_seen": 332922880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265476429287864, + "loss": 1.5842, + "theoretical_loss": 4.124868233088054, + "tokens_seen": 332988416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265315947843532, + "loss": 1.506, + "theoretical_loss": 4.124775251130455, + "tokens_seen": 333053952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007265155466399198, + "loss": 1.5667, + "theoretical_loss": 4.124682292589212, + "tokens_seen": 333119488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264994984954866, + "loss": 1.5985, + "theoretical_loss": 4.124589357453823, + "tokens_seen": 333185024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264834503510532, + "loss": 1.623, + "theoretical_loss": 4.124496445713792, + "tokens_seen": 333250560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00072646740220662, + "loss": 1.5873, + "theoretical_loss": 4.12440355735863, + "tokens_seen": 333316096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264513540621866, + "loss": 1.7098, + "theoretical_loss": 4.124310692377857, + "tokens_seen": 333381632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264353059177534, + "loss": 1.5656, + "theoretical_loss": 4.124217850760996, + "tokens_seen": 333447168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00072641925777332, + "loss": 1.6564, + "theoretical_loss": 4.124125032497578, + "tokens_seen": 333512704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007264032096288868, + "loss": 1.6811, + "theoretical_loss": 4.124032237577143, + "tokens_seen": 333578240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263871614844534, + "loss": 1.4363, + "theoretical_loss": 4.1239394659892366, + "tokens_seen": 333643776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263711133400202, + "loss": 1.638, + "theoretical_loss": 4.12384671772341, + "tokens_seen": 333709312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263550651955868, + "loss": 1.7131, + "theoretical_loss": 4.12375399276922, + "tokens_seen": 333774848 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263390170511536, + "loss": 1.5945, + "theoretical_loss": 4.1236612911162345, + "tokens_seen": 333840384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007263229689067202, + "loss": 1.6497, + "theoretical_loss": 4.123568612754024, + "tokens_seen": 333905920 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726306920762287, + "loss": 1.6644, + "theoretical_loss": 4.123475957672168, + "tokens_seen": 333971456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262908726178536, + "loss": 1.5124, + "theoretical_loss": 4.123383325860252, + "tokens_seen": 334036992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262748244734204, + "loss": 1.6337, + "theoretical_loss": 4.123290717307867, + "tokens_seen": 334102528 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726258776328987, + "loss": 1.6667, + "theoretical_loss": 4.123198132004614, + "tokens_seen": 334168064 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 130593, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.6924984455108643, + "objective/train/theoretical_loss": 4.123105569940097, + "objective/train/tokens_used": 354693600, + "theoretical_loss": 4.123105569940097, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262427281845538, + "loss": 1.6925, + "theoretical_loss": 4.123105569940097, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262266800401204, + "loss": 1.6058, + "theoretical_loss": 4.123013031103927, + "tokens_seen": 334299136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007262106318956872, + "loss": 1.5292, + "theoretical_loss": 4.122920515485726, + "tokens_seen": 334364672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261945837512538, + "loss": 1.6724, + "theoretical_loss": 4.122828023075117, + "tokens_seen": 334430208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261785356068206, + "loss": 1.529, + "theoretical_loss": 4.122735553861733, + "tokens_seen": 334495744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261624874623872, + "loss": 1.542, + "theoretical_loss": 4.122643107835213, + "tokens_seen": 334561280 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726146439317954, + "loss": 1.6993, + "theoretical_loss": 4.122550684985203, + "tokens_seen": 334626816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261303911735206, + "loss": 1.5968, + "theoretical_loss": 4.122458285301355, + "tokens_seen": 334692352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007261143430290873, + "loss": 1.5314, + "theoretical_loss": 4.122365908773327, + "tokens_seen": 334757888 + }, + { + "epoch": 0.1, + "learning_rate": 0.000726098294884654, + "loss": 1.6674, + "theoretical_loss": 4.1222735553907865, + "tokens_seen": 334823424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260822467402207, + "loss": 1.5624, + "theoretical_loss": 4.122181225143404, + "tokens_seen": 334888960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260661985957874, + "loss": 1.6051, + "theoretical_loss": 4.122088918020857, + "tokens_seen": 334954496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260501504513541, + "loss": 1.5872, + "theoretical_loss": 4.121996634012834, + "tokens_seen": 335020032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260341023069208, + "loss": 1.6442, + "theoretical_loss": 4.121904373109024, + "tokens_seen": 335085568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260180541624875, + "loss": 1.5371, + "theoretical_loss": 4.121812135299129, + "tokens_seen": 335151104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007260020060180541, + "loss": 1.6379, + "theoretical_loss": 4.1217199205728505, + "tokens_seen": 335216640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259859578736209, + "loss": 1.5505, + "theoretical_loss": 4.121627728919903, + "tokens_seen": 335282176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259699097291875, + "loss": 1.6807, + "theoretical_loss": 4.121535560330004, + "tokens_seen": 335347712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259538615847543, + "loss": 1.6171, + "theoretical_loss": 4.1214434147928785, + "tokens_seen": 335413248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259378134403209, + "loss": 1.6394, + "theoretical_loss": 4.121351292298257, + "tokens_seen": 335478784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259217652958877, + "loss": 1.5916, + "theoretical_loss": 4.12125919283588, + "tokens_seen": 335544320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007259057171514543, + "loss": 1.7589, + "theoretical_loss": 4.12116711639549, + "tokens_seen": 335609856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258896690070211, + "loss": 1.5513, + "theoretical_loss": 4.12107506296684, + "tokens_seen": 335675392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258736208625877, + "loss": 1.6355, + "theoretical_loss": 4.120983032539687, + "tokens_seen": 335740928 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258575727181545, + "loss": 1.5022, + "theoretical_loss": 4.120891025103796, + "tokens_seen": 335806464 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258415245737211, + "loss": 1.6189, + "theoretical_loss": 4.1207990406489365, + "tokens_seen": 335872000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258254764292879, + "loss": 1.6445, + "theoretical_loss": 4.120707079164887, + "tokens_seen": 335937536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007258094282848545, + "loss": 1.6455, + "theoretical_loss": 4.120615140641433, + "tokens_seen": 336003072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257933801404213, + "loss": 1.6334, + "theoretical_loss": 4.1205232250683625, + "tokens_seen": 336068608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257773319959879, + "loss": 1.703, + "theoretical_loss": 4.120431332435475, + "tokens_seen": 336134144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257612838515547, + "loss": 1.5706, + "theoretical_loss": 4.120339462732572, + "tokens_seen": 336199680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257452357071213, + "loss": 1.646, + "theoretical_loss": 4.120247615949465, + "tokens_seen": 336265216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257291875626881, + "loss": 1.721, + "theoretical_loss": 4.12015579207597, + "tokens_seen": 336330752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007257131394182547, + "loss": 1.5871, + "theoretical_loss": 4.12006399110191, + "tokens_seen": 336396288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256970912738215, + "loss": 1.708, + "theoretical_loss": 4.119972213017116, + "tokens_seen": 336461824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256810431293881, + "loss": 1.5485, + "theoretical_loss": 4.119880457811424, + "tokens_seen": 336527360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256649949849549, + "loss": 1.6568, + "theoretical_loss": 4.119788725474675, + "tokens_seen": 336592896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256489468405215, + "loss": 1.6263, + "theoretical_loss": 4.119697015996719, + "tokens_seen": 336658432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256328986960883, + "loss": 1.5893, + "theoretical_loss": 4.119605329367413, + "tokens_seen": 336723968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256168505516549, + "loss": 1.6002, + "theoretical_loss": 4.119513665576617, + "tokens_seen": 336789504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007256008024072217, + "loss": 1.6765, + "theoretical_loss": 4.119422024614201, + "tokens_seen": 336855040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255847542627883, + "loss": 1.5966, + "theoretical_loss": 4.11933040647004, + "tokens_seen": 336920576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255687061183551, + "loss": 1.5932, + "theoretical_loss": 4.119238811134014, + "tokens_seen": 336986112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255526579739217, + "loss": 1.6461, + "theoretical_loss": 4.119147238596014, + "tokens_seen": 337051648 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255366098294885, + "loss": 1.6594, + "theoretical_loss": 4.119055688845931, + "tokens_seen": 337117184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255205616850552, + "loss": 1.6131, + "theoretical_loss": 4.118964161873667, + "tokens_seen": 337182720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007255045135406219, + "loss": 1.6731, + "theoretical_loss": 4.118872657669131, + "tokens_seen": 337248256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254884653961886, + "loss": 1.5342, + "theoretical_loss": 4.118781176222235, + "tokens_seen": 337313792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254724172517553, + "loss": 1.5934, + "theoretical_loss": 4.1186897175229, + "tokens_seen": 337379328 + }, + { + "epoch": 0.1, + "learning_rate": 0.000725456369107322, + "loss": 1.6158, + "theoretical_loss": 4.1185982815610505, + "tokens_seen": 337444864 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254403209628887, + "loss": 1.5802, + "theoretical_loss": 4.118506868326623, + "tokens_seen": 337510400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254242728184554, + "loss": 1.6301, + "theoretical_loss": 4.118415477809553, + "tokens_seen": 337575936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007254082246740221, + "loss": 1.6133, + "theoretical_loss": 4.11832410999979, + "tokens_seen": 337641472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253921765295888, + "loss": 1.6438, + "theoretical_loss": 4.118232764887285, + "tokens_seen": 337707008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253761283851555, + "loss": 1.5859, + "theoretical_loss": 4.118141442461995, + "tokens_seen": 337772544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253600802407223, + "loss": 1.5689, + "theoretical_loss": 4.118050142713886, + "tokens_seen": 337838080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253440320962889, + "loss": 1.6378, + "theoretical_loss": 4.1179588656329305, + "tokens_seen": 337903616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253279839518557, + "loss": 1.6218, + "theoretical_loss": 4.117867611209104, + "tokens_seen": 337969152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007253119358074223, + "loss": 1.5666, + "theoretical_loss": 4.117776379432392, + "tokens_seen": 338034688 + }, + { + "epoch": 0.1, + "learning_rate": 0.000725295887662989, + "loss": 1.6658, + "theoretical_loss": 4.117685170292786, + "tokens_seen": 338100224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252798395185557, + "loss": 1.6368, + "theoretical_loss": 4.1175939837802815, + "tokens_seen": 338165760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252637913741225, + "loss": 1.5659, + "theoretical_loss": 4.117502819884882, + "tokens_seen": 338231296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252477432296891, + "loss": 1.5796, + "theoretical_loss": 4.117411678596596, + "tokens_seen": 338296832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252316950852559, + "loss": 1.5486, + "theoretical_loss": 4.117320559905441, + "tokens_seen": 338362368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007252156469408225, + "loss": 1.5603, + "theoretical_loss": 4.117229463801439, + "tokens_seen": 338427904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251995987963893, + "loss": 1.6417, + "theoretical_loss": 4.117138390274619, + "tokens_seen": 338493440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251835506519559, + "loss": 1.6901, + "theoretical_loss": 4.117047339315014, + "tokens_seen": 338558976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251675025075227, + "loss": 1.581, + "theoretical_loss": 4.116956310912666, + "tokens_seen": 338624512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251514543630893, + "loss": 1.6113, + "theoretical_loss": 4.116865305057625, + "tokens_seen": 338690048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251354062186561, + "loss": 1.6759, + "theoretical_loss": 4.116774321739942, + "tokens_seen": 338755584 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251193580742227, + "loss": 1.6598, + "theoretical_loss": 4.116683360949678, + "tokens_seen": 338821120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007251033099297895, + "loss": 1.6576, + "theoretical_loss": 4.1165924226769, + "tokens_seen": 338886656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250872617853561, + "loss": 1.6715, + "theoretical_loss": 4.11650150691168, + "tokens_seen": 338952192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250712136409229, + "loss": 1.5988, + "theoretical_loss": 4.116410613644098, + "tokens_seen": 339017728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250551654964895, + "loss": 1.6035, + "theoretical_loss": 4.116319742864239, + "tokens_seen": 339083264 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250391173520563, + "loss": 1.5839, + "theoretical_loss": 4.1162288945621945, + "tokens_seen": 339148800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250230692076229, + "loss": 1.5613, + "theoretical_loss": 4.116138068728063, + "tokens_seen": 339214336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007250070210631897, + "loss": 1.467, + "theoretical_loss": 4.1160472653519475, + "tokens_seen": 339279872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249909729187563, + "loss": 1.5754, + "theoretical_loss": 4.11595648442396, + "tokens_seen": 339345408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249749247743231, + "loss": 1.6362, + "theoretical_loss": 4.115865725934217, + "tokens_seen": 339410944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249588766298897, + "loss": 1.5956, + "theoretical_loss": 4.11577498987284, + "tokens_seen": 339476480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249428284854565, + "loss": 1.5675, + "theoretical_loss": 4.11568427622996, + "tokens_seen": 339542016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249267803410231, + "loss": 1.679, + "theoretical_loss": 4.115593584995712, + "tokens_seen": 339607552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007249107321965899, + "loss": 1.6227, + "theoretical_loss": 4.115502916160237, + "tokens_seen": 339673088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248946840521565, + "loss": 1.6039, + "theoretical_loss": 4.115412269713684, + "tokens_seen": 339738624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248786359077233, + "loss": 1.6046, + "theoretical_loss": 4.115321645646207, + "tokens_seen": 339804160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248625877632899, + "loss": 1.5707, + "theoretical_loss": 4.115231043947968, + "tokens_seen": 339869696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248465396188567, + "loss": 1.6426, + "theoretical_loss": 4.11514046460913, + "tokens_seen": 339935232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248304914744233, + "loss": 1.5191, + "theoretical_loss": 4.115049907619868, + "tokens_seen": 340000768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007248144433299901, + "loss": 1.5964, + "theoretical_loss": 4.114959372970362, + "tokens_seen": 340066304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247983951855567, + "loss": 1.7014, + "theoretical_loss": 4.114868860650796, + "tokens_seen": 340131840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247823470411235, + "loss": 1.5722, + "theoretical_loss": 4.114778370651361, + "tokens_seen": 340197376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247662988966901, + "loss": 1.5813, + "theoretical_loss": 4.114687902962258, + "tokens_seen": 340262912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247502507522569, + "loss": 1.6241, + "theoretical_loss": 4.114597457573687, + "tokens_seen": 340328448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247342026078235, + "loss": 1.6508, + "theoretical_loss": 4.11450703447586, + "tokens_seen": 340393984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247181544633902, + "loss": 1.6059, + "theoretical_loss": 4.114416633658992, + "tokens_seen": 340459520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007247021063189569, + "loss": 1.5488, + "theoretical_loss": 4.114326255113308, + "tokens_seen": 340525056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246860581745236, + "loss": 1.7082, + "theoretical_loss": 4.114235898829034, + "tokens_seen": 340590592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246700100300903, + "loss": 1.5225, + "theoretical_loss": 4.114145564796406, + "tokens_seen": 340656128 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724653961885657, + "loss": 1.6291, + "theoretical_loss": 4.114055253005665, + "tokens_seen": 340721664 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 132897, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5375351905822754, + "objective/train/theoretical_loss": 4.113964963447058, + "objective/train/tokens_used": 361247200, + "theoretical_loss": 4.113964963447058, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246379137412237, + "loss": 1.5375, + "theoretical_loss": 4.113964963447058, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246218655967904, + "loss": 1.6709, + "theoretical_loss": 4.113874696110838, + "tokens_seen": 340852736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007246058174523571, + "loss": 1.6858, + "theoretical_loss": 4.113784450987264, + "tokens_seen": 340918272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245897693079238, + "loss": 1.5364, + "theoretical_loss": 4.113694228066604, + "tokens_seen": 340983808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245737211634904, + "loss": 1.6789, + "theoretical_loss": 4.113604027339126, + "tokens_seen": 341049344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245576730190572, + "loss": 1.6804, + "theoretical_loss": 4.11351384879511, + "tokens_seen": 341114880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245416248746238, + "loss": 1.5358, + "theoretical_loss": 4.11342369242484, + "tokens_seen": 341180416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245255767301906, + "loss": 1.6287, + "theoretical_loss": 4.1133335582186055, + "tokens_seen": 341245952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007245095285857572, + "loss": 1.5099, + "theoretical_loss": 4.1132434461667025, + "tokens_seen": 341311488 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724493480441324, + "loss": 1.5011, + "theoretical_loss": 4.113153356259433, + "tokens_seen": 341377024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007244774322968906, + "loss": 1.571, + "theoretical_loss": 4.1130632884871074, + "tokens_seen": 341442560 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007244613841524574, + "loss": 1.579, + "theoretical_loss": 4.112973242840038, + "tokens_seen": 341508096 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724445336008024, + "loss": 1.6051, + "theoretical_loss": 4.1128832193085465, + "tokens_seen": 341573632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007244292878635908, + "loss": 1.7012, + "theoretical_loss": 4.11279321788296, + "tokens_seen": 341639168 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007244132397191574, + "loss": 1.6724, + "theoretical_loss": 4.112703238553609, + "tokens_seen": 341704704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243971915747242, + "loss": 1.6606, + "theoretical_loss": 4.112613281310835, + "tokens_seen": 341770240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243811434302908, + "loss": 1.6357, + "theoretical_loss": 4.1125233461449815, + "tokens_seen": 341835776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243650952858576, + "loss": 1.5951, + "theoretical_loss": 4.1124334330464, + "tokens_seen": 341901312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243490471414242, + "loss": 1.6979, + "theoretical_loss": 4.112343542005447, + "tokens_seen": 341966848 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724332998996991, + "loss": 1.6491, + "theoretical_loss": 4.112253673012486, + "tokens_seen": 342032384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243169508525577, + "loss": 1.6642, + "theoretical_loss": 4.112163826057886, + "tokens_seen": 342097920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007243009027081244, + "loss": 1.5889, + "theoretical_loss": 4.112074001132023, + "tokens_seen": 342163456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242848545636911, + "loss": 1.6556, + "theoretical_loss": 4.111984198225278, + "tokens_seen": 342228992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242688064192578, + "loss": 1.5829, + "theoretical_loss": 4.1118944173280365, + "tokens_seen": 342294528 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242527582748245, + "loss": 1.5425, + "theoretical_loss": 4.111804658430694, + "tokens_seen": 342360064 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242367101303912, + "loss": 1.6801, + "theoretical_loss": 4.11171492152365, + "tokens_seen": 342425600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242206619859579, + "loss": 1.6416, + "theoretical_loss": 4.111625206597309, + "tokens_seen": 342491136 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007242046138415246, + "loss": 1.6267, + "theoretical_loss": 4.111535513642082, + "tokens_seen": 342556672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241885656970913, + "loss": 1.6407, + "theoretical_loss": 4.111445842648387, + "tokens_seen": 342622208 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724172517552658, + "loss": 1.5106, + "theoretical_loss": 4.111356193606648, + "tokens_seen": 342687744 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241564694082247, + "loss": 1.5066, + "theoretical_loss": 4.111266566507294, + "tokens_seen": 342753280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241404212637914, + "loss": 1.5674, + "theoretical_loss": 4.11117696134076, + "tokens_seen": 342818816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241243731193581, + "loss": 1.6451, + "theoretical_loss": 4.111087378097488, + "tokens_seen": 342884352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007241083249749248, + "loss": 1.4945, + "theoretical_loss": 4.1109978167679255, + "tokens_seen": 342949888 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007240922768304915, + "loss": 1.6544, + "theoretical_loss": 4.110908277342526, + "tokens_seen": 343015424 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007240762286860582, + "loss": 1.6921, + "theoretical_loss": 4.110818759811748, + "tokens_seen": 343080960 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724060180541625, + "loss": 1.5826, + "theoretical_loss": 4.110729264166058, + "tokens_seen": 343146496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007240441323971916, + "loss": 1.6388, + "theoretical_loss": 4.110639790395926, + "tokens_seen": 343212032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007240280842527583, + "loss": 1.5861, + "theoretical_loss": 4.110550338491832, + "tokens_seen": 343277568 + }, + { + "epoch": 0.1, + "learning_rate": 0.000724012036108325, + "loss": 1.5436, + "theoretical_loss": 4.1104609084442565, + "tokens_seen": 343343104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239959879638917, + "loss": 1.6422, + "theoretical_loss": 4.11037150024369, + "tokens_seen": 343408640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239799398194584, + "loss": 1.6607, + "theoretical_loss": 4.110282113880627, + "tokens_seen": 343474176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239638916750252, + "loss": 1.5877, + "theoretical_loss": 4.11019274934557, + "tokens_seen": 343539712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239478435305918, + "loss": 1.5196, + "theoretical_loss": 4.110103406629026, + "tokens_seen": 343605248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239317953861586, + "loss": 1.6279, + "theoretical_loss": 4.110014085721506, + "tokens_seen": 343670784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007239157472417252, + "loss": 1.5936, + "theoretical_loss": 4.109924786613531, + "tokens_seen": 343736320 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723899699097292, + "loss": 1.5042, + "theoretical_loss": 4.109835509295625, + "tokens_seen": 343801856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238836509528586, + "loss": 1.5742, + "theoretical_loss": 4.10974625375832, + "tokens_seen": 343867392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238676028084254, + "loss": 1.6809, + "theoretical_loss": 4.1096570199921505, + "tokens_seen": 343932928 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723851554663992, + "loss": 1.592, + "theoretical_loss": 4.109567807987661, + "tokens_seen": 343998464 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238355065195588, + "loss": 1.6294, + "theoretical_loss": 4.109478617735399, + "tokens_seen": 344064000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238194583751254, + "loss": 1.5755, + "theoretical_loss": 4.10938944922592, + "tokens_seen": 344129536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007238034102306922, + "loss": 1.5997, + "theoretical_loss": 4.109300302449784, + "tokens_seen": 344195072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237873620862588, + "loss": 1.6682, + "theoretical_loss": 4.109211177397556, + "tokens_seen": 344260608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237713139418256, + "loss": 1.5688, + "theoretical_loss": 4.109122074059811, + "tokens_seen": 344326144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237552657973922, + "loss": 1.5819, + "theoretical_loss": 4.1090329924271245, + "tokens_seen": 344391680 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723739217652959, + "loss": 1.5167, + "theoretical_loss": 4.108943932490082, + "tokens_seen": 344457216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237231695085256, + "loss": 1.5922, + "theoretical_loss": 4.108854894239271, + "tokens_seen": 344522752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007237071213640924, + "loss": 1.5999, + "theoretical_loss": 4.10876587766529, + "tokens_seen": 344588288 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723691073219659, + "loss": 1.647, + "theoretical_loss": 4.108676882758738, + "tokens_seen": 344653824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236750250752258, + "loss": 1.5694, + "theoretical_loss": 4.1085879095102245, + "tokens_seen": 344719360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236589769307924, + "loss": 1.5775, + "theoretical_loss": 4.108498957910362, + "tokens_seen": 344784896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236429287863592, + "loss": 1.4493, + "theoretical_loss": 4.108410027949769, + "tokens_seen": 344850432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236268806419258, + "loss": 1.4936, + "theoretical_loss": 4.10832111961907, + "tokens_seen": 344915968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007236108324974926, + "loss": 1.5584, + "theoretical_loss": 4.1082322329088985, + "tokens_seen": 344981504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007235947843530592, + "loss": 1.5863, + "theoretical_loss": 4.108143367809888, + "tokens_seen": 345047040 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723578736208626, + "loss": 1.6147, + "theoretical_loss": 4.108054524312683, + "tokens_seen": 345112576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007235626880641926, + "loss": 1.5275, + "theoretical_loss": 4.107965702407931, + "tokens_seen": 345178112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007235466399197594, + "loss": 1.6587, + "theoretical_loss": 4.107876902086286, + "tokens_seen": 345243648 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723530591775326, + "loss": 1.6409, + "theoretical_loss": 4.107788123338408, + "tokens_seen": 345309184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007235145436308928, + "loss": 1.7015, + "theoretical_loss": 4.1076993661549634, + "tokens_seen": 345374720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234984954864594, + "loss": 1.5596, + "theoretical_loss": 4.107610630526622, + "tokens_seen": 345440256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234824473420262, + "loss": 1.5441, + "theoretical_loss": 4.107521916444064, + "tokens_seen": 345505792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234663991975928, + "loss": 1.6876, + "theoretical_loss": 4.10743322389797, + "tokens_seen": 345571328 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234503510531596, + "loss": 1.4999, + "theoretical_loss": 4.107344552879029, + "tokens_seen": 345636864 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234343029087262, + "loss": 1.562, + "theoretical_loss": 4.107255903377938, + "tokens_seen": 345702400 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723418254764293, + "loss": 1.609, + "theoretical_loss": 4.107167275385395, + "tokens_seen": 345767936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007234022066198596, + "loss": 1.4823, + "theoretical_loss": 4.107078668892108, + "tokens_seen": 345833472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233861584754264, + "loss": 1.5689, + "theoretical_loss": 4.106990083888789, + "tokens_seen": 345899008 + }, + { + "epoch": 0.1, + "learning_rate": 0.000723370110330993, + "loss": 1.5869, + "theoretical_loss": 4.106901520366154, + "tokens_seen": 345964544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233540621865598, + "loss": 1.5317, + "theoretical_loss": 4.106812978314929, + "tokens_seen": 346030080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233380140421264, + "loss": 1.606, + "theoretical_loss": 4.106724457725842, + "tokens_seen": 346095616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233219658976931, + "loss": 1.539, + "theoretical_loss": 4.106635958589628, + "tokens_seen": 346161152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007233059177532598, + "loss": 1.5996, + "theoretical_loss": 4.106547480897029, + "tokens_seen": 346226688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007232898696088265, + "loss": 1.5941, + "theoretical_loss": 4.106459024638791, + "tokens_seen": 346292224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007232738214643932, + "loss": 1.5349, + "theoretical_loss": 4.106370589805666, + "tokens_seen": 346357760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007232577733199599, + "loss": 1.5177, + "theoretical_loss": 4.1062821763884125, + "tokens_seen": 346423296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0007232417251755266, + "loss": 1.6398, + "theoretical_loss": 4.106193784377794, + "tokens_seen": 346488832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007232256770310933, + "loss": 1.584, + "theoretical_loss": 4.106105413764581, + "tokens_seen": 346554368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00072320962888666, + "loss": 1.6424, + "theoretical_loss": 4.106017064539548, + "tokens_seen": 346619904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231935807422267, + "loss": 1.5905, + "theoretical_loss": 4.1059287366934765, + "tokens_seen": 346685440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231775325977933, + "loss": 1.5433, + "theoretical_loss": 4.105840430217153, + "tokens_seen": 346750976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231614844533601, + "loss": 1.5812, + "theoretical_loss": 4.10575214510137, + "tokens_seen": 346816512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231454363089267, + "loss": 1.5674, + "theoretical_loss": 4.105663881336925, + "tokens_seen": 346882048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231293881644935, + "loss": 1.6391, + "theoretical_loss": 4.105575638914623, + "tokens_seen": 346947584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007231133400200602, + "loss": 1.5769, + "theoretical_loss": 4.105487417825273, + "tokens_seen": 347013120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230972918756269, + "loss": 1.4786, + "theoretical_loss": 4.10539921805969, + "tokens_seen": 347078656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230812437311936, + "loss": 1.576, + "theoretical_loss": 4.105311039608695, + "tokens_seen": 347144192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230651955867603, + "loss": 1.6604, + "theoretical_loss": 4.105222882463115, + "tokens_seen": 347209728 + }, + { + "epoch": 0.11, + "learning_rate": 0.000723049147442327, + "loss": 1.6055, + "theoretical_loss": 4.105134746613783, + "tokens_seen": 347275264 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 135262, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.6616510152816772, + "objective/train/theoretical_loss": 4.105046632051535, + "objective/train/tokens_used": 367800800, + "theoretical_loss": 4.105046632051535, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230330992978937, + "loss": 1.6617, + "theoretical_loss": 4.105046632051535, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230170511534604, + "loss": 1.5624, + "theoretical_loss": 4.1049585387672165, + "tokens_seen": 347406336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007230010030090271, + "loss": 1.5592, + "theoretical_loss": 4.1048704667516756, + "tokens_seen": 347471872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229849548645938, + "loss": 1.6141, + "theoretical_loss": 4.104782415995768, + "tokens_seen": 347537408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229689067201605, + "loss": 1.4403, + "theoretical_loss": 4.104694386490354, + "tokens_seen": 347602944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229528585757272, + "loss": 1.5317, + "theoretical_loss": 4.1046063782263, + "tokens_seen": 347668480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229368104312939, + "loss": 1.4454, + "theoretical_loss": 4.104518391194477, + "tokens_seen": 347734016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229207622868606, + "loss": 1.6714, + "theoretical_loss": 4.104430425385765, + "tokens_seen": 347799552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007229047141424273, + "loss": 1.601, + "theoretical_loss": 4.104342480791044, + "tokens_seen": 347865088 + }, + { + "epoch": 0.11, + "learning_rate": 0.000722888665997994, + "loss": 1.6325, + "theoretical_loss": 4.104254557401205, + "tokens_seen": 347930624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228726178535607, + "loss": 1.6168, + "theoretical_loss": 4.104166655207142, + "tokens_seen": 347996160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228565697091274, + "loss": 1.6514, + "theoretical_loss": 4.104078774199754, + "tokens_seen": 348061696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228405215646941, + "loss": 1.5455, + "theoretical_loss": 4.10399091436995, + "tokens_seen": 348127232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228244734202608, + "loss": 1.6219, + "theoretical_loss": 4.103903075708637, + "tokens_seen": 348192768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007228084252758275, + "loss": 1.6193, + "theoretical_loss": 4.103815258206735, + "tokens_seen": 348258304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227923771313942, + "loss": 1.578, + "theoretical_loss": 4.103727461855165, + "tokens_seen": 348323840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227763289869609, + "loss": 1.49, + "theoretical_loss": 4.103639686644856, + "tokens_seen": 348389376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227602808425276, + "loss": 1.6916, + "theoretical_loss": 4.103551932566741, + "tokens_seen": 348454912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227442326980943, + "loss": 1.6533, + "theoretical_loss": 4.103464199611759, + "tokens_seen": 348520448 + }, + { + "epoch": 0.11, + "learning_rate": 0.000722728184553661, + "loss": 1.5429, + "theoretical_loss": 4.1033764877708565, + "tokens_seen": 348585984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007227121364092277, + "loss": 1.4656, + "theoretical_loss": 4.103288797034983, + "tokens_seen": 348651520 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226960882647944, + "loss": 1.6337, + "theoretical_loss": 4.103201127395095, + "tokens_seen": 348717056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226800401203611, + "loss": 1.636, + "theoretical_loss": 4.103113478842153, + "tokens_seen": 348782592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226639919759278, + "loss": 1.5892, + "theoretical_loss": 4.103025851367125, + "tokens_seen": 348848128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226479438314945, + "loss": 1.5842, + "theoretical_loss": 4.102938244960985, + "tokens_seen": 348913664 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226318956870612, + "loss": 1.663, + "theoretical_loss": 4.102850659614709, + "tokens_seen": 348979200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007226158475426279, + "loss": 1.799, + "theoretical_loss": 4.102763095319282, + "tokens_seen": 349044736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225997993981947, + "loss": 1.61, + "theoretical_loss": 4.102675552065694, + "tokens_seen": 349110272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225837512537613, + "loss": 1.5611, + "theoretical_loss": 4.10258802984494, + "tokens_seen": 349175808 + }, + { + "epoch": 0.11, + "learning_rate": 0.000722567703109328, + "loss": 1.6167, + "theoretical_loss": 4.102500528648019, + "tokens_seen": 349241344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225516549648947, + "loss": 1.553, + "theoretical_loss": 4.102413048465938, + "tokens_seen": 349306880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225356068204615, + "loss": 1.5393, + "theoretical_loss": 4.10232558928971, + "tokens_seen": 349372416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225195586760281, + "loss": 1.6027, + "theoretical_loss": 4.10223815111035, + "tokens_seen": 349437952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007225035105315949, + "loss": 1.6247, + "theoretical_loss": 4.102150733918882, + "tokens_seen": 349503488 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224874623871615, + "loss": 1.6615, + "theoretical_loss": 4.102063337706333, + "tokens_seen": 349569024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224714142427283, + "loss": 1.6731, + "theoretical_loss": 4.101975962463738, + "tokens_seen": 349634560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224553660982949, + "loss": 1.5679, + "theoretical_loss": 4.101888608182136, + "tokens_seen": 349700096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224393179538617, + "loss": 1.6098, + "theoretical_loss": 4.101801274852571, + "tokens_seen": 349765632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224232698094283, + "loss": 1.5087, + "theoretical_loss": 4.1017139624660945, + "tokens_seen": 349831168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007224072216649951, + "loss": 1.6347, + "theoretical_loss": 4.1016266710137606, + "tokens_seen": 349896704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223911735205617, + "loss": 1.7227, + "theoretical_loss": 4.101539400486631, + "tokens_seen": 349962240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223751253761285, + "loss": 1.5378, + "theoretical_loss": 4.101452150875773, + "tokens_seen": 350027776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223590772316951, + "loss": 1.5825, + "theoretical_loss": 4.101364922172259, + "tokens_seen": 350093312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223430290872619, + "loss": 1.5989, + "theoretical_loss": 4.101277714367166, + "tokens_seen": 350158848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223269809428285, + "loss": 1.6306, + "theoretical_loss": 4.1011905274515765, + "tokens_seen": 350224384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007223109327983953, + "loss": 1.6313, + "theoretical_loss": 4.101103361416581, + "tokens_seen": 350289920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222948846539619, + "loss": 1.5476, + "theoretical_loss": 4.101016216253273, + "tokens_seen": 350355456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222788365095287, + "loss": 1.7066, + "theoretical_loss": 4.10092909195275, + "tokens_seen": 350420992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222627883650953, + "loss": 1.5159, + "theoretical_loss": 4.10084198850612, + "tokens_seen": 350486528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222467402206621, + "loss": 1.5857, + "theoretical_loss": 4.100754905904491, + "tokens_seen": 350552064 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222306920762287, + "loss": 1.5857, + "theoretical_loss": 4.100667844138981, + "tokens_seen": 350617600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007222146439317955, + "loss": 1.5756, + "theoretical_loss": 4.10058080320071, + "tokens_seen": 350683136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221985957873621, + "loss": 1.7532, + "theoretical_loss": 4.100493783080806, + "tokens_seen": 350748672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221825476429289, + "loss": 1.5905, + "theoretical_loss": 4.1004067837704, + "tokens_seen": 350814208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221664994984955, + "loss": 1.6247, + "theoretical_loss": 4.1003198052606304, + "tokens_seen": 350879744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221504513540623, + "loss": 1.465, + "theoretical_loss": 4.10023284754264, + "tokens_seen": 350945280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221344032096289, + "loss": 1.6729, + "theoretical_loss": 4.100145910607576, + "tokens_seen": 351010816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221183550651957, + "loss": 1.565, + "theoretical_loss": 4.100058994446596, + "tokens_seen": 351076352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007221023069207623, + "loss": 1.5775, + "theoretical_loss": 4.099972099050856, + "tokens_seen": 351141888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220862587763291, + "loss": 1.5194, + "theoretical_loss": 4.099885224411522, + "tokens_seen": 351207424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220702106318957, + "loss": 1.571, + "theoretical_loss": 4.099798370519764, + "tokens_seen": 351272960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220541624874625, + "loss": 1.5893, + "theoretical_loss": 4.099711537366758, + "tokens_seen": 351338496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220381143430291, + "loss": 1.5867, + "theoretical_loss": 4.099624724943685, + "tokens_seen": 351404032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220220661985959, + "loss": 1.5471, + "theoretical_loss": 4.099537933241731, + "tokens_seen": 351469568 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007220060180541625, + "loss": 1.5796, + "theoretical_loss": 4.0994511622520875, + "tokens_seen": 351535104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219899699097293, + "loss": 1.6227, + "theoretical_loss": 4.099364411965952, + "tokens_seen": 351600640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219739217652959, + "loss": 1.6629, + "theoretical_loss": 4.0992776823745265, + "tokens_seen": 351666176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219578736208627, + "loss": 1.5689, + "theoretical_loss": 4.09919097346902, + "tokens_seen": 351731712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219418254764293, + "loss": 1.5766, + "theoretical_loss": 4.099104285240646, + "tokens_seen": 351797248 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721925777331996, + "loss": 1.6236, + "theoretical_loss": 4.099017617680621, + "tokens_seen": 351862784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007219097291875627, + "loss": 1.584, + "theoretical_loss": 4.098930970780172, + "tokens_seen": 351928320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218936810431294, + "loss": 1.619, + "theoretical_loss": 4.098844344530526, + "tokens_seen": 351993856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218776328986961, + "loss": 1.4421, + "theoretical_loss": 4.098757738922918, + "tokens_seen": 352059392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218615847542628, + "loss": 1.5487, + "theoretical_loss": 4.09867115394859, + "tokens_seen": 352124928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218455366098295, + "loss": 1.4885, + "theoretical_loss": 4.098584589598786, + "tokens_seen": 352190464 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218294884653962, + "loss": 1.7185, + "theoretical_loss": 4.098498045864756, + "tokens_seen": 352256000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007218134403209629, + "loss": 1.6247, + "theoretical_loss": 4.0984115227377576, + "tokens_seen": 352321536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217973921765296, + "loss": 1.5532, + "theoretical_loss": 4.098325020209051, + "tokens_seen": 352387072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217813440320963, + "loss": 1.6228, + "theoretical_loss": 4.098238538269904, + "tokens_seen": 352452608 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721765295887663, + "loss": 1.5536, + "theoretical_loss": 4.098152076911588, + "tokens_seen": 352518144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217492477432297, + "loss": 1.5069, + "theoretical_loss": 4.098065636125381, + "tokens_seen": 352583680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217331995987964, + "loss": 1.6023, + "theoretical_loss": 4.097979215902566, + "tokens_seen": 352649216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217171514543631, + "loss": 1.6239, + "theoretical_loss": 4.097892816234429, + "tokens_seen": 352714752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007217011033099298, + "loss": 1.6094, + "theoretical_loss": 4.097806437112267, + "tokens_seen": 352780288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216850551654965, + "loss": 1.6432, + "theoretical_loss": 4.0977200785273755, + "tokens_seen": 352845824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216690070210632, + "loss": 1.5343, + "theoretical_loss": 4.09763374047106, + "tokens_seen": 352911360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216529588766299, + "loss": 1.6171, + "theoretical_loss": 4.097547422934629, + "tokens_seen": 352976896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216369107321966, + "loss": 1.5649, + "theoretical_loss": 4.097461125909398, + "tokens_seen": 353042432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007216208625877633, + "loss": 1.5358, + "theoretical_loss": 4.097374849386685, + "tokens_seen": 353107968 + }, + { + "epoch": 0.11, + "learning_rate": 0.00072160481444333, + "loss": 1.6765, + "theoretical_loss": 4.097288593357817, + "tokens_seen": 353173504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215887662988967, + "loss": 1.6335, + "theoretical_loss": 4.097202357814124, + "tokens_seen": 353239040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215727181544634, + "loss": 1.6223, + "theoretical_loss": 4.097116142746941, + "tokens_seen": 353304576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215566700100301, + "loss": 1.5255, + "theoretical_loss": 4.09702994814761, + "tokens_seen": 353370112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215406218655968, + "loss": 1.5861, + "theoretical_loss": 4.096943774007476, + "tokens_seen": 353435648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215245737211635, + "loss": 1.6104, + "theoretical_loss": 4.096857620317892, + "tokens_seen": 353501184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007215085255767302, + "loss": 1.6289, + "theoretical_loss": 4.096771487070214, + "tokens_seen": 353566720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214924774322969, + "loss": 1.6283, + "theoretical_loss": 4.096685374255803, + "tokens_seen": 353632256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214764292878636, + "loss": 1.5793, + "theoretical_loss": 4.096599281866028, + "tokens_seen": 353697792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214603811434303, + "loss": 1.5995, + "theoretical_loss": 4.09651320989226, + "tokens_seen": 353763328 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721444332998997, + "loss": 1.5182, + "theoretical_loss": 4.096427158325877, + "tokens_seen": 353828864 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 138080, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5173215866088867, + "objective/train/theoretical_loss": 4.096341127158263, + "objective/train/tokens_used": 374354400, + "theoretical_loss": 4.096341127158263, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214282848545637, + "loss": 1.5173, + "theoretical_loss": 4.096341127158263, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007214122367101304, + "loss": 1.5777, + "theoretical_loss": 4.096255116380806, + "tokens_seen": 353959936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213961885656971, + "loss": 1.4943, + "theoretical_loss": 4.096169125984899, + "tokens_seen": 354025472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213801404212638, + "loss": 1.6381, + "theoretical_loss": 4.0960831559619395, + "tokens_seen": 354091008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213640922768305, + "loss": 1.6237, + "theoretical_loss": 4.095997206303333, + "tokens_seen": 354156544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213480441323972, + "loss": 1.6363, + "theoretical_loss": 4.095911277000488, + "tokens_seen": 354222080 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721331995987964, + "loss": 1.6552, + "theoretical_loss": 4.095825368044819, + "tokens_seen": 354287616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007213159478435306, + "loss": 1.5586, + "theoretical_loss": 4.0957394794277455, + "tokens_seen": 354353152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212998996990973, + "loss": 1.5788, + "theoretical_loss": 4.095653611140692, + "tokens_seen": 354418688 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721283851554664, + "loss": 1.4686, + "theoretical_loss": 4.09556776317509, + "tokens_seen": 354484224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212678034102307, + "loss": 1.6332, + "theoretical_loss": 4.095481935522372, + "tokens_seen": 354549760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212517552657974, + "loss": 1.4976, + "theoretical_loss": 4.09539612817398, + "tokens_seen": 354615296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212357071213642, + "loss": 1.5706, + "theoretical_loss": 4.095310341121358, + "tokens_seen": 354680832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212196589769308, + "loss": 1.4539, + "theoretical_loss": 4.095224574355958, + "tokens_seen": 354746368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007212036108324976, + "loss": 1.5821, + "theoretical_loss": 4.095138827869237, + "tokens_seen": 354811904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007211875626880642, + "loss": 1.5724, + "theoretical_loss": 4.0950531016526535, + "tokens_seen": 354877440 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721171514543631, + "loss": 1.5285, + "theoretical_loss": 4.094967395697675, + "tokens_seen": 354942976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007211554663991976, + "loss": 1.6447, + "theoretical_loss": 4.094881709995773, + "tokens_seen": 355008512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007211394182547644, + "loss": 1.4995, + "theoretical_loss": 4.094796044538423, + "tokens_seen": 355074048 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721123370110331, + "loss": 1.5935, + "theoretical_loss": 4.094710399317109, + "tokens_seen": 355139584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007211073219658978, + "loss": 1.6351, + "theoretical_loss": 4.094624774323315, + "tokens_seen": 355205120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210912738214644, + "loss": 1.626, + "theoretical_loss": 4.094539169548535, + "tokens_seen": 355270656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210752256770312, + "loss": 1.5341, + "theoretical_loss": 4.094453584984265, + "tokens_seen": 355336192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210591775325978, + "loss": 1.5873, + "theoretical_loss": 4.094368020622008, + "tokens_seen": 355401728 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210431293881646, + "loss": 1.6657, + "theoretical_loss": 4.094282476453272, + "tokens_seen": 355467264 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007210270812437312, + "loss": 1.5531, + "theoretical_loss": 4.094196952469568, + "tokens_seen": 355532800 + }, + { + "epoch": 0.11, + "learning_rate": 0.000721011033099298, + "loss": 1.6383, + "theoretical_loss": 4.0941114486624155, + "tokens_seen": 355598336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209949849548646, + "loss": 1.6545, + "theoretical_loss": 4.094025965023335, + "tokens_seen": 355663872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209789368104314, + "loss": 1.6333, + "theoretical_loss": 4.093940501543857, + "tokens_seen": 355729408 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720962888665998, + "loss": 1.6294, + "theoretical_loss": 4.093855058215513, + "tokens_seen": 355794944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209468405215648, + "loss": 1.5252, + "theoretical_loss": 4.093769635029841, + "tokens_seen": 355860480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209307923771314, + "loss": 1.5511, + "theoretical_loss": 4.0936842319783855, + "tokens_seen": 355926016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007209147442326982, + "loss": 1.5631, + "theoretical_loss": 4.093598849052694, + "tokens_seen": 355991552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208986960882648, + "loss": 1.4806, + "theoretical_loss": 4.093513486244321, + "tokens_seen": 356057088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208826479438316, + "loss": 1.5344, + "theoretical_loss": 4.093428143544823, + "tokens_seen": 356122624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208665997993982, + "loss": 1.6201, + "theoretical_loss": 4.093342820945766, + "tokens_seen": 356188160 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720850551654965, + "loss": 1.5237, + "theoretical_loss": 4.093257518438718, + "tokens_seen": 356253696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208345035105316, + "loss": 1.5561, + "theoretical_loss": 4.093172236015253, + "tokens_seen": 356319232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007208184553660984, + "loss": 1.541, + "theoretical_loss": 4.093086973666949, + "tokens_seen": 356384768 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720802407221665, + "loss": 1.4872, + "theoretical_loss": 4.0930017313853915, + "tokens_seen": 356450304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207863590772318, + "loss": 1.5454, + "theoretical_loss": 4.092916509162169, + "tokens_seen": 356515840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207703109327984, + "loss": 1.5057, + "theoretical_loss": 4.092831306988875, + "tokens_seen": 356581376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207542627883652, + "loss": 1.6614, + "theoretical_loss": 4.0927461248571095, + "tokens_seen": 356646912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207382146439318, + "loss": 1.5735, + "theoretical_loss": 4.092660962758477, + "tokens_seen": 356712448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207221664994986, + "loss": 1.58, + "theoretical_loss": 4.092575820684586, + "tokens_seen": 356777984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007207061183550653, + "loss": 1.5812, + "theoretical_loss": 4.092490698627052, + "tokens_seen": 356843520 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720690070210632, + "loss": 1.5704, + "theoretical_loss": 4.0924055965774935, + "tokens_seen": 356909056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007206740220661987, + "loss": 1.509, + "theoretical_loss": 4.092320514527535, + "tokens_seen": 356974592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007206579739217654, + "loss": 1.5247, + "theoretical_loss": 4.092235452468808, + "tokens_seen": 357040128 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720641925777332, + "loss": 1.5446, + "theoretical_loss": 4.092150410392945, + "tokens_seen": 357105664 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007206258776328988, + "loss": 1.5341, + "theoretical_loss": 4.092065388291585, + "tokens_seen": 357171200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007206098294884654, + "loss": 1.4889, + "theoretical_loss": 4.091980386156376, + "tokens_seen": 357236736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205937813440322, + "loss": 1.6212, + "theoretical_loss": 4.091895403978965, + "tokens_seen": 357302272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205777331995988, + "loss": 1.538, + "theoretical_loss": 4.091810441751007, + "tokens_seen": 357367808 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205616850551656, + "loss": 1.5175, + "theoretical_loss": 4.091725499464163, + "tokens_seen": 357433344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205456369107322, + "loss": 1.6118, + "theoretical_loss": 4.091640577110096, + "tokens_seen": 357498880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205295887662989, + "loss": 1.4938, + "theoretical_loss": 4.091555674680476, + "tokens_seen": 357564416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007205135406218656, + "loss": 1.661, + "theoretical_loss": 4.09147079216698, + "tokens_seen": 357629952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204974924774323, + "loss": 1.519, + "theoretical_loss": 4.0913859295612856, + "tokens_seen": 357695488 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720481444332999, + "loss": 1.5618, + "theoretical_loss": 4.091301086855078, + "tokens_seen": 357761024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204653961885657, + "loss": 1.715, + "theoretical_loss": 4.091216264040048, + "tokens_seen": 357826560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204493480441324, + "loss": 1.5829, + "theoretical_loss": 4.091131461107889, + "tokens_seen": 357892096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204332998996991, + "loss": 1.4462, + "theoretical_loss": 4.091046678050301, + "tokens_seen": 357957632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204172517552658, + "loss": 1.6307, + "theoretical_loss": 4.090961914858989, + "tokens_seen": 358023168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007204012036108325, + "loss": 1.608, + "theoretical_loss": 4.090877171525663, + "tokens_seen": 358088704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203851554663992, + "loss": 1.6068, + "theoretical_loss": 4.090792448042038, + "tokens_seen": 358154240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203691073219659, + "loss": 1.5972, + "theoretical_loss": 4.0907077443998325, + "tokens_seen": 358219776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203530591775326, + "loss": 1.471, + "theoretical_loss": 4.090623060590772, + "tokens_seen": 358285312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203370110330993, + "loss": 1.5695, + "theoretical_loss": 4.090538396606585, + "tokens_seen": 358350848 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720320962888666, + "loss": 1.5249, + "theoretical_loss": 4.090453752439008, + "tokens_seen": 358416384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007203049147442327, + "loss": 1.5363, + "theoretical_loss": 4.09036912807978, + "tokens_seen": 358481920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202888665997994, + "loss": 1.5885, + "theoretical_loss": 4.090284523520644, + "tokens_seen": 358547456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202728184553661, + "loss": 1.6194, + "theoretical_loss": 4.09019993875335, + "tokens_seen": 358612992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202567703109328, + "loss": 1.6017, + "theoretical_loss": 4.090115373769653, + "tokens_seen": 358678528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202407221664995, + "loss": 1.6503, + "theoretical_loss": 4.0900308285613125, + "tokens_seen": 358744064 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202246740220662, + "loss": 1.6323, + "theoretical_loss": 4.089946303120092, + "tokens_seen": 358809600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007202086258776329, + "loss": 1.5246, + "theoretical_loss": 4.089861797437761, + "tokens_seen": 358875136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201925777331996, + "loss": 1.5373, + "theoretical_loss": 4.089777311506094, + "tokens_seen": 358940672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201765295887663, + "loss": 1.5308, + "theoretical_loss": 4.089692845316868, + "tokens_seen": 359006208 + }, + { + "epoch": 0.11, + "learning_rate": 0.000720160481444333, + "loss": 1.6408, + "theoretical_loss": 4.08960839886187, + "tokens_seen": 359071744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201444332998997, + "loss": 1.5076, + "theoretical_loss": 4.089523972132886, + "tokens_seen": 359137280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201283851554664, + "loss": 1.5593, + "theoretical_loss": 4.089439565121712, + "tokens_seen": 359202816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007201123370110331, + "loss": 1.5845, + "theoretical_loss": 4.089355177820146, + "tokens_seen": 359268352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200962888665998, + "loss": 1.6598, + "theoretical_loss": 4.0892708102199915, + "tokens_seen": 359333888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200802407221665, + "loss": 1.5196, + "theoretical_loss": 4.089186462313056, + "tokens_seen": 359399424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200641925777332, + "loss": 1.4496, + "theoretical_loss": 4.089102134091155, + "tokens_seen": 359464960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200481444332999, + "loss": 1.6177, + "theoretical_loss": 4.0890178255461045, + "tokens_seen": 359530496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200320962888666, + "loss": 1.583, + "theoretical_loss": 4.08893353666973, + "tokens_seen": 359596032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007200160481444333, + "loss": 1.7186, + "theoretical_loss": 4.088849267453858, + "tokens_seen": 359661568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00072, + "loss": 1.5987, + "theoretical_loss": 4.088765017890322, + "tokens_seen": 359727104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199839518555667, + "loss": 1.6169, + "theoretical_loss": 4.08868078797096, + "tokens_seen": 359792640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199679037111334, + "loss": 1.5529, + "theoretical_loss": 4.088596577687614, + "tokens_seen": 359858176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199518555667001, + "loss": 1.5777, + "theoretical_loss": 4.088512387032132, + "tokens_seen": 359923712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199358074222668, + "loss": 1.5217, + "theoretical_loss": 4.088428215996366, + "tokens_seen": 359989248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199197592778335, + "loss": 1.5887, + "theoretical_loss": 4.0883440645721745, + "tokens_seen": 360054784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007199037111334002, + "loss": 1.6308, + "theoretical_loss": 4.088259932751419, + "tokens_seen": 360120320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198876629889669, + "loss": 1.5625, + "theoretical_loss": 4.088175820525966, + "tokens_seen": 360185856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198716148445337, + "loss": 1.6057, + "theoretical_loss": 4.088091727887688, + "tokens_seen": 360251392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198555667001003, + "loss": 1.6449, + "theoretical_loss": 4.088007654828463, + "tokens_seen": 360316928 + }, + { + "epoch": 0.11, + "learning_rate": 0.000719839518555667, + "loss": 1.6278, + "theoretical_loss": 4.087923601340169, + "tokens_seen": 360382464 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 140424, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.508183240890503, + "objective/train/theoretical_loss": 4.0878395674146955, + "objective/train/tokens_used": 380908000, + "theoretical_loss": 4.0878395674146955, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198234704112337, + "loss": 1.5082, + "theoretical_loss": 4.0878395674146955, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007198074222668005, + "loss": 1.5526, + "theoretical_loss": 4.087755553043934, + "tokens_seen": 360513536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197913741223671, + "loss": 1.6577, + "theoretical_loss": 4.087671558219778, + "tokens_seen": 360579072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197753259779339, + "loss": 1.5613, + "theoretical_loss": 4.087587582934131, + "tokens_seen": 360644608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197592778335005, + "loss": 1.6302, + "theoretical_loss": 4.087503627178897, + "tokens_seen": 360710144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197432296890673, + "loss": 1.6309, + "theoretical_loss": 4.087419690945988, + "tokens_seen": 360775680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197271815446339, + "loss": 1.6427, + "theoretical_loss": 4.087335774227318, + "tokens_seen": 360841216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007197111334002007, + "loss": 1.6041, + "theoretical_loss": 4.0872518770148085, + "tokens_seen": 360906752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196950852557673, + "loss": 1.5403, + "theoretical_loss": 4.087167999300384, + "tokens_seen": 360972288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196790371113341, + "loss": 1.6773, + "theoretical_loss": 4.087084141075974, + "tokens_seen": 361037824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196629889669007, + "loss": 1.5938, + "theoretical_loss": 4.087000302333513, + "tokens_seen": 361103360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196469408224675, + "loss": 1.5875, + "theoretical_loss": 4.086916483064941, + "tokens_seen": 361168896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196308926780341, + "loss": 1.5214, + "theoretical_loss": 4.086832683262203, + "tokens_seen": 361234432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007196148445336009, + "loss": 1.5343, + "theoretical_loss": 4.086748902917246, + "tokens_seen": 361299968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195987963891675, + "loss": 1.5412, + "theoretical_loss": 4.086665142022025, + "tokens_seen": 361365504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195827482447343, + "loss": 1.5219, + "theoretical_loss": 4.086581400568499, + "tokens_seen": 361431040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195667001003009, + "loss": 1.5523, + "theoretical_loss": 4.086497678548631, + "tokens_seen": 361496576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195506519558677, + "loss": 1.5436, + "theoretical_loss": 4.086413975954389, + "tokens_seen": 361562112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195346038114343, + "loss": 1.5144, + "theoretical_loss": 4.086330292777745, + "tokens_seen": 361627648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195185556670011, + "loss": 1.5775, + "theoretical_loss": 4.086246629010679, + "tokens_seen": 361693184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007195025075225678, + "loss": 1.5851, + "theoretical_loss": 4.086162984645172, + "tokens_seen": 361758720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194864593781345, + "loss": 1.5277, + "theoretical_loss": 4.086079359673212, + "tokens_seen": 361824256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194704112337012, + "loss": 1.612, + "theoretical_loss": 4.085995754086789, + "tokens_seen": 361889792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194543630892679, + "loss": 1.6035, + "theoretical_loss": 4.085912167877902, + "tokens_seen": 361955328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194383149448346, + "loss": 1.586, + "theoretical_loss": 4.085828601038552, + "tokens_seen": 362020864 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007194222668004013, + "loss": 1.5204, + "theoretical_loss": 4.0857450535607445, + "tokens_seen": 362086400 + }, + { + "epoch": 0.11, + "learning_rate": 0.000719406218655968, + "loss": 1.622, + "theoretical_loss": 4.085661525436492, + "tokens_seen": 362151936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193901705115347, + "loss": 1.5449, + "theoretical_loss": 4.0855780166578075, + "tokens_seen": 362217472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193741223671014, + "loss": 1.5697, + "theoretical_loss": 4.0854945272167145, + "tokens_seen": 362283008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193580742226681, + "loss": 1.6722, + "theoretical_loss": 4.085411057105237, + "tokens_seen": 362348544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193420260782348, + "loss": 1.6218, + "theoretical_loss": 4.0853276063154045, + "tokens_seen": 362414080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193259779338015, + "loss": 1.5914, + "theoretical_loss": 4.085244174839252, + "tokens_seen": 362479616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007193099297893682, + "loss": 1.5573, + "theoretical_loss": 4.085160762668819, + "tokens_seen": 362545152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192938816449349, + "loss": 1.7303, + "theoretical_loss": 4.085077369796149, + "tokens_seen": 362610688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192778335005016, + "loss": 1.6232, + "theoretical_loss": 4.084993996213292, + "tokens_seen": 362676224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192617853560683, + "loss": 1.5843, + "theoretical_loss": 4.084910641912301, + "tokens_seen": 362741760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192457372116349, + "loss": 1.6754, + "theoretical_loss": 4.084827306885234, + "tokens_seen": 362807296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192296890672017, + "loss": 1.6248, + "theoretical_loss": 4.084743991124155, + "tokens_seen": 362872832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007192136409227683, + "loss": 1.5227, + "theoretical_loss": 4.08466069462113, + "tokens_seen": 362938368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191975927783351, + "loss": 1.685, + "theoretical_loss": 4.084577417368232, + "tokens_seen": 363003904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191815446339017, + "loss": 1.6786, + "theoretical_loss": 4.084494159357538, + "tokens_seen": 363069440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191654964894685, + "loss": 1.5773, + "theoretical_loss": 4.0844109205811305, + "tokens_seen": 363134976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191494483450351, + "loss": 1.5465, + "theoretical_loss": 4.084327701031095, + "tokens_seen": 363200512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191334002006019, + "loss": 1.5521, + "theoretical_loss": 4.084244500699524, + "tokens_seen": 363266048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191173520561685, + "loss": 1.5312, + "theoretical_loss": 4.084161319578511, + "tokens_seen": 363331584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007191013039117352, + "loss": 1.5154, + "theoretical_loss": 4.084078157660159, + "tokens_seen": 363397120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190852557673019, + "loss": 1.555, + "theoretical_loss": 4.083995014936571, + "tokens_seen": 363462656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190692076228686, + "loss": 1.6144, + "theoretical_loss": 4.0839118913998576, + "tokens_seen": 363528192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190531594784353, + "loss": 1.6106, + "theoretical_loss": 4.083828787042134, + "tokens_seen": 363593728 + }, + { + "epoch": 0.11, + "learning_rate": 0.000719037111334002, + "loss": 1.5385, + "theoretical_loss": 4.083745701855518, + "tokens_seen": 363659264 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190210631895687, + "loss": 1.5139, + "theoretical_loss": 4.083662635832135, + "tokens_seen": 363724800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007190050150451354, + "loss": 1.6588, + "theoretical_loss": 4.0835795889641116, + "tokens_seen": 363790336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189889669007021, + "loss": 1.4868, + "theoretical_loss": 4.0834965612435825, + "tokens_seen": 363855872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189729187562688, + "loss": 1.5978, + "theoretical_loss": 4.083413552662685, + "tokens_seen": 363921408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189568706118355, + "loss": 1.5763, + "theoretical_loss": 4.08333056321356, + "tokens_seen": 363986944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189408224674022, + "loss": 1.5981, + "theoretical_loss": 4.083247592888357, + "tokens_seen": 364052480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189247743229689, + "loss": 1.6278, + "theoretical_loss": 4.083164641679225, + "tokens_seen": 364118016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007189087261785356, + "loss": 1.5343, + "theoretical_loss": 4.083081709578323, + "tokens_seen": 364183552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188926780341023, + "loss": 1.5713, + "theoretical_loss": 4.082998796577811, + "tokens_seen": 364249088 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718876629889669, + "loss": 1.5053, + "theoretical_loss": 4.082915902669853, + "tokens_seen": 364314624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188605817452357, + "loss": 1.591, + "theoretical_loss": 4.082833027846622, + "tokens_seen": 364380160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188445336008024, + "loss": 1.6401, + "theoretical_loss": 4.08275017210029, + "tokens_seen": 364445696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188284854563691, + "loss": 1.6523, + "theoretical_loss": 4.082667335423038, + "tokens_seen": 364511232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007188124373119358, + "loss": 1.6441, + "theoretical_loss": 4.0825845178070495, + "tokens_seen": 364576768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187963891675025, + "loss": 1.6448, + "theoretical_loss": 4.0825017192445126, + "tokens_seen": 364642304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187803410230692, + "loss": 1.6102, + "theoretical_loss": 4.082418939727622, + "tokens_seen": 364707840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187642928786359, + "loss": 1.5225, + "theoretical_loss": 4.082336179248574, + "tokens_seen": 364773376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187482447342026, + "loss": 1.5513, + "theoretical_loss": 4.082253437799572, + "tokens_seen": 364838912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187321965897693, + "loss": 1.5953, + "theoretical_loss": 4.082170715372823, + "tokens_seen": 364904448 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718716148445336, + "loss": 1.6993, + "theoretical_loss": 4.082088011960538, + "tokens_seen": 364969984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007187001003009027, + "loss": 1.6455, + "theoretical_loss": 4.082005327554934, + "tokens_seen": 365035520 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186840521564694, + "loss": 1.7432, + "theoretical_loss": 4.081922662148231, + "tokens_seen": 365101056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186680040120361, + "loss": 1.582, + "theoretical_loss": 4.081840015732654, + "tokens_seen": 365166592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186519558676028, + "loss": 1.6128, + "theoretical_loss": 4.081757388300435, + "tokens_seen": 365232128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186359077231695, + "loss": 1.588, + "theoretical_loss": 4.081674779843806, + "tokens_seen": 365297664 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007186198595787362, + "loss": 1.6869, + "theoretical_loss": 4.081592190355008, + "tokens_seen": 365363200 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718603811434303, + "loss": 1.5364, + "theoretical_loss": 4.081509619826283, + "tokens_seen": 365428736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185877632898696, + "loss": 1.5534, + "theoretical_loss": 4.081427068249881, + "tokens_seen": 365494272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185717151454363, + "loss": 1.7178, + "theoretical_loss": 4.081344535618053, + "tokens_seen": 365559808 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718555667001003, + "loss": 1.5657, + "theoretical_loss": 4.081262021923058, + "tokens_seen": 365625344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185396188565697, + "loss": 1.6522, + "theoretical_loss": 4.081179527157156, + "tokens_seen": 365690880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185235707121364, + "loss": 1.5949, + "theoretical_loss": 4.081097051312614, + "tokens_seen": 365756416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007185075225677032, + "loss": 1.5894, + "theoretical_loss": 4.081014594381705, + "tokens_seen": 365821952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184914744232698, + "loss": 1.679, + "theoretical_loss": 4.0809321563567025, + "tokens_seen": 365887488 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184754262788366, + "loss": 1.623, + "theoretical_loss": 4.080849737229887, + "tokens_seen": 365953024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184593781344032, + "loss": 1.5734, + "theoretical_loss": 4.080767336993542, + "tokens_seen": 366018560 + }, + { + "epoch": 0.11, + "learning_rate": 0.00071844332998997, + "loss": 1.5694, + "theoretical_loss": 4.080684955639958, + "tokens_seen": 366084096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184272818455366, + "loss": 1.5741, + "theoretical_loss": 4.080602593161428, + "tokens_seen": 366149632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007184112337011034, + "loss": 1.5871, + "theoretical_loss": 4.0805202495502515, + "tokens_seen": 366215168 + }, + { + "epoch": 0.11, + "learning_rate": 0.00071839518555667, + "loss": 1.647, + "theoretical_loss": 4.080437924798729, + "tokens_seen": 366280704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183791374122368, + "loss": 1.5974, + "theoretical_loss": 4.080355618899169, + "tokens_seen": 366346240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183630892678034, + "loss": 1.5611, + "theoretical_loss": 4.080273331843884, + "tokens_seen": 366411776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183470411233702, + "loss": 1.5434, + "theoretical_loss": 4.080191063625188, + "tokens_seen": 366477312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183309929789368, + "loss": 1.5918, + "theoretical_loss": 4.080108814235404, + "tokens_seen": 366542848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007183149448345036, + "loss": 1.5994, + "theoretical_loss": 4.080026583666855, + "tokens_seen": 366608384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182988966900703, + "loss": 1.6065, + "theoretical_loss": 4.079944371911871, + "tokens_seen": 366673920 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718282848545637, + "loss": 1.5472, + "theoretical_loss": 4.07986217896279, + "tokens_seen": 366739456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182668004012037, + "loss": 1.5965, + "theoretical_loss": 4.079780004811946, + "tokens_seen": 366804992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182507522567704, + "loss": 1.5225, + "theoretical_loss": 4.079697849451684, + "tokens_seen": 366870528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182347041123371, + "loss": 1.3532, + "theoretical_loss": 4.079615712874352, + "tokens_seen": 366936064 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 142675, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.6665570735931396, + "objective/train/theoretical_loss": 4.079533595072301, + "objective/train/tokens_used": 387461600, + "theoretical_loss": 4.079533595072301, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182186559679038, + "loss": 1.6666, + "theoretical_loss": 4.079533595072301, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007182026078234705, + "loss": 1.5206, + "theoretical_loss": 4.0794514960378905, + "tokens_seen": 367067136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181865596790372, + "loss": 1.6253, + "theoretical_loss": 4.079369415763479, + "tokens_seen": 367132672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181705115346039, + "loss": 1.592, + "theoretical_loss": 4.079287354241432, + "tokens_seen": 367198208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181544633901706, + "loss": 1.7526, + "theoretical_loss": 4.07920531146412, + "tokens_seen": 367263744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181384152457373, + "loss": 1.6287, + "theoretical_loss": 4.0791232874239185, + "tokens_seen": 367329280 + }, + { + "epoch": 0.11, + "learning_rate": 0.000718122367101304, + "loss": 1.6725, + "theoretical_loss": 4.079041282113206, + "tokens_seen": 367394816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007181063189568707, + "loss": 1.6728, + "theoretical_loss": 4.078959295524365, + "tokens_seen": 367460352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180902708124374, + "loss": 1.5385, + "theoretical_loss": 4.078877327649785, + "tokens_seen": 367525888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180742226680041, + "loss": 1.5009, + "theoretical_loss": 4.078795378481858, + "tokens_seen": 367591424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180581745235708, + "loss": 1.5996, + "theoretical_loss": 4.07871344801298, + "tokens_seen": 367656960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180421263791375, + "loss": 1.5471, + "theoretical_loss": 4.0786315362355525, + "tokens_seen": 367722496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180260782347042, + "loss": 1.4668, + "theoretical_loss": 4.078549643141981, + "tokens_seen": 367788032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007180100300902709, + "loss": 1.6105, + "theoretical_loss": 4.078467768724677, + "tokens_seen": 367853568 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179939819458376, + "loss": 1.4909, + "theoretical_loss": 4.078385912976053, + "tokens_seen": 367919104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179779338014043, + "loss": 1.5897, + "theoretical_loss": 4.078304075888529, + "tokens_seen": 367984640 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717961885656971, + "loss": 1.6694, + "theoretical_loss": 4.078222257454529, + "tokens_seen": 368050176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179458375125377, + "loss": 1.564, + "theoretical_loss": 4.07814045766648, + "tokens_seen": 368115712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179297893681044, + "loss": 1.6268, + "theoretical_loss": 4.0780586765168145, + "tokens_seen": 368181248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007179137412236711, + "loss": 1.5299, + "theoretical_loss": 4.07797691399797, + "tokens_seen": 368246784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178976930792378, + "loss": 1.6779, + "theoretical_loss": 4.077895170102386, + "tokens_seen": 368312320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178816449348045, + "loss": 1.5011, + "theoretical_loss": 4.077813444822509, + "tokens_seen": 368377856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178655967903712, + "loss": 1.6999, + "theoretical_loss": 4.077731738150789, + "tokens_seen": 368443392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178495486459378, + "loss": 1.6138, + "theoretical_loss": 4.077650050079681, + "tokens_seen": 368508928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178335005015046, + "loss": 1.6833, + "theoretical_loss": 4.077568380601642, + "tokens_seen": 368574464 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007178174523570712, + "loss": 1.5291, + "theoretical_loss": 4.077486729709136, + "tokens_seen": 368640000 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717801404212638, + "loss": 1.5386, + "theoretical_loss": 4.077405097394631, + "tokens_seen": 368705536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177853560682046, + "loss": 1.5819, + "theoretical_loss": 4.077323483650598, + "tokens_seen": 368771072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177693079237714, + "loss": 1.5466, + "theoretical_loss": 4.077241888469514, + "tokens_seen": 368836608 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717753259779338, + "loss": 1.6678, + "theoretical_loss": 4.077160311843858, + "tokens_seen": 368902144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177372116349048, + "loss": 1.5159, + "theoretical_loss": 4.077078753766118, + "tokens_seen": 368967680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177211634904714, + "loss": 1.6022, + "theoretical_loss": 4.076997214228782, + "tokens_seen": 369033216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007177051153460381, + "loss": 1.7694, + "theoretical_loss": 4.076915693224343, + "tokens_seen": 369098752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176890672016048, + "loss": 1.5662, + "theoretical_loss": 4.0768341907453, + "tokens_seen": 369164288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176730190571715, + "loss": 1.4227, + "theoretical_loss": 4.0767527067841565, + "tokens_seen": 369229824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176569709127382, + "loss": 1.5903, + "theoretical_loss": 4.076671241333417, + "tokens_seen": 369295360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176409227683049, + "loss": 1.5129, + "theoretical_loss": 4.076589794385596, + "tokens_seen": 369360896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176248746238716, + "loss": 1.6636, + "theoretical_loss": 4.076508365933206, + "tokens_seen": 369426432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007176088264794383, + "loss": 1.6538, + "theoretical_loss": 4.076426955968769, + "tokens_seen": 369491968 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717592778335005, + "loss": 1.5841, + "theoretical_loss": 4.076345564484807, + "tokens_seen": 369557504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175767301905717, + "loss": 1.5383, + "theoretical_loss": 4.076264191473852, + "tokens_seen": 369623040 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175606820461384, + "loss": 1.606, + "theoretical_loss": 4.076182836928435, + "tokens_seen": 369688576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175446339017051, + "loss": 1.5958, + "theoretical_loss": 4.0761015008410935, + "tokens_seen": 369754112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175285857572718, + "loss": 1.6833, + "theoretical_loss": 4.07602018320437, + "tokens_seen": 369819648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007175125376128385, + "loss": 1.6659, + "theoretical_loss": 4.075938884010809, + "tokens_seen": 369885184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174964894684052, + "loss": 1.5919, + "theoretical_loss": 4.075857603252963, + "tokens_seen": 369950720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174804413239719, + "loss": 1.5458, + "theoretical_loss": 4.075776340923385, + "tokens_seen": 370016256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174643931795386, + "loss": 1.6362, + "theoretical_loss": 4.075695097014635, + "tokens_seen": 370081792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174483450351053, + "loss": 1.6066, + "theoretical_loss": 4.075613871519276, + "tokens_seen": 370147328 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717432296890672, + "loss": 1.5889, + "theoretical_loss": 4.0755326644298755, + "tokens_seen": 370212864 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174162487462387, + "loss": 1.5128, + "theoretical_loss": 4.075451475739007, + "tokens_seen": 370278400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007174002006018054, + "loss": 1.6007, + "theoretical_loss": 4.075370305439243, + "tokens_seen": 370343936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173841524573721, + "loss": 1.6355, + "theoretical_loss": 4.075289153523168, + "tokens_seen": 370409472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173681043129388, + "loss": 1.6158, + "theoretical_loss": 4.075208019983366, + "tokens_seen": 370475008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173520561685055, + "loss": 1.585, + "theoretical_loss": 4.075126904812425, + "tokens_seen": 370540544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173360080240722, + "loss": 1.5947, + "theoretical_loss": 4.0750458080029395, + "tokens_seen": 370606080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173199598796389, + "loss": 1.5537, + "theoretical_loss": 4.074964729547506, + "tokens_seen": 370671616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007173039117352056, + "loss": 1.6195, + "theoretical_loss": 4.074883669438729, + "tokens_seen": 370737152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172878635907723, + "loss": 1.6526, + "theoretical_loss": 4.074802627669213, + "tokens_seen": 370802688 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717271815446339, + "loss": 1.5895, + "theoretical_loss": 4.07472160423157, + "tokens_seen": 370868224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172557673019057, + "loss": 1.597, + "theoretical_loss": 4.074640599118413, + "tokens_seen": 370933760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172397191574724, + "loss": 1.6072, + "theoretical_loss": 4.074559612322363, + "tokens_seen": 370999296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172236710130391, + "loss": 1.6019, + "theoretical_loss": 4.074478643836043, + "tokens_seen": 371064832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007172076228686058, + "loss": 1.5784, + "theoretical_loss": 4.074397693652081, + "tokens_seen": 371130368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171915747241725, + "loss": 1.4801, + "theoretical_loss": 4.074316761763107, + "tokens_seen": 371195904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171755265797392, + "loss": 1.7117, + "theoretical_loss": 4.074235848161761, + "tokens_seen": 371261440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171594784353059, + "loss": 1.5646, + "theoretical_loss": 4.074154952840681, + "tokens_seen": 371326976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171434302908726, + "loss": 1.5484, + "theoretical_loss": 4.074074075792513, + "tokens_seen": 371392512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007171273821464393, + "loss": 1.559, + "theoretical_loss": 4.073993217009905, + "tokens_seen": 371458048 + }, + { + "epoch": 0.11, + "learning_rate": 0.000717111334002006, + "loss": 1.6053, + "theoretical_loss": 4.073912376485511, + "tokens_seen": 371523584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170952858575728, + "loss": 1.5723, + "theoretical_loss": 4.073831554211989, + "tokens_seen": 371589120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170792377131395, + "loss": 1.5524, + "theoretical_loss": 4.0737507501820005, + "tokens_seen": 371654656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170631895687062, + "loss": 1.542, + "theoretical_loss": 4.073669964388212, + "tokens_seen": 371720192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170471414242729, + "loss": 1.5857, + "theoretical_loss": 4.073589196823293, + "tokens_seen": 371785728 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170310932798396, + "loss": 1.5204, + "theoretical_loss": 4.073508447479918, + "tokens_seen": 371851264 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007170150451354063, + "loss": 1.6921, + "theoretical_loss": 4.073427716350767, + "tokens_seen": 371916800 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716998996990973, + "loss": 1.5335, + "theoretical_loss": 4.0733470034285215, + "tokens_seen": 371982336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169829488465397, + "loss": 1.5488, + "theoretical_loss": 4.07326630870587, + "tokens_seen": 372047872 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169669007021064, + "loss": 1.6071, + "theoretical_loss": 4.073185632175504, + "tokens_seen": 372113408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169508525576731, + "loss": 1.5816, + "theoretical_loss": 4.073104973830118, + "tokens_seen": 372178944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169348044132398, + "loss": 1.5847, + "theoretical_loss": 4.073024333662413, + "tokens_seen": 372244480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169187562688065, + "loss": 1.6069, + "theoretical_loss": 4.072943711665093, + "tokens_seen": 372310016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007169027081243732, + "loss": 1.6071, + "theoretical_loss": 4.072863107830866, + "tokens_seen": 372375552 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168866599799399, + "loss": 1.6336, + "theoretical_loss": 4.072782522152444, + "tokens_seen": 372441088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168706118355066, + "loss": 1.4765, + "theoretical_loss": 4.072701954622546, + "tokens_seen": 372506624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168545636910733, + "loss": 1.6, + "theoretical_loss": 4.072621405233891, + "tokens_seen": 372572160 + }, + { + "epoch": 0.11, + "learning_rate": 0.00071683851554664, + "loss": 1.6165, + "theoretical_loss": 4.072540873979205, + "tokens_seen": 372637696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168224674022067, + "loss": 1.6396, + "theoretical_loss": 4.072460360851217, + "tokens_seen": 372703232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007168064192577734, + "loss": 1.556, + "theoretical_loss": 4.07237986584266, + "tokens_seen": 372768768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167903711133401, + "loss": 1.6744, + "theoretical_loss": 4.0722993889462735, + "tokens_seen": 372834304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167743229689068, + "loss": 1.5303, + "theoretical_loss": 4.072218930154797, + "tokens_seen": 372899840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167582748244735, + "loss": 1.6165, + "theoretical_loss": 4.072138489460979, + "tokens_seen": 372965376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167422266800402, + "loss": 1.6034, + "theoretical_loss": 4.0720580668575685, + "tokens_seen": 373030912 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167261785356069, + "loss": 1.6118, + "theoretical_loss": 4.0719776623373205, + "tokens_seen": 373096448 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007167101303911736, + "loss": 1.6188, + "theoretical_loss": 4.071897275892994, + "tokens_seen": 373161984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166940822467403, + "loss": 1.6016, + "theoretical_loss": 4.071816907517349, + "tokens_seen": 373227520 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716678034102307, + "loss": 1.575, + "theoretical_loss": 4.071736557203157, + "tokens_seen": 373293056 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166619859578737, + "loss": 1.5539, + "theoretical_loss": 4.071656224943186, + "tokens_seen": 373358592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166459378134404, + "loss": 1.5981, + "theoretical_loss": 4.071575910730212, + "tokens_seen": 373424128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166298896690071, + "loss": 1.613, + "theoretical_loss": 4.071495614557016, + "tokens_seen": 373489664 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 144870, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.635735034942627, + "objective/train/theoretical_loss": 4.071415336416378, + "objective/train/tokens_used": 394015200, + "theoretical_loss": 4.071415336416378, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007166138415245738, + "loss": 1.6357, + "theoretical_loss": 4.071415336416378, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165977933801405, + "loss": 1.5696, + "theoretical_loss": 4.071335076301089, + "tokens_seen": 373620736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165817452357072, + "loss": 1.5439, + "theoretical_loss": 4.071254834203941, + "tokens_seen": 373686272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165656970912739, + "loss": 1.5904, + "theoretical_loss": 4.071174610117729, + "tokens_seen": 373751808 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165496489468406, + "loss": 1.5567, + "theoretical_loss": 4.071094404035253, + "tokens_seen": 373817344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165336008024073, + "loss": 1.5475, + "theoretical_loss": 4.071014215949317, + "tokens_seen": 373882880 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716517552657974, + "loss": 1.6128, + "theoretical_loss": 4.070934045852731, + "tokens_seen": 373948416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007165015045135407, + "loss": 1.6391, + "theoretical_loss": 4.070853893738307, + "tokens_seen": 374013952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164854563691074, + "loss": 1.4634, + "theoretical_loss": 4.070773759598861, + "tokens_seen": 374079488 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164694082246741, + "loss": 1.5947, + "theoretical_loss": 4.070693643427215, + "tokens_seen": 374145024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164533600802407, + "loss": 1.6175, + "theoretical_loss": 4.070613545216193, + "tokens_seen": 374210560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164373119358075, + "loss": 1.5641, + "theoretical_loss": 4.070533464958626, + "tokens_seen": 374276096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164212637913741, + "loss": 1.5614, + "theoretical_loss": 4.070453402647344, + "tokens_seen": 374341632 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007164052156469409, + "loss": 1.5804, + "theoretical_loss": 4.070373358275187, + "tokens_seen": 374407168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163891675025075, + "loss": 1.6213, + "theoretical_loss": 4.070293331834996, + "tokens_seen": 374472704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163731193580743, + "loss": 1.4412, + "theoretical_loss": 4.070213323319616, + "tokens_seen": 374538240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163570712136409, + "loss": 1.5962, + "theoretical_loss": 4.070133332721897, + "tokens_seen": 374603776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163410230692077, + "loss": 1.5521, + "theoretical_loss": 4.070053360034693, + "tokens_seen": 374669312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007163249749247743, + "loss": 1.5172, + "theoretical_loss": 4.069973405250862, + "tokens_seen": 374734848 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716308926780341, + "loss": 1.5655, + "theoretical_loss": 4.069893468363265, + "tokens_seen": 374800384 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162928786359077, + "loss": 1.5791, + "theoretical_loss": 4.06981354936477, + "tokens_seen": 374865920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162768304914744, + "loss": 1.6519, + "theoretical_loss": 4.069733648248246, + "tokens_seen": 374931456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162607823470411, + "loss": 1.498, + "theoretical_loss": 4.069653765006565, + "tokens_seen": 374996992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162447342026078, + "loss": 1.5561, + "theoretical_loss": 4.06957389963261, + "tokens_seen": 375062528 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162286860581745, + "loss": 1.5651, + "theoretical_loss": 4.069494052119261, + "tokens_seen": 375128064 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007162126379137412, + "loss": 1.4672, + "theoretical_loss": 4.069414222459403, + "tokens_seen": 375193600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161965897693079, + "loss": 1.575, + "theoretical_loss": 4.069334410645928, + "tokens_seen": 375259136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161805416248746, + "loss": 1.5902, + "theoretical_loss": 4.069254616671732, + "tokens_seen": 375324672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161644934804413, + "loss": 1.7298, + "theoretical_loss": 4.069174840529712, + "tokens_seen": 375390208 + }, + { + "epoch": 0.11, + "learning_rate": 0.000716148445336008, + "loss": 1.6308, + "theoretical_loss": 4.069095082212772, + "tokens_seen": 375455744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161323971915747, + "loss": 1.6533, + "theoretical_loss": 4.0690153417138175, + "tokens_seen": 375521280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161163490471414, + "loss": 1.6314, + "theoretical_loss": 4.06893561902576, + "tokens_seen": 375586816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007161003009027081, + "loss": 1.5225, + "theoretical_loss": 4.068855914141515, + "tokens_seen": 375652352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160842527582748, + "loss": 1.576, + "theoretical_loss": 4.068776227054, + "tokens_seen": 375717888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160682046138415, + "loss": 1.6257, + "theoretical_loss": 4.06869655775614, + "tokens_seen": 375783424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160521564694082, + "loss": 1.5834, + "theoretical_loss": 4.068616906240861, + "tokens_seen": 375848960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160361083249749, + "loss": 1.6287, + "theoretical_loss": 4.068537272501095, + "tokens_seen": 375914496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160200601805416, + "loss": 1.5467, + "theoretical_loss": 4.068457656529775, + "tokens_seen": 375980032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007160040120361083, + "loss": 1.6462, + "theoretical_loss": 4.068378058319843, + "tokens_seen": 376045568 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715987963891675, + "loss": 1.5426, + "theoretical_loss": 4.068298477864241, + "tokens_seen": 376111104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159719157472417, + "loss": 1.6969, + "theoretical_loss": 4.068218915155917, + "tokens_seen": 376176640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159558676028084, + "loss": 1.6886, + "theoretical_loss": 4.068139370187821, + "tokens_seen": 376242176 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159398194583751, + "loss": 1.5676, + "theoretical_loss": 4.06805984295291, + "tokens_seen": 376307712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159237713139418, + "loss": 1.5628, + "theoretical_loss": 4.067980333444142, + "tokens_seen": 376373248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007159077231695085, + "loss": 1.5888, + "theoretical_loss": 4.067900841654481, + "tokens_seen": 376438784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158916750250753, + "loss": 1.6268, + "theoretical_loss": 4.067821367576894, + "tokens_seen": 376504320 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715875626880642, + "loss": 1.6326, + "theoretical_loss": 4.067741911204354, + "tokens_seen": 376569856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158595787362087, + "loss": 1.4978, + "theoretical_loss": 4.067662472529834, + "tokens_seen": 376635392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158435305917753, + "loss": 1.5713, + "theoretical_loss": 4.067583051546316, + "tokens_seen": 376700928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158274824473421, + "loss": 1.5713, + "theoretical_loss": 4.067503648246781, + "tokens_seen": 376766464 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007158114343029087, + "loss": 1.5302, + "theoretical_loss": 4.067424262624218, + "tokens_seen": 376832000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157953861584755, + "loss": 1.5568, + "theoretical_loss": 4.06734489467162, + "tokens_seen": 376897536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157793380140421, + "loss": 1.6152, + "theoretical_loss": 4.0672655443819785, + "tokens_seen": 376963072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157632898696089, + "loss": 1.5911, + "theoretical_loss": 4.067186211748297, + "tokens_seen": 377028608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157472417251756, + "loss": 1.6492, + "theoretical_loss": 4.067106896763575, + "tokens_seen": 377094144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007157311935807423, + "loss": 1.6437, + "theoretical_loss": 4.067027599420824, + "tokens_seen": 377159680 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715715145436309, + "loss": 1.5367, + "theoretical_loss": 4.0669483197130525, + "tokens_seen": 377225216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156990972918757, + "loss": 1.5343, + "theoretical_loss": 4.066869057633277, + "tokens_seen": 377290752 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156830491474424, + "loss": 1.637, + "theoretical_loss": 4.066789813174517, + "tokens_seen": 377356288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156670010030091, + "loss": 1.6174, + "theoretical_loss": 4.066710586329796, + "tokens_seen": 377421824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156509528585758, + "loss": 1.6421, + "theoretical_loss": 4.066631377092141, + "tokens_seen": 377487360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156349047141425, + "loss": 1.6096, + "theoretical_loss": 4.066552185454582, + "tokens_seen": 377552896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156188565697092, + "loss": 1.5523, + "theoretical_loss": 4.066473011410157, + "tokens_seen": 377618432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007156028084252759, + "loss": 1.6377, + "theoretical_loss": 4.066393854951903, + "tokens_seen": 377683968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155867602808426, + "loss": 1.5204, + "theoretical_loss": 4.066314716072864, + "tokens_seen": 377749504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155707121364093, + "loss": 1.5818, + "theoretical_loss": 4.066235594766088, + "tokens_seen": 377815040 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715554663991976, + "loss": 1.5265, + "theoretical_loss": 4.066156491024625, + "tokens_seen": 377880576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155386158475427, + "loss": 1.7267, + "theoretical_loss": 4.06607740484153, + "tokens_seen": 377946112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155225677031094, + "loss": 1.6535, + "theoretical_loss": 4.065998336209863, + "tokens_seen": 378011648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007155065195586761, + "loss": 1.6532, + "theoretical_loss": 4.065919285122686, + "tokens_seen": 378077184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154904714142428, + "loss": 1.5823, + "theoretical_loss": 4.065840251573066, + "tokens_seen": 378142720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154744232698095, + "loss": 1.6299, + "theoretical_loss": 4.065761235554075, + "tokens_seen": 378208256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154583751253762, + "loss": 1.4748, + "theoretical_loss": 4.065682237058787, + "tokens_seen": 378273792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154423269809429, + "loss": 1.6441, + "theoretical_loss": 4.06560325608028, + "tokens_seen": 378339328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154262788365096, + "loss": 1.5606, + "theoretical_loss": 4.065524292611637, + "tokens_seen": 378404864 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007154102306920763, + "loss": 1.5982, + "theoretical_loss": 4.065445346645945, + "tokens_seen": 378470400 + }, + { + "epoch": 0.11, + "learning_rate": 0.000715394182547643, + "loss": 1.5258, + "theoretical_loss": 4.0653664181762945, + "tokens_seen": 378535936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153781344032097, + "loss": 1.5836, + "theoretical_loss": 4.06528750719578, + "tokens_seen": 378601472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153620862587764, + "loss": 1.5484, + "theoretical_loss": 4.0652086136975, + "tokens_seen": 378667008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153460381143431, + "loss": 1.5613, + "theoretical_loss": 4.065129737674556, + "tokens_seen": 378732544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153299899699098, + "loss": 1.5142, + "theoretical_loss": 4.065050879120054, + "tokens_seen": 378798080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007153139418254765, + "loss": 1.5505, + "theoretical_loss": 4.064972038027106, + "tokens_seen": 378863616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152978936810432, + "loss": 1.6849, + "theoretical_loss": 4.064893214388825, + "tokens_seen": 378929152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152818455366099, + "loss": 1.5803, + "theoretical_loss": 4.064814408198327, + "tokens_seen": 378994688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152657973921766, + "loss": 1.6275, + "theoretical_loss": 4.0647356194487365, + "tokens_seen": 379060224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152497492477433, + "loss": 1.6681, + "theoretical_loss": 4.064656848133178, + "tokens_seen": 379125760 + }, + { + "epoch": 0.11, + "learning_rate": 0.00071523370110331, + "loss": 1.6286, + "theoretical_loss": 4.064578094244782, + "tokens_seen": 379191296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152176529588767, + "loss": 1.5823, + "theoretical_loss": 4.06449935777668, + "tokens_seen": 379256832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007152016048144434, + "loss": 1.483, + "theoretical_loss": 4.0644206387220105, + "tokens_seen": 379322368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007151855566700101, + "loss": 1.7101, + "theoretical_loss": 4.064341937073916, + "tokens_seen": 379387904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0007151695085255768, + "loss": 1.5285, + "theoretical_loss": 4.0642632528255405, + "tokens_seen": 379453440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007151534603811435, + "loss": 1.5496, + "theoretical_loss": 4.064184585970033, + "tokens_seen": 379518976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007151374122367102, + "loss": 1.5875, + "theoretical_loss": 4.064105936500547, + "tokens_seen": 379584512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007151213640922769, + "loss": 1.5715, + "theoretical_loss": 4.064027304410239, + "tokens_seen": 379650048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007151053159478436, + "loss": 1.4562, + "theoretical_loss": 4.063948689692269, + "tokens_seen": 379715584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007150892678034103, + "loss": 1.5498, + "theoretical_loss": 4.063870092339803, + "tokens_seen": 379781120 + }, + { + "epoch": 0.12, + "learning_rate": 0.000715073219658977, + "loss": 1.556, + "theoretical_loss": 4.0637915123460076, + "tokens_seen": 379846656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007150571715145436, + "loss": 1.5482, + "theoretical_loss": 4.0637129497040565, + "tokens_seen": 379912192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007150411233701104, + "loss": 1.6552, + "theoretical_loss": 4.063634404407126, + "tokens_seen": 379977728 + }, + { + "epoch": 0.12, + "learning_rate": 0.000715025075225677, + "loss": 1.5666, + "theoretical_loss": 4.063555876448395, + "tokens_seen": 380043264 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 147136, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.6120753288269043, + "objective/train/theoretical_loss": 4.063477365821048, + "objective/train/tokens_used": 400568800, + "theoretical_loss": 4.063477365821048, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007150090270812438, + "loss": 1.6121, + "theoretical_loss": 4.063477365821048, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149929789368104, + "loss": 1.4885, + "theoretical_loss": 4.063398872518273, + "tokens_seen": 380174336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149769307923772, + "loss": 1.6225, + "theoretical_loss": 4.0633203965332605, + "tokens_seen": 380239872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149608826479438, + "loss": 1.692, + "theoretical_loss": 4.063241937859207, + "tokens_seen": 380305408 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149448345035106, + "loss": 1.6794, + "theoretical_loss": 4.063163496489312, + "tokens_seen": 380370944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149287863590772, + "loss": 1.6206, + "theoretical_loss": 4.0630850724167775, + "tokens_seen": 380436480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007149127382146439, + "loss": 1.5479, + "theoretical_loss": 4.063006665634811, + "tokens_seen": 380502016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148966900702106, + "loss": 1.5644, + "theoretical_loss": 4.062928276136624, + "tokens_seen": 380567552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148806419257773, + "loss": 1.5028, + "theoretical_loss": 4.0628499039154296, + "tokens_seen": 380633088 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714864593781344, + "loss": 1.6273, + "theoretical_loss": 4.062771548964447, + "tokens_seen": 380698624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148485456369107, + "loss": 1.6775, + "theoretical_loss": 4.062693211276899, + "tokens_seen": 380764160 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148324974924774, + "loss": 1.6853, + "theoretical_loss": 4.062614890846012, + "tokens_seen": 380829696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148164493480441, + "loss": 1.5306, + "theoretical_loss": 4.062536587665014, + "tokens_seen": 380895232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007148004012036108, + "loss": 1.5554, + "theoretical_loss": 4.0624583017271405, + "tokens_seen": 380960768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147843530591775, + "loss": 1.6352, + "theoretical_loss": 4.062380033025629, + "tokens_seen": 381026304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147683049147442, + "loss": 1.6092, + "theoretical_loss": 4.062301781553719, + "tokens_seen": 381091840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147522567703109, + "loss": 1.5803, + "theoretical_loss": 4.062223547304658, + "tokens_seen": 381157376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147362086258776, + "loss": 1.62, + "theoretical_loss": 4.0621453302716946, + "tokens_seen": 381222912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007147201604814443, + "loss": 1.4979, + "theoretical_loss": 4.062067130448081, + "tokens_seen": 381288448 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714704112337011, + "loss": 1.5743, + "theoretical_loss": 4.061988947827073, + "tokens_seen": 381353984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146880641925778, + "loss": 1.6731, + "theoretical_loss": 4.061910782401933, + "tokens_seen": 381419520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146720160481444, + "loss": 1.5153, + "theoretical_loss": 4.061832634165923, + "tokens_seen": 381485056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146559679037112, + "loss": 1.6036, + "theoretical_loss": 4.061754503112313, + "tokens_seen": 381550592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146399197592778, + "loss": 1.6045, + "theoretical_loss": 4.061676389234375, + "tokens_seen": 381616128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146238716148446, + "loss": 1.5651, + "theoretical_loss": 4.061598292525382, + "tokens_seen": 381681664 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007146078234704112, + "loss": 1.5677, + "theoretical_loss": 4.061520212978616, + "tokens_seen": 381747200 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714591775325978, + "loss": 1.5722, + "theoretical_loss": 4.061442150587358, + "tokens_seen": 381812736 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007145757271815446, + "loss": 1.6635, + "theoretical_loss": 4.0613641053448974, + "tokens_seen": 381878272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007145596790371114, + "loss": 1.585, + "theoretical_loss": 4.061286077244523, + "tokens_seen": 381943808 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714543630892678, + "loss": 1.5448, + "theoretical_loss": 4.06120806627953, + "tokens_seen": 382009344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007145275827482448, + "loss": 1.5069, + "theoretical_loss": 4.061130072443216, + "tokens_seen": 382074880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007145115346038114, + "loss": 1.5988, + "theoretical_loss": 4.0610520957288845, + "tokens_seen": 382140416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144954864593782, + "loss": 1.5636, + "theoretical_loss": 4.06097413612984, + "tokens_seen": 382205952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144794383149448, + "loss": 1.5992, + "theoretical_loss": 4.060896193639392, + "tokens_seen": 382271488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144633901705116, + "loss": 1.5965, + "theoretical_loss": 4.060818268250855, + "tokens_seen": 382337024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144473420260782, + "loss": 1.6481, + "theoretical_loss": 4.060740359957546, + "tokens_seen": 382402560 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714431293881645, + "loss": 1.5452, + "theoretical_loss": 4.060662468752784, + "tokens_seen": 382468096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007144152457372116, + "loss": 1.6091, + "theoretical_loss": 4.060584594629895, + "tokens_seen": 382533632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143991975927784, + "loss": 1.6555, + "theoretical_loss": 4.060506737582208, + "tokens_seen": 382599168 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714383149448345, + "loss": 1.5669, + "theoretical_loss": 4.060428897603054, + "tokens_seen": 382664704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143671013039118, + "loss": 1.6971, + "theoretical_loss": 4.060351074685769, + "tokens_seen": 382730240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143510531594785, + "loss": 1.5216, + "theoretical_loss": 4.060273268823693, + "tokens_seen": 382795776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143350050150452, + "loss": 1.6712, + "theoretical_loss": 4.060195480010169, + "tokens_seen": 382861312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143189568706119, + "loss": 1.6195, + "theoretical_loss": 4.060117708238544, + "tokens_seen": 382926848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007143029087261786, + "loss": 1.5318, + "theoretical_loss": 4.060039953502169, + "tokens_seen": 382992384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142868605817453, + "loss": 1.6185, + "theoretical_loss": 4.059962215794399, + "tokens_seen": 383057920 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714270812437312, + "loss": 1.6063, + "theoretical_loss": 4.059884495108591, + "tokens_seen": 383123456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142547642928787, + "loss": 1.685, + "theoretical_loss": 4.059806791438108, + "tokens_seen": 383188992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142387161484454, + "loss": 1.5573, + "theoretical_loss": 4.059729104776316, + "tokens_seen": 383254528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142226680040121, + "loss": 1.6274, + "theoretical_loss": 4.059651435116583, + "tokens_seen": 383320064 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007142066198595788, + "loss": 1.6083, + "theoretical_loss": 4.059573782452284, + "tokens_seen": 383385600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141905717151455, + "loss": 1.682, + "theoretical_loss": 4.0594961467767945, + "tokens_seen": 383451136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141745235707122, + "loss": 1.699, + "theoretical_loss": 4.059418528083495, + "tokens_seen": 383516672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141584754262789, + "loss": 1.5119, + "theoretical_loss": 4.0593409263657705, + "tokens_seen": 383582208 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141424272818456, + "loss": 1.6047, + "theoretical_loss": 4.059263341617009, + "tokens_seen": 383647744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007141263791374123, + "loss": 1.6323, + "theoretical_loss": 4.059185773830602, + "tokens_seen": 383713280 + }, + { + "epoch": 0.12, + "learning_rate": 0.000714110330992979, + "loss": 1.5862, + "theoretical_loss": 4.059108222999944, + "tokens_seen": 383778816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140942828485457, + "loss": 1.5736, + "theoretical_loss": 4.059030689118436, + "tokens_seen": 383844352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140782347041124, + "loss": 1.5046, + "theoretical_loss": 4.058953172179479, + "tokens_seen": 383909888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140621865596791, + "loss": 1.6811, + "theoretical_loss": 4.05887567217648, + "tokens_seen": 383975424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140461384152458, + "loss": 1.5871, + "theoretical_loss": 4.05879818910285, + "tokens_seen": 384040960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140300902708125, + "loss": 1.6244, + "theoretical_loss": 4.058720722952002, + "tokens_seen": 384106496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007140140421263792, + "loss": 1.4743, + "theoretical_loss": 4.058643273717355, + "tokens_seen": 384172032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139979939819459, + "loss": 1.5426, + "theoretical_loss": 4.058565841392328, + "tokens_seen": 384237568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139819458375126, + "loss": 1.583, + "theoretical_loss": 4.058488425970347, + "tokens_seen": 384303104 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139658976930793, + "loss": 1.4633, + "theoretical_loss": 4.058411027444841, + "tokens_seen": 384368640 + }, + { + "epoch": 0.12, + "learning_rate": 0.000713949849548646, + "loss": 1.5868, + "theoretical_loss": 4.05833364580924, + "tokens_seen": 384434176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139338014042127, + "loss": 1.5693, + "theoretical_loss": 4.058256281056984, + "tokens_seen": 384499712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139177532597794, + "loss": 1.409, + "theoretical_loss": 4.058178933181509, + "tokens_seen": 384565248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007139017051153461, + "loss": 1.4899, + "theoretical_loss": 4.05810160217626, + "tokens_seen": 384630784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138856569709128, + "loss": 1.6796, + "theoretical_loss": 4.058024288034684, + "tokens_seen": 384696320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138696088264795, + "loss": 1.6137, + "theoretical_loss": 4.057946990750231, + "tokens_seen": 384761856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138535606820462, + "loss": 1.5878, + "theoretical_loss": 4.057869710316355, + "tokens_seen": 384827392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138375125376129, + "loss": 1.5176, + "theoretical_loss": 4.0577924467265145, + "tokens_seen": 384892928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138214643931796, + "loss": 1.6999, + "theoretical_loss": 4.057715199974171, + "tokens_seen": 384958464 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007138054162487463, + "loss": 1.3855, + "theoretical_loss": 4.05763797005279, + "tokens_seen": 385024000 + }, + { + "epoch": 0.12, + "learning_rate": 0.000713789368104313, + "loss": 1.5799, + "theoretical_loss": 4.05756075695584, + "tokens_seen": 385089536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137733199598797, + "loss": 1.6051, + "theoretical_loss": 4.0574835606767925, + "tokens_seen": 385155072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137572718154464, + "loss": 1.5242, + "theoretical_loss": 4.057406381209126, + "tokens_seen": 385220608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137412236710131, + "loss": 1.6672, + "theoretical_loss": 4.057329218546318, + "tokens_seen": 385286144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137251755265798, + "loss": 1.5213, + "theoretical_loss": 4.0572520726818535, + "tokens_seen": 385351680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007137091273821465, + "loss": 1.54, + "theoretical_loss": 4.057174943609218, + "tokens_seen": 385417216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136930792377132, + "loss": 1.6338, + "theoretical_loss": 4.057097831321904, + "tokens_seen": 385482752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136770310932799, + "loss": 1.564, + "theoretical_loss": 4.057020735813405, + "tokens_seen": 385548288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136609829488466, + "loss": 1.4348, + "theoretical_loss": 4.056943657077219, + "tokens_seen": 385613824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136449348044133, + "loss": 1.8022, + "theoretical_loss": 4.056866595106847, + "tokens_seen": 385679360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136288866599799, + "loss": 1.5342, + "theoretical_loss": 4.056789549895795, + "tokens_seen": 385744896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007136128385155467, + "loss": 1.5946, + "theoretical_loss": 4.056712521437571, + "tokens_seen": 385810432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135967903711133, + "loss": 1.5653, + "theoretical_loss": 4.056635509725689, + "tokens_seen": 385875968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135807422266801, + "loss": 1.6365, + "theoretical_loss": 4.056558514753664, + "tokens_seen": 385941504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135646940822467, + "loss": 1.6195, + "theoretical_loss": 4.056481536515014, + "tokens_seen": 386007040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135486459378135, + "loss": 1.5882, + "theoretical_loss": 4.056404575003266, + "tokens_seen": 386072576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135325977933801, + "loss": 1.6211, + "theoretical_loss": 4.0563276302119435, + "tokens_seen": 386138112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135165496489468, + "loss": 1.6054, + "theoretical_loss": 4.056250702134578, + "tokens_seen": 386203648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007135005015045135, + "loss": 1.7561, + "theoretical_loss": 4.0561737907647055, + "tokens_seen": 386269184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134844533600803, + "loss": 1.5714, + "theoretical_loss": 4.056096896095861, + "tokens_seen": 386334720 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134684052156469, + "loss": 1.547, + "theoretical_loss": 4.056020018121587, + "tokens_seen": 386400256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134523570712137, + "loss": 1.5879, + "theoretical_loss": 4.055943156835428, + "tokens_seen": 386465792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134363089267803, + "loss": 1.5852, + "theoretical_loss": 4.055866312230933, + "tokens_seen": 386531328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134202607823471, + "loss": 1.63, + "theoretical_loss": 4.055789484301653, + "tokens_seen": 386596864 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 149397, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.6475073099136353, + "objective/train/theoretical_loss": 4.055712673041144, + "objective/train/tokens_used": 407122400, + "theoretical_loss": 4.055712673041144, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007134042126379137, + "loss": 1.6475, + "theoretical_loss": 4.055712673041144, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133881644934805, + "loss": 1.6519, + "theoretical_loss": 4.055635878442967, + "tokens_seen": 386727936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133721163490471, + "loss": 1.5662, + "theoretical_loss": 4.055559100500682, + "tokens_seen": 386793472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133560682046139, + "loss": 1.6529, + "theoretical_loss": 4.055482339207858, + "tokens_seen": 386859008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133400200601805, + "loss": 1.5962, + "theoretical_loss": 4.055405594558062, + "tokens_seen": 386924544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133239719157473, + "loss": 1.5063, + "theoretical_loss": 4.05532886654487, + "tokens_seen": 386990080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007133079237713139, + "loss": 1.6101, + "theoretical_loss": 4.055252155161858, + "tokens_seen": 387055616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132918756268807, + "loss": 1.542, + "theoretical_loss": 4.055175460402607, + "tokens_seen": 387121152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132758274824473, + "loss": 1.6014, + "theoretical_loss": 4.055098782260701, + "tokens_seen": 387186688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132597793380141, + "loss": 1.5145, + "theoretical_loss": 4.055022120729728, + "tokens_seen": 387252224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132437311935807, + "loss": 1.6753, + "theoretical_loss": 4.054945475803278, + "tokens_seen": 387317760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132276830491475, + "loss": 1.4869, + "theoretical_loss": 4.054868847474948, + "tokens_seen": 387383296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007132116349047141, + "loss": 1.6354, + "theoretical_loss": 4.054792235738335, + "tokens_seen": 387448832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131955867602809, + "loss": 1.5686, + "theoretical_loss": 4.054715640587042, + "tokens_seen": 387514368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131795386158475, + "loss": 1.5895, + "theoretical_loss": 4.054639062014673, + "tokens_seen": 387579904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131634904714143, + "loss": 1.6742, + "theoretical_loss": 4.054562500014839, + "tokens_seen": 387645440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131474423269809, + "loss": 1.5421, + "theoretical_loss": 4.054485954581152, + "tokens_seen": 387710976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131313941825477, + "loss": 1.5601, + "theoretical_loss": 4.054409425707227, + "tokens_seen": 387776512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007131153460381143, + "loss": 1.5785, + "theoretical_loss": 4.054332913386686, + "tokens_seen": 387842048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130992978936811, + "loss": 1.5537, + "theoretical_loss": 4.054256417613149, + "tokens_seen": 387907584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130832497492477, + "loss": 1.7037, + "theoretical_loss": 4.054179938380246, + "tokens_seen": 387973120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130672016048145, + "loss": 1.5813, + "theoretical_loss": 4.054103475681605, + "tokens_seen": 388038656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130511534603811, + "loss": 1.577, + "theoretical_loss": 4.054027029510861, + "tokens_seen": 388104192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130351053159479, + "loss": 1.4928, + "theoretical_loss": 4.053950599861651, + "tokens_seen": 388169728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130190571715146, + "loss": 1.5789, + "theoretical_loss": 4.053874186727617, + "tokens_seen": 388235264 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007130030090270813, + "loss": 1.4436, + "theoretical_loss": 4.053797790102402, + "tokens_seen": 388300800 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712986960882648, + "loss": 1.4588, + "theoretical_loss": 4.053721409979654, + "tokens_seen": 388366336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129709127382147, + "loss": 1.5483, + "theoretical_loss": 4.053645046353025, + "tokens_seen": 388431872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129548645937814, + "loss": 1.5597, + "theoretical_loss": 4.05356869921617, + "tokens_seen": 388497408 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129388164493481, + "loss": 1.6699, + "theoretical_loss": 4.053492368562747, + "tokens_seen": 388562944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129227683049148, + "loss": 1.6458, + "theoretical_loss": 4.0534160543864175, + "tokens_seen": 388628480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007129067201604815, + "loss": 1.588, + "theoretical_loss": 4.053339756680849, + "tokens_seen": 388694016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128906720160482, + "loss": 1.5798, + "theoretical_loss": 4.053263475439707, + "tokens_seen": 388759552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128746238716149, + "loss": 1.5001, + "theoretical_loss": 4.053187210656668, + "tokens_seen": 388825088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128585757271816, + "loss": 1.4575, + "theoretical_loss": 4.053110962325406, + "tokens_seen": 388890624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128425275827483, + "loss": 1.5752, + "theoretical_loss": 4.053034730439601, + "tokens_seen": 388956160 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712826479438315, + "loss": 1.5883, + "theoretical_loss": 4.052958514992934, + "tokens_seen": 389021696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007128104312938817, + "loss": 1.4732, + "theoretical_loss": 4.052882315979094, + "tokens_seen": 389087232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127943831494484, + "loss": 1.502, + "theoretical_loss": 4.052806133391769, + "tokens_seen": 389152768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127783350050151, + "loss": 1.4581, + "theoretical_loss": 4.052729967224654, + "tokens_seen": 389218304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127622868605818, + "loss": 1.5934, + "theoretical_loss": 4.0526538174714455, + "tokens_seen": 389283840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127462387161485, + "loss": 1.5532, + "theoretical_loss": 4.052577684125843, + "tokens_seen": 389349376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127301905717152, + "loss": 1.5983, + "theoretical_loss": 4.052501567181552, + "tokens_seen": 389414912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007127141424272819, + "loss": 1.5752, + "theoretical_loss": 4.052425466632278, + "tokens_seen": 389480448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126980942828486, + "loss": 1.5496, + "theoretical_loss": 4.052349382471734, + "tokens_seen": 389545984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126820461384153, + "loss": 1.5285, + "theoretical_loss": 4.052273314693632, + "tokens_seen": 389611520 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712665997993982, + "loss": 1.5299, + "theoretical_loss": 4.05219726329169, + "tokens_seen": 389677056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126499498495487, + "loss": 1.5573, + "theoretical_loss": 4.052121228259631, + "tokens_seen": 389742592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126339017051154, + "loss": 1.6006, + "theoretical_loss": 4.052045209591179, + "tokens_seen": 389808128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126178535606821, + "loss": 1.5693, + "theoretical_loss": 4.05196920728006, + "tokens_seen": 389873664 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007126018054162488, + "loss": 1.6375, + "theoretical_loss": 4.051893221320009, + "tokens_seen": 389939200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125857572718155, + "loss": 1.754, + "theoretical_loss": 4.05181725170476, + "tokens_seen": 390004736 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125697091273822, + "loss": 1.5083, + "theoretical_loss": 4.051741298428049, + "tokens_seen": 390070272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125536609829489, + "loss": 1.5868, + "theoretical_loss": 4.0516653614836216, + "tokens_seen": 390135808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125376128385156, + "loss": 1.6428, + "theoretical_loss": 4.05158944086522, + "tokens_seen": 390201344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007125215646940823, + "loss": 1.6121, + "theoretical_loss": 4.0515135365665955, + "tokens_seen": 390266880 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712505516549649, + "loss": 1.7383, + "theoretical_loss": 4.0514376485815, + "tokens_seen": 390332416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124894684052157, + "loss": 1.6227, + "theoretical_loss": 4.051361776903688, + "tokens_seen": 390397952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124734202607824, + "loss": 1.5682, + "theoretical_loss": 4.051285921526919, + "tokens_seen": 390463488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124573721163491, + "loss": 1.5637, + "theoretical_loss": 4.051210082444956, + "tokens_seen": 390529024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124413239719158, + "loss": 1.5864, + "theoretical_loss": 4.051134259651565, + "tokens_seen": 390594560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124252758274825, + "loss": 1.52, + "theoretical_loss": 4.0510584531405165, + "tokens_seen": 390660096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007124092276830492, + "loss": 1.5601, + "theoretical_loss": 4.050982662905581, + "tokens_seen": 390725632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123931795386159, + "loss": 1.4908, + "theoretical_loss": 4.0509068889405375, + "tokens_seen": 390791168 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123771313941826, + "loss": 1.6032, + "theoretical_loss": 4.050831131239164, + "tokens_seen": 390856704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123610832497493, + "loss": 1.5559, + "theoretical_loss": 4.050755389795244, + "tokens_seen": 390922240 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712345035105316, + "loss": 1.5972, + "theoretical_loss": 4.050679664602564, + "tokens_seen": 390987776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123289869608827, + "loss": 1.533, + "theoretical_loss": 4.050603955654914, + "tokens_seen": 391053312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007123129388164494, + "loss": 1.6338, + "theoretical_loss": 4.050528262946088, + "tokens_seen": 391118848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122968906720161, + "loss": 1.5616, + "theoretical_loss": 4.050452586469882, + "tokens_seen": 391184384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122808425275828, + "loss": 1.5888, + "theoretical_loss": 4.050376926220098, + "tokens_seen": 391249920 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122647943831495, + "loss": 1.5333, + "theoretical_loss": 4.050301282190537, + "tokens_seen": 391315456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122487462387162, + "loss": 1.6047, + "theoretical_loss": 4.050225654375008, + "tokens_seen": 391380992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122326980942828, + "loss": 1.5639, + "theoretical_loss": 4.050150042767321, + "tokens_seen": 391446528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122166499498496, + "loss": 1.5325, + "theoretical_loss": 4.050074447361289, + "tokens_seen": 391512064 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007122006018054162, + "loss": 1.6786, + "theoretical_loss": 4.04999886815073, + "tokens_seen": 391577600 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712184553660983, + "loss": 1.6675, + "theoretical_loss": 4.0499233051294645, + "tokens_seen": 391643136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007121685055165496, + "loss": 1.5738, + "theoretical_loss": 4.049847758291317, + "tokens_seen": 391708672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007121524573721164, + "loss": 1.6631, + "theoretical_loss": 4.049772227630114, + "tokens_seen": 391774208 + }, + { + "epoch": 0.12, + "learning_rate": 0.000712136409227683, + "loss": 1.598, + "theoretical_loss": 4.049696713139687, + "tokens_seen": 391839744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007121203610832498, + "loss": 1.659, + "theoretical_loss": 4.049621214813869, + "tokens_seen": 391905280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007121043129388164, + "loss": 1.5147, + "theoretical_loss": 4.0495457326465, + "tokens_seen": 391970816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120882647943832, + "loss": 1.5579, + "theoretical_loss": 4.049470266631419, + "tokens_seen": 392036352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120722166499498, + "loss": 1.6694, + "theoretical_loss": 4.04939481676247, + "tokens_seen": 392101888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120561685055166, + "loss": 1.582, + "theoretical_loss": 4.0493193830335015, + "tokens_seen": 392167424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120401203610832, + "loss": 1.519, + "theoretical_loss": 4.0492439654383645, + "tokens_seen": 392232960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00071202407221665, + "loss": 1.5524, + "theoretical_loss": 4.049168563970914, + "tokens_seen": 392298496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007120080240722166, + "loss": 1.7572, + "theoretical_loss": 4.049093178625007, + "tokens_seen": 392364032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119919759277834, + "loss": 1.5717, + "theoretical_loss": 4.049017809394506, + "tokens_seen": 392429568 + }, + { + "epoch": 0.12, + "learning_rate": 0.00071197592778335, + "loss": 1.5631, + "theoretical_loss": 4.048942456273274, + "tokens_seen": 392495104 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119598796389168, + "loss": 1.5754, + "theoretical_loss": 4.048867119255178, + "tokens_seen": 392560640 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119438314944834, + "loss": 1.4863, + "theoretical_loss": 4.048791798334092, + "tokens_seen": 392626176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119277833500502, + "loss": 1.6165, + "theoretical_loss": 4.048716493503888, + "tokens_seen": 392691712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007119117352056168, + "loss": 1.6519, + "theoretical_loss": 4.048641204758446, + "tokens_seen": 392757248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007118956870611836, + "loss": 1.5396, + "theoretical_loss": 4.048565932091646, + "tokens_seen": 392822784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007118796389167502, + "loss": 1.5964, + "theoretical_loss": 4.048490675497373, + "tokens_seen": 392888320 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711863590772317, + "loss": 1.518, + "theoretical_loss": 4.048415434969516, + "tokens_seen": 392953856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007118475426278836, + "loss": 1.6275, + "theoretical_loss": 4.048340210501965, + "tokens_seen": 393019392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007118314944834504, + "loss": 1.6406, + "theoretical_loss": 4.048265002088614, + "tokens_seen": 393084928 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711815446339017, + "loss": 1.7019, + "theoretical_loss": 4.048189809723364, + "tokens_seen": 393150464 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 152158, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5794931650161743, + "objective/train/theoretical_loss": 4.048114633400113, + "objective/train/tokens_used": 413676000, + "theoretical_loss": 4.048114633400113, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117993981945838, + "loss": 1.5795, + "theoretical_loss": 4.048114633400113, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117833500501504, + "loss": 1.6456, + "theoretical_loss": 4.048039473112768, + "tokens_seen": 393281536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117673019057172, + "loss": 1.6227, + "theoretical_loss": 4.047964328855237, + "tokens_seen": 393347072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117512537612838, + "loss": 1.691, + "theoretical_loss": 4.047889200621428, + "tokens_seen": 393412608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117352056168506, + "loss": 1.6047, + "theoretical_loss": 4.047814088405261, + "tokens_seen": 393478144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007117191574724172, + "loss": 1.5286, + "theoretical_loss": 4.0477389922006495, + "tokens_seen": 393543680 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711703109327984, + "loss": 1.5547, + "theoretical_loss": 4.047663912001518, + "tokens_seen": 393609216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116870611835506, + "loss": 1.6003, + "theoretical_loss": 4.047588847801788, + "tokens_seen": 393674752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116710130391174, + "loss": 1.5363, + "theoretical_loss": 4.04751379959539, + "tokens_seen": 393740288 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711654964894684, + "loss": 1.4706, + "theoretical_loss": 4.047438767376255, + "tokens_seen": 393805824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116389167502508, + "loss": 1.6442, + "theoretical_loss": 4.047363751138316, + "tokens_seen": 393871360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116228686058175, + "loss": 1.5837, + "theoretical_loss": 4.047288750875512, + "tokens_seen": 393936896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007116068204613842, + "loss": 1.5422, + "theoretical_loss": 4.047213766581785, + "tokens_seen": 394002432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115907723169509, + "loss": 1.5594, + "theoretical_loss": 4.047138798251078, + "tokens_seen": 394067968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115747241725176, + "loss": 1.5742, + "theoretical_loss": 4.047063845877339, + "tokens_seen": 394133504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115586760280843, + "loss": 1.5488, + "theoretical_loss": 4.04698890945452, + "tokens_seen": 394199040 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711542627883651, + "loss": 1.5365, + "theoretical_loss": 4.046913988976575, + "tokens_seen": 394264576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115265797392177, + "loss": 1.5225, + "theoretical_loss": 4.046839084437463, + "tokens_seen": 394330112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007115105315947844, + "loss": 1.6615, + "theoretical_loss": 4.046764195831142, + "tokens_seen": 394395648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114944834503511, + "loss": 1.5474, + "theoretical_loss": 4.046689323151578, + "tokens_seen": 394461184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114784353059178, + "loss": 1.6094, + "theoretical_loss": 4.0466144663927395, + "tokens_seen": 394526720 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114623871614845, + "loss": 1.5698, + "theoretical_loss": 4.046539625548595, + "tokens_seen": 394592256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114463390170512, + "loss": 1.5722, + "theoretical_loss": 4.046464800613121, + "tokens_seen": 394657792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114302908726179, + "loss": 1.5475, + "theoretical_loss": 4.046389991580294, + "tokens_seen": 394723328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007114142427281846, + "loss": 1.6544, + "theoretical_loss": 4.046315198444094, + "tokens_seen": 394788864 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113981945837513, + "loss": 1.6017, + "theoretical_loss": 4.046240421198506, + "tokens_seen": 394854400 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711382146439318, + "loss": 1.4742, + "theoretical_loss": 4.046165659837517, + "tokens_seen": 394919936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113660982948847, + "loss": 1.5972, + "theoretical_loss": 4.046090914355117, + "tokens_seen": 394985472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113500501504514, + "loss": 1.4823, + "theoretical_loss": 4.046016184745302, + "tokens_seen": 395051008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113340020060181, + "loss": 1.6604, + "theoretical_loss": 4.045941471002066, + "tokens_seen": 395116544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113179538615848, + "loss": 1.632, + "theoretical_loss": 4.0458667731194105, + "tokens_seen": 395182080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007113019057171515, + "loss": 1.7032, + "theoretical_loss": 4.045792091091339, + "tokens_seen": 395247616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112858575727182, + "loss": 1.4971, + "theoretical_loss": 4.04571742491186, + "tokens_seen": 395313152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112698094282849, + "loss": 1.6019, + "theoretical_loss": 4.045642774574981, + "tokens_seen": 395378688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112537612838516, + "loss": 1.5502, + "theoretical_loss": 4.045568140074717, + "tokens_seen": 395444224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112377131394183, + "loss": 1.5816, + "theoretical_loss": 4.045493521405085, + "tokens_seen": 395509760 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711221664994985, + "loss": 1.6757, + "theoretical_loss": 4.045418918560104, + "tokens_seen": 395575296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007112056168505517, + "loss": 1.6892, + "theoretical_loss": 4.045344331533796, + "tokens_seen": 395640832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111895687061184, + "loss": 1.6011, + "theoretical_loss": 4.04526976032019, + "tokens_seen": 395706368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111735205616851, + "loss": 1.7308, + "theoretical_loss": 4.0451952049133135, + "tokens_seen": 395771904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111574724172518, + "loss": 1.6607, + "theoretical_loss": 4.0451206653072, + "tokens_seen": 395837440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111414242728185, + "loss": 1.5932, + "theoretical_loss": 4.045046141495887, + "tokens_seen": 395902976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007111253761283852, + "loss": 1.5839, + "theoretical_loss": 4.044971633473412, + "tokens_seen": 395968512 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711109327983952, + "loss": 1.5876, + "theoretical_loss": 4.044897141233818, + "tokens_seen": 396034048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110932798395186, + "loss": 1.4768, + "theoretical_loss": 4.044822664771151, + "tokens_seen": 396099584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110772316950854, + "loss": 1.5088, + "theoretical_loss": 4.044748204079459, + "tokens_seen": 396165120 + }, + { + "epoch": 0.12, + "learning_rate": 0.000711061183550652, + "loss": 1.5454, + "theoretical_loss": 4.044673759152797, + "tokens_seen": 396230656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110451354062188, + "loss": 1.4653, + "theoretical_loss": 4.0445993299852185, + "tokens_seen": 396296192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110290872617854, + "loss": 1.5134, + "theoretical_loss": 4.044524916570782, + "tokens_seen": 396361728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007110130391173522, + "loss": 1.5726, + "theoretical_loss": 4.04445051890355, + "tokens_seen": 396427264 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109969909729188, + "loss": 1.6622, + "theoretical_loss": 4.044376136977588, + "tokens_seen": 396492800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109809428284855, + "loss": 1.6644, + "theoretical_loss": 4.044301770786964, + "tokens_seen": 396558336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109648946840522, + "loss": 1.5216, + "theoretical_loss": 4.044227420325749, + "tokens_seen": 396623872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109488465396189, + "loss": 1.5431, + "theoretical_loss": 4.044153085588018, + "tokens_seen": 396689408 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109327983951856, + "loss": 1.5712, + "theoretical_loss": 4.044078766567851, + "tokens_seen": 396754944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007109167502507523, + "loss": 1.5974, + "theoretical_loss": 4.044004463259327, + "tokens_seen": 396820480 + }, + { + "epoch": 0.12, + "learning_rate": 0.000710900702106319, + "loss": 1.5131, + "theoretical_loss": 4.043930175656531, + "tokens_seen": 396886016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108846539618857, + "loss": 1.5501, + "theoretical_loss": 4.043855903753551, + "tokens_seen": 396951552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108686058174524, + "loss": 1.4939, + "theoretical_loss": 4.043781647544477, + "tokens_seen": 397017088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108525576730191, + "loss": 1.4825, + "theoretical_loss": 4.043707407023404, + "tokens_seen": 397082624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108365095285857, + "loss": 1.5249, + "theoretical_loss": 4.043633182184428, + "tokens_seen": 397148160 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108204613841525, + "loss": 1.549, + "theoretical_loss": 4.043558973021652, + "tokens_seen": 397213696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007108044132397191, + "loss": 1.5316, + "theoretical_loss": 4.043484779529176, + "tokens_seen": 397279232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107883650952859, + "loss": 1.5164, + "theoretical_loss": 4.04341060170111, + "tokens_seen": 397344768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107723169508525, + "loss": 1.5737, + "theoretical_loss": 4.043336439531561, + "tokens_seen": 397410304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107562688064193, + "loss": 1.6715, + "theoretical_loss": 4.043262293014646, + "tokens_seen": 397475840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107402206619859, + "loss": 1.5088, + "theoretical_loss": 4.043188162144478, + "tokens_seen": 397541376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107241725175527, + "loss": 1.5586, + "theoretical_loss": 4.0431140469151785, + "tokens_seen": 397606912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007107081243731193, + "loss": 1.5918, + "theoretical_loss": 4.043039947320869, + "tokens_seen": 397672448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106920762286861, + "loss": 1.5685, + "theoretical_loss": 4.0429658633556755, + "tokens_seen": 397737984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106760280842527, + "loss": 1.6876, + "theoretical_loss": 4.042891795013729, + "tokens_seen": 397803520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106599799398195, + "loss": 1.6017, + "theoretical_loss": 4.042817742289159, + "tokens_seen": 397869056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106439317953861, + "loss": 1.5494, + "theoretical_loss": 4.042743705176102, + "tokens_seen": 397934592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106278836509529, + "loss": 1.5331, + "theoretical_loss": 4.042669683668698, + "tokens_seen": 398000128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007106118355065195, + "loss": 1.5438, + "theoretical_loss": 4.042595677761087, + "tokens_seen": 398065664 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105957873620863, + "loss": 1.4694, + "theoretical_loss": 4.0425216874474135, + "tokens_seen": 398131200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105797392176529, + "loss": 1.519, + "theoretical_loss": 4.0424477127218275, + "tokens_seen": 398196736 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105636910732197, + "loss": 1.6717, + "theoretical_loss": 4.042373753578479, + "tokens_seen": 398262272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105476429287863, + "loss": 1.6016, + "theoretical_loss": 4.042299810011523, + "tokens_seen": 398327808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105315947843531, + "loss": 1.5246, + "theoretical_loss": 4.042225882015115, + "tokens_seen": 398393344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007105155466399197, + "loss": 1.6573, + "theoretical_loss": 4.0421519695834185, + "tokens_seen": 398458880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104994984954865, + "loss": 1.5101, + "theoretical_loss": 4.042078072710597, + "tokens_seen": 398524416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104834503510531, + "loss": 1.6694, + "theoretical_loss": 4.042004191390816, + "tokens_seen": 398589952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104674022066199, + "loss": 1.5666, + "theoretical_loss": 4.041930325618246, + "tokens_seen": 398655488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104513540621865, + "loss": 1.5017, + "theoretical_loss": 4.041856475387061, + "tokens_seen": 398721024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104353059177533, + "loss": 1.5841, + "theoretical_loss": 4.041782640691437, + "tokens_seen": 398786560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104192577733199, + "loss": 1.4684, + "theoretical_loss": 4.041708821525554, + "tokens_seen": 398852096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007104032096288867, + "loss": 1.7083, + "theoretical_loss": 4.0416350178835945, + "tokens_seen": 398917632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103871614844533, + "loss": 1.7087, + "theoretical_loss": 4.041561229759743, + "tokens_seen": 398983168 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103711133400201, + "loss": 1.489, + "theoretical_loss": 4.0414874571481905, + "tokens_seen": 399048704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103550651955867, + "loss": 1.6111, + "theoretical_loss": 4.041413700043128, + "tokens_seen": 399114240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103390170511535, + "loss": 1.6118, + "theoretical_loss": 4.041339958438751, + "tokens_seen": 399179776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103229689067201, + "loss": 1.6473, + "theoretical_loss": 4.041266232329257, + "tokens_seen": 399245312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007103069207622869, + "loss": 1.5204, + "theoretical_loss": 4.041192521708849, + "tokens_seen": 399310848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102908726178535, + "loss": 1.5711, + "theoretical_loss": 4.041118826571731, + "tokens_seen": 399376384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102748244734203, + "loss": 1.6319, + "theoretical_loss": 4.041045146912111, + "tokens_seen": 399441920 + }, + { + "epoch": 0.12, + "learning_rate": 0.000710258776328987, + "loss": 1.6238, + "theoretical_loss": 4.040971482724199, + "tokens_seen": 399507456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102427281845537, + "loss": 1.5368, + "theoretical_loss": 4.040897834002209, + "tokens_seen": 399572992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102266800401204, + "loss": 1.5823, + "theoretical_loss": 4.040824200740358, + "tokens_seen": 399638528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007102106318956871, + "loss": 1.568, + "theoretical_loss": 4.040750582932868, + "tokens_seen": 399704064 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 154461, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5243334770202637, + "objective/train/theoretical_loss": 4.04067698057396, + "objective/train/tokens_used": 420229600, + "theoretical_loss": 4.04067698057396, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101945837512538, + "loss": 1.5243, + "theoretical_loss": 4.04067698057396, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101785356068205, + "loss": 1.4795, + "theoretical_loss": 4.040603393657862, + "tokens_seen": 399835136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101624874623872, + "loss": 1.6792, + "theoretical_loss": 4.040529822178802, + "tokens_seen": 399900672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101464393179539, + "loss": 1.5578, + "theoretical_loss": 4.040456266131013, + "tokens_seen": 399966208 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101303911735206, + "loss": 1.5527, + "theoretical_loss": 4.0403827255087315, + "tokens_seen": 400031744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007101143430290873, + "loss": 1.5599, + "theoretical_loss": 4.040309200306195, + "tokens_seen": 400097280 + }, + { + "epoch": 0.12, + "learning_rate": 0.000710098294884654, + "loss": 1.6103, + "theoretical_loss": 4.0402356905176475, + "tokens_seen": 400162816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100822467402207, + "loss": 1.7009, + "theoretical_loss": 4.040162196137332, + "tokens_seen": 400228352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100661985957874, + "loss": 1.4815, + "theoretical_loss": 4.040088717159497, + "tokens_seen": 400293888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100501504513541, + "loss": 1.6785, + "theoretical_loss": 4.040015253578394, + "tokens_seen": 400359424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100341023069208, + "loss": 1.544, + "theoretical_loss": 4.039941805388277, + "tokens_seen": 400424960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100180541624875, + "loss": 1.5608, + "theoretical_loss": 4.0398683725834035, + "tokens_seen": 400490496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007100020060180542, + "loss": 1.4589, + "theoretical_loss": 4.039794955158033, + "tokens_seen": 400556032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099859578736209, + "loss": 1.551, + "theoretical_loss": 4.039721553106431, + "tokens_seen": 400621568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099699097291876, + "loss": 1.6083, + "theoretical_loss": 4.039648166422863, + "tokens_seen": 400687104 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099538615847543, + "loss": 1.5064, + "theoretical_loss": 4.039574795101597, + "tokens_seen": 400752640 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709937813440321, + "loss": 1.5414, + "theoretical_loss": 4.039501439136908, + "tokens_seen": 400818176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099217652958877, + "loss": 1.6257, + "theoretical_loss": 4.0394280985230715, + "tokens_seen": 400883712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007099057171514545, + "loss": 1.579, + "theoretical_loss": 4.039354773254365, + "tokens_seen": 400949248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098896690070211, + "loss": 1.579, + "theoretical_loss": 4.0392814633250715, + "tokens_seen": 401014784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098736208625879, + "loss": 1.5996, + "theoretical_loss": 4.039208168729475, + "tokens_seen": 401080320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098575727181545, + "loss": 1.5902, + "theoretical_loss": 4.039134889461866, + "tokens_seen": 401145856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098415245737213, + "loss": 1.5628, + "theoretical_loss": 4.039061625516532, + "tokens_seen": 401211392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098254764292879, + "loss": 1.6383, + "theoretical_loss": 4.038988376887771, + "tokens_seen": 401276928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007098094282848547, + "loss": 1.6232, + "theoretical_loss": 4.038915143569876, + "tokens_seen": 401342464 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097933801404213, + "loss": 1.6, + "theoretical_loss": 4.038841925557151, + "tokens_seen": 401408000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097773319959881, + "loss": 1.626, + "theoretical_loss": 4.038768722843898, + "tokens_seen": 401473536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097612838515547, + "loss": 1.6323, + "theoretical_loss": 4.038695535424422, + "tokens_seen": 401539072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097452357071215, + "loss": 1.601, + "theoretical_loss": 4.038622363293034, + "tokens_seen": 401604608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097291875626881, + "loss": 1.5631, + "theoretical_loss": 4.0385492064440465, + "tokens_seen": 401670144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007097131394182549, + "loss": 1.6532, + "theoretical_loss": 4.038476064871775, + "tokens_seen": 401735680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096970912738215, + "loss": 1.4557, + "theoretical_loss": 4.0384029385705364, + "tokens_seen": 401801216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096810431293883, + "loss": 1.6542, + "theoretical_loss": 4.038329827534654, + "tokens_seen": 401866752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096649949849549, + "loss": 1.4734, + "theoretical_loss": 4.038256731758453, + "tokens_seen": 401932288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096489468405217, + "loss": 1.5659, + "theoretical_loss": 4.038183651236259, + "tokens_seen": 401997824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096328986960883, + "loss": 1.5709, + "theoretical_loss": 4.038110585962404, + "tokens_seen": 402063360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096168505516551, + "loss": 1.5486, + "theoretical_loss": 4.038037535931221, + "tokens_seen": 402128896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007096008024072217, + "loss": 1.6004, + "theoretical_loss": 4.037964501137048, + "tokens_seen": 402194432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095847542627884, + "loss": 1.5491, + "theoretical_loss": 4.037891481574224, + "tokens_seen": 402259968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095687061183551, + "loss": 1.6316, + "theoretical_loss": 4.037818477237091, + "tokens_seen": 402325504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095526579739218, + "loss": 1.6106, + "theoretical_loss": 4.037745488119996, + "tokens_seen": 402391040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095366098294885, + "loss": 1.5746, + "theoretical_loss": 4.037672514217288, + "tokens_seen": 402456576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095205616850552, + "loss": 1.5434, + "theoretical_loss": 4.037599555523318, + "tokens_seen": 402522112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007095045135406219, + "loss": 1.5773, + "theoretical_loss": 4.037526612032441, + "tokens_seen": 402587648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007094884653961886, + "loss": 1.5463, + "theoretical_loss": 4.037453683739015, + "tokens_seen": 402653184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007094724172517553, + "loss": 1.6682, + "theoretical_loss": 4.037380770637402, + "tokens_seen": 402718720 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709456369107322, + "loss": 1.529, + "theoretical_loss": 4.037307872721964, + "tokens_seen": 402784256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007094403209628886, + "loss": 1.5318, + "theoretical_loss": 4.037234989987069, + "tokens_seen": 402849792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007094242728184554, + "loss": 1.5061, + "theoretical_loss": 4.0371621224270875, + "tokens_seen": 402915328 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709408224674022, + "loss": 1.5479, + "theoretical_loss": 4.037089270036391, + "tokens_seen": 402980864 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093921765295888, + "loss": 1.5867, + "theoretical_loss": 4.037016432809357, + "tokens_seen": 403046400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093761283851554, + "loss": 1.4335, + "theoretical_loss": 4.036943610740362, + "tokens_seen": 403111936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093600802407222, + "loss": 1.5164, + "theoretical_loss": 4.036870803823791, + "tokens_seen": 403177472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093440320962888, + "loss": 1.6218, + "theoretical_loss": 4.036798012054027, + "tokens_seen": 403243008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093279839518556, + "loss": 1.472, + "theoretical_loss": 4.036725235425458, + "tokens_seen": 403308544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007093119358074222, + "loss": 1.578, + "theoretical_loss": 4.0366524739324765, + "tokens_seen": 403374080 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709295887662989, + "loss": 1.5596, + "theoretical_loss": 4.036579727569475, + "tokens_seen": 403439616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007092798395185556, + "loss": 1.6448, + "theoretical_loss": 4.036506996330849, + "tokens_seen": 403505152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007092637913741224, + "loss": 1.678, + "theoretical_loss": 4.036434280211001, + "tokens_seen": 403570688 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709247743229689, + "loss": 1.6295, + "theoretical_loss": 4.036361579204334, + "tokens_seen": 403636224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007092316950852558, + "loss": 1.5692, + "theoretical_loss": 4.036288893305251, + "tokens_seen": 403701760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007092156469408224, + "loss": 1.5479, + "theoretical_loss": 4.036216222508163, + "tokens_seen": 403767296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091995987963892, + "loss": 1.4827, + "theoretical_loss": 4.036143566807481, + "tokens_seen": 403832832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091835506519558, + "loss": 1.5575, + "theoretical_loss": 4.03607092619762, + "tokens_seen": 403898368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091675025075226, + "loss": 1.5299, + "theoretical_loss": 4.035998300672999, + "tokens_seen": 403963904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091514543630892, + "loss": 1.5802, + "theoretical_loss": 4.035925690228035, + "tokens_seen": 404029440 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709135406218656, + "loss": 1.583, + "theoretical_loss": 4.035853094857156, + "tokens_seen": 404094976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091193580742226, + "loss": 1.5788, + "theoretical_loss": 4.035780514554787, + "tokens_seen": 404160512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007091033099297894, + "loss": 1.5586, + "theoretical_loss": 4.0357079493153565, + "tokens_seen": 404226048 + }, + { + "epoch": 0.12, + "learning_rate": 0.000709087261785356, + "loss": 1.585, + "theoretical_loss": 4.035635399133298, + "tokens_seen": 404291584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090712136409228, + "loss": 1.5105, + "theoretical_loss": 4.035562864003047, + "tokens_seen": 404357120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090551654964894, + "loss": 1.5765, + "theoretical_loss": 4.035490343919042, + "tokens_seen": 404422656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090391173520562, + "loss": 1.6182, + "theoretical_loss": 4.035417838875725, + "tokens_seen": 404488192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090230692076228, + "loss": 1.5339, + "theoretical_loss": 4.035345348867539, + "tokens_seen": 404553728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007090070210631896, + "loss": 1.5832, + "theoretical_loss": 4.035272873888932, + "tokens_seen": 404619264 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007089909729187562, + "loss": 1.6442, + "theoretical_loss": 4.035200413934355, + "tokens_seen": 404684800 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708974924774323, + "loss": 1.5942, + "theoretical_loss": 4.03512796899826, + "tokens_seen": 404750336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007089588766298896, + "loss": 1.6477, + "theoretical_loss": 4.035055539075104, + "tokens_seen": 404815872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007089428284854564, + "loss": 1.6691, + "theoretical_loss": 4.034983124159346, + "tokens_seen": 404881408 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708926780341023, + "loss": 1.5859, + "theoretical_loss": 4.0349107242454485, + "tokens_seen": 404946944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007089107321965898, + "loss": 1.504, + "theoretical_loss": 4.034838339327876, + "tokens_seen": 405012480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088946840521565, + "loss": 1.6081, + "theoretical_loss": 4.034765969401096, + "tokens_seen": 405078016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088786359077232, + "loss": 1.5384, + "theoretical_loss": 4.03469361445958, + "tokens_seen": 405143552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088625877632899, + "loss": 1.5811, + "theoretical_loss": 4.034621274497802, + "tokens_seen": 405209088 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088465396188566, + "loss": 1.592, + "theoretical_loss": 4.0345489495102385, + "tokens_seen": 405274624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007088304914744233, + "loss": 1.5492, + "theoretical_loss": 4.034476639491368, + "tokens_seen": 405340160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00070881444332999, + "loss": 1.5726, + "theoretical_loss": 4.034404344435676, + "tokens_seen": 405405696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087983951855567, + "loss": 1.5586, + "theoretical_loss": 4.0343320643376455, + "tokens_seen": 405471232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087823470411234, + "loss": 1.4824, + "theoretical_loss": 4.034259799191765, + "tokens_seen": 405536768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087662988966901, + "loss": 1.5647, + "theoretical_loss": 4.034187548992527, + "tokens_seen": 405602304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087502507522568, + "loss": 1.6055, + "theoretical_loss": 4.034115313734426, + "tokens_seen": 405667840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087342026078235, + "loss": 1.5044, + "theoretical_loss": 4.034043093411958, + "tokens_seen": 405733376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007087181544633902, + "loss": 1.5433, + "theoretical_loss": 4.033970888019624, + "tokens_seen": 405798912 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708702106318957, + "loss": 1.6041, + "theoretical_loss": 4.033898697551926, + "tokens_seen": 405864448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086860581745236, + "loss": 1.6329, + "theoretical_loss": 4.0338265220033716, + "tokens_seen": 405929984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086700100300904, + "loss": 1.5586, + "theoretical_loss": 4.0337543613684685, + "tokens_seen": 405995520 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708653961885657, + "loss": 1.5343, + "theoretical_loss": 4.033682215641729, + "tokens_seen": 406061056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086379137412238, + "loss": 1.6231, + "theoretical_loss": 4.033610084817667, + "tokens_seen": 406126592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086218655967904, + "loss": 1.49, + "theoretical_loss": 4.0335379688908, + "tokens_seen": 406192128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007086058174523572, + "loss": 1.5436, + "theoretical_loss": 4.03346586785565, + "tokens_seen": 406257664 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 156877, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.6139613389968872, + "objective/train/theoretical_loss": 4.03339378170674, + "objective/train/tokens_used": 426783200, + "theoretical_loss": 4.03339378170674, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085897693079238, + "loss": 1.614, + "theoretical_loss": 4.03339378170674, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085737211634906, + "loss": 1.5568, + "theoretical_loss": 4.033321710438594, + "tokens_seen": 406388736 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085576730190572, + "loss": 1.573, + "theoretical_loss": 4.033249654045743, + "tokens_seen": 406454272 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708541624874624, + "loss": 1.4725, + "theoretical_loss": 4.033177612522721, + "tokens_seen": 406519808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085255767301906, + "loss": 1.6681, + "theoretical_loss": 4.0331055858640585, + "tokens_seen": 406585344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007085095285857574, + "loss": 1.6627, + "theoretical_loss": 4.033033574064297, + "tokens_seen": 406650880 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708493480441324, + "loss": 1.5569, + "theoretical_loss": 4.032961577117976, + "tokens_seen": 406716416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084774322968908, + "loss": 1.55, + "theoretical_loss": 4.032889595019638, + "tokens_seen": 406781952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084613841524574, + "loss": 1.5956, + "theoretical_loss": 4.032817627763832, + "tokens_seen": 406847488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084453360080242, + "loss": 1.6012, + "theoretical_loss": 4.0327456753451045, + "tokens_seen": 406913024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084292878635908, + "loss": 1.5403, + "theoretical_loss": 4.03267373775801, + "tokens_seen": 406978560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007084132397191576, + "loss": 1.5224, + "theoretical_loss": 4.032601814997102, + "tokens_seen": 407044096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083971915747242, + "loss": 1.5572, + "theoretical_loss": 4.03252990705694, + "tokens_seen": 407109632 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708381143430291, + "loss": 1.6601, + "theoretical_loss": 4.032458013932084, + "tokens_seen": 407175168 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083650952858576, + "loss": 1.5601, + "theoretical_loss": 4.032386135617098, + "tokens_seen": 407240704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083490471414244, + "loss": 1.5083, + "theoretical_loss": 4.032314272106548, + "tokens_seen": 407306240 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708332998996991, + "loss": 1.5387, + "theoretical_loss": 4.0322424233950045, + "tokens_seen": 407371776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083169508525578, + "loss": 1.6333, + "theoretical_loss": 4.032170589477039, + "tokens_seen": 407437312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007083009027081244, + "loss": 1.6525, + "theoretical_loss": 4.032098770347228, + "tokens_seen": 407502848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082848545636912, + "loss": 1.5364, + "theoretical_loss": 4.0320269660001475, + "tokens_seen": 407568384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082688064192578, + "loss": 1.4939, + "theoretical_loss": 4.0319551764303805, + "tokens_seen": 407633920 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082527582748246, + "loss": 1.6321, + "theoretical_loss": 4.03188340163251, + "tokens_seen": 407699456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082367101303912, + "loss": 1.5849, + "theoretical_loss": 4.031811641601122, + "tokens_seen": 407764992 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708220661985958, + "loss": 1.5032, + "theoretical_loss": 4.031739896330807, + "tokens_seen": 407830528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007082046138415246, + "loss": 1.538, + "theoretical_loss": 4.031668165816156, + "tokens_seen": 407896064 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081885656970914, + "loss": 1.5937, + "theoretical_loss": 4.031596450051766, + "tokens_seen": 407961600 + }, + { + "epoch": 0.12, + "learning_rate": 0.000708172517552658, + "loss": 1.6138, + "theoretical_loss": 4.031524749032234, + "tokens_seen": 408027136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081564694082247, + "loss": 1.6729, + "theoretical_loss": 4.031453062752161, + "tokens_seen": 408092672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081404212637914, + "loss": 1.5282, + "theoretical_loss": 4.031381391206151, + "tokens_seen": 408158208 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081243731193581, + "loss": 1.5871, + "theoretical_loss": 4.03130973438881, + "tokens_seen": 408223744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007081083249749248, + "loss": 1.6104, + "theoretical_loss": 4.031238092294748, + "tokens_seen": 408289280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080922768304915, + "loss": 1.5787, + "theoretical_loss": 4.031166464918576, + "tokens_seen": 408354816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080762286860582, + "loss": 1.5776, + "theoretical_loss": 4.0310948522549115, + "tokens_seen": 408420352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080601805416249, + "loss": 1.5568, + "theoretical_loss": 4.031023254298371, + "tokens_seen": 408485888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080441323971915, + "loss": 1.5924, + "theoretical_loss": 4.030951671043574, + "tokens_seen": 408551424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080280842527583, + "loss": 1.5248, + "theoretical_loss": 4.030880102485147, + "tokens_seen": 408616960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007080120361083249, + "loss": 1.5255, + "theoretical_loss": 4.030808548617714, + "tokens_seen": 408682496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079959879638917, + "loss": 1.5901, + "theoretical_loss": 4.030737009435906, + "tokens_seen": 408748032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079799398194583, + "loss": 1.5547, + "theoretical_loss": 4.030665484934353, + "tokens_seen": 408813568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079638916750251, + "loss": 1.4937, + "theoretical_loss": 4.030593975107692, + "tokens_seen": 408879104 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079478435305917, + "loss": 1.5198, + "theoretical_loss": 4.03052247995056, + "tokens_seen": 408944640 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079317953861585, + "loss": 1.601, + "theoretical_loss": 4.030450999457596, + "tokens_seen": 409010176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007079157472417251, + "loss": 1.5058, + "theoretical_loss": 4.030379533623445, + "tokens_seen": 409075712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078996990972919, + "loss": 1.6373, + "theoretical_loss": 4.030308082442753, + "tokens_seen": 409141248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078836509528585, + "loss": 1.556, + "theoretical_loss": 4.030236645910169, + "tokens_seen": 409206784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078676028084253, + "loss": 1.5949, + "theoretical_loss": 4.030165224020345, + "tokens_seen": 409272320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078515546639919, + "loss": 1.6309, + "theoretical_loss": 4.030093816767934, + "tokens_seen": 409337856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078355065195587, + "loss": 1.6374, + "theoretical_loss": 4.030022424147595, + "tokens_seen": 409403392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078194583751253, + "loss": 1.4966, + "theoretical_loss": 4.029951046153988, + "tokens_seen": 409468928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007078034102306921, + "loss": 1.6398, + "theoretical_loss": 4.029879682781776, + "tokens_seen": 409534464 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077873620862587, + "loss": 1.6454, + "theoretical_loss": 4.029808334025624, + "tokens_seen": 409600000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077713139418255, + "loss": 1.5449, + "theoretical_loss": 4.0297369998802015, + "tokens_seen": 409665536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077552657973921, + "loss": 1.4696, + "theoretical_loss": 4.0296656803401785, + "tokens_seen": 409731072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077392176529589, + "loss": 1.505, + "theoretical_loss": 4.029594375400231, + "tokens_seen": 409796608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077231695085255, + "loss": 1.5956, + "theoretical_loss": 4.029523085055036, + "tokens_seen": 409862144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007077071213640923, + "loss": 1.4953, + "theoretical_loss": 4.029451809299273, + "tokens_seen": 409927680 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076910732196589, + "loss": 1.4677, + "theoretical_loss": 4.029380548127622, + "tokens_seen": 409993216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076750250752257, + "loss": 1.557, + "theoretical_loss": 4.029309301534772, + "tokens_seen": 410058752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076589769307923, + "loss": 1.509, + "theoretical_loss": 4.02923806951541, + "tokens_seen": 410124288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076429287863591, + "loss": 1.5434, + "theoretical_loss": 4.029166852064225, + "tokens_seen": 410189824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076268806419257, + "loss": 1.5171, + "theoretical_loss": 4.029095649175913, + "tokens_seen": 410255360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007076108324974925, + "loss": 1.5361, + "theoretical_loss": 4.02902446084517, + "tokens_seen": 410320896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075947843530591, + "loss": 1.6106, + "theoretical_loss": 4.028953287066695, + "tokens_seen": 410386432 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075787362086259, + "loss": 1.5217, + "theoretical_loss": 4.02888212783519, + "tokens_seen": 410451968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075626880641925, + "loss": 1.61, + "theoretical_loss": 4.028810983145361, + "tokens_seen": 410517504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075466399197593, + "loss": 1.4772, + "theoretical_loss": 4.0287398529919125, + "tokens_seen": 410583040 + }, + { + "epoch": 0.12, + "learning_rate": 0.000707530591775326, + "loss": 1.4708, + "theoretical_loss": 4.028668737369558, + "tokens_seen": 410648576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007075145436308927, + "loss": 1.5223, + "theoretical_loss": 4.028597636273009, + "tokens_seen": 410714112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074984954864595, + "loss": 1.5519, + "theoretical_loss": 4.028526549696982, + "tokens_seen": 410779648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074824473420261, + "loss": 1.5352, + "theoretical_loss": 4.0284554776361965, + "tokens_seen": 410845184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074663991975929, + "loss": 1.5438, + "theoretical_loss": 4.028384420085372, + "tokens_seen": 410910720 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074503510531595, + "loss": 1.4767, + "theoretical_loss": 4.028313377039234, + "tokens_seen": 410976256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074343029087263, + "loss": 1.5331, + "theoretical_loss": 4.028242348492509, + "tokens_seen": 411041792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074182547642929, + "loss": 1.4952, + "theoretical_loss": 4.028171334439927, + "tokens_seen": 411107328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007074022066198597, + "loss": 1.6354, + "theoretical_loss": 4.02810033487622, + "tokens_seen": 411172864 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073861584754263, + "loss": 1.4612, + "theoretical_loss": 4.028029349796125, + "tokens_seen": 411238400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073701103309931, + "loss": 1.6012, + "theoretical_loss": 4.027958379194376, + "tokens_seen": 411303936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073540621865597, + "loss": 1.5418, + "theoretical_loss": 4.027887423065717, + "tokens_seen": 411369472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073380140421265, + "loss": 1.6623, + "theoretical_loss": 4.027816481404892, + "tokens_seen": 411435008 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073219658976931, + "loss": 1.5026, + "theoretical_loss": 4.0277455542066445, + "tokens_seen": 411500544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007073059177532599, + "loss": 1.5137, + "theoretical_loss": 4.027674641465725, + "tokens_seen": 411566080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072898696088265, + "loss": 1.5731, + "theoretical_loss": 4.027603743176885, + "tokens_seen": 411631616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072738214643933, + "loss": 1.5084, + "theoretical_loss": 4.02753285933488, + "tokens_seen": 411697152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072577733199599, + "loss": 1.5717, + "theoretical_loss": 4.027461989934466, + "tokens_seen": 411762688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072417251755267, + "loss": 1.4779, + "theoretical_loss": 4.027391134970402, + "tokens_seen": 411828224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072256770310933, + "loss": 1.4737, + "theoretical_loss": 4.027320294437452, + "tokens_seen": 411893760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007072096288866601, + "loss": 1.5757, + "theoretical_loss": 4.027249468330381, + "tokens_seen": 411959296 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071935807422267, + "loss": 1.4463, + "theoretical_loss": 4.027178656643957, + "tokens_seen": 412024832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071775325977935, + "loss": 1.5847, + "theoretical_loss": 4.0271078593729515, + "tokens_seen": 412090368 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071614844533601, + "loss": 1.4867, + "theoretical_loss": 4.027037076512137, + "tokens_seen": 412155904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071454363089269, + "loss": 1.677, + "theoretical_loss": 4.026966308056291, + "tokens_seen": 412221440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071293881644935, + "loss": 1.5667, + "theoretical_loss": 4.026895554000192, + "tokens_seen": 412286976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007071133400200603, + "loss": 1.6299, + "theoretical_loss": 4.026824814338621, + "tokens_seen": 412352512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007070972918756269, + "loss": 1.6339, + "theoretical_loss": 4.026754089066364, + "tokens_seen": 412418048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0007070812437311937, + "loss": 1.544, + "theoretical_loss": 4.026683378178206, + "tokens_seen": 412483584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070651955867603, + "loss": 1.6636, + "theoretical_loss": 4.026612681668939, + "tokens_seen": 412549120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070491474423271, + "loss": 1.6522, + "theoretical_loss": 4.026541999533355, + "tokens_seen": 412614656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070330992978937, + "loss": 1.5452, + "theoretical_loss": 4.026471331766249, + "tokens_seen": 412680192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070170511534605, + "loss": 1.5873, + "theoretical_loss": 4.026400678362418, + "tokens_seen": 412745728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007070010030090271, + "loss": 1.4774, + "theoretical_loss": 4.026330039316665, + "tokens_seen": 412811264 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 159288, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4372012615203857, + "objective/train/theoretical_loss": 4.026259414623792, + "objective/train/tokens_used": 433336800, + "theoretical_loss": 4.026259414623792, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069849548645939, + "loss": 1.4372, + "theoretical_loss": 4.026259414623792, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069689067201605, + "loss": 1.5881, + "theoretical_loss": 4.026188804278605, + "tokens_seen": 412942336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069528585757273, + "loss": 1.5112, + "theoretical_loss": 4.026118208275914, + "tokens_seen": 413007872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069368104312939, + "loss": 1.5162, + "theoretical_loss": 4.02604762661053, + "tokens_seen": 413073408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069207622868607, + "loss": 1.4671, + "theoretical_loss": 4.025977059277267, + "tokens_seen": 413138944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007069047141424273, + "loss": 1.5576, + "theoretical_loss": 4.025906506270941, + "tokens_seen": 413204480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068886659979941, + "loss": 1.5743, + "theoretical_loss": 4.0258359675863735, + "tokens_seen": 413270016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068726178535607, + "loss": 1.5947, + "theoretical_loss": 4.025765443218386, + "tokens_seen": 413335552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068565697091275, + "loss": 1.55, + "theoretical_loss": 4.025694933161804, + "tokens_seen": 413401088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068405215646941, + "loss": 1.5963, + "theoretical_loss": 4.025624437411455, + "tokens_seen": 413466624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068244734202609, + "loss": 1.5899, + "theoretical_loss": 4.0255539559621685, + "tokens_seen": 413532160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007068084252758275, + "loss": 1.5798, + "theoretical_loss": 4.025483488808779, + "tokens_seen": 413597696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067923771313943, + "loss": 1.5636, + "theoretical_loss": 4.025413035946121, + "tokens_seen": 413663232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067763289869609, + "loss": 1.4993, + "theoretical_loss": 4.025342597369034, + "tokens_seen": 413728768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067602808425276, + "loss": 1.5002, + "theoretical_loss": 4.025272173072359, + "tokens_seen": 413794304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067442326980943, + "loss": 1.6247, + "theoretical_loss": 4.025201763050939, + "tokens_seen": 413859840 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706728184553661, + "loss": 1.4499, + "theoretical_loss": 4.025131367299621, + "tokens_seen": 413925376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007067121364092277, + "loss": 1.5985, + "theoretical_loss": 4.025060985813255, + "tokens_seen": 413990912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066960882647944, + "loss": 1.5398, + "theoretical_loss": 4.024990618586692, + "tokens_seen": 414056448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066800401203611, + "loss": 1.5233, + "theoretical_loss": 4.024920265614787, + "tokens_seen": 414121984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066639919759278, + "loss": 1.6466, + "theoretical_loss": 4.024849926892396, + "tokens_seen": 414187520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066479438314945, + "loss": 1.6483, + "theoretical_loss": 4.02477960241438, + "tokens_seen": 414253056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066318956870612, + "loss": 1.5632, + "theoretical_loss": 4.024709292175602, + "tokens_seen": 414318592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007066158475426278, + "loss": 1.5182, + "theoretical_loss": 4.024638996170927, + "tokens_seen": 414384128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065997993981946, + "loss": 1.6799, + "theoretical_loss": 4.024568714395221, + "tokens_seen": 414449664 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065837512537612, + "loss": 1.5202, + "theoretical_loss": 4.0244984468433564, + "tokens_seen": 414515200 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706567703109328, + "loss": 1.5717, + "theoretical_loss": 4.0244281935102055, + "tokens_seen": 414580736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065516549648946, + "loss": 1.568, + "theoretical_loss": 4.024357954390645, + "tokens_seen": 414646272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065356068204614, + "loss": 1.5404, + "theoretical_loss": 4.0242877294795525, + "tokens_seen": 414711808 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706519558676028, + "loss": 1.591, + "theoretical_loss": 4.0242175187718106, + "tokens_seen": 414777344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007065035105315948, + "loss": 1.5315, + "theoretical_loss": 4.024147322262301, + "tokens_seen": 414842880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064874623871614, + "loss": 1.5635, + "theoretical_loss": 4.024077139945912, + "tokens_seen": 414908416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064714142427282, + "loss": 1.6063, + "theoretical_loss": 4.024006971817532, + "tokens_seen": 414973952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064553660982948, + "loss": 1.5373, + "theoretical_loss": 4.023936817872053, + "tokens_seen": 415039488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064393179538616, + "loss": 1.6443, + "theoretical_loss": 4.023866678104368, + "tokens_seen": 415105024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007064232698094282, + "loss": 1.5737, + "theoretical_loss": 4.023796552509376, + "tokens_seen": 415170560 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706407221664995, + "loss": 1.556, + "theoretical_loss": 4.023726441081976, + "tokens_seen": 415236096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063911735205616, + "loss": 1.6566, + "theoretical_loss": 4.02365634381707, + "tokens_seen": 415301632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063751253761284, + "loss": 1.4228, + "theoretical_loss": 4.023586260709564, + "tokens_seen": 415367168 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706359077231695, + "loss": 1.5799, + "theoretical_loss": 4.023516191754363, + "tokens_seen": 415432704 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063430290872618, + "loss": 1.6765, + "theoretical_loss": 4.02344613694638, + "tokens_seen": 415498240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063269809428284, + "loss": 1.5918, + "theoretical_loss": 4.023376096280527, + "tokens_seen": 415563776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007063109327983952, + "loss": 1.5878, + "theoretical_loss": 4.02330606975172, + "tokens_seen": 415629312 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706294884653962, + "loss": 1.5748, + "theoretical_loss": 4.023236057354875, + "tokens_seen": 415694848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007062788365095286, + "loss": 1.5453, + "theoretical_loss": 4.023166059084916, + "tokens_seen": 415760384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007062627883650954, + "loss": 1.5237, + "theoretical_loss": 4.023096074936764, + "tokens_seen": 415825920 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706246740220662, + "loss": 1.5419, + "theoretical_loss": 4.0230261049053455, + "tokens_seen": 415891456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007062306920762288, + "loss": 1.5327, + "theoretical_loss": 4.022956148985589, + "tokens_seen": 415956992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007062146439317954, + "loss": 1.4798, + "theoretical_loss": 4.0228862071724265, + "tokens_seen": 416022528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061985957873622, + "loss": 1.6593, + "theoretical_loss": 4.022816279460791, + "tokens_seen": 416088064 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061825476429288, + "loss": 1.5114, + "theoretical_loss": 4.02274636584562, + "tokens_seen": 416153600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061664994984956, + "loss": 1.5893, + "theoretical_loss": 4.022676466321853, + "tokens_seen": 416219136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061504513540622, + "loss": 1.6279, + "theoretical_loss": 4.022606580884429, + "tokens_seen": 416284672 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706134403209629, + "loss": 1.5497, + "theoretical_loss": 4.0225367095282945, + "tokens_seen": 416350208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061183550651956, + "loss": 1.5376, + "theoretical_loss": 4.022466852248396, + "tokens_seen": 416415744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007061023069207624, + "loss": 1.6056, + "theoretical_loss": 4.0223970090396834, + "tokens_seen": 416481280 + }, + { + "epoch": 0.13, + "learning_rate": 0.000706086258776329, + "loss": 1.5805, + "theoretical_loss": 4.0223271798971085, + "tokens_seen": 416546816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060702106318958, + "loss": 1.525, + "theoretical_loss": 4.0222573648156255, + "tokens_seen": 416612352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060541624874624, + "loss": 1.5295, + "theoretical_loss": 4.022187563790193, + "tokens_seen": 416677888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060381143430292, + "loss": 1.556, + "theoretical_loss": 4.02211777681577, + "tokens_seen": 416743424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060220661985958, + "loss": 1.5227, + "theoretical_loss": 4.022048003887319, + "tokens_seen": 416808960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007060060180541626, + "loss": 1.4419, + "theoretical_loss": 4.021978244999806, + "tokens_seen": 416874496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007059899699097292, + "loss": 1.4929, + "theoretical_loss": 4.021908500148198, + "tokens_seen": 416940032 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705973921765296, + "loss": 1.5446, + "theoretical_loss": 4.0218387693274655, + "tokens_seen": 417005568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007059578736208626, + "loss": 1.6637, + "theoretical_loss": 4.021769052532582, + "tokens_seen": 417071104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007059418254764294, + "loss": 1.429, + "theoretical_loss": 4.0216993497585225, + "tokens_seen": 417136640 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705925777331996, + "loss": 1.562, + "theoretical_loss": 4.021629661000265, + "tokens_seen": 417202176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007059097291875628, + "loss": 1.5289, + "theoretical_loss": 4.02155998625279, + "tokens_seen": 417267712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058936810431294, + "loss": 1.673, + "theoretical_loss": 4.021490325511082, + "tokens_seen": 417333248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058776328986962, + "loss": 1.5095, + "theoretical_loss": 4.021420678770125, + "tokens_seen": 417398784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058615847542628, + "loss": 1.5587, + "theoretical_loss": 4.02135104602491, + "tokens_seen": 417464320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058455366098296, + "loss": 1.5996, + "theoretical_loss": 4.021281427270425, + "tokens_seen": 417529856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007058294884653962, + "loss": 1.5527, + "theoretical_loss": 4.021211822501666, + "tokens_seen": 417595392 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705813440320963, + "loss": 1.5381, + "theoretical_loss": 4.021142231713629, + "tokens_seen": 417660928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057973921765296, + "loss": 1.4925, + "theoretical_loss": 4.021072654901312, + "tokens_seen": 417726464 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057813440320964, + "loss": 1.6603, + "theoretical_loss": 4.021003092059716, + "tokens_seen": 417792000 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705765295887663, + "loss": 1.4737, + "theoretical_loss": 4.020933543183845, + "tokens_seen": 417857536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057492477432298, + "loss": 1.5272, + "theoretical_loss": 4.020864008268706, + "tokens_seen": 417923072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057331995987964, + "loss": 1.5189, + "theoretical_loss": 4.020794487309308, + "tokens_seen": 417988608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057171514543632, + "loss": 1.5304, + "theoretical_loss": 4.020724980300663, + "tokens_seen": 418054144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007057011033099298, + "loss": 1.5501, + "theoretical_loss": 4.020655487237785, + "tokens_seen": 418119680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007056850551654966, + "loss": 1.6046, + "theoretical_loss": 4.02058600811569, + "tokens_seen": 418185216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007056690070210632, + "loss": 1.6652, + "theoretical_loss": 4.020516542929398, + "tokens_seen": 418250752 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070565295887663, + "loss": 1.577, + "theoretical_loss": 4.0204470916739306, + "tokens_seen": 418316288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007056369107321966, + "loss": 1.5276, + "theoretical_loss": 4.020377654344312, + "tokens_seen": 418381824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007056208625877634, + "loss": 1.5332, + "theoretical_loss": 4.020308230935569, + "tokens_seen": 418447360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070560481444333, + "loss": 1.6317, + "theoretical_loss": 4.020238821442733, + "tokens_seen": 418512896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055887662988968, + "loss": 1.5705, + "theoretical_loss": 4.020169425860834, + "tokens_seen": 418578432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055727181544634, + "loss": 1.5996, + "theoretical_loss": 4.0201000441849075, + "tokens_seen": 418643968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055566700100302, + "loss": 1.6797, + "theoretical_loss": 4.0200306764099905, + "tokens_seen": 418709504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055406218655968, + "loss": 1.5467, + "theoretical_loss": 4.019961322531122, + "tokens_seen": 418775040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055245737211636, + "loss": 1.6259, + "theoretical_loss": 4.019891982543346, + "tokens_seen": 418840576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007055085255767302, + "loss": 1.4581, + "theoretical_loss": 4.019822656441706, + "tokens_seen": 418906112 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705492477432297, + "loss": 1.5658, + "theoretical_loss": 4.01975334422125, + "tokens_seen": 418971648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007054764292878636, + "loss": 1.595, + "theoretical_loss": 4.019684045877027, + "tokens_seen": 419037184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007054603811434304, + "loss": 1.551, + "theoretical_loss": 4.019614761404091, + "tokens_seen": 419102720 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705444332998997, + "loss": 1.5646, + "theoretical_loss": 4.019545490797495, + "tokens_seen": 419168256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007054282848545638, + "loss": 1.5488, + "theoretical_loss": 4.0194762340522985, + "tokens_seen": 419233792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007054122367101304, + "loss": 1.5734, + "theoretical_loss": 4.01940699116356, + "tokens_seen": 419299328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053961885656972, + "loss": 1.6452, + "theoretical_loss": 4.019337762126343, + "tokens_seen": 419364864 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 162117, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.646161675453186, + "objective/train/theoretical_loss": 4.019268546935713, + "objective/train/tokens_used": 439890400, + "theoretical_loss": 4.019268546935713, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053801404212638, + "loss": 1.6462, + "theoretical_loss": 4.019268546935713, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053640922768305, + "loss": 1.5112, + "theoretical_loss": 4.0191993455867365, + "tokens_seen": 419495936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053480441323972, + "loss": 1.5107, + "theoretical_loss": 4.019130158074484, + "tokens_seen": 419561472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053319959879639, + "loss": 1.6006, + "theoretical_loss": 4.019060984394028, + "tokens_seen": 419627008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007053159478435306, + "loss": 1.6259, + "theoretical_loss": 4.018991824540445, + "tokens_seen": 419692544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052998996990973, + "loss": 1.5075, + "theoretical_loss": 4.018922678508812, + "tokens_seen": 419758080 + }, + { + "epoch": 0.13, + "learning_rate": 0.000705283851554664, + "loss": 1.6408, + "theoretical_loss": 4.018853546294208, + "tokens_seen": 419823616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052678034102307, + "loss": 1.5506, + "theoretical_loss": 4.018784427891718, + "tokens_seen": 419889152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052517552657974, + "loss": 1.6745, + "theoretical_loss": 4.018715323296425, + "tokens_seen": 419954688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052357071213641, + "loss": 1.583, + "theoretical_loss": 4.018646232503419, + "tokens_seen": 420020224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052196589769307, + "loss": 1.6676, + "theoretical_loss": 4.0185771555077885, + "tokens_seen": 420085760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007052036108324975, + "loss": 1.5991, + "theoretical_loss": 4.018508092304628, + "tokens_seen": 420151296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051875626880641, + "loss": 1.6253, + "theoretical_loss": 4.018439042889032, + "tokens_seen": 420216832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051715145436309, + "loss": 1.532, + "theoretical_loss": 4.018370007256098, + "tokens_seen": 420282368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051554663991975, + "loss": 1.516, + "theoretical_loss": 4.018300985400927, + "tokens_seen": 420347904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051394182547643, + "loss": 1.5455, + "theoretical_loss": 4.018231977318622, + "tokens_seen": 420413440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051233701103309, + "loss": 1.5668, + "theoretical_loss": 4.018162983004287, + "tokens_seen": 420478976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007051073219658977, + "loss": 1.6067, + "theoretical_loss": 4.018094002453031, + "tokens_seen": 420544512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050912738214644, + "loss": 1.5067, + "theoretical_loss": 4.018025035659965, + "tokens_seen": 420610048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050752256770311, + "loss": 1.6879, + "theoretical_loss": 4.017956082620201, + "tokens_seen": 420675584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050591775325978, + "loss": 1.5074, + "theoretical_loss": 4.0178871433288545, + "tokens_seen": 420741120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050431293881645, + "loss": 1.5969, + "theoretical_loss": 4.017818217781044, + "tokens_seen": 420806656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050270812437312, + "loss": 1.4653, + "theoretical_loss": 4.017749305971889, + "tokens_seen": 420872192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007050110330992979, + "loss": 1.6144, + "theoretical_loss": 4.0176804078965125, + "tokens_seen": 420937728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049949849548647, + "loss": 1.571, + "theoretical_loss": 4.017611523550039, + "tokens_seen": 421003264 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049789368104313, + "loss": 1.478, + "theoretical_loss": 4.017542652927599, + "tokens_seen": 421068800 + }, + { + "epoch": 0.13, + "learning_rate": 0.000704962888665998, + "loss": 1.5921, + "theoretical_loss": 4.01747379602432, + "tokens_seen": 421134336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049468405215647, + "loss": 1.627, + "theoretical_loss": 4.017404952835336, + "tokens_seen": 421199872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049307923771315, + "loss": 1.5851, + "theoretical_loss": 4.017336123355783, + "tokens_seen": 421265408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007049147442326981, + "loss": 1.5435, + "theoretical_loss": 4.017267307580798, + "tokens_seen": 421330944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048986960882649, + "loss": 1.6383, + "theoretical_loss": 4.017198505505521, + "tokens_seen": 421396480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048826479438315, + "loss": 1.6266, + "theoretical_loss": 4.017129717125094, + "tokens_seen": 421462016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048665997993983, + "loss": 1.5394, + "theoretical_loss": 4.017060942434664, + "tokens_seen": 421527552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048505516549649, + "loss": 1.4649, + "theoretical_loss": 4.016992181429377, + "tokens_seen": 421593088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048345035105317, + "loss": 1.6191, + "theoretical_loss": 4.016923434104386, + "tokens_seen": 421658624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048184553660983, + "loss": 1.605, + "theoretical_loss": 4.01685470045484, + "tokens_seen": 421724160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007048024072216651, + "loss": 1.582, + "theoretical_loss": 4.0167859804758965, + "tokens_seen": 421789696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047863590772317, + "loss": 1.6607, + "theoretical_loss": 4.016717274162712, + "tokens_seen": 421855232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047703109327985, + "loss": 1.5215, + "theoretical_loss": 4.016648581510448, + "tokens_seen": 421920768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047542627883651, + "loss": 1.5648, + "theoretical_loss": 4.016579902514264, + "tokens_seen": 421986304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047382146439319, + "loss": 1.5774, + "theoretical_loss": 4.016511237169329, + "tokens_seen": 422051840 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047221664994985, + "loss": 1.5083, + "theoretical_loss": 4.016442585470807, + "tokens_seen": 422117376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007047061183550653, + "loss": 1.6081, + "theoretical_loss": 4.01637394741387, + "tokens_seen": 422182912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046900702106319, + "loss": 1.5452, + "theoretical_loss": 4.016305322993689, + "tokens_seen": 422248448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046740220661987, + "loss": 1.5755, + "theoretical_loss": 4.016236712205439, + "tokens_seen": 422313984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046579739217653, + "loss": 1.5275, + "theoretical_loss": 4.016168115044298, + "tokens_seen": 422379520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046419257773321, + "loss": 1.543, + "theoretical_loss": 4.016099531505445, + "tokens_seen": 422445056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046258776328987, + "loss": 1.5375, + "theoretical_loss": 4.016030961584063, + "tokens_seen": 422510592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007046098294884655, + "loss": 1.4548, + "theoretical_loss": 4.015962405275336, + "tokens_seen": 422576128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045937813440321, + "loss": 1.4848, + "theoretical_loss": 4.015893862574451, + "tokens_seen": 422641664 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045777331995989, + "loss": 1.5465, + "theoretical_loss": 4.015825333476598, + "tokens_seen": 422707200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045616850551655, + "loss": 1.5631, + "theoretical_loss": 4.0157568179769685, + "tokens_seen": 422772736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045456369107323, + "loss": 1.5731, + "theoretical_loss": 4.015688316070757, + "tokens_seen": 422838272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045295887662989, + "loss": 1.5612, + "theoretical_loss": 4.01561982775316, + "tokens_seen": 422903808 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007045135406218657, + "loss": 1.4995, + "theoretical_loss": 4.015551353019378, + "tokens_seen": 422969344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044974924774323, + "loss": 1.6303, + "theoretical_loss": 4.015482891864612, + "tokens_seen": 423034880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044814443329991, + "loss": 1.5179, + "theoretical_loss": 4.015414444284065, + "tokens_seen": 423100416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044653961885657, + "loss": 1.6042, + "theoretical_loss": 4.015346010272946, + "tokens_seen": 423165952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044493480441325, + "loss": 1.5085, + "theoretical_loss": 4.0152775898264625, + "tokens_seen": 423231488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044332998996991, + "loss": 1.5327, + "theoretical_loss": 4.0152091829398255, + "tokens_seen": 423297024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044172517552659, + "loss": 1.4237, + "theoretical_loss": 4.0151407896082505, + "tokens_seen": 423362560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007044012036108325, + "loss": 1.5754, + "theoretical_loss": 4.015072409826953, + "tokens_seen": 423428096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043851554663993, + "loss": 1.5892, + "theoretical_loss": 4.015004043591151, + "tokens_seen": 423493632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043691073219659, + "loss": 1.5286, + "theoretical_loss": 4.014935690896067, + "tokens_seen": 423559168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043530591775327, + "loss": 1.5252, + "theoretical_loss": 4.014867351736924, + "tokens_seen": 423624704 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043370110330993, + "loss": 1.6296, + "theoretical_loss": 4.014799026108948, + "tokens_seen": 423690240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043209628886661, + "loss": 1.5628, + "theoretical_loss": 4.014730714007367, + "tokens_seen": 423755776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007043049147442327, + "loss": 1.6137, + "theoretical_loss": 4.014662415427413, + "tokens_seen": 423821312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042888665997995, + "loss": 1.56, + "theoretical_loss": 4.014594130364319, + "tokens_seen": 423886848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042728184553661, + "loss": 1.5045, + "theoretical_loss": 4.01452585881332, + "tokens_seen": 423952384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042567703109329, + "loss": 1.5957, + "theoretical_loss": 4.014457600769655, + "tokens_seen": 424017920 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042407221664995, + "loss": 1.4292, + "theoretical_loss": 4.0143893562285635, + "tokens_seen": 424083456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042246740220663, + "loss": 1.6194, + "theoretical_loss": 4.014321125185289, + "tokens_seen": 424148992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007042086258776329, + "loss": 1.4971, + "theoretical_loss": 4.014252907635077, + "tokens_seen": 424214528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041925777331997, + "loss": 1.4971, + "theoretical_loss": 4.014184703573176, + "tokens_seen": 424280064 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041765295887663, + "loss": 1.4143, + "theoretical_loss": 4.014116512994835, + "tokens_seen": 424345600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041604814443331, + "loss": 1.5918, + "theoretical_loss": 4.014048335895307, + "tokens_seen": 424411136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041444332998997, + "loss": 1.643, + "theoretical_loss": 4.0139801722698465, + "tokens_seen": 424476672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041283851554665, + "loss": 1.6272, + "theoretical_loss": 4.013912022113711, + "tokens_seen": 424542208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007041123370110331, + "loss": 1.4814, + "theoretical_loss": 4.013843885422162, + "tokens_seen": 424607744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040962888665999, + "loss": 1.5679, + "theoretical_loss": 4.01377576219046, + "tokens_seen": 424673280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040802407221665, + "loss": 1.6492, + "theoretical_loss": 4.01370765241387, + "tokens_seen": 424738816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040641925777333, + "loss": 1.5711, + "theoretical_loss": 4.013639556087659, + "tokens_seen": 424804352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040481444332999, + "loss": 1.5728, + "theoretical_loss": 4.0135714732070955, + "tokens_seen": 424869888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040320962888667, + "loss": 1.4865, + "theoretical_loss": 4.013503403767453, + "tokens_seen": 424935424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040160481444333, + "loss": 1.5567, + "theoretical_loss": 4.013435347764005, + "tokens_seen": 425000960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007040000000000001, + "loss": 1.4682, + "theoretical_loss": 4.013367305192027, + "tokens_seen": 425066496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039839518555667, + "loss": 1.5965, + "theoretical_loss": 4.0132992760468, + "tokens_seen": 425132032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039679037111334, + "loss": 1.4913, + "theoretical_loss": 4.013231260323603, + "tokens_seen": 425197568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039518555667001, + "loss": 1.4986, + "theoretical_loss": 4.0131632580177214, + "tokens_seen": 425263104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039358074222668, + "loss": 1.5391, + "theoretical_loss": 4.01309526912444, + "tokens_seen": 425328640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039197592778335, + "loss": 1.482, + "theoretical_loss": 4.013027293639049, + "tokens_seen": 425394176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007039037111334002, + "loss": 1.5102, + "theoretical_loss": 4.0129593315568375, + "tokens_seen": 425459712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038876629889669, + "loss": 1.6137, + "theoretical_loss": 4.0128913828731, + "tokens_seen": 425525248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038716148445336, + "loss": 1.5185, + "theoretical_loss": 4.012823447583132, + "tokens_seen": 425590784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038555667001003, + "loss": 1.4315, + "theoretical_loss": 4.012755525682231, + "tokens_seen": 425656320 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703839518555667, + "loss": 1.5509, + "theoretical_loss": 4.012687617165698, + "tokens_seen": 425721856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038234704112337, + "loss": 1.5081, + "theoretical_loss": 4.012619722028835, + "tokens_seen": 425787392 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007038074222668004, + "loss": 1.4857, + "theoretical_loss": 4.012551840266948, + "tokens_seen": 425852928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037913741223671, + "loss": 1.5158, + "theoretical_loss": 4.0124839718753424, + "tokens_seen": 425918464 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 164536, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5723100900650024, + "objective/train/theoretical_loss": 4.01241611684933, + "objective/train/tokens_used": 446444000, + "theoretical_loss": 4.01241611684933, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037753259779338, + "loss": 1.5723, + "theoretical_loss": 4.01241611684933, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037592778335005, + "loss": 1.6059, + "theoretical_loss": 4.012348275184223, + "tokens_seen": 426049536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037432296890672, + "loss": 1.5708, + "theoretical_loss": 4.012280446875336, + "tokens_seen": 426115072 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703727181544634, + "loss": 1.5438, + "theoretical_loss": 4.012212631917986, + "tokens_seen": 426180608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007037111334002006, + "loss": 1.6134, + "theoretical_loss": 4.012144830307491, + "tokens_seen": 426246144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036950852557673, + "loss": 1.5824, + "theoretical_loss": 4.012077042039174, + "tokens_seen": 426311680 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703679037111334, + "loss": 1.5548, + "theoretical_loss": 4.012009267108358, + "tokens_seen": 426377216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036629889669007, + "loss": 1.604, + "theoretical_loss": 4.01194150551037, + "tokens_seen": 426442752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036469408224674, + "loss": 1.6663, + "theoretical_loss": 4.011873757240539, + "tokens_seen": 426508288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036308926780342, + "loss": 1.4639, + "theoretical_loss": 4.011806022294196, + "tokens_seen": 426573824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007036148445336008, + "loss": 1.4404, + "theoretical_loss": 4.011738300666674, + "tokens_seen": 426639360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035987963891676, + "loss": 1.5368, + "theoretical_loss": 4.011670592353309, + "tokens_seen": 426704896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035827482447342, + "loss": 1.491, + "theoretical_loss": 4.011602897349439, + "tokens_seen": 426770432 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703566700100301, + "loss": 1.4685, + "theoretical_loss": 4.011535215650405, + "tokens_seen": 426835968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035506519558676, + "loss": 1.552, + "theoretical_loss": 4.0114675472515495, + "tokens_seen": 426901504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035346038114344, + "loss": 1.5757, + "theoretical_loss": 4.011399892148218, + "tokens_seen": 426967040 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703518555667001, + "loss": 1.6017, + "theoretical_loss": 4.011332250335758, + "tokens_seen": 427032576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007035025075225678, + "loss": 1.4918, + "theoretical_loss": 4.011264621809518, + "tokens_seen": 427098112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034864593781344, + "loss": 1.504, + "theoretical_loss": 4.011197006564853, + "tokens_seen": 427163648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034704112337012, + "loss": 1.5398, + "theoretical_loss": 4.011129404597114, + "tokens_seen": 427229184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034543630892678, + "loss": 1.5175, + "theoretical_loss": 4.011061815901661, + "tokens_seen": 427294720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034383149448346, + "loss": 1.475, + "theoretical_loss": 4.010994240473853, + "tokens_seen": 427360256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007034222668004012, + "loss": 1.519, + "theoretical_loss": 4.0109266783090485, + "tokens_seen": 427425792 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703406218655968, + "loss": 1.5062, + "theoretical_loss": 4.010859129402615, + "tokens_seen": 427491328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033901705115346, + "loss": 1.5876, + "theoretical_loss": 4.010791593749916, + "tokens_seen": 427556864 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033741223671014, + "loss": 1.5328, + "theoretical_loss": 4.010724071346322, + "tokens_seen": 427622400 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703358074222668, + "loss": 1.5617, + "theoretical_loss": 4.0106565621872035, + "tokens_seen": 427687936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033420260782348, + "loss": 1.5451, + "theoretical_loss": 4.010589066267933, + "tokens_seen": 427753472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033259779338014, + "loss": 1.5432, + "theoretical_loss": 4.010521583583886, + "tokens_seen": 427819008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007033099297893682, + "loss": 1.5287, + "theoretical_loss": 4.010454114130441, + "tokens_seen": 427884544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032938816449348, + "loss": 1.4988, + "theoretical_loss": 4.010386657902979, + "tokens_seen": 427950080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032778335005016, + "loss": 1.6294, + "theoretical_loss": 4.01031921489688, + "tokens_seen": 428015616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032617853560682, + "loss": 1.6991, + "theoretical_loss": 4.01025178510753, + "tokens_seen": 428081152 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703245737211635, + "loss": 1.602, + "theoretical_loss": 4.010184368530318, + "tokens_seen": 428146688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032296890672016, + "loss": 1.5621, + "theoretical_loss": 4.0101169651606305, + "tokens_seen": 428212224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007032136409227684, + "loss": 1.5289, + "theoretical_loss": 4.010049574993861, + "tokens_seen": 428277760 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703197592778335, + "loss": 1.5677, + "theoretical_loss": 4.009982198025404, + "tokens_seen": 428343296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031815446339018, + "loss": 1.578, + "theoretical_loss": 4.0099148342506545, + "tokens_seen": 428408832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031654964894684, + "loss": 1.4671, + "theoretical_loss": 4.009847483665011, + "tokens_seen": 428474368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031494483450352, + "loss": 1.6482, + "theoretical_loss": 4.009780146263877, + "tokens_seen": 428539904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031334002006018, + "loss": 1.6687, + "theoretical_loss": 4.009712822042652, + "tokens_seen": 428605440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031173520561686, + "loss": 1.629, + "theoretical_loss": 4.009645510996744, + "tokens_seen": 428670976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007031013039117352, + "loss": 1.4694, + "theoretical_loss": 4.0095782131215625, + "tokens_seen": 428736512 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703085255767302, + "loss": 1.5511, + "theoretical_loss": 4.0095109284125146, + "tokens_seen": 428802048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007030692076228686, + "loss": 1.5534, + "theoretical_loss": 4.009443656865013, + "tokens_seen": 428867584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007030531594784354, + "loss": 1.5525, + "theoretical_loss": 4.009376398474475, + "tokens_seen": 428933120 + }, + { + "epoch": 0.13, + "learning_rate": 0.000703037111334002, + "loss": 1.4294, + "theoretical_loss": 4.009309153236316, + "tokens_seen": 428998656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007030210631895688, + "loss": 1.4977, + "theoretical_loss": 4.009241921145956, + "tokens_seen": 429064192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007030050150451354, + "loss": 1.675, + "theoretical_loss": 4.009174702198816, + "tokens_seen": 429129728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029889669007022, + "loss": 1.5143, + "theoretical_loss": 4.009107496390321, + "tokens_seen": 429195264 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029729187562688, + "loss": 1.5592, + "theoretical_loss": 4.009040303715897, + "tokens_seen": 429260800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029568706118356, + "loss": 1.6147, + "theoretical_loss": 4.008973124170971, + "tokens_seen": 429326336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029408224674022, + "loss": 1.5866, + "theoretical_loss": 4.008905957750976, + "tokens_seen": 429391872 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702924774322969, + "loss": 1.6269, + "theoretical_loss": 4.0088388044513446, + "tokens_seen": 429457408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007029087261785356, + "loss": 1.558, + "theoretical_loss": 4.0087716642675115, + "tokens_seen": 429522944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028926780341024, + "loss": 1.4511, + "theoretical_loss": 4.008704537194916, + "tokens_seen": 429588480 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702876629889669, + "loss": 1.609, + "theoretical_loss": 4.008637423228996, + "tokens_seen": 429654016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028605817452358, + "loss": 1.6015, + "theoretical_loss": 4.008570322365195, + "tokens_seen": 429719552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028445336008024, + "loss": 1.4859, + "theoretical_loss": 4.008503234598958, + "tokens_seen": 429785088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028284854563692, + "loss": 1.6017, + "theoretical_loss": 4.008436159925731, + "tokens_seen": 429850624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007028124373119358, + "loss": 1.5273, + "theoretical_loss": 4.008369098340963, + "tokens_seen": 429916160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027963891675026, + "loss": 1.6092, + "theoretical_loss": 4.008302049840106, + "tokens_seen": 429981696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027803410230692, + "loss": 1.5006, + "theoretical_loss": 4.008235014418614, + "tokens_seen": 430047232 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702764292878636, + "loss": 1.5609, + "theoretical_loss": 4.008167992071941, + "tokens_seen": 430112768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027482447342026, + "loss": 1.5817, + "theoretical_loss": 4.008100982795549, + "tokens_seen": 430178304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027321965897694, + "loss": 1.5328, + "theoretical_loss": 4.008033986584895, + "tokens_seen": 430243840 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702716148445336, + "loss": 1.5055, + "theoretical_loss": 4.007967003435443, + "tokens_seen": 430309376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007027001003009028, + "loss": 1.4671, + "theoretical_loss": 4.007900033342658, + "tokens_seen": 430374912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026840521564694, + "loss": 1.4425, + "theoretical_loss": 4.007833076302006, + "tokens_seen": 430440448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026680040120362, + "loss": 1.5726, + "theoretical_loss": 4.00776613230896, + "tokens_seen": 430505984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026519558676028, + "loss": 1.5256, + "theoretical_loss": 4.007699201358989, + "tokens_seen": 430571520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026359077231696, + "loss": 1.6005, + "theoretical_loss": 4.007632283447567, + "tokens_seen": 430637056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007026198595787362, + "loss": 1.5577, + "theoretical_loss": 4.007565378570171, + "tokens_seen": 430702592 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702603811434303, + "loss": 1.4706, + "theoretical_loss": 4.00749848672228, + "tokens_seen": 430768128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025877632898696, + "loss": 1.4797, + "theoretical_loss": 4.007431607899375, + "tokens_seen": 430833664 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025717151454363, + "loss": 1.5136, + "theoretical_loss": 4.007364742096938, + "tokens_seen": 430899200 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702555667001003, + "loss": 1.611, + "theoretical_loss": 4.007297889310455, + "tokens_seen": 430964736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025396188565697, + "loss": 1.5215, + "theoretical_loss": 4.007231049535413, + "tokens_seen": 431030272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025235707121364, + "loss": 1.5787, + "theoretical_loss": 4.007164222767304, + "tokens_seen": 431095808 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007025075225677031, + "loss": 1.4663, + "theoretical_loss": 4.0070974090016165, + "tokens_seen": 431161344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024914744232698, + "loss": 1.5774, + "theoretical_loss": 4.007030608233848, + "tokens_seen": 431226880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024754262788365, + "loss": 1.4571, + "theoretical_loss": 4.006963820459494, + "tokens_seen": 431292416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024593781344032, + "loss": 1.4446, + "theoretical_loss": 4.006897045674053, + "tokens_seen": 431357952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024433299899699, + "loss": 1.5262, + "theoretical_loss": 4.006830283873027, + "tokens_seen": 431423488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024272818455366, + "loss": 1.4291, + "theoretical_loss": 4.006763535051918, + "tokens_seen": 431489024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007024112337011033, + "loss": 1.5031, + "theoretical_loss": 4.006696799206233, + "tokens_seen": 431554560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070239518555667, + "loss": 1.5797, + "theoretical_loss": 4.006630076331478, + "tokens_seen": 431620096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023791374122367, + "loss": 1.5945, + "theoretical_loss": 4.0065633664231655, + "tokens_seen": 431685632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023630892678034, + "loss": 1.5357, + "theoretical_loss": 4.006496669476805, + "tokens_seen": 431751168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023470411233701, + "loss": 1.6997, + "theoretical_loss": 4.006429985487913, + "tokens_seen": 431816704 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023309929789368, + "loss": 1.4693, + "theoretical_loss": 4.006363314452007, + "tokens_seen": 431882240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007023149448345035, + "loss": 1.492, + "theoretical_loss": 4.006296656364603, + "tokens_seen": 431947776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022988966900702, + "loss": 1.5278, + "theoretical_loss": 4.006230011221225, + "tokens_seen": 432013312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022828485456369, + "loss": 1.5508, + "theoretical_loss": 4.006163379017394, + "tokens_seen": 432078848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022668004012037, + "loss": 1.4927, + "theoretical_loss": 4.0060967597486385, + "tokens_seen": 432144384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022507522567703, + "loss": 1.5925, + "theoretical_loss": 4.006030153410483, + "tokens_seen": 432209920 + }, + { + "epoch": 0.13, + "learning_rate": 0.000702234704112337, + "loss": 1.5988, + "theoretical_loss": 4.005963559998461, + "tokens_seen": 432275456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022186559679037, + "loss": 1.5938, + "theoretical_loss": 4.005896979508103, + "tokens_seen": 432340992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007022026078234705, + "loss": 1.5983, + "theoretical_loss": 4.005830411934944, + "tokens_seen": 432406528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021865596790371, + "loss": 1.5777, + "theoretical_loss": 4.005763857274521, + "tokens_seen": 432472064 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 166832, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5703991651535034, + "objective/train/theoretical_loss": 4.005697315522371, + "objective/train/tokens_used": 452997600, + "theoretical_loss": 4.005697315522371, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021705115346039, + "loss": 1.5704, + "theoretical_loss": 4.005697315522371, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021544633901705, + "loss": 1.5658, + "theoretical_loss": 4.005630786674039, + "tokens_seen": 432603136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021384152457373, + "loss": 1.589, + "theoretical_loss": 4.005564270725065, + "tokens_seen": 432668672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021223671013039, + "loss": 1.5895, + "theoretical_loss": 4.005497767670996, + "tokens_seen": 432734208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007021063189568707, + "loss": 1.4925, + "theoretical_loss": 4.0054312775073795, + "tokens_seen": 432799744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020902708124373, + "loss": 1.5391, + "theoretical_loss": 4.005364800229766, + "tokens_seen": 432865280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020742226680041, + "loss": 1.5513, + "theoretical_loss": 4.005298335833708, + "tokens_seen": 432930816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020581745235707, + "loss": 1.6322, + "theoretical_loss": 4.005231884314759, + "tokens_seen": 432996352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020421263791375, + "loss": 1.6409, + "theoretical_loss": 4.005165445668477, + "tokens_seen": 433061888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020260782347041, + "loss": 1.4646, + "theoretical_loss": 4.0050990198904195, + "tokens_seen": 433127424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007020100300902709, + "loss": 1.5576, + "theoretical_loss": 4.005032606976149, + "tokens_seen": 433192960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019939819458375, + "loss": 1.643, + "theoretical_loss": 4.004966206921227, + "tokens_seen": 433258496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019779338014043, + "loss": 1.5911, + "theoretical_loss": 4.004899819721221, + "tokens_seen": 433324032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019618856569709, + "loss": 1.4882, + "theoretical_loss": 4.004833445371697, + "tokens_seen": 433389568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019458375125377, + "loss": 1.6259, + "theoretical_loss": 4.004767083868226, + "tokens_seen": 433455104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019297893681043, + "loss": 1.6064, + "theoretical_loss": 4.004700735206381, + "tokens_seen": 433520640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007019137412236711, + "loss": 1.6409, + "theoretical_loss": 4.004634399381733, + "tokens_seen": 433586176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018976930792377, + "loss": 1.5818, + "theoretical_loss": 4.0045680763898615, + "tokens_seen": 433651712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018816449348045, + "loss": 1.5335, + "theoretical_loss": 4.004501766226344, + "tokens_seen": 433717248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018655967903711, + "loss": 1.5425, + "theoretical_loss": 4.004435468886761, + "tokens_seen": 433782784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018495486459379, + "loss": 1.4265, + "theoretical_loss": 4.0043691843666975, + "tokens_seen": 433848320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018335005015045, + "loss": 1.5845, + "theoretical_loss": 4.004302912661736, + "tokens_seen": 433913856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018174523570713, + "loss": 1.5734, + "theoretical_loss": 4.0042366537674665, + "tokens_seen": 433979392 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007018014042126379, + "loss": 1.6141, + "theoretical_loss": 4.004170407679476, + "tokens_seen": 434044928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017853560682047, + "loss": 1.4705, + "theoretical_loss": 4.004104174393359, + "tokens_seen": 434110464 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017693079237713, + "loss": 1.4868, + "theoretical_loss": 4.004037953904708, + "tokens_seen": 434176000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017532597793381, + "loss": 1.553, + "theoretical_loss": 4.003971746209119, + "tokens_seen": 434241536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017372116349047, + "loss": 1.4536, + "theoretical_loss": 4.003905551302191, + "tokens_seen": 434307072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017211634904715, + "loss": 1.5679, + "theoretical_loss": 4.0038393691795235, + "tokens_seen": 434372608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007017051153460381, + "loss": 1.5152, + "theoretical_loss": 4.003773199836721, + "tokens_seen": 434438144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016890672016049, + "loss": 1.5495, + "theoretical_loss": 4.003707043269387, + "tokens_seen": 434503680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016730190571715, + "loss": 1.6212, + "theoretical_loss": 4.003640899473128, + "tokens_seen": 434569216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016569709127383, + "loss": 1.5025, + "theoretical_loss": 4.0035747684435545, + "tokens_seen": 434634752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016409227683049, + "loss": 1.4926, + "theoretical_loss": 4.003508650176278, + "tokens_seen": 434700288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016248746238717, + "loss": 1.6495, + "theoretical_loss": 4.003442544666911, + "tokens_seen": 434765824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007016088264794383, + "loss": 1.6077, + "theoretical_loss": 4.003376451911071, + "tokens_seen": 434831360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015927783350051, + "loss": 1.6396, + "theoretical_loss": 4.003310371904373, + "tokens_seen": 434896896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015767301905717, + "loss": 1.6277, + "theoretical_loss": 4.00324430464244, + "tokens_seen": 434962432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015606820461385, + "loss": 1.4191, + "theoretical_loss": 4.003178250120892, + "tokens_seen": 435027968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015446339017051, + "loss": 1.4487, + "theoretical_loss": 4.003112208335355, + "tokens_seen": 435093504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015285857572719, + "loss": 1.4458, + "theoretical_loss": 4.003046179281455, + "tokens_seen": 435159040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007015125376128385, + "loss": 1.6631, + "theoretical_loss": 4.00298016295482, + "tokens_seen": 435224576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014964894684053, + "loss": 1.5753, + "theoretical_loss": 4.002914159351082, + "tokens_seen": 435290112 + }, + { + "epoch": 0.13, + "learning_rate": 0.000701480441323972, + "loss": 1.5673, + "theoretical_loss": 4.0028481684658725, + "tokens_seen": 435355648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014643931795387, + "loss": 1.4775, + "theoretical_loss": 4.002782190294829, + "tokens_seen": 435421184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014483450351054, + "loss": 1.6062, + "theoretical_loss": 4.002716224833588, + "tokens_seen": 435486720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014322968906721, + "loss": 1.5555, + "theoretical_loss": 4.002650272077787, + "tokens_seen": 435552256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014162487462388, + "loss": 1.538, + "theoretical_loss": 4.00258433202307, + "tokens_seen": 435617792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007014002006018055, + "loss": 1.6039, + "theoretical_loss": 4.0025184046650795, + "tokens_seen": 435683328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013841524573722, + "loss": 1.5413, + "theoretical_loss": 4.002452489999463, + "tokens_seen": 435748864 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013681043129389, + "loss": 1.6083, + "theoretical_loss": 4.002386588021867, + "tokens_seen": 435814400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013520561685056, + "loss": 1.4699, + "theoretical_loss": 4.002320698727942, + "tokens_seen": 435879936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013360080240723, + "loss": 1.5256, + "theoretical_loss": 4.002254822113341, + "tokens_seen": 435945472 + }, + { + "epoch": 0.13, + "learning_rate": 0.000701319959879639, + "loss": 1.5015, + "theoretical_loss": 4.0021889581737184, + "tokens_seen": 436011008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007013039117352057, + "loss": 1.5673, + "theoretical_loss": 4.00212310690473, + "tokens_seen": 436076544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012878635907723, + "loss": 1.4924, + "theoretical_loss": 4.0020572683020355, + "tokens_seen": 436142080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012718154463391, + "loss": 1.5541, + "theoretical_loss": 4.0019914423612954, + "tokens_seen": 436207616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012557673019057, + "loss": 1.4258, + "theoretical_loss": 4.0019256290781735, + "tokens_seen": 436273152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012397191574725, + "loss": 1.6071, + "theoretical_loss": 4.001859828448334, + "tokens_seen": 436338688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012236710130391, + "loss": 1.4929, + "theoretical_loss": 4.001794040467446, + "tokens_seen": 436404224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007012076228686059, + "loss": 1.6593, + "theoretical_loss": 4.0017282651311765, + "tokens_seen": 436469760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011915747241725, + "loss": 1.4546, + "theoretical_loss": 4.001662502435199, + "tokens_seen": 436535296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011755265797393, + "loss": 1.4627, + "theoretical_loss": 4.001596752375186, + "tokens_seen": 436600832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011594784353059, + "loss": 1.5625, + "theoretical_loss": 4.001531014946815, + "tokens_seen": 436666368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011434302908726, + "loss": 1.5899, + "theoretical_loss": 4.001465290145763, + "tokens_seen": 436731904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007011273821464393, + "loss": 1.4815, + "theoretical_loss": 4.00139957796771, + "tokens_seen": 436797440 + }, + { + "epoch": 0.13, + "learning_rate": 0.000701111334002006, + "loss": 1.4776, + "theoretical_loss": 4.001333878408339, + "tokens_seen": 436862976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010952858575727, + "loss": 1.516, + "theoretical_loss": 4.001268191463334, + "tokens_seen": 436928512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010792377131394, + "loss": 1.5179, + "theoretical_loss": 4.00120251712838, + "tokens_seen": 436994048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010631895687061, + "loss": 1.5433, + "theoretical_loss": 4.001136855399168, + "tokens_seen": 437059584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010471414242728, + "loss": 1.5898, + "theoretical_loss": 4.001071206271388, + "tokens_seen": 437125120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010310932798395, + "loss": 1.5636, + "theoretical_loss": 4.001005569740732, + "tokens_seen": 437190656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007010150451354062, + "loss": 1.6252, + "theoretical_loss": 4.000939945802896, + "tokens_seen": 437256192 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700998996990973, + "loss": 1.4857, + "theoretical_loss": 4.000874334453577, + "tokens_seen": 437321728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009829488465396, + "loss": 1.4687, + "theoretical_loss": 4.000808735688474, + "tokens_seen": 437387264 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009669007021063, + "loss": 1.4804, + "theoretical_loss": 4.000743149503288, + "tokens_seen": 437452800 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700950852557673, + "loss": 1.4974, + "theoretical_loss": 4.000677575893723, + "tokens_seen": 437518336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009348044132397, + "loss": 1.5203, + "theoretical_loss": 4.000612014855484, + "tokens_seen": 437583872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009187562688064, + "loss": 1.5411, + "theoretical_loss": 4.00054646638428, + "tokens_seen": 437649408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007009027081243731, + "loss": 1.532, + "theoretical_loss": 4.000480930475819, + "tokens_seen": 437714944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008866599799398, + "loss": 1.4639, + "theoretical_loss": 4.000415407125814, + "tokens_seen": 437780480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008706118355066, + "loss": 1.5364, + "theoretical_loss": 4.000349896329979, + "tokens_seen": 437846016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008545636910732, + "loss": 1.5157, + "theoretical_loss": 4.0002843980840295, + "tokens_seen": 437911552 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070083851554664, + "loss": 1.5553, + "theoretical_loss": 4.0002189123836835, + "tokens_seen": 437977088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008224674022066, + "loss": 1.4489, + "theoretical_loss": 4.000153439224662, + "tokens_seen": 438042624 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007008064192577734, + "loss": 1.489, + "theoretical_loss": 4.0000879786026875, + "tokens_seen": 438108160 + }, + { + "epoch": 0.13, + "learning_rate": 0.00070079037111334, + "loss": 1.562, + "theoretical_loss": 4.000022530513484, + "tokens_seen": 438173696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007743229689068, + "loss": 1.502, + "theoretical_loss": 3.999957094952779, + "tokens_seen": 438239232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007582748244734, + "loss": 1.4577, + "theoretical_loss": 3.9998916719163002, + "tokens_seen": 438304768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007422266800402, + "loss": 1.5305, + "theoretical_loss": 3.999826261399778, + "tokens_seen": 438370304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007261785356068, + "loss": 1.4927, + "theoretical_loss": 3.9997608633989463, + "tokens_seen": 438435840 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007007101303911736, + "loss": 1.527, + "theoretical_loss": 3.9996954779095395, + "tokens_seen": 438501376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007006940822467402, + "loss": 1.5402, + "theoretical_loss": 3.999630104927295, + "tokens_seen": 438566912 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700678034102307, + "loss": 1.4227, + "theoretical_loss": 3.999564744447952, + "tokens_seen": 438632448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007006619859578736, + "loss": 1.4515, + "theoretical_loss": 3.9994993964672516, + "tokens_seen": 438697984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007006459378134404, + "loss": 1.5086, + "theoretical_loss": 3.9994340609809367, + "tokens_seen": 438763520 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700629889669007, + "loss": 1.508, + "theoretical_loss": 3.999368737984754, + "tokens_seen": 438829056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007006138415245738, + "loss": 1.51, + "theoretical_loss": 3.999303427474449, + "tokens_seen": 438894592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005977933801404, + "loss": 1.4687, + "theoretical_loss": 3.9992381294457733, + "tokens_seen": 438960128 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005817452357072, + "loss": 1.5765, + "theoretical_loss": 3.9991728438944767, + "tokens_seen": 439025664 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 168939, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.587876558303833, + "objective/train/theoretical_loss": 3.9991075708163146, + "objective/train/tokens_used": 459551200, + "theoretical_loss": 3.9991075708163146, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005656970912738, + "loss": 1.5879, + "theoretical_loss": 3.9991075708163146, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005496489468406, + "loss": 1.5072, + "theoretical_loss": 3.9990423102070416, + "tokens_seen": 439156736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005336008024072, + "loss": 1.6051, + "theoretical_loss": 3.9989770620624165, + "tokens_seen": 439222272 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700517552657974, + "loss": 1.5635, + "theoretical_loss": 3.9989118263781984, + "tokens_seen": 439287808 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007005015045135406, + "loss": 1.4929, + "theoretical_loss": 3.99884660315015, + "tokens_seen": 439353344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004854563691074, + "loss": 1.551, + "theoretical_loss": 3.9987813923740356, + "tokens_seen": 439418880 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700469408224674, + "loss": 1.4987, + "theoretical_loss": 3.9987161940456204, + "tokens_seen": 439484416 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004533600802408, + "loss": 1.5697, + "theoretical_loss": 3.9986510081606736, + "tokens_seen": 439549952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004373119358074, + "loss": 1.4425, + "theoretical_loss": 3.998585834714965, + "tokens_seen": 439615488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004212637913742, + "loss": 1.4781, + "theoretical_loss": 3.9985206737042676, + "tokens_seen": 439681024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007004052156469408, + "loss": 1.4281, + "theoretical_loss": 3.998455525124355, + "tokens_seen": 439746560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007003891675025076, + "loss": 1.4825, + "theoretical_loss": 3.9983903889710044, + "tokens_seen": 439812096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007003731193580742, + "loss": 1.4466, + "theoretical_loss": 3.998325265239994, + "tokens_seen": 439877632 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700357071213641, + "loss": 1.5189, + "theoretical_loss": 3.998260153927105, + "tokens_seen": 439943168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007003410230692076, + "loss": 1.4568, + "theoretical_loss": 3.998195055028119, + "tokens_seen": 440008704 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007003249749247744, + "loss": 1.4916, + "theoretical_loss": 3.998129968538822, + "tokens_seen": 440074240 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700308926780341, + "loss": 1.4337, + "theoretical_loss": 3.998064894455, + "tokens_seen": 440139776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002928786359078, + "loss": 1.516, + "theoretical_loss": 3.997999832772443, + "tokens_seen": 440205312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002768304914745, + "loss": 1.5958, + "theoretical_loss": 3.997934783486941, + "tokens_seen": 440270848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002607823470412, + "loss": 1.4797, + "theoretical_loss": 3.997869746594287, + "tokens_seen": 440336384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002447342026079, + "loss": 1.5088, + "theoretical_loss": 3.9978047220902764, + "tokens_seen": 440401920 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002286860581746, + "loss": 1.5202, + "theoretical_loss": 3.997739709970706, + "tokens_seen": 440467456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007002126379137413, + "loss": 1.5361, + "theoretical_loss": 3.997674710231375, + "tokens_seen": 440532992 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700196589769308, + "loss": 1.5617, + "theoretical_loss": 3.997609722868085, + "tokens_seen": 440598528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001805416248747, + "loss": 1.5416, + "theoretical_loss": 3.9975447478766393, + "tokens_seen": 440664064 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001644934804414, + "loss": 1.6983, + "theoretical_loss": 3.997479785252842, + "tokens_seen": 440729600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001484453360081, + "loss": 1.4706, + "theoretical_loss": 3.997414834992502, + "tokens_seen": 440795136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001323971915748, + "loss": 1.5702, + "theoretical_loss": 3.9973498970914285, + "tokens_seen": 440860672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001163490471415, + "loss": 1.4714, + "theoretical_loss": 3.9972849715454317, + "tokens_seen": 440926208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007001003009027082, + "loss": 1.5184, + "theoretical_loss": 3.9972200583503263, + "tokens_seen": 440991744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000842527582749, + "loss": 1.4428, + "theoretical_loss": 3.9971551575019273, + "tokens_seen": 441057280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000682046138416, + "loss": 1.5048, + "theoretical_loss": 3.997090268996052, + "tokens_seen": 441122816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000521564694083, + "loss": 1.4647, + "theoretical_loss": 3.9970253928285215, + "tokens_seen": 441188352 + }, + { + "epoch": 0.13, + "learning_rate": 0.000700036108324975, + "loss": 1.4516, + "theoretical_loss": 3.996960528995155, + "tokens_seen": 441253888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000200601805417, + "loss": 1.4828, + "theoretical_loss": 3.996895677491778, + "tokens_seen": 441319424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0007000040120361084, + "loss": 1.6635, + "theoretical_loss": 3.996830838314216, + "tokens_seen": 441384960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999879638916751, + "loss": 1.4694, + "theoretical_loss": 3.996766011458296, + "tokens_seen": 441450496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999719157472418, + "loss": 1.4891, + "theoretical_loss": 3.9967011969198483, + "tokens_seen": 441516032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999558676028085, + "loss": 1.5424, + "theoretical_loss": 3.996636394694705, + "tokens_seen": 441581568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999398194583752, + "loss": 1.4673, + "theoretical_loss": 3.9965716047786985, + "tokens_seen": 441647104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999237713139419, + "loss": 1.6674, + "theoretical_loss": 3.996506827167667, + "tokens_seen": 441712640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006999077231695086, + "loss": 1.4392, + "theoretical_loss": 3.9964420618574454, + "tokens_seen": 441778176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006998916750250752, + "loss": 1.5038, + "theoretical_loss": 3.9963773088438765, + "tokens_seen": 441843712 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699875626880642, + "loss": 1.5878, + "theoretical_loss": 3.996312568122801, + "tokens_seen": 441909248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006998595787362086, + "loss": 1.4169, + "theoretical_loss": 3.9962478396900627, + "tokens_seen": 441974784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006998435305917754, + "loss": 1.487, + "theoretical_loss": 3.996183123541507, + "tokens_seen": 442040320 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699827482447342, + "loss": 1.635, + "theoretical_loss": 3.9961184196729835, + "tokens_seen": 442105856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006998114343029088, + "loss": 1.4991, + "theoretical_loss": 3.9960537280803408, + "tokens_seen": 442171392 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997953861584754, + "loss": 1.5549, + "theoretical_loss": 3.995989048759432, + "tokens_seen": 442236928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997793380140422, + "loss": 1.6127, + "theoretical_loss": 3.99592438170611, + "tokens_seen": 442302464 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997632898696088, + "loss": 1.4228, + "theoretical_loss": 3.9958597269162315, + "tokens_seen": 442368000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997472417251755, + "loss": 1.5859, + "theoretical_loss": 3.9957950843856542, + "tokens_seen": 442433536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997311935807422, + "loss": 1.4444, + "theoretical_loss": 3.9957304541102396, + "tokens_seen": 442499072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006997151454363089, + "loss": 1.4267, + "theoretical_loss": 3.995665836085848, + "tokens_seen": 442564608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996990972918756, + "loss": 1.5101, + "theoretical_loss": 3.9956012303083446, + "tokens_seen": 442630144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996830491474423, + "loss": 1.4524, + "theoretical_loss": 3.9955366367735947, + "tokens_seen": 442695680 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699667001003009, + "loss": 1.4763, + "theoretical_loss": 3.995472055477467, + "tokens_seen": 442761216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996509528585757, + "loss": 1.5469, + "theoretical_loss": 3.9954074864158313, + "tokens_seen": 442826752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996349047141424, + "loss": 1.4735, + "theoretical_loss": 3.9953429295845604, + "tokens_seen": 442892288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996188565697091, + "loss": 1.5536, + "theoretical_loss": 3.9952783849795277, + "tokens_seen": 442957824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006996028084252758, + "loss": 1.5109, + "theoretical_loss": 3.99521385259661, + "tokens_seen": 443023360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995867602808425, + "loss": 1.4835, + "theoretical_loss": 3.9951493324316845, + "tokens_seen": 443088896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995707121364092, + "loss": 1.5134, + "theoretical_loss": 3.9950848244806316, + "tokens_seen": 443154432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995546639919759, + "loss": 1.4988, + "theoretical_loss": 3.9950203287393347, + "tokens_seen": 443219968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995386158475426, + "loss": 1.4969, + "theoretical_loss": 3.994955845203676, + "tokens_seen": 443285504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006995225677031093, + "loss": 1.4761, + "theoretical_loss": 3.994891373869543, + "tokens_seen": 443351040 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699506519558676, + "loss": 1.5215, + "theoretical_loss": 3.9948269147328235, + "tokens_seen": 443416576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994904714142427, + "loss": 1.4577, + "theoretical_loss": 3.994762467789407, + "tokens_seen": 443482112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994744232698095, + "loss": 1.5919, + "theoretical_loss": 3.9946980330351862, + "tokens_seen": 443547648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994583751253761, + "loss": 1.6184, + "theoretical_loss": 3.994633610466056, + "tokens_seen": 443613184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994423269809429, + "loss": 1.5094, + "theoretical_loss": 3.994569200077911, + "tokens_seen": 443678720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994262788365095, + "loss": 1.6159, + "theoretical_loss": 3.9945048018666496, + "tokens_seen": 443744256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006994102306920763, + "loss": 1.5415, + "theoretical_loss": 3.9944404158281723, + "tokens_seen": 443809792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993941825476429, + "loss": 1.5069, + "theoretical_loss": 3.994376041958381, + "tokens_seen": 443875328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993781344032097, + "loss": 1.442, + "theoretical_loss": 3.9943116802531797, + "tokens_seen": 443940864 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993620862587763, + "loss": 1.4373, + "theoretical_loss": 3.9942473307084745, + "tokens_seen": 444006400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993460381143431, + "loss": 1.5545, + "theoretical_loss": 3.9941829933201736, + "tokens_seen": 444071936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993299899699097, + "loss": 1.5242, + "theoretical_loss": 3.9941186680841865, + "tokens_seen": 444137472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006993139418254765, + "loss": 1.5771, + "theoretical_loss": 3.9940543549964254, + "tokens_seen": 444203008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992978936810431, + "loss": 1.5284, + "theoretical_loss": 3.993990054052804, + "tokens_seen": 444268544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992818455366099, + "loss": 1.4537, + "theoretical_loss": 3.993925765249239, + "tokens_seen": 444334080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992657973921765, + "loss": 1.4964, + "theoretical_loss": 3.993861488581647, + "tokens_seen": 444399616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992497492477433, + "loss": 1.5097, + "theoretical_loss": 3.9937972240459487, + "tokens_seen": 444465152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992337011033099, + "loss": 1.4987, + "theoretical_loss": 3.9937329716380665, + "tokens_seen": 444530688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992176529588767, + "loss": 1.498, + "theoretical_loss": 3.9936687313539228, + "tokens_seen": 444596224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006992016048144433, + "loss": 1.5015, + "theoretical_loss": 3.9936045031894443, + "tokens_seen": 444661760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991855566700101, + "loss": 1.4908, + "theoretical_loss": 3.9935402871405588, + "tokens_seen": 444727296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991695085255767, + "loss": 1.4961, + "theoretical_loss": 3.9934760832031957, + "tokens_seen": 444792832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991534603811435, + "loss": 1.5685, + "theoretical_loss": 3.9934118913732863, + "tokens_seen": 444858368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991374122367101, + "loss": 1.5466, + "theoretical_loss": 3.9933477116467646, + "tokens_seen": 444923904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991213640922769, + "loss": 1.4745, + "theoretical_loss": 3.993283544019567, + "tokens_seen": 444989440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006991053159478435, + "loss": 1.5951, + "theoretical_loss": 3.9932193884876304, + "tokens_seen": 445054976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990892678034103, + "loss": 1.4568, + "theoretical_loss": 3.9931552450468937, + "tokens_seen": 445120512 + }, + { + "epoch": 0.13, + "learning_rate": 0.000699073219658977, + "loss": 1.5503, + "theoretical_loss": 3.9930911136933, + "tokens_seen": 445186048 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990571715145437, + "loss": 1.5399, + "theoretical_loss": 3.9930269944227907, + "tokens_seen": 445251584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990411233701104, + "loss": 1.5799, + "theoretical_loss": 3.9929628872313128, + "tokens_seen": 445317120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990250752256771, + "loss": 1.5266, + "theoretical_loss": 3.9928987921148127, + "tokens_seen": 445382656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0006990090270812438, + "loss": 1.5878, + "theoretical_loss": 3.992834709069241, + "tokens_seen": 445448192 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989929789368105, + "loss": 1.5647, + "theoretical_loss": 3.9927706380905477, + "tokens_seen": 445513728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989769307923772, + "loss": 1.5352, + "theoretical_loss": 3.992706579174686, + "tokens_seen": 445579264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 171375, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.6052347421646118, + "objective/train/theoretical_loss": 3.9926425323176122, + "objective/train/tokens_used": 466104800, + "theoretical_loss": 3.9926425323176122, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989608826479439, + "loss": 1.6052, + "theoretical_loss": 3.9926425323176122, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989448345035106, + "loss": 1.4536, + "theoretical_loss": 3.992578497515283, + "tokens_seen": 445710336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006989287863590773, + "loss": 1.3648, + "theoretical_loss": 3.9925144747636576, + "tokens_seen": 445775872 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698912738214644, + "loss": 1.5392, + "theoretical_loss": 3.992450464058696, + "tokens_seen": 445841408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988966900702107, + "loss": 1.5498, + "theoretical_loss": 3.9923864653963625, + "tokens_seen": 445906944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988806419257774, + "loss": 1.447, + "theoretical_loss": 3.9923224787726213, + "tokens_seen": 445972480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988645937813441, + "loss": 1.527, + "theoretical_loss": 3.9922585041834395, + "tokens_seen": 446038016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988485456369108, + "loss": 1.4865, + "theoretical_loss": 3.9921945416247855, + "tokens_seen": 446103552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988324974924775, + "loss": 1.5938, + "theoretical_loss": 3.9921305910926312, + "tokens_seen": 446169088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988164493480442, + "loss": 1.536, + "theoretical_loss": 3.992066652582948, + "tokens_seen": 446234624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006988004012036109, + "loss": 1.5059, + "theoretical_loss": 3.9920027260917115, + "tokens_seen": 446300160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987843530591776, + "loss": 1.4532, + "theoretical_loss": 3.9919388116148977, + "tokens_seen": 446365696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987683049147443, + "loss": 1.6235, + "theoretical_loss": 3.991874909148485, + "tokens_seen": 446431232 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698752256770311, + "loss": 1.4589, + "theoretical_loss": 3.9918110186884546, + "tokens_seen": 446496768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987362086258777, + "loss": 1.687, + "theoretical_loss": 3.991747140230788, + "tokens_seen": 446562304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987201604814444, + "loss": 1.6214, + "theoretical_loss": 3.9916832737714705, + "tokens_seen": 446627840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006987041123370111, + "loss": 1.504, + "theoretical_loss": 3.9916194193064873, + "tokens_seen": 446693376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986880641925778, + "loss": 1.5632, + "theoretical_loss": 3.9915555768318276, + "tokens_seen": 446758912 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986720160481445, + "loss": 1.5959, + "theoretical_loss": 3.9914917463434807, + "tokens_seen": 446824448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986559679037112, + "loss": 1.5479, + "theoretical_loss": 3.9914279278374387, + "tokens_seen": 446889984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986399197592779, + "loss": 1.4653, + "theoretical_loss": 3.9913641213096964, + "tokens_seen": 446955520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986238716148446, + "loss": 1.5469, + "theoretical_loss": 3.9913003267562486, + "tokens_seen": 447021056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006986078234704113, + "loss": 1.5028, + "theoretical_loss": 3.9912365441730944, + "tokens_seen": 447086592 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698591775325978, + "loss": 1.5987, + "theoretical_loss": 3.9911727735562326, + "tokens_seen": 447152128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985757271815447, + "loss": 1.5711, + "theoretical_loss": 3.9911090149016646, + "tokens_seen": 447217664 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985596790371114, + "loss": 1.6229, + "theoretical_loss": 3.9910452682053954, + "tokens_seen": 447283200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985436308926781, + "loss": 1.5326, + "theoretical_loss": 3.9909815334634287, + "tokens_seen": 447348736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985275827482448, + "loss": 1.5106, + "theoretical_loss": 3.990917810671773, + "tokens_seen": 447414272 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006985115346038115, + "loss": 1.5088, + "theoretical_loss": 3.9908540998264375, + "tokens_seen": 447479808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984954864593781, + "loss": 1.5141, + "theoretical_loss": 3.9907904009234336, + "tokens_seen": 447545344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984794383149449, + "loss": 1.5221, + "theoretical_loss": 3.9907267139587743, + "tokens_seen": 447610880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984633901705115, + "loss": 1.6763, + "theoretical_loss": 3.9906630389284747, + "tokens_seen": 447676416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984473420260783, + "loss": 1.4744, + "theoretical_loss": 3.990599375828552, + "tokens_seen": 447741952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984312938816449, + "loss": 1.4899, + "theoretical_loss": 3.9905357246550253, + "tokens_seen": 447807488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006984152457372117, + "loss": 1.4999, + "theoretical_loss": 3.9904720854039137, + "tokens_seen": 447873024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983991975927783, + "loss": 1.418, + "theoretical_loss": 3.9904084580712427, + "tokens_seen": 447938560 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983831494483451, + "loss": 1.5501, + "theoretical_loss": 3.9903448426530352, + "tokens_seen": 448004096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983671013039117, + "loss": 1.3928, + "theoretical_loss": 3.990281239145318, + "tokens_seen": 448069632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983510531594784, + "loss": 1.5261, + "theoretical_loss": 3.99021764754412, + "tokens_seen": 448135168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983350050150451, + "loss": 1.4993, + "theoretical_loss": 3.990154067845472, + "tokens_seen": 448200704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983189568706118, + "loss": 1.4759, + "theoretical_loss": 3.990090500045404, + "tokens_seen": 448266240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006983029087261785, + "loss": 1.4739, + "theoretical_loss": 3.990026944139953, + "tokens_seen": 448331776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982868605817452, + "loss": 1.4565, + "theoretical_loss": 3.989963400125153, + "tokens_seen": 448397312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982708124373119, + "loss": 1.4874, + "theoretical_loss": 3.9898998679970434, + "tokens_seen": 448462848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982547642928786, + "loss": 1.6287, + "theoretical_loss": 3.9898363477516634, + "tokens_seen": 448528384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982387161484453, + "loss": 1.4659, + "theoretical_loss": 3.9897728393850547, + "tokens_seen": 448593920 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698222668004012, + "loss": 1.5338, + "theoretical_loss": 3.9897093428932613, + "tokens_seen": 448659456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006982066198595787, + "loss": 1.4943, + "theoretical_loss": 3.989645858272328, + "tokens_seen": 448724992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981905717151454, + "loss": 1.4744, + "theoretical_loss": 3.989582385518304, + "tokens_seen": 448790528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981745235707121, + "loss": 1.5788, + "theoretical_loss": 3.989518924627237, + "tokens_seen": 448856064 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981584754262788, + "loss": 1.5368, + "theoretical_loss": 3.9894554755951788, + "tokens_seen": 448921600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981424272818456, + "loss": 1.4918, + "theoretical_loss": 3.989392038418182, + "tokens_seen": 448987136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006981263791374122, + "loss": 1.4835, + "theoretical_loss": 3.9893286130923027, + "tokens_seen": 449052672 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698110330992979, + "loss": 1.5563, + "theoretical_loss": 3.989265199613597, + "tokens_seen": 449118208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980942828485456, + "loss": 1.5204, + "theoretical_loss": 3.989201797978124, + "tokens_seen": 449183744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980782347041124, + "loss": 1.546, + "theoretical_loss": 3.989138408181944, + "tokens_seen": 449249280 + }, + { + "epoch": 0.14, + "learning_rate": 0.000698062186559679, + "loss": 1.4209, + "theoretical_loss": 3.9890750302211204, + "tokens_seen": 449314816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980461384152458, + "loss": 1.4304, + "theoretical_loss": 3.9890116640917173, + "tokens_seen": 449380352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980300902708124, + "loss": 1.5169, + "theoretical_loss": 3.9889483097898006, + "tokens_seen": 449445888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006980140421263792, + "loss": 1.4068, + "theoretical_loss": 3.9888849673114386, + "tokens_seen": 449511424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979979939819458, + "loss": 1.5146, + "theoretical_loss": 3.988821636652702, + "tokens_seen": 449576960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979819458375126, + "loss": 1.5135, + "theoretical_loss": 3.988758317809662, + "tokens_seen": 449642496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979658976930792, + "loss": 1.4979, + "theoretical_loss": 3.988695010778393, + "tokens_seen": 449708032 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697949849548646, + "loss": 1.4246, + "theoretical_loss": 3.9886317155549706, + "tokens_seen": 449773568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979338014042126, + "loss": 1.5666, + "theoretical_loss": 3.988568432135472, + "tokens_seen": 449839104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006979177532597794, + "loss": 1.5269, + "theoretical_loss": 3.988505160515978, + "tokens_seen": 449904640 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697901705115346, + "loss": 1.5746, + "theoretical_loss": 3.9884419006925684, + "tokens_seen": 449970176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978856569709128, + "loss": 1.4625, + "theoretical_loss": 3.9883786526613267, + "tokens_seen": 450035712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978696088264795, + "loss": 1.6329, + "theoretical_loss": 3.988315416418339, + "tokens_seen": 450101248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978535606820462, + "loss": 1.5033, + "theoretical_loss": 3.988252191959692, + "tokens_seen": 450166784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978375125376129, + "loss": 1.5938, + "theoretical_loss": 3.9881889792814733, + "tokens_seen": 450232320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978214643931796, + "loss": 1.5604, + "theoretical_loss": 3.9881257783797746, + "tokens_seen": 450297856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006978054162487463, + "loss": 1.6214, + "theoretical_loss": 3.9880625892506885, + "tokens_seen": 450363392 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697789368104313, + "loss": 1.502, + "theoretical_loss": 3.9879994118903093, + "tokens_seen": 450428928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977733199598797, + "loss": 1.4641, + "theoretical_loss": 3.9879362462947334, + "tokens_seen": 450494464 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977572718154464, + "loss": 1.4461, + "theoretical_loss": 3.9878730924600583, + "tokens_seen": 450560000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977412236710131, + "loss": 1.5824, + "theoretical_loss": 3.9878099503823847, + "tokens_seen": 450625536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977251755265798, + "loss": 1.5345, + "theoretical_loss": 3.9877468200578146, + "tokens_seen": 450691072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006977091273821465, + "loss": 1.5363, + "theoretical_loss": 3.9876837014824513, + "tokens_seen": 450756608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976930792377132, + "loss": 1.532, + "theoretical_loss": 3.9876205946524, + "tokens_seen": 450822144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976770310932799, + "loss": 1.6342, + "theoretical_loss": 3.9875574995637697, + "tokens_seen": 450887680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976609829488466, + "loss": 1.4844, + "theoretical_loss": 3.987494416212668, + "tokens_seen": 450953216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976449348044133, + "loss": 1.4881, + "theoretical_loss": 3.987431344595207, + "tokens_seen": 451018752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00069762888665998, + "loss": 1.5449, + "theoretical_loss": 3.9873682847074994, + "tokens_seen": 451084288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006976128385155467, + "loss": 1.5949, + "theoretical_loss": 3.9873052365456605, + "tokens_seen": 451149824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975967903711134, + "loss": 1.4422, + "theoretical_loss": 3.9872422001058063, + "tokens_seen": 451215360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975807422266801, + "loss": 1.5327, + "theoretical_loss": 3.987179175384056, + "tokens_seen": 451280896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975646940822468, + "loss": 1.5012, + "theoretical_loss": 3.9871161623765294, + "tokens_seen": 451346432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975486459378135, + "loss": 1.4703, + "theoretical_loss": 3.9870531610793494, + "tokens_seen": 451411968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975325977933802, + "loss": 1.4929, + "theoretical_loss": 3.9869901714886398, + "tokens_seen": 451477504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975165496489469, + "loss": 1.5338, + "theoretical_loss": 3.9869271936005264, + "tokens_seen": 451543040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006975005015045136, + "loss": 1.46, + "theoretical_loss": 3.9868642274111377, + "tokens_seen": 451608576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974844533600803, + "loss": 1.4993, + "theoretical_loss": 3.986801272916603, + "tokens_seen": 451674112 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697468405215647, + "loss": 1.5069, + "theoretical_loss": 3.9867383301130532, + "tokens_seen": 451739648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974523570712137, + "loss": 1.5506, + "theoretical_loss": 3.9866753989966224, + "tokens_seen": 451805184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974363089267804, + "loss": 1.5058, + "theoretical_loss": 3.986612479563445, + "tokens_seen": 451870720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974202607823471, + "loss": 1.5182, + "theoretical_loss": 3.9865495718096584, + "tokens_seen": 451936256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006974042126379138, + "loss": 1.5674, + "theoretical_loss": 3.9864866757314017, + "tokens_seen": 452001792 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973881644934805, + "loss": 1.4775, + "theoretical_loss": 3.9864237913248157, + "tokens_seen": 452067328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973721163490472, + "loss": 1.5393, + "theoretical_loss": 3.9863609185860422, + "tokens_seen": 452132864 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 173483, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.569876790046692, + "objective/train/theoretical_loss": 3.9862980575112257, + "objective/train/tokens_used": 472658400, + "theoretical_loss": 3.9862980575112257, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973560682046139, + "loss": 1.5699, + "theoretical_loss": 3.9862980575112257, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973400200601806, + "loss": 1.4943, + "theoretical_loss": 3.986235208096513, + "tokens_seen": 452263936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006973239719157473, + "loss": 1.6535, + "theoretical_loss": 3.9861723703380516, + "tokens_seen": 452329472 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697307923771314, + "loss": 1.5864, + "theoretical_loss": 3.986109544231992, + "tokens_seen": 452395008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972918756268807, + "loss": 1.4552, + "theoretical_loss": 3.9860467297744844, + "tokens_seen": 452460544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972758274824474, + "loss": 1.5841, + "theoretical_loss": 3.985983926961684, + "tokens_seen": 452526080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972597793380141, + "loss": 1.5327, + "theoretical_loss": 3.985921135789745, + "tokens_seen": 452591616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972437311935808, + "loss": 1.4091, + "theoretical_loss": 3.9858583562548247, + "tokens_seen": 452657152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972276830491475, + "loss": 1.5554, + "theoretical_loss": 3.9857955883530827, + "tokens_seen": 452722688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006972116349047142, + "loss": 1.5018, + "theoretical_loss": 3.985732832080679, + "tokens_seen": 452788224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971955867602809, + "loss": 1.3814, + "theoretical_loss": 3.985670087433777, + "tokens_seen": 452853760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971795386158476, + "loss": 1.5006, + "theoretical_loss": 3.9856073544085406, + "tokens_seen": 452919296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971634904714143, + "loss": 1.452, + "theoretical_loss": 3.9855446330011364, + "tokens_seen": 452984832 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697147442326981, + "loss": 1.401, + "theoretical_loss": 3.9854819232077316, + "tokens_seen": 453050368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971313941825477, + "loss": 1.419, + "theoretical_loss": 3.985419225024498, + "tokens_seen": 453115904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006971153460381144, + "loss": 1.6285, + "theoretical_loss": 3.985356538447605, + "tokens_seen": 453181440 + }, + { + "epoch": 0.14, + "learning_rate": 0.000697099297893681, + "loss": 1.5325, + "theoretical_loss": 3.9852938634732276, + "tokens_seen": 453246976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970832497492478, + "loss": 1.5741, + "theoretical_loss": 3.985231200097541, + "tokens_seen": 453312512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970672016048144, + "loss": 1.4757, + "theoretical_loss": 3.9851685483167225, + "tokens_seen": 453378048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970511534603812, + "loss": 1.5295, + "theoretical_loss": 3.9851059081269504, + "tokens_seen": 453443584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970351053159478, + "loss": 1.4699, + "theoretical_loss": 3.9850432795244055, + "tokens_seen": 453509120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970190571715146, + "loss": 1.4964, + "theoretical_loss": 3.9849806625052713, + "tokens_seen": 453574656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006970030090270812, + "loss": 1.4018, + "theoretical_loss": 3.984918057065731, + "tokens_seen": 453640192 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696986960882648, + "loss": 1.5985, + "theoretical_loss": 3.984855463201972, + "tokens_seen": 453705728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006969709127382146, + "loss": 1.5649, + "theoretical_loss": 3.984792880910182, + "tokens_seen": 453771264 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006969548645937813, + "loss": 1.5455, + "theoretical_loss": 3.98473031018655, + "tokens_seen": 453836800 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696938816449348, + "loss": 1.5706, + "theoretical_loss": 3.984667751027268, + "tokens_seen": 453902336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006969227683049147, + "loss": 1.4126, + "theoretical_loss": 3.98460520342853, + "tokens_seen": 453967872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006969067201604814, + "loss": 1.5714, + "theoretical_loss": 3.984542667386531, + "tokens_seen": 454033408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968906720160481, + "loss": 1.5239, + "theoretical_loss": 3.9844801428974685, + "tokens_seen": 454098944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968746238716148, + "loss": 1.5103, + "theoretical_loss": 3.98441762995754, + "tokens_seen": 454164480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968585757271815, + "loss": 1.5197, + "theoretical_loss": 3.984355128562947, + "tokens_seen": 454230016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968425275827482, + "loss": 1.5499, + "theoretical_loss": 3.984292638709892, + "tokens_seen": 454295552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968264794383149, + "loss": 1.4613, + "theoretical_loss": 3.9842301603945787, + "tokens_seen": 454361088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006968104312938816, + "loss": 1.4049, + "theoretical_loss": 3.9841676936132133, + "tokens_seen": 454426624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967943831494483, + "loss": 1.4664, + "theoretical_loss": 3.984105238362005, + "tokens_seen": 454492160 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696778335005015, + "loss": 1.5506, + "theoretical_loss": 3.984042794637161, + "tokens_seen": 454557696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967622868605817, + "loss": 1.5416, + "theoretical_loss": 3.9839803624348944, + "tokens_seen": 454623232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967462387161485, + "loss": 1.4757, + "theoretical_loss": 3.983917941751418, + "tokens_seen": 454688768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967301905717151, + "loss": 1.4974, + "theoretical_loss": 3.983855532582947, + "tokens_seen": 454754304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006967141424272819, + "loss": 1.5443, + "theoretical_loss": 3.983793134925697, + "tokens_seen": 454819840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966980942828485, + "loss": 1.4687, + "theoretical_loss": 3.9837307487758884, + "tokens_seen": 454885376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966820461384153, + "loss": 1.6184, + "theoretical_loss": 3.9836683741297403, + "tokens_seen": 454950912 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696665997993982, + "loss": 1.5434, + "theoretical_loss": 3.9836060109834746, + "tokens_seen": 455016448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966499498495487, + "loss": 1.5531, + "theoretical_loss": 3.983543659333316, + "tokens_seen": 455081984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966339017051154, + "loss": 1.6237, + "theoretical_loss": 3.9834813191754908, + "tokens_seen": 455147520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966178535606821, + "loss": 1.4922, + "theoretical_loss": 3.983418990506225, + "tokens_seen": 455213056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006966018054162488, + "loss": 1.6329, + "theoretical_loss": 3.9833566733217483, + "tokens_seen": 455278592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965857572718155, + "loss": 1.5283, + "theoretical_loss": 3.983294367618292, + "tokens_seen": 455344128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965697091273822, + "loss": 1.5971, + "theoretical_loss": 3.983232073392089, + "tokens_seen": 455409664 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965536609829489, + "loss": 1.5391, + "theoretical_loss": 3.983169790639374, + "tokens_seen": 455475200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965376128385156, + "loss": 1.5567, + "theoretical_loss": 3.983107519356383, + "tokens_seen": 455540736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006965215646940823, + "loss": 1.5627, + "theoretical_loss": 3.9830452595393546, + "tokens_seen": 455606272 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696505516549649, + "loss": 1.6076, + "theoretical_loss": 3.9829830111845284, + "tokens_seen": 455671808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964894684052157, + "loss": 1.5826, + "theoretical_loss": 3.9829207742881456, + "tokens_seen": 455737344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964734202607824, + "loss": 1.666, + "theoretical_loss": 3.982858548846451, + "tokens_seen": 455802880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964573721163491, + "loss": 1.4704, + "theoretical_loss": 3.982796334855689, + "tokens_seen": 455868416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964413239719158, + "loss": 1.4583, + "theoretical_loss": 3.9827341323121064, + "tokens_seen": 455933952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964252758274825, + "loss": 1.4122, + "theoretical_loss": 3.9826719412119527, + "tokens_seen": 455999488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006964092276830492, + "loss": 1.4852, + "theoretical_loss": 3.982609761551478, + "tokens_seen": 456065024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963931795386159, + "loss": 1.5479, + "theoretical_loss": 3.982547593326934, + "tokens_seen": 456130560 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963771313941826, + "loss": 1.6689, + "theoretical_loss": 3.982485436534576, + "tokens_seen": 456196096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963610832497493, + "loss": 1.5739, + "theoretical_loss": 3.98242329117066, + "tokens_seen": 456261632 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696345035105316, + "loss": 1.4532, + "theoretical_loss": 3.982361157231442, + "tokens_seen": 456327168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963289869608827, + "loss": 1.5289, + "theoretical_loss": 3.982299034713183, + "tokens_seen": 456392704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006963129388164494, + "loss": 1.5715, + "theoretical_loss": 3.9822369236121427, + "tokens_seen": 456458240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962968906720161, + "loss": 1.4796, + "theoretical_loss": 3.982174823924586, + "tokens_seen": 456523776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962808425275828, + "loss": 1.5233, + "theoretical_loss": 3.9821127356467754, + "tokens_seen": 456589312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962647943831495, + "loss": 1.527, + "theoretical_loss": 3.9820506587749787, + "tokens_seen": 456654848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962487462387162, + "loss": 1.4464, + "theoretical_loss": 3.9819885933054637, + "tokens_seen": 456720384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962326980942829, + "loss": 1.4648, + "theoretical_loss": 3.9819265392345, + "tokens_seen": 456785920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962166499498496, + "loss": 1.4259, + "theoretical_loss": 3.9818644965583596, + "tokens_seen": 456851456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006962006018054163, + "loss": 1.499, + "theoretical_loss": 3.981802465273316, + "tokens_seen": 456916992 + }, + { + "epoch": 0.14, + "learning_rate": 0.000696184553660983, + "loss": 1.5404, + "theoretical_loss": 3.9817404453756446, + "tokens_seen": 456982528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961685055165497, + "loss": 1.5461, + "theoretical_loss": 3.981678436861622, + "tokens_seen": 457048064 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961524573721164, + "loss": 1.5975, + "theoretical_loss": 3.981616439727527, + "tokens_seen": 457113600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961364092276831, + "loss": 1.4844, + "theoretical_loss": 3.9815544539696397, + "tokens_seen": 457179136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961203610832498, + "loss": 1.5654, + "theoretical_loss": 3.981492479584243, + "tokens_seen": 457244672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006961043129388165, + "loss": 1.5671, + "theoretical_loss": 3.9814305165676203, + "tokens_seen": 457310208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960882647943832, + "loss": 1.5522, + "theoretical_loss": 3.9813685649160577, + "tokens_seen": 457375744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960722166499499, + "loss": 1.5942, + "theoretical_loss": 3.981306624625842, + "tokens_seen": 457441280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960561685055166, + "loss": 1.5251, + "theoretical_loss": 3.9812446956932632, + "tokens_seen": 457506816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960401203610833, + "loss": 1.5832, + "theoretical_loss": 3.9811827781146123, + "tokens_seen": 457572352 + }, + { + "epoch": 0.14, + "learning_rate": 0.00069602407221665, + "loss": 1.5439, + "theoretical_loss": 3.981120871886181, + "tokens_seen": 457637888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006960080240722167, + "loss": 1.6228, + "theoretical_loss": 3.9810589770042637, + "tokens_seen": 457703424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959919759277834, + "loss": 1.5636, + "theoretical_loss": 3.9809970934651577, + "tokens_seen": 457768960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959759277833501, + "loss": 1.5494, + "theoretical_loss": 3.9809352212651596, + "tokens_seen": 457834496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959598796389168, + "loss": 1.5647, + "theoretical_loss": 3.9808733604005706, + "tokens_seen": 457900032 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959438314944835, + "loss": 1.4363, + "theoretical_loss": 3.9808115108676905, + "tokens_seen": 457965568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959277833500502, + "loss": 1.4064, + "theoretical_loss": 3.9807496726628235, + "tokens_seen": 458031104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006959117352056169, + "loss": 1.3768, + "theoretical_loss": 3.9806878457822736, + "tokens_seen": 458096640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958956870611836, + "loss": 1.4324, + "theoretical_loss": 3.9806260302223477, + "tokens_seen": 458162176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958796389167503, + "loss": 1.5745, + "theoretical_loss": 3.9805642259793546, + "tokens_seen": 458227712 + }, + { + "epoch": 0.14, + "learning_rate": 0.000695863590772317, + "loss": 1.5488, + "theoretical_loss": 3.9805024330496037, + "tokens_seen": 458293248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958475426278837, + "loss": 1.5386, + "theoretical_loss": 3.980440651429407, + "tokens_seen": 458358784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958314944834504, + "loss": 1.5966, + "theoretical_loss": 3.9803788811150778, + "tokens_seen": 458424320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006958154463390171, + "loss": 1.5591, + "theoretical_loss": 3.980317122102931, + "tokens_seen": 458489856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957993981945838, + "loss": 1.5356, + "theoretical_loss": 3.9802553743892846, + "tokens_seen": 458555392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957833500501505, + "loss": 1.4651, + "theoretical_loss": 3.9801936379704568, + "tokens_seen": 458620928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957673019057172, + "loss": 1.633, + "theoretical_loss": 3.9801319128427677, + "tokens_seen": 458686464 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 176189, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5714786052703857, + "objective/train/theoretical_loss": 3.9800701990025393, + "objective/train/tokens_used": 479212000, + "theoretical_loss": 3.9800701990025393, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957512537612839, + "loss": 1.5715, + "theoretical_loss": 3.9800701990025393, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957352056168506, + "loss": 1.6602, + "theoretical_loss": 3.9800084964460964, + "tokens_seen": 458817536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957191574724173, + "loss": 1.6253, + "theoretical_loss": 3.9799468051697633, + "tokens_seen": 458883072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006957031093279839, + "loss": 1.6018, + "theoretical_loss": 3.9798851251698686, + "tokens_seen": 458948608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956870611835507, + "loss": 1.5467, + "theoretical_loss": 3.97982345644274, + "tokens_seen": 459014144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956710130391173, + "loss": 1.5417, + "theoretical_loss": 3.9797617989847094, + "tokens_seen": 459079680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956549648946841, + "loss": 1.5148, + "theoretical_loss": 3.979700152792108, + "tokens_seen": 459145216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956389167502507, + "loss": 1.5933, + "theoretical_loss": 3.9796385178612717, + "tokens_seen": 459210752 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956228686058175, + "loss": 1.5859, + "theoretical_loss": 3.9795768941885346, + "tokens_seen": 459276288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006956068204613841, + "loss": 1.6, + "theoretical_loss": 3.9795152817702357, + "tokens_seen": 459341824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955907723169509, + "loss": 1.5086, + "theoretical_loss": 3.979453680602713, + "tokens_seen": 459407360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955747241725175, + "loss": 1.5996, + "theoretical_loss": 3.9793920906823086, + "tokens_seen": 459472896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955586760280842, + "loss": 1.5435, + "theoretical_loss": 3.979330512005365, + "tokens_seen": 459538432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955426278836509, + "loss": 1.5717, + "theoretical_loss": 3.9792689445682257, + "tokens_seen": 459603968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955265797392176, + "loss": 1.5963, + "theoretical_loss": 3.9792073883672385, + "tokens_seen": 459669504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006955105315947843, + "loss": 1.6129, + "theoretical_loss": 3.9791458433987503, + "tokens_seen": 459735040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954944834503511, + "loss": 1.6797, + "theoretical_loss": 3.97908430965911, + "tokens_seen": 459800576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954784353059177, + "loss": 1.6161, + "theoretical_loss": 3.979022787144671, + "tokens_seen": 459866112 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954623871614845, + "loss": 1.6426, + "theoretical_loss": 3.9789612758517836, + "tokens_seen": 459931648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954463390170511, + "loss": 1.6343, + "theoretical_loss": 3.9788997757768043, + "tokens_seen": 459997184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954302908726179, + "loss": 1.6977, + "theoretical_loss": 3.9788382869160897, + "tokens_seen": 460062720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006954142427281846, + "loss": 1.597, + "theoretical_loss": 3.9787768092659963, + "tokens_seen": 460128256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953981945837513, + "loss": 1.5465, + "theoretical_loss": 3.978715342822885, + "tokens_seen": 460193792 + }, + { + "epoch": 0.14, + "learning_rate": 0.000695382146439318, + "loss": 1.615, + "theoretical_loss": 3.9786538875831168, + "tokens_seen": 460259328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953660982948847, + "loss": 1.6, + "theoretical_loss": 3.9785924435430555, + "tokens_seen": 460324864 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953500501504514, + "loss": 1.6258, + "theoretical_loss": 3.978531010699066, + "tokens_seen": 460390400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953340020060181, + "loss": 1.666, + "theoretical_loss": 3.9784695890475144, + "tokens_seen": 460455936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953179538615848, + "loss": 1.7152, + "theoretical_loss": 3.9784081785847687, + "tokens_seen": 460521472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006953019057171515, + "loss": 1.7028, + "theoretical_loss": 3.9783467793071994, + "tokens_seen": 460587008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952858575727182, + "loss": 1.5478, + "theoretical_loss": 3.978285391211178, + "tokens_seen": 460652544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952698094282849, + "loss": 1.5711, + "theoretical_loss": 3.9782240142930787, + "tokens_seen": 460718080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952537612838516, + "loss": 1.5385, + "theoretical_loss": 3.978162648549275, + "tokens_seen": 460783616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952377131394183, + "loss": 1.5657, + "theoretical_loss": 3.9781012939761453, + "tokens_seen": 460849152 + }, + { + "epoch": 0.14, + "learning_rate": 0.000695221664994985, + "loss": 1.6169, + "theoretical_loss": 3.978039950570067, + "tokens_seen": 460914688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006952056168505517, + "loss": 1.5825, + "theoretical_loss": 3.9779786183274197, + "tokens_seen": 460980224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951895687061184, + "loss": 1.5666, + "theoretical_loss": 3.977917297244587, + "tokens_seen": 461045760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951735205616851, + "loss": 1.6134, + "theoretical_loss": 3.9778559873179504, + "tokens_seen": 461111296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951574724172518, + "loss": 1.6112, + "theoretical_loss": 3.977794688543897, + "tokens_seen": 461176832 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951414242728185, + "loss": 1.556, + "theoretical_loss": 3.9777334009188117, + "tokens_seen": 461242368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951253761283852, + "loss": 1.5927, + "theoretical_loss": 3.9776721244390845, + "tokens_seen": 461307904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006951093279839519, + "loss": 1.4586, + "theoretical_loss": 3.9776108591011052, + "tokens_seen": 461373440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950932798395186, + "loss": 1.538, + "theoretical_loss": 3.9775496049012657, + "tokens_seen": 461438976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950772316950853, + "loss": 1.4766, + "theoretical_loss": 3.9774883618359596, + "tokens_seen": 461504512 + }, + { + "epoch": 0.14, + "learning_rate": 0.000695061183550652, + "loss": 1.7255, + "theoretical_loss": 3.9774271299015824, + "tokens_seen": 461570048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950451354062187, + "loss": 1.5657, + "theoretical_loss": 3.977365909094531, + "tokens_seen": 461635584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950290872617854, + "loss": 1.5224, + "theoretical_loss": 3.977304699411204, + "tokens_seen": 461701120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006950130391173521, + "loss": 1.6465, + "theoretical_loss": 3.977243500848001, + "tokens_seen": 461766656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949969909729188, + "loss": 1.6044, + "theoretical_loss": 3.977182313401325, + "tokens_seen": 461832192 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949809428284855, + "loss": 1.5936, + "theoretical_loss": 3.977121137067579, + "tokens_seen": 461897728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949648946840522, + "loss": 1.612, + "theoretical_loss": 3.9770599718431687, + "tokens_seen": 461963264 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949488465396189, + "loss": 1.4658, + "theoretical_loss": 3.9769988177245015, + "tokens_seen": 462028800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949327983951856, + "loss": 1.6042, + "theoretical_loss": 3.976937674707985, + "tokens_seen": 462094336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006949167502507523, + "loss": 1.5804, + "theoretical_loss": 3.9768765427900306, + "tokens_seen": 462159872 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694900702106319, + "loss": 1.6919, + "theoretical_loss": 3.97681542196705, + "tokens_seen": 462225408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948846539618857, + "loss": 1.4906, + "theoretical_loss": 3.976754312235456, + "tokens_seen": 462290944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948686058174524, + "loss": 1.5243, + "theoretical_loss": 3.9766932135916653, + "tokens_seen": 462356480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948525576730191, + "loss": 1.5798, + "theoretical_loss": 3.9766321260320945, + "tokens_seen": 462422016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948365095285858, + "loss": 1.5613, + "theoretical_loss": 3.976571049553162, + "tokens_seen": 462487552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948204613841525, + "loss": 1.6249, + "theoretical_loss": 3.9765099841512885, + "tokens_seen": 462553088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006948044132397192, + "loss": 1.6278, + "theoretical_loss": 3.976448929822895, + "tokens_seen": 462618624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947883650952859, + "loss": 1.569, + "theoretical_loss": 3.9763878865644067, + "tokens_seen": 462684160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947723169508526, + "loss": 1.5272, + "theoretical_loss": 3.9763268543722483, + "tokens_seen": 462749696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947562688064193, + "loss": 1.4232, + "theoretical_loss": 3.976265833242846, + "tokens_seen": 462815232 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694740220661986, + "loss": 1.5524, + "theoretical_loss": 3.9762048231726297, + "tokens_seen": 462880768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947241725175527, + "loss": 1.5326, + "theoretical_loss": 3.9761438241580294, + "tokens_seen": 462946304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006947081243731194, + "loss": 1.6068, + "theoretical_loss": 3.976082836195477, + "tokens_seen": 463011840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946920762286861, + "loss": 1.5389, + "theoretical_loss": 3.976021859281406, + "tokens_seen": 463077376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946760280842528, + "loss": 1.4789, + "theoretical_loss": 3.975960893412251, + "tokens_seen": 463142912 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946599799398195, + "loss": 1.5005, + "theoretical_loss": 3.9758999385844502, + "tokens_seen": 463208448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946439317953862, + "loss": 1.5717, + "theoretical_loss": 3.975838994794441, + "tokens_seen": 463273984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946278836509529, + "loss": 1.6051, + "theoretical_loss": 3.975778062038665, + "tokens_seen": 463339520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006946118355065196, + "loss": 1.5792, + "theoretical_loss": 3.975717140313564, + "tokens_seen": 463405056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945957873620863, + "loss": 1.4498, + "theoretical_loss": 3.9756562296155797, + "tokens_seen": 463470592 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694579739217653, + "loss": 1.5979, + "theoretical_loss": 3.975595329941159, + "tokens_seen": 463536128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945636910732197, + "loss": 1.6151, + "theoretical_loss": 3.975534441286748, + "tokens_seen": 463601664 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945476429287864, + "loss": 1.4495, + "theoretical_loss": 3.975473563648796, + "tokens_seen": 463667200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945315947843531, + "loss": 1.5228, + "theoretical_loss": 3.975412697023753, + "tokens_seen": 463732736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006945155466399198, + "loss": 1.5302, + "theoretical_loss": 3.9753518414080693, + "tokens_seen": 463798272 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944994984954865, + "loss": 1.7371, + "theoretical_loss": 3.9752909967982, + "tokens_seen": 463863808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944834503510532, + "loss": 1.5405, + "theoretical_loss": 3.9752301631906, + "tokens_seen": 463929344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944674022066199, + "loss": 1.5157, + "theoretical_loss": 3.9751693405817248, + "tokens_seen": 463994880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944513540621866, + "loss": 1.5892, + "theoretical_loss": 3.9751085289680343, + "tokens_seen": 464060416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944353059177533, + "loss": 1.4875, + "theoretical_loss": 3.975047728345987, + "tokens_seen": 464125952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00069441925777332, + "loss": 1.5242, + "theoretical_loss": 3.9749869387120462, + "tokens_seen": 464191488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006944032096288867, + "loss": 1.4979, + "theoretical_loss": 3.974926160062674, + "tokens_seen": 464257024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943871614844534, + "loss": 1.5567, + "theoretical_loss": 3.9748653923943356, + "tokens_seen": 464322560 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943711133400201, + "loss": 1.592, + "theoretical_loss": 3.9748046357034976, + "tokens_seen": 464388096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943550651955868, + "loss": 1.6049, + "theoretical_loss": 3.9747438899866276, + "tokens_seen": 464453632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943390170511535, + "loss": 1.5489, + "theoretical_loss": 3.9746831552401964, + "tokens_seen": 464519168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006943229689067202, + "loss": 1.5158, + "theoretical_loss": 3.9746224314606753, + "tokens_seen": 464584704 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694306920762287, + "loss": 1.4733, + "theoretical_loss": 3.9745617186445372, + "tokens_seen": 464650240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006942908726178536, + "loss": 1.489, + "theoretical_loss": 3.974501016788256, + "tokens_seen": 464715776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006942748244734202, + "loss": 1.5355, + "theoretical_loss": 3.9744403258883096, + "tokens_seen": 464781312 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694258776328987, + "loss": 1.5733, + "theoretical_loss": 3.974379645941174, + "tokens_seen": 464846848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006942427281845536, + "loss": 1.4746, + "theoretical_loss": 3.974318976943331, + "tokens_seen": 464912384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006942266800401204, + "loss": 1.5705, + "theoretical_loss": 3.974258318891261, + "tokens_seen": 464977920 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694210631895687, + "loss": 1.5576, + "theoretical_loss": 3.974197671781446, + "tokens_seen": 465043456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941945837512538, + "loss": 1.646, + "theoretical_loss": 3.9741370356103713, + "tokens_seen": 465108992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941785356068204, + "loss": 1.6522, + "theoretical_loss": 3.9740764103745225, + "tokens_seen": 465174528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941624874623872, + "loss": 1.5674, + "theoretical_loss": 3.974015796070388, + "tokens_seen": 465240064 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 178655, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.6087545156478882, + "objective/train/theoretical_loss": 3.973955192694457, + "objective/train/tokens_used": 485765600, + "theoretical_loss": 3.973955192694457, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941464393179538, + "loss": 1.6088, + "theoretical_loss": 3.973955192694457, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941303911735206, + "loss": 1.4672, + "theoretical_loss": 3.97389460024322, + "tokens_seen": 465371136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006941143430290872, + "loss": 1.5984, + "theoretical_loss": 3.9738340187131698, + "tokens_seen": 465436672 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694098294884654, + "loss": 1.548, + "theoretical_loss": 3.973773448100801, + "tokens_seen": 465502208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940822467402206, + "loss": 1.4505, + "theoretical_loss": 3.973712888402609, + "tokens_seen": 465567744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940661985957874, + "loss": 1.6133, + "theoretical_loss": 3.9736523396150916, + "tokens_seen": 465633280 + }, + { + "epoch": 0.14, + "learning_rate": 0.000694050150451354, + "loss": 1.6009, + "theoretical_loss": 3.973591801734748, + "tokens_seen": 465698816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940341023069208, + "loss": 1.5129, + "theoretical_loss": 3.9735312747580775, + "tokens_seen": 465764352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940180541624875, + "loss": 1.5096, + "theoretical_loss": 3.973470758681584, + "tokens_seen": 465829888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006940020060180542, + "loss": 1.4872, + "theoretical_loss": 3.9734102535017715, + "tokens_seen": 465895424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939859578736209, + "loss": 1.6523, + "theoretical_loss": 3.9733497592151443, + "tokens_seen": 465960960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939699097291876, + "loss": 1.5173, + "theoretical_loss": 3.9732892758182103, + "tokens_seen": 466026496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939538615847543, + "loss": 1.5132, + "theoretical_loss": 3.973228803307478, + "tokens_seen": 466092032 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693937813440321, + "loss": 1.49, + "theoretical_loss": 3.9731683416794583, + "tokens_seen": 466157568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939217652958877, + "loss": 1.5465, + "theoretical_loss": 3.9731078909306623, + "tokens_seen": 466223104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006939057171514544, + "loss": 1.4384, + "theoretical_loss": 3.9730474510576044, + "tokens_seen": 466288640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938896690070211, + "loss": 1.4888, + "theoretical_loss": 3.9729870220567993, + "tokens_seen": 466354176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938736208625878, + "loss": 1.5602, + "theoretical_loss": 3.9729266039247637, + "tokens_seen": 466419712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938575727181545, + "loss": 1.5388, + "theoretical_loss": 3.9728661966580163, + "tokens_seen": 466485248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938415245737212, + "loss": 1.5098, + "theoretical_loss": 3.9728058002530773, + "tokens_seen": 466550784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938254764292879, + "loss": 1.6561, + "theoretical_loss": 3.972745414706468, + "tokens_seen": 466616320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006938094282848546, + "loss": 1.4798, + "theoretical_loss": 3.972685040014711, + "tokens_seen": 466681856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937933801404213, + "loss": 1.4671, + "theoretical_loss": 3.9726246761743322, + "tokens_seen": 466747392 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693777331995988, + "loss": 1.4287, + "theoretical_loss": 3.9725643231818575, + "tokens_seen": 466812928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937612838515547, + "loss": 1.5203, + "theoretical_loss": 3.9725039810338147, + "tokens_seen": 466878464 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937452357071214, + "loss": 1.4584, + "theoretical_loss": 3.972443649726734, + "tokens_seen": 466944000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937291875626881, + "loss": 1.5224, + "theoretical_loss": 3.9723833292571458, + "tokens_seen": 467009536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006937131394182548, + "loss": 1.5539, + "theoretical_loss": 3.9723230196215837, + "tokens_seen": 467075072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936970912738215, + "loss": 1.4898, + "theoretical_loss": 3.9722627208165817, + "tokens_seen": 467140608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936810431293882, + "loss": 1.4993, + "theoretical_loss": 3.9722024328386754, + "tokens_seen": 467206144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936649949849549, + "loss": 1.4426, + "theoretical_loss": 3.9721421556844034, + "tokens_seen": 467271680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936489468405216, + "loss": 1.5722, + "theoretical_loss": 3.972081889350304, + "tokens_seen": 467337216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936328986960883, + "loss": 1.517, + "theoretical_loss": 3.9720216338329184, + "tokens_seen": 467402752 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693616850551655, + "loss": 1.4703, + "theoretical_loss": 3.971961389128789, + "tokens_seen": 467468288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006936008024072217, + "loss": 1.5694, + "theoretical_loss": 3.97190115523446, + "tokens_seen": 467533824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935847542627884, + "loss": 1.3955, + "theoretical_loss": 3.9718409321464754, + "tokens_seen": 467599360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935687061183551, + "loss": 1.4686, + "theoretical_loss": 3.9717807198613837, + "tokens_seen": 467664896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935526579739218, + "loss": 1.5093, + "theoretical_loss": 3.971720518375734, + "tokens_seen": 467730432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935366098294885, + "loss": 1.5434, + "theoretical_loss": 3.971660327686075, + "tokens_seen": 467795968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935205616850552, + "loss": 1.6034, + "theoretical_loss": 3.97160014778896, + "tokens_seen": 467861504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006935045135406219, + "loss": 1.476, + "theoretical_loss": 3.9715399786809424, + "tokens_seen": 467927040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934884653961886, + "loss": 1.6572, + "theoretical_loss": 3.971479820358576, + "tokens_seen": 467992576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934724172517553, + "loss": 1.4584, + "theoretical_loss": 3.9714196728184197, + "tokens_seen": 468058112 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693456369107322, + "loss": 1.4966, + "theoretical_loss": 3.971359536057029, + "tokens_seen": 468123648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934403209628887, + "loss": 1.4544, + "theoretical_loss": 3.9712994100709653, + "tokens_seen": 468189184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934242728184554, + "loss": 1.4978, + "theoretical_loss": 3.97123929485679, + "tokens_seen": 468254720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006934082246740221, + "loss": 1.5642, + "theoretical_loss": 3.971179190411066, + "tokens_seen": 468320256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933921765295888, + "loss": 1.5341, + "theoretical_loss": 3.9711190967303565, + "tokens_seen": 468385792 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933761283851555, + "loss": 1.4775, + "theoretical_loss": 3.971059013811229, + "tokens_seen": 468451328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933600802407222, + "loss": 1.6577, + "theoretical_loss": 3.9709989416502514, + "tokens_seen": 468516864 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933440320962889, + "loss": 1.4512, + "theoretical_loss": 3.9709388802439918, + "tokens_seen": 468582400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933279839518556, + "loss": 1.6747, + "theoretical_loss": 3.9708788295890223, + "tokens_seen": 468647936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006933119358074223, + "loss": 1.4636, + "theoretical_loss": 3.9708187896819136, + "tokens_seen": 468713472 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693295887662989, + "loss": 1.523, + "theoretical_loss": 3.9707587605192414, + "tokens_seen": 468779008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932798395185557, + "loss": 1.5071, + "theoretical_loss": 3.9706987420975803, + "tokens_seen": 468844544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932637913741224, + "loss": 1.485, + "theoretical_loss": 3.970638734413508, + "tokens_seen": 468910080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932477432296891, + "loss": 1.5651, + "theoretical_loss": 3.970578737463603, + "tokens_seen": 468975616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932316950852558, + "loss": 1.5342, + "theoretical_loss": 3.9705187512444446, + "tokens_seen": 469041152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006932156469408225, + "loss": 1.532, + "theoretical_loss": 3.9704587757526157, + "tokens_seen": 469106688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931995987963892, + "loss": 1.4314, + "theoretical_loss": 3.9703988109846997, + "tokens_seen": 469172224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931835506519559, + "loss": 1.5136, + "theoretical_loss": 3.970338856937281, + "tokens_seen": 469237760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931675025075226, + "loss": 1.49, + "theoretical_loss": 3.970278913606947, + "tokens_seen": 469303296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931514543630893, + "loss": 1.5094, + "theoretical_loss": 3.9702189809902846, + "tokens_seen": 469368832 + }, + { + "epoch": 0.14, + "learning_rate": 0.000693135406218656, + "loss": 1.4447, + "theoretical_loss": 3.9701590590838842, + "tokens_seen": 469434368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931193580742227, + "loss": 1.4604, + "theoretical_loss": 3.970099147884336, + "tokens_seen": 469499904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006931033099297894, + "loss": 1.543, + "theoretical_loss": 3.9700392473882347, + "tokens_seen": 469565440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930872617853562, + "loss": 1.5127, + "theoretical_loss": 3.969979357592173, + "tokens_seen": 469630976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930712136409228, + "loss": 1.4286, + "theoretical_loss": 3.969919478492747, + "tokens_seen": 469696512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930551654964896, + "loss": 1.4533, + "theoretical_loss": 3.9698596100865546, + "tokens_seen": 469762048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930391173520562, + "loss": 1.5532, + "theoretical_loss": 3.9697997523701947, + "tokens_seen": 469827584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930230692076229, + "loss": 1.5467, + "theoretical_loss": 3.969739905340268, + "tokens_seen": 469893120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006930070210631896, + "loss": 1.5425, + "theoretical_loss": 3.9696800689933758, + "tokens_seen": 469958656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929909729187563, + "loss": 1.4773, + "theoretical_loss": 3.9696202433261227, + "tokens_seen": 470024192 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692974924774323, + "loss": 1.5549, + "theoretical_loss": 3.9695604283351136, + "tokens_seen": 470089728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929588766298897, + "loss": 1.4935, + "theoretical_loss": 3.9695006240169546, + "tokens_seen": 470155264 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929428284854564, + "loss": 1.5408, + "theoretical_loss": 3.9694408303682556, + "tokens_seen": 470220800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929267803410231, + "loss": 1.5161, + "theoretical_loss": 3.9693810473856246, + "tokens_seen": 470286336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006929107321965898, + "loss": 1.4718, + "theoretical_loss": 3.969321275065674, + "tokens_seen": 470351872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928946840521565, + "loss": 1.5758, + "theoretical_loss": 3.969261513405017, + "tokens_seen": 470417408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928786359077231, + "loss": 1.4547, + "theoretical_loss": 3.9692017624002682, + "tokens_seen": 470482944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928625877632899, + "loss": 1.5734, + "theoretical_loss": 3.9691420220480422, + "tokens_seen": 470548480 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928465396188565, + "loss": 1.5057, + "theoretical_loss": 3.9690822923449582, + "tokens_seen": 470614016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928304914744233, + "loss": 1.5712, + "theoretical_loss": 3.969022573287635, + "tokens_seen": 470679552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006928144433299899, + "loss": 1.5685, + "theoretical_loss": 3.968962864872693, + "tokens_seen": 470745088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927983951855567, + "loss": 1.5355, + "theoretical_loss": 3.9689031670967543, + "tokens_seen": 470810624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927823470411233, + "loss": 1.5351, + "theoretical_loss": 3.9688434799564427, + "tokens_seen": 470876160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927662988966901, + "loss": 1.5651, + "theoretical_loss": 3.9687838034483844, + "tokens_seen": 470941696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927502507522567, + "loss": 1.4879, + "theoretical_loss": 3.9687241375692053, + "tokens_seen": 471007232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927342026078235, + "loss": 1.5786, + "theoretical_loss": 3.9686644823155337, + "tokens_seen": 471072768 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927181544633901, + "loss": 1.4935, + "theoretical_loss": 3.9686048376840004, + "tokens_seen": 471138304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006927021063189569, + "loss": 1.62, + "theoretical_loss": 3.968545203671236, + "tokens_seen": 471203840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926860581745235, + "loss": 1.5814, + "theoretical_loss": 3.9684855802738745, + "tokens_seen": 471269376 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926700100300903, + "loss": 1.4297, + "theoretical_loss": 3.9684259674885496, + "tokens_seen": 471334912 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692653961885657, + "loss": 1.4324, + "theoretical_loss": 3.9683663653118977, + "tokens_seen": 471400448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926379137412237, + "loss": 1.6158, + "theoretical_loss": 3.968306773740556, + "tokens_seen": 471465984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926218655967904, + "loss": 1.4514, + "theoretical_loss": 3.9682471927711642, + "tokens_seen": 471531520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006926058174523571, + "loss": 1.5359, + "theoretical_loss": 3.9681876224003627, + "tokens_seen": 471597056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925897693079238, + "loss": 1.5795, + "theoretical_loss": 3.9681280626247943, + "tokens_seen": 471662592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925737211634905, + "loss": 1.6455, + "theoretical_loss": 3.9680685134411022, + "tokens_seen": 471728128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925576730190572, + "loss": 1.5507, + "theoretical_loss": 3.9680089748459313, + "tokens_seen": 471793664 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 180997, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.603252649307251, + "objective/train/theoretical_loss": 3.967949446835929, + "objective/train/tokens_used": 492319200, + "theoretical_loss": 3.967949446835929, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925416248746239, + "loss": 1.6033, + "theoretical_loss": 3.967949446835929, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925255767301906, + "loss": 1.5063, + "theoretical_loss": 3.967889929407743, + "tokens_seen": 471924736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006925095285857573, + "loss": 1.5918, + "theoretical_loss": 3.9678304225580248, + "tokens_seen": 471990272 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692493480441324, + "loss": 1.4148, + "theoretical_loss": 3.967770926283424, + "tokens_seen": 472055808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924774322968907, + "loss": 1.5649, + "theoretical_loss": 3.9677114405805938, + "tokens_seen": 472121344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924613841524574, + "loss": 1.5074, + "theoretical_loss": 3.967651965446189, + "tokens_seen": 472186880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924453360080241, + "loss": 1.5488, + "theoretical_loss": 3.9675925008768655, + "tokens_seen": 472252416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924292878635908, + "loss": 1.4436, + "theoretical_loss": 3.9675330468692804, + "tokens_seen": 472317952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006924132397191575, + "loss": 1.5743, + "theoretical_loss": 3.9674736034200935, + "tokens_seen": 472383488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923971915747242, + "loss": 1.5415, + "theoretical_loss": 3.967414170525965, + "tokens_seen": 472449024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923811434302909, + "loss": 1.436, + "theoretical_loss": 3.9673547481835567, + "tokens_seen": 472514560 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923650952858576, + "loss": 1.5276, + "theoretical_loss": 3.9672953363895322, + "tokens_seen": 472580096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923490471414243, + "loss": 1.4866, + "theoretical_loss": 3.9672359351405566, + "tokens_seen": 472645632 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692332998996991, + "loss": 1.4293, + "theoretical_loss": 3.967176544433296, + "tokens_seen": 472711168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923169508525577, + "loss": 1.5664, + "theoretical_loss": 3.96711716426442, + "tokens_seen": 472776704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006923009027081244, + "loss": 1.5168, + "theoretical_loss": 3.9670577946305965, + "tokens_seen": 472842240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922848545636911, + "loss": 1.599, + "theoretical_loss": 3.9669984355284975, + "tokens_seen": 472907776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922688064192578, + "loss": 1.4253, + "theoretical_loss": 3.9669390869547954, + "tokens_seen": 472973312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922527582748245, + "loss": 1.5621, + "theoretical_loss": 3.966879748906165, + "tokens_seen": 473038848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922367101303912, + "loss": 1.5583, + "theoretical_loss": 3.966820421379281, + "tokens_seen": 473104384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922206619859579, + "loss": 1.423, + "theoretical_loss": 3.966761104370821, + "tokens_seen": 473169920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006922046138415246, + "loss": 1.5407, + "theoretical_loss": 3.9667017978774632, + "tokens_seen": 473235456 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921885656970913, + "loss": 1.5654, + "theoretical_loss": 3.966642501895889, + "tokens_seen": 473300992 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692172517552658, + "loss": 1.5257, + "theoretical_loss": 3.9665832164227792, + "tokens_seen": 473366528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921564694082247, + "loss": 1.5916, + "theoretical_loss": 3.966523941454817, + "tokens_seen": 473432064 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921404212637914, + "loss": 1.5673, + "theoretical_loss": 3.9664646769886875, + "tokens_seen": 473497600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921243731193581, + "loss": 1.6244, + "theoretical_loss": 3.966405423021077, + "tokens_seen": 473563136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006921083249749248, + "loss": 1.4931, + "theoretical_loss": 3.9663461795486725, + "tokens_seen": 473628672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920922768304915, + "loss": 1.4226, + "theoretical_loss": 3.9662869465681636, + "tokens_seen": 473694208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920762286860582, + "loss": 1.4949, + "theoretical_loss": 3.9662277240762416, + "tokens_seen": 473759744 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920601805416249, + "loss": 1.5331, + "theoretical_loss": 3.9661685120695975, + "tokens_seen": 473825280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920441323971916, + "loss": 1.5394, + "theoretical_loss": 3.9661093105449265, + "tokens_seen": 473890816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006920280842527583, + "loss": 1.5547, + "theoretical_loss": 3.966050119498923, + "tokens_seen": 473956352 + }, + { + "epoch": 0.14, + "learning_rate": 0.000692012036108325, + "loss": 1.4748, + "theoretical_loss": 3.9659909389282832, + "tokens_seen": 474021888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919959879638917, + "loss": 1.5044, + "theoretical_loss": 3.9659317688297064, + "tokens_seen": 474087424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919799398194584, + "loss": 1.5826, + "theoretical_loss": 3.965872609199892, + "tokens_seen": 474152960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919638916750251, + "loss": 1.628, + "theoretical_loss": 3.965813460035541, + "tokens_seen": 474218496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919478435305918, + "loss": 1.5189, + "theoretical_loss": 3.965754321333356, + "tokens_seen": 474284032 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919317953861585, + "loss": 1.5367, + "theoretical_loss": 3.965695193090042, + "tokens_seen": 474349568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006919157472417252, + "loss": 1.5717, + "theoretical_loss": 3.9656360753023034, + "tokens_seen": 474415104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918996990972919, + "loss": 1.5585, + "theoretical_loss": 3.9655769679668484, + "tokens_seen": 474480640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918836509528587, + "loss": 1.4593, + "theoretical_loss": 3.965517871080386, + "tokens_seen": 474546176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918676028084253, + "loss": 1.4349, + "theoretical_loss": 3.965458784639625, + "tokens_seen": 474611712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918515546639921, + "loss": 1.5558, + "theoretical_loss": 3.965399708641278, + "tokens_seen": 474677248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918355065195587, + "loss": 1.4936, + "theoretical_loss": 3.9653406430820586, + "tokens_seen": 474742784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918194583751255, + "loss": 1.5491, + "theoretical_loss": 3.9652815879586805, + "tokens_seen": 474808320 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006918034102306921, + "loss": 1.5014, + "theoretical_loss": 3.96522254326786, + "tokens_seen": 474873856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917873620862589, + "loss": 1.5681, + "theoretical_loss": 3.9651635090063153, + "tokens_seen": 474939392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917713139418255, + "loss": 1.6158, + "theoretical_loss": 3.965104485170765, + "tokens_seen": 475004928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917552657973923, + "loss": 1.5403, + "theoretical_loss": 3.9650454717579295, + "tokens_seen": 475070464 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917392176529589, + "loss": 1.5289, + "theoretical_loss": 3.9649864687645318, + "tokens_seen": 475136000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917231695085257, + "loss": 1.6642, + "theoretical_loss": 3.9649274761872944, + "tokens_seen": 475201536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006917071213640923, + "loss": 1.583, + "theoretical_loss": 3.9648684940229426, + "tokens_seen": 475267072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916910732196591, + "loss": 1.503, + "theoretical_loss": 3.9648095222682036, + "tokens_seen": 475332608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916750250752257, + "loss": 1.6172, + "theoretical_loss": 3.9647505609198044, + "tokens_seen": 475398144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916589769307925, + "loss": 1.4547, + "theoretical_loss": 3.9646916099744756, + "tokens_seen": 475463680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916429287863591, + "loss": 1.5818, + "theoretical_loss": 3.9646326694289473, + "tokens_seen": 475529216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916268806419258, + "loss": 1.4325, + "theoretical_loss": 3.9645737392799516, + "tokens_seen": 475594752 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006916108324974925, + "loss": 1.5487, + "theoretical_loss": 3.964514819524224, + "tokens_seen": 475660288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006915947843530592, + "loss": 1.5089, + "theoretical_loss": 3.964455910158498, + "tokens_seen": 475725824 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691578736208626, + "loss": 1.4216, + "theoretical_loss": 3.9643970111795115, + "tokens_seen": 475791360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006915626880641926, + "loss": 1.4196, + "theoretical_loss": 3.964338122584003, + "tokens_seen": 475856896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006915466399197593, + "loss": 1.5528, + "theoretical_loss": 3.964279244368712, + "tokens_seen": 475922432 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691530591775326, + "loss": 1.4964, + "theoretical_loss": 3.9642203765303794, + "tokens_seen": 475987968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006915145436308928, + "loss": 1.5784, + "theoretical_loss": 3.9641615190657484, + "tokens_seen": 476053504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914984954864594, + "loss": 1.5367, + "theoretical_loss": 3.9641026719715633, + "tokens_seen": 476119040 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691482447342026, + "loss": 1.6148, + "theoretical_loss": 3.9640438352445697, + "tokens_seen": 476184576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914663991975928, + "loss": 1.6588, + "theoretical_loss": 3.9639850088815143, + "tokens_seen": 476250112 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914503510531594, + "loss": 1.4635, + "theoretical_loss": 3.9639261928791463, + "tokens_seen": 476315648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914343029087262, + "loss": 1.524, + "theoretical_loss": 3.963867387234216, + "tokens_seen": 476381184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914182547642928, + "loss": 1.5689, + "theoretical_loss": 3.963808591943474, + "tokens_seen": 476446720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006914022066198596, + "loss": 1.4803, + "theoretical_loss": 3.9637498070036745, + "tokens_seen": 476512256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006913861584754262, + "loss": 1.4908, + "theoretical_loss": 3.963691032411571, + "tokens_seen": 476577792 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691370110330993, + "loss": 1.6481, + "theoretical_loss": 3.9636322681639196, + "tokens_seen": 476643328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006913540621865596, + "loss": 1.4734, + "theoretical_loss": 3.9635735142574786, + "tokens_seen": 476708864 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006913380140421264, + "loss": 1.5567, + "theoretical_loss": 3.963514770689006, + "tokens_seen": 476774400 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691321965897693, + "loss": 1.5122, + "theoretical_loss": 3.9634560374552623, + "tokens_seen": 476839936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006913059177532598, + "loss": 1.5072, + "theoretical_loss": 3.96339731455301, + "tokens_seen": 476905472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912898696088265, + "loss": 1.4483, + "theoretical_loss": 3.963338601979011, + "tokens_seen": 476971008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912738214643932, + "loss": 1.6579, + "theoretical_loss": 3.9632798997300314, + "tokens_seen": 477036544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912577733199599, + "loss": 1.5416, + "theoretical_loss": 3.963221207802836, + "tokens_seen": 477102080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912417251755266, + "loss": 1.4728, + "theoretical_loss": 3.9631625261941945, + "tokens_seen": 477167616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006912256770310933, + "loss": 1.5535, + "theoretical_loss": 3.963103854900874, + "tokens_seen": 477233152 + }, + { + "epoch": 0.14, + "learning_rate": 0.00069120962888666, + "loss": 1.5115, + "theoretical_loss": 3.963045193919646, + "tokens_seen": 477298688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911935807422267, + "loss": 1.5282, + "theoretical_loss": 3.9629865432472817, + "tokens_seen": 477364224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911775325977934, + "loss": 1.4732, + "theoretical_loss": 3.9629279028805553, + "tokens_seen": 477429760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911614844533601, + "loss": 1.5549, + "theoretical_loss": 3.962869272816242, + "tokens_seen": 477495296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911454363089268, + "loss": 1.6219, + "theoretical_loss": 3.9628106530511173, + "tokens_seen": 477560832 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911293881644935, + "loss": 1.612, + "theoretical_loss": 3.9627520435819594, + "tokens_seen": 477626368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006911133400200602, + "loss": 1.5136, + "theoretical_loss": 3.9626934444055477, + "tokens_seen": 477691904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910972918756269, + "loss": 1.5133, + "theoretical_loss": 3.962634855518663, + "tokens_seen": 477757440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910812437311936, + "loss": 1.4191, + "theoretical_loss": 3.962576276918087, + "tokens_seen": 477822976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910651955867603, + "loss": 1.6126, + "theoretical_loss": 3.962517708600603, + "tokens_seen": 477888512 + }, + { + "epoch": 0.14, + "learning_rate": 0.000691049147442327, + "loss": 1.6248, + "theoretical_loss": 3.9624591505629976, + "tokens_seen": 477954048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910330992978937, + "loss": 1.5054, + "theoretical_loss": 3.962400602802056, + "tokens_seen": 478019584 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910170511534604, + "loss": 1.4844, + "theoretical_loss": 3.962342065314566, + "tokens_seen": 478085120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006910010030090271, + "loss": 1.5378, + "theoretical_loss": 3.962283538097318, + "tokens_seen": 478150656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909849548645938, + "loss": 1.4851, + "theoretical_loss": 3.962225021147102, + "tokens_seen": 478216192 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909689067201605, + "loss": 1.5837, + "theoretical_loss": 3.9621665144607103, + "tokens_seen": 478281728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909528585757272, + "loss": 1.4537, + "theoretical_loss": 3.9621080180349373, + "tokens_seen": 478347264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 183452, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5442909002304077, + "objective/train/theoretical_loss": 3.9620495318665774, + "objective/train/tokens_used": 498872800, + "theoretical_loss": 3.9620495318665774, + "tokens_seen": 478412800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909368104312939, + "loss": 1.5443, + "theoretical_loss": 3.9620495318665774, + "tokens_seen": 478412800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0006909207622868606, + "loss": 1.5828, + "theoretical_loss": 3.9619910559524274, + "tokens_seen": 478478336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006909047141424273, + "loss": 1.521, + "theoretical_loss": 3.9619325902892855, + "tokens_seen": 478543872 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690888665997994, + "loss": 1.6962, + "theoretical_loss": 3.961874134873951, + "tokens_seen": 478609408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908726178535607, + "loss": 1.5424, + "theoretical_loss": 3.9618156897032257, + "tokens_seen": 478674944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908565697091274, + "loss": 1.6259, + "theoretical_loss": 3.961757254773911, + "tokens_seen": 478740480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908405215646941, + "loss": 1.6806, + "theoretical_loss": 3.9616988300828098, + "tokens_seen": 478806016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908244734202608, + "loss": 1.5474, + "theoretical_loss": 3.9616404156267286, + "tokens_seen": 478871552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006908084252758275, + "loss": 1.6407, + "theoretical_loss": 3.961582011402475, + "tokens_seen": 478937088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907923771313942, + "loss": 1.5113, + "theoretical_loss": 3.9615236174068547, + "tokens_seen": 479002624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907763289869609, + "loss": 1.5974, + "theoretical_loss": 3.961465233636679, + "tokens_seen": 479068160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907602808425276, + "loss": 1.5773, + "theoretical_loss": 3.9614068600887578, + "tokens_seen": 479133696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907442326980943, + "loss": 1.659, + "theoretical_loss": 3.9613484967599044, + "tokens_seen": 479199232 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690728184553661, + "loss": 1.5473, + "theoretical_loss": 3.961290143646932, + "tokens_seen": 479264768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006907121364092277, + "loss": 1.5356, + "theoretical_loss": 3.961231800746656, + "tokens_seen": 479330304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906960882647944, + "loss": 1.5348, + "theoretical_loss": 3.961173468055893, + "tokens_seen": 479395840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906800401203612, + "loss": 1.6171, + "theoretical_loss": 3.961115145571461, + "tokens_seen": 479461376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906639919759278, + "loss": 1.4489, + "theoretical_loss": 3.96105683329018, + "tokens_seen": 479526912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906479438314946, + "loss": 1.5477, + "theoretical_loss": 3.9609985312088707, + "tokens_seen": 479592448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006906318956870612, + "loss": 1.5041, + "theoretical_loss": 3.960940239324355, + "tokens_seen": 479657984 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690615847542628, + "loss": 1.7069, + "theoretical_loss": 3.960881957633457, + "tokens_seen": 479723520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905997993981946, + "loss": 1.5411, + "theoretical_loss": 3.9608236861330024, + "tokens_seen": 479789056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905837512537614, + "loss": 1.4837, + "theoretical_loss": 3.960765424819817, + "tokens_seen": 479854592 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690567703109328, + "loss": 1.6034, + "theoretical_loss": 3.9607071736907296, + "tokens_seen": 479920128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905516549648948, + "loss": 1.446, + "theoretical_loss": 3.9606489327425694, + "tokens_seen": 479985664 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905356068204614, + "loss": 1.5508, + "theoretical_loss": 3.960590701972167, + "tokens_seen": 480051200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905195586760282, + "loss": 1.4526, + "theoretical_loss": 3.960532481376355, + "tokens_seen": 480116736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006905035105315948, + "loss": 1.5166, + "theoretical_loss": 3.960474270951967, + "tokens_seen": 480182272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006904874623871616, + "loss": 1.4288, + "theoretical_loss": 3.9604160706958385, + "tokens_seen": 480247808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006904714142427282, + "loss": 1.5271, + "theoretical_loss": 3.960357880604806, + "tokens_seen": 480313344 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690455366098295, + "loss": 1.4782, + "theoretical_loss": 3.9602997006757072, + "tokens_seen": 480378880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006904393179538616, + "loss": 1.4914, + "theoretical_loss": 3.960241530905382, + "tokens_seen": 480444416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006904232698094284, + "loss": 1.4161, + "theoretical_loss": 3.96018337129067, + "tokens_seen": 480509952 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690407221664995, + "loss": 1.6271, + "theoretical_loss": 3.960125221828415, + "tokens_seen": 480575488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903911735205618, + "loss": 1.5727, + "theoretical_loss": 3.9600670825154594, + "tokens_seen": 480641024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903751253761284, + "loss": 1.5346, + "theoretical_loss": 3.960008953348649, + "tokens_seen": 480706560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903590772316952, + "loss": 1.6175, + "theoretical_loss": 3.9599508343248297, + "tokens_seen": 480772096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903430290872618, + "loss": 1.5171, + "theoretical_loss": 3.9598927254408505, + "tokens_seen": 480837632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903269809428286, + "loss": 1.4718, + "theoretical_loss": 3.9598346266935596, + "tokens_seen": 480903168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006903109327983952, + "loss": 1.6093, + "theoretical_loss": 3.959776538079808, + "tokens_seen": 480968704 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690294884653962, + "loss": 1.5958, + "theoretical_loss": 3.9597184595964476, + "tokens_seen": 481034240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006902788365095286, + "loss": 1.4514, + "theoretical_loss": 3.959660391240332, + "tokens_seen": 481099776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006902627883650954, + "loss": 1.5174, + "theoretical_loss": 3.9596023330083163, + "tokens_seen": 481165312 + }, + { + "epoch": 0.15, + "learning_rate": 0.000690246740220662, + "loss": 1.5143, + "theoretical_loss": 3.9595442848972566, + "tokens_seen": 481230848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006902306920762287, + "loss": 1.6373, + "theoretical_loss": 3.959486246904011, + "tokens_seen": 481296384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006902146439317954, + "loss": 1.6214, + "theoretical_loss": 3.959428219025438, + "tokens_seen": 481361920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901985957873621, + "loss": 1.5131, + "theoretical_loss": 3.9593702012583987, + "tokens_seen": 481427456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901825476429288, + "loss": 1.5421, + "theoretical_loss": 3.9593121935997546, + "tokens_seen": 481492992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901664994984955, + "loss": 1.4499, + "theoretical_loss": 3.95925419604637, + "tokens_seen": 481558528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901504513540622, + "loss": 1.4724, + "theoretical_loss": 3.959196208595108, + "tokens_seen": 481624064 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901344032096289, + "loss": 1.5084, + "theoretical_loss": 3.959138231242836, + "tokens_seen": 481689600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901183550651957, + "loss": 1.4378, + "theoretical_loss": 3.959080263986421, + "tokens_seen": 481755136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006901023069207623, + "loss": 1.442, + "theoretical_loss": 3.9590223068227317, + "tokens_seen": 481820672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900862587763289, + "loss": 1.4555, + "theoretical_loss": 3.9589643597486393, + "tokens_seen": 481886208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900702106318957, + "loss": 1.4756, + "theoretical_loss": 3.9589064227610145, + "tokens_seen": 481951744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900541624874623, + "loss": 1.4963, + "theoretical_loss": 3.9588484958567314, + "tokens_seen": 482017280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900381143430291, + "loss": 1.4515, + "theoretical_loss": 3.958790579032663, + "tokens_seen": 482082816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900220661985957, + "loss": 1.5883, + "theoretical_loss": 3.958732672285687, + "tokens_seen": 482148352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006900060180541625, + "loss": 1.5478, + "theoretical_loss": 3.95867477561268, + "tokens_seen": 482213888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899899699097291, + "loss": 1.4071, + "theoretical_loss": 3.95861688901052, + "tokens_seen": 482279424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899739217652959, + "loss": 1.5907, + "theoretical_loss": 3.958559012476088, + "tokens_seen": 482344960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899578736208625, + "loss": 1.5229, + "theoretical_loss": 3.958501146006265, + "tokens_seen": 482410496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899418254764293, + "loss": 1.4597, + "theoretical_loss": 3.958443289597934, + "tokens_seen": 482476032 + }, + { + "epoch": 0.15, + "learning_rate": 0.000689925777331996, + "loss": 1.6134, + "theoretical_loss": 3.958385443247979, + "tokens_seen": 482541568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006899097291875627, + "loss": 1.6374, + "theoretical_loss": 3.958327606953286, + "tokens_seen": 482607104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898936810431294, + "loss": 1.5936, + "theoretical_loss": 3.958269780710742, + "tokens_seen": 482672640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898776328986961, + "loss": 1.4493, + "theoretical_loss": 3.9582119645172344, + "tokens_seen": 482738176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898615847542628, + "loss": 1.6693, + "theoretical_loss": 3.9581541583696547, + "tokens_seen": 482803712 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898455366098295, + "loss": 1.5794, + "theoretical_loss": 3.958096362264893, + "tokens_seen": 482869248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898294884653962, + "loss": 1.4834, + "theoretical_loss": 3.958038576199842, + "tokens_seen": 482934784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006898134403209629, + "loss": 1.5811, + "theoretical_loss": 3.957980800171396, + "tokens_seen": 483000320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897973921765296, + "loss": 1.6002, + "theoretical_loss": 3.9579230341764493, + "tokens_seen": 483065856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897813440320963, + "loss": 1.4357, + "theoretical_loss": 3.9578652782118997, + "tokens_seen": 483131392 + }, + { + "epoch": 0.15, + "learning_rate": 0.000689765295887663, + "loss": 1.3866, + "theoretical_loss": 3.957807532274645, + "tokens_seen": 483196928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897492477432297, + "loss": 1.4499, + "theoretical_loss": 3.957749796361585, + "tokens_seen": 483262464 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897331995987964, + "loss": 1.6061, + "theoretical_loss": 3.9576920704696192, + "tokens_seen": 483328000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897171514543631, + "loss": 1.5137, + "theoretical_loss": 3.957634354595651, + "tokens_seen": 483393536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006897011033099298, + "loss": 1.5327, + "theoretical_loss": 3.957576648736584, + "tokens_seen": 483459072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896850551654965, + "loss": 1.5925, + "theoretical_loss": 3.9575189528893233, + "tokens_seen": 483524608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896690070210632, + "loss": 1.5758, + "theoretical_loss": 3.957461267050774, + "tokens_seen": 483590144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896529588766299, + "loss": 1.5964, + "theoretical_loss": 3.957403591217845, + "tokens_seen": 483655680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896369107321966, + "loss": 1.569, + "theoretical_loss": 3.9573459253874455, + "tokens_seen": 483721216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006896208625877633, + "loss": 1.5378, + "theoretical_loss": 3.9572882695564853, + "tokens_seen": 483786752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068960481444333, + "loss": 1.4906, + "theoretical_loss": 3.9572306237218764, + "tokens_seen": 483852288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895887662988967, + "loss": 1.5088, + "theoretical_loss": 3.957172987880532, + "tokens_seen": 483917824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895727181544634, + "loss": 1.6218, + "theoretical_loss": 3.9571153620293673, + "tokens_seen": 483983360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895566700100301, + "loss": 1.5368, + "theoretical_loss": 3.9570577461652974, + "tokens_seen": 484048896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895406218655968, + "loss": 1.5992, + "theoretical_loss": 3.95700014028524, + "tokens_seen": 484114432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895245737211635, + "loss": 1.567, + "theoretical_loss": 3.956942544386114, + "tokens_seen": 484179968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006895085255767302, + "loss": 1.6582, + "theoretical_loss": 3.9568849584648396, + "tokens_seen": 484245504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894924774322969, + "loss": 1.4959, + "theoretical_loss": 3.9568273825183375, + "tokens_seen": 484311040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894764292878637, + "loss": 1.6113, + "theoretical_loss": 3.9567698165435305, + "tokens_seen": 484376576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894603811434303, + "loss": 1.5103, + "theoretical_loss": 3.9567122605373437, + "tokens_seen": 484442112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894443329989971, + "loss": 1.4807, + "theoretical_loss": 3.956654714496702, + "tokens_seen": 484507648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894282848545637, + "loss": 1.5781, + "theoretical_loss": 3.9565971784185328, + "tokens_seen": 484573184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006894122367101305, + "loss": 1.4845, + "theoretical_loss": 3.9565396522997633, + "tokens_seen": 484638720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893961885656971, + "loss": 1.5237, + "theoretical_loss": 3.9564821361373244, + "tokens_seen": 484704256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893801404212639, + "loss": 1.5238, + "theoretical_loss": 3.956424629928146, + "tokens_seen": 484769792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893640922768305, + "loss": 1.5249, + "theoretical_loss": 3.9563671336691613, + "tokens_seen": 484835328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893480441323973, + "loss": 1.481, + "theoretical_loss": 3.956309647357303, + "tokens_seen": 484900864 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 185723, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5091984272003174, + "objective/train/theoretical_loss": 3.956252170989507, + "objective/train/tokens_used": 505426400, + "theoretical_loss": 3.956252170989507, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893319959879639, + "loss": 1.5092, + "theoretical_loss": 3.956252170989507, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006893159478435307, + "loss": 1.4572, + "theoretical_loss": 3.9561947045627095, + "tokens_seen": 485031936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892998996990973, + "loss": 1.5348, + "theoretical_loss": 3.9561372480738486, + "tokens_seen": 485097472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892838515546641, + "loss": 1.4964, + "theoretical_loss": 3.956079801519863, + "tokens_seen": 485163008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892678034102307, + "loss": 1.4513, + "theoretical_loss": 3.9560223648976924, + "tokens_seen": 485228544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892517552657975, + "loss": 1.5588, + "theoretical_loss": 3.9559649382042803, + "tokens_seen": 485294080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892357071213641, + "loss": 1.5369, + "theoretical_loss": 3.955907521436569, + "tokens_seen": 485359616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892196589769309, + "loss": 1.473, + "theoretical_loss": 3.9558501145915033, + "tokens_seen": 485425152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006892036108324975, + "loss": 1.5747, + "theoretical_loss": 3.9557927176660286, + "tokens_seen": 485490688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891875626880643, + "loss": 1.4841, + "theoretical_loss": 3.9557353306570926, + "tokens_seen": 485556224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891715145436309, + "loss": 1.6007, + "theoretical_loss": 3.9556779535616444, + "tokens_seen": 485621760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891554663991977, + "loss": 1.5341, + "theoretical_loss": 3.955620586376633, + "tokens_seen": 485687296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891394182547643, + "loss": 1.4777, + "theoretical_loss": 3.9555632290990097, + "tokens_seen": 485752832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891233701103311, + "loss": 1.5709, + "theoretical_loss": 3.9555058817257285, + "tokens_seen": 485818368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006891073219658977, + "loss": 1.5596, + "theoretical_loss": 3.9554485442537417, + "tokens_seen": 485883904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890912738214645, + "loss": 1.4986, + "theoretical_loss": 3.955391216680006, + "tokens_seen": 485949440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890752256770311, + "loss": 1.4487, + "theoretical_loss": 3.9553338990014772, + "tokens_seen": 486014976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890591775325979, + "loss": 1.4888, + "theoretical_loss": 3.9552765912151138, + "tokens_seen": 486080512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890431293881645, + "loss": 1.4521, + "theoretical_loss": 3.9552192933178745, + "tokens_seen": 486146048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890270812437313, + "loss": 1.5154, + "theoretical_loss": 3.9551620053067214, + "tokens_seen": 486211584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006890110330992979, + "loss": 1.541, + "theoretical_loss": 3.9551047271786155, + "tokens_seen": 486277120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889949849548647, + "loss": 1.4366, + "theoretical_loss": 3.9550474589305202, + "tokens_seen": 486342656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889789368104313, + "loss": 1.4734, + "theoretical_loss": 3.954990200559401, + "tokens_seen": 486408192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889628886659981, + "loss": 1.5113, + "theoretical_loss": 3.9549329520622236, + "tokens_seen": 486473728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889468405215647, + "loss": 1.543, + "theoretical_loss": 3.954875713435955, + "tokens_seen": 486539264 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889307923771315, + "loss": 1.4912, + "theoretical_loss": 3.9548184846775647, + "tokens_seen": 486604800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006889147442326981, + "loss": 1.4411, + "theoretical_loss": 3.9547612657840228, + "tokens_seen": 486670336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888986960882649, + "loss": 1.5094, + "theoretical_loss": 3.9547040567523, + "tokens_seen": 486735872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888826479438315, + "loss": 1.5652, + "theoretical_loss": 3.9546468575793696, + "tokens_seen": 486801408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888665997993983, + "loss": 1.5311, + "theoretical_loss": 3.9545896682622064, + "tokens_seen": 486866944 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688850551654965, + "loss": 1.5545, + "theoretical_loss": 3.954532488797785, + "tokens_seen": 486932480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888345035105317, + "loss": 1.5898, + "theoretical_loss": 3.954475319183082, + "tokens_seen": 486998016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006888184553660983, + "loss": 1.5182, + "theoretical_loss": 3.9544181594150762, + "tokens_seen": 487063552 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688802407221665, + "loss": 1.4349, + "theoretical_loss": 3.9543610094907464, + "tokens_seen": 487129088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887863590772317, + "loss": 1.573, + "theoretical_loss": 3.9543038694070747, + "tokens_seen": 487194624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887703109327984, + "loss": 1.5117, + "theoretical_loss": 3.9542467391610416, + "tokens_seen": 487260160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887542627883652, + "loss": 1.466, + "theoretical_loss": 3.9541896187496315, + "tokens_seen": 487325696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887382146439318, + "loss": 1.5429, + "theoretical_loss": 3.9541325081698293, + "tokens_seen": 487391232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887221664994986, + "loss": 1.6194, + "theoretical_loss": 3.954075407418621, + "tokens_seen": 487456768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006887061183550652, + "loss": 1.5898, + "theoretical_loss": 3.954018316492994, + "tokens_seen": 487522304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886900702106318, + "loss": 1.61, + "theoretical_loss": 3.9539612353899365, + "tokens_seen": 487587840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886740220661986, + "loss": 1.4679, + "theoretical_loss": 3.9539041641064396, + "tokens_seen": 487653376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886579739217652, + "loss": 1.54, + "theoretical_loss": 3.953847102639494, + "tokens_seen": 487718912 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688641925777332, + "loss": 1.5792, + "theoretical_loss": 3.9537900509860933, + "tokens_seen": 487784448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886258776328986, + "loss": 1.5688, + "theoretical_loss": 3.953733009143231, + "tokens_seen": 487849984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006886098294884654, + "loss": 1.4593, + "theoretical_loss": 3.9536759771079018, + "tokens_seen": 487915520 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688593781344032, + "loss": 1.4885, + "theoretical_loss": 3.9536189548771037, + "tokens_seen": 487981056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885777331995988, + "loss": 1.4351, + "theoretical_loss": 3.953561942447834, + "tokens_seen": 488046592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885616850551655, + "loss": 1.5066, + "theoretical_loss": 3.953504939817093, + "tokens_seen": 488112128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885456369107322, + "loss": 1.5392, + "theoretical_loss": 3.953447946981881, + "tokens_seen": 488177664 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885295887662989, + "loss": 1.5247, + "theoretical_loss": 3.9533909639391993, + "tokens_seen": 488243200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006885135406218656, + "loss": 1.4818, + "theoretical_loss": 3.953333990686051, + "tokens_seen": 488308736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884974924774323, + "loss": 1.5048, + "theoretical_loss": 3.9532770272194426, + "tokens_seen": 488374272 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688481444332999, + "loss": 1.547, + "theoretical_loss": 3.9532200735363787, + "tokens_seen": 488439808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884653961885657, + "loss": 1.6154, + "theoretical_loss": 3.953163129633867, + "tokens_seen": 488505344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884493480441324, + "loss": 1.4032, + "theoretical_loss": 3.9531061955089157, + "tokens_seen": 488570880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884332998996991, + "loss": 1.4734, + "theoretical_loss": 3.9530492711585357, + "tokens_seen": 488636416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884172517552658, + "loss": 1.5429, + "theoretical_loss": 3.9529923565797374, + "tokens_seen": 488701952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006884012036108325, + "loss": 1.4302, + "theoretical_loss": 3.952935451769533, + "tokens_seen": 488767488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883851554663992, + "loss": 1.5539, + "theoretical_loss": 3.9528785567249374, + "tokens_seen": 488833024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883691073219659, + "loss": 1.5634, + "theoretical_loss": 3.9528216714429654, + "tokens_seen": 488898560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883530591775326, + "loss": 1.468, + "theoretical_loss": 3.9527647959206336, + "tokens_seen": 488964096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883370110330993, + "loss": 1.5018, + "theoretical_loss": 3.9527079301549595, + "tokens_seen": 489029632 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688320962888666, + "loss": 1.4775, + "theoretical_loss": 3.9526510741429624, + "tokens_seen": 489095168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006883049147442327, + "loss": 1.5398, + "theoretical_loss": 3.9525942278816633, + "tokens_seen": 489160704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882888665997994, + "loss": 1.6012, + "theoretical_loss": 3.952537391368083, + "tokens_seen": 489226240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882728184553662, + "loss": 1.5567, + "theoretical_loss": 3.9524805645992447, + "tokens_seen": 489291776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882567703109328, + "loss": 1.6264, + "theoretical_loss": 3.9524237475721726, + "tokens_seen": 489357312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882407221664996, + "loss": 1.6461, + "theoretical_loss": 3.952366940283894, + "tokens_seen": 489422848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006882246740220662, + "loss": 1.5331, + "theoretical_loss": 3.952310142731434, + "tokens_seen": 489488384 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688208625877633, + "loss": 1.4402, + "theoretical_loss": 3.9522533549118215, + "tokens_seen": 489553920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881925777331996, + "loss": 1.5865, + "theoretical_loss": 3.952196576822086, + "tokens_seen": 489619456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881765295887664, + "loss": 1.5268, + "theoretical_loss": 3.9521398084592585, + "tokens_seen": 489684992 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688160481444333, + "loss": 1.513, + "theoretical_loss": 3.952083049820372, + "tokens_seen": 489750528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881444332998998, + "loss": 1.4614, + "theoretical_loss": 3.9520263009024585, + "tokens_seen": 489816064 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881283851554664, + "loss": 1.562, + "theoretical_loss": 3.951969561702554, + "tokens_seen": 489881600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006881123370110332, + "loss": 1.4838, + "theoretical_loss": 3.9519128322176935, + "tokens_seen": 489947136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880962888665998, + "loss": 1.5069, + "theoretical_loss": 3.951856112444915, + "tokens_seen": 490012672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880802407221666, + "loss": 1.5892, + "theoretical_loss": 3.951799402381258, + "tokens_seen": 490078208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880641925777332, + "loss": 1.3615, + "theoretical_loss": 3.951742702023761, + "tokens_seen": 490143744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880481444333, + "loss": 1.5228, + "theoretical_loss": 3.951686011369466, + "tokens_seen": 490209280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880320962888666, + "loss": 1.6299, + "theoretical_loss": 3.9516293304154155, + "tokens_seen": 490274816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006880160481444334, + "loss": 1.5785, + "theoretical_loss": 3.9515726591586544, + "tokens_seen": 490340352 + }, + { + "epoch": 0.15, + "learning_rate": 0.000688, + "loss": 1.4361, + "theoretical_loss": 3.9515159975962257, + "tokens_seen": 490405888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879839518555668, + "loss": 1.5663, + "theoretical_loss": 3.951459345725178, + "tokens_seen": 490471424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879679037111334, + "loss": 1.5153, + "theoretical_loss": 3.9514027035425583, + "tokens_seen": 490536960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879518555667002, + "loss": 1.4287, + "theoretical_loss": 3.951346071045415, + "tokens_seen": 490602496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879358074222668, + "loss": 1.5787, + "theoretical_loss": 3.951289448230799, + "tokens_seen": 490668032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879197592778336, + "loss": 1.5076, + "theoretical_loss": 3.9512328350957624, + "tokens_seen": 490733568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006879037111334002, + "loss": 1.5472, + "theoretical_loss": 3.9511762316373575, + "tokens_seen": 490799104 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687887662988967, + "loss": 1.5685, + "theoretical_loss": 3.951119637852639, + "tokens_seen": 490864640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006878716148445336, + "loss": 1.4781, + "theoretical_loss": 3.951063053738662, + "tokens_seen": 490930176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006878555667001004, + "loss": 1.5648, + "theoretical_loss": 3.951006479292484, + "tokens_seen": 490995712 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687839518555667, + "loss": 1.6309, + "theoretical_loss": 3.9509499145111624, + "tokens_seen": 491061248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006878234704112338, + "loss": 1.605, + "theoretical_loss": 3.950893359391756, + "tokens_seen": 491126784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006878074222668004, + "loss": 1.5809, + "theoretical_loss": 3.9508368139313275, + "tokens_seen": 491192320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877913741223672, + "loss": 1.5764, + "theoretical_loss": 3.950780278126937, + "tokens_seen": 491257856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877753259779338, + "loss": 1.4766, + "theoretical_loss": 3.9507237519756484, + "tokens_seen": 491323392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877592778335006, + "loss": 1.3444, + "theoretical_loss": 3.9506672354745263, + "tokens_seen": 491388928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877432296890672, + "loss": 1.5173, + "theoretical_loss": 3.950610728620636, + "tokens_seen": 491454464 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 188044, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.498471736907959, + "objective/train/theoretical_loss": 3.950554231411046, + "objective/train/tokens_used": 511980000, + "theoretical_loss": 3.950554231411046, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687727181544634, + "loss": 1.4985, + "theoretical_loss": 3.950554231411046, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006877111334002006, + "loss": 1.4874, + "theoretical_loss": 3.9504977438428233, + "tokens_seen": 491585536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876950852557674, + "loss": 1.5478, + "theoretical_loss": 3.950441265913038, + "tokens_seen": 491651072 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687679037111334, + "loss": 1.4837, + "theoretical_loss": 3.950384797618761, + "tokens_seen": 491716608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876629889669008, + "loss": 1.4839, + "theoretical_loss": 3.950328338957064, + "tokens_seen": 491782144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876469408224674, + "loss": 1.5039, + "theoretical_loss": 3.950271889925022, + "tokens_seen": 491847680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876308926780342, + "loss": 1.4563, + "theoretical_loss": 3.9502154505197087, + "tokens_seen": 491913216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006876148445336008, + "loss": 1.5554, + "theoretical_loss": 3.9501590207382, + "tokens_seen": 491978752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875987963891676, + "loss": 1.5749, + "theoretical_loss": 3.9501026005775732, + "tokens_seen": 492044288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875827482447342, + "loss": 1.6107, + "theoretical_loss": 3.9500461900349078, + "tokens_seen": 492109824 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687566700100301, + "loss": 1.5203, + "theoretical_loss": 3.949989789107283, + "tokens_seen": 492175360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875506519558676, + "loss": 1.663, + "theoretical_loss": 3.9499333977917805, + "tokens_seen": 492240896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875346038114344, + "loss": 1.5451, + "theoretical_loss": 3.949877016085482, + "tokens_seen": 492306432 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687518555667001, + "loss": 1.4595, + "theoretical_loss": 3.949820643985472, + "tokens_seen": 492371968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006875025075225678, + "loss": 1.5336, + "theoretical_loss": 3.9497642814888345, + "tokens_seen": 492437504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874864593781344, + "loss": 1.5868, + "theoretical_loss": 3.949707928592657, + "tokens_seen": 492503040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874704112337012, + "loss": 1.5068, + "theoretical_loss": 3.949651585294026, + "tokens_seen": 492568576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874543630892678, + "loss": 1.5023, + "theoretical_loss": 3.9495952515900306, + "tokens_seen": 492634112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874383149448346, + "loss": 1.5565, + "theoretical_loss": 3.9495389274777613, + "tokens_seen": 492699648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874222668004012, + "loss": 1.4768, + "theoretical_loss": 3.9494826129543084, + "tokens_seen": 492765184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006874062186559679, + "loss": 1.5537, + "theoretical_loss": 3.9494263080167658, + "tokens_seen": 492830720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873901705115347, + "loss": 1.5246, + "theoretical_loss": 3.949370012662227, + "tokens_seen": 492896256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873741223671013, + "loss": 1.5869, + "theoretical_loss": 3.9493137268877865, + "tokens_seen": 492961792 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687358074222668, + "loss": 1.558, + "theoretical_loss": 3.9492574506905416, + "tokens_seen": 493027328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873420260782347, + "loss": 1.4085, + "theoretical_loss": 3.949201184067589, + "tokens_seen": 493092864 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873259779338015, + "loss": 1.5637, + "theoretical_loss": 3.949144927016029, + "tokens_seen": 493158400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006873099297893681, + "loss": 1.4535, + "theoretical_loss": 3.9490886795329603, + "tokens_seen": 493223936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872938816449349, + "loss": 1.5059, + "theoretical_loss": 3.9490324416154854, + "tokens_seen": 493289472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872778335005015, + "loss": 1.5403, + "theoretical_loss": 3.9489762132607065, + "tokens_seen": 493355008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872617853560681, + "loss": 1.5657, + "theoretical_loss": 3.948919994465728, + "tokens_seen": 493420544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872457372116349, + "loss": 1.4838, + "theoretical_loss": 3.9488637852276547, + "tokens_seen": 493486080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872296890672015, + "loss": 1.5458, + "theoretical_loss": 3.9488075855435936, + "tokens_seen": 493551616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006872136409227683, + "loss": 1.5213, + "theoretical_loss": 3.9487513954106523, + "tokens_seen": 493617152 + }, + { + "epoch": 0.15, + "learning_rate": 0.000687197592778335, + "loss": 1.5807, + "theoretical_loss": 3.94869521482594, + "tokens_seen": 493682688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871815446339017, + "loss": 1.577, + "theoretical_loss": 3.9486390437865664, + "tokens_seen": 493748224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871654964894684, + "loss": 1.55, + "theoretical_loss": 3.948582882289644, + "tokens_seen": 493813760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871494483450351, + "loss": 1.5151, + "theoretical_loss": 3.9485267303322846, + "tokens_seen": 493879296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871334002006018, + "loss": 1.4614, + "theoretical_loss": 3.9484705879116033, + "tokens_seen": 493944832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871173520561685, + "loss": 1.4307, + "theoretical_loss": 3.9484144550247144, + "tokens_seen": 494010368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006871013039117352, + "loss": 1.5147, + "theoretical_loss": 3.948358331668735, + "tokens_seen": 494075904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870852557673019, + "loss": 1.5127, + "theoretical_loss": 3.9483022178407827, + "tokens_seen": 494141440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870692076228687, + "loss": 1.4983, + "theoretical_loss": 3.9482461135379774, + "tokens_seen": 494206976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870531594784353, + "loss": 1.542, + "theoretical_loss": 3.9481900187574386, + "tokens_seen": 494272512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870371113340021, + "loss": 1.5766, + "theoretical_loss": 3.948133933496288, + "tokens_seen": 494338048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870210631895687, + "loss": 1.4443, + "theoretical_loss": 3.948077857751649, + "tokens_seen": 494403584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006870050150451355, + "loss": 1.453, + "theoretical_loss": 3.948021791520645, + "tokens_seen": 494469120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869889669007021, + "loss": 1.4175, + "theoretical_loss": 3.9479657348004014, + "tokens_seen": 494534656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869729187562689, + "loss": 1.6324, + "theoretical_loss": 3.9479096875880453, + "tokens_seen": 494600192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869568706118355, + "loss": 1.3906, + "theoretical_loss": 3.947853649880704, + "tokens_seen": 494665728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869408224674023, + "loss": 1.5448, + "theoretical_loss": 3.947797621675507, + "tokens_seen": 494731264 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869247743229689, + "loss": 1.4851, + "theoretical_loss": 3.947741602969585, + "tokens_seen": 494796800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006869087261785357, + "loss": 1.5923, + "theoretical_loss": 3.9476855937600686, + "tokens_seen": 494862336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868926780341023, + "loss": 1.4741, + "theoretical_loss": 3.9476295940440913, + "tokens_seen": 494927872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868766298896691, + "loss": 1.6255, + "theoretical_loss": 3.9475736038187867, + "tokens_seen": 494993408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868605817452357, + "loss": 1.6426, + "theoretical_loss": 3.947517623081291, + "tokens_seen": 495058944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868445336008025, + "loss": 1.5871, + "theoretical_loss": 3.9474616518287395, + "tokens_seen": 495124480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868284854563691, + "loss": 1.5123, + "theoretical_loss": 3.9474056900582717, + "tokens_seen": 495190016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006868124373119359, + "loss": 1.5652, + "theoretical_loss": 3.9473497377670244, + "tokens_seen": 495255552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867963891675025, + "loss": 1.4239, + "theoretical_loss": 3.94729379495214, + "tokens_seen": 495321088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867803410230693, + "loss": 1.4911, + "theoretical_loss": 3.947237861610759, + "tokens_seen": 495386624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867642928786359, + "loss": 1.5142, + "theoretical_loss": 3.9471819377400243, + "tokens_seen": 495452160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867482447342027, + "loss": 1.5046, + "theoretical_loss": 3.94712602333708, + "tokens_seen": 495517696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867321965897693, + "loss": 1.6441, + "theoretical_loss": 3.9470701183990715, + "tokens_seen": 495583232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867161484453361, + "loss": 1.4662, + "theoretical_loss": 3.947014222923145, + "tokens_seen": 495648768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006867001003009027, + "loss": 1.4744, + "theoretical_loss": 3.946958336906449, + "tokens_seen": 495714304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866840521564695, + "loss": 1.4727, + "theoretical_loss": 3.9469024603461307, + "tokens_seen": 495779840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866680040120361, + "loss": 1.4566, + "theoretical_loss": 3.946846593239342, + "tokens_seen": 495845376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866519558676029, + "loss": 1.5441, + "theoretical_loss": 3.946790735583234, + "tokens_seen": 495910912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866359077231695, + "loss": 1.4912, + "theoretical_loss": 3.9467348873749586, + "tokens_seen": 495976448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866198595787363, + "loss": 1.4336, + "theoretical_loss": 3.9466790486116707, + "tokens_seen": 496041984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006866038114343029, + "loss": 1.5128, + "theoretical_loss": 3.946623219290525, + "tokens_seen": 496107520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865877632898697, + "loss": 1.4828, + "theoretical_loss": 3.9465673994086776, + "tokens_seen": 496173056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865717151454363, + "loss": 1.4968, + "theoretical_loss": 3.9465115889632862, + "tokens_seen": 496238592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865556670010031, + "loss": 1.6369, + "theoretical_loss": 3.9464557879515105, + "tokens_seen": 496304128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865396188565697, + "loss": 1.4091, + "theoretical_loss": 3.946399996370509, + "tokens_seen": 496369664 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865235707121365, + "loss": 1.4526, + "theoretical_loss": 3.9463442142174445, + "tokens_seen": 496435200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006865075225677031, + "loss": 1.5737, + "theoretical_loss": 3.9462884414894788, + "tokens_seen": 496500736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864914744232699, + "loss": 1.6132, + "theoretical_loss": 3.946232678183775, + "tokens_seen": 496566272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864754262788365, + "loss": 1.4111, + "theoretical_loss": 3.9461769242974998, + "tokens_seen": 496631808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864593781344033, + "loss": 1.5397, + "theoretical_loss": 3.946121179827818, + "tokens_seen": 496697344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864433299899699, + "loss": 1.5683, + "theoretical_loss": 3.9460654447718975, + "tokens_seen": 496762880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864272818455367, + "loss": 1.5419, + "theoretical_loss": 3.9460097191269066, + "tokens_seen": 496828416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006864112337011033, + "loss": 1.4989, + "theoretical_loss": 3.9459540028900157, + "tokens_seen": 496893952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863951855566701, + "loss": 1.5217, + "theoretical_loss": 3.945898296058396, + "tokens_seen": 496959488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863791374122367, + "loss": 1.3938, + "theoretical_loss": 3.9458425986292194, + "tokens_seen": 497025024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863630892678035, + "loss": 1.5193, + "theoretical_loss": 3.9457869105996597, + "tokens_seen": 497090560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863470411233701, + "loss": 1.5502, + "theoretical_loss": 3.9457312319668914, + "tokens_seen": 497156096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863309929789369, + "loss": 1.408, + "theoretical_loss": 3.9456755627280904, + "tokens_seen": 497221632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006863149448345035, + "loss": 1.5934, + "theoretical_loss": 3.945619902880434, + "tokens_seen": 497287168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862988966900703, + "loss": 1.5139, + "theoretical_loss": 3.945564252421102, + "tokens_seen": 497352704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862828485456369, + "loss": 1.547, + "theoretical_loss": 3.9455086113472717, + "tokens_seen": 497418240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862668004012037, + "loss": 1.587, + "theoretical_loss": 3.9454529796561255, + "tokens_seen": 497483776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862507522567703, + "loss": 1.5478, + "theoretical_loss": 3.9453973573448455, + "tokens_seen": 497549312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862347041123371, + "loss": 1.4873, + "theoretical_loss": 3.945341744410615, + "tokens_seen": 497614848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862186559679037, + "loss": 1.5189, + "theoretical_loss": 3.9452861408506172, + "tokens_seen": 497680384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006862026078234705, + "loss": 1.5402, + "theoretical_loss": 3.945230546662039, + "tokens_seen": 497745920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861865596790371, + "loss": 1.5443, + "theoretical_loss": 3.945174961842068, + "tokens_seen": 497811456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861705115346039, + "loss": 1.5253, + "theoretical_loss": 3.9451193863878906, + "tokens_seen": 497876992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861544633901705, + "loss": 1.4372, + "theoretical_loss": 3.945063820296698, + "tokens_seen": 497942528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861384152457373, + "loss": 1.4311, + "theoretical_loss": 3.9450082635656796, + "tokens_seen": 498008064 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 190526, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4806199073791504, + "objective/train/theoretical_loss": 3.9449527161920273, + "objective/train/tokens_used": 518533600, + "theoretical_loss": 3.9449527161920273, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.000686122367101304, + "loss": 1.4806, + "theoretical_loss": 3.9449527161920273, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006861063189568707, + "loss": 1.4629, + "theoretical_loss": 3.944897178172935, + "tokens_seen": 498139136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860902708124373, + "loss": 1.4226, + "theoretical_loss": 3.9448416495055962, + "tokens_seen": 498204672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860742226680041, + "loss": 1.456, + "theoretical_loss": 3.9447861301872065, + "tokens_seen": 498270208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860581745235707, + "loss": 1.5504, + "theoretical_loss": 3.944730620214962, + "tokens_seen": 498335744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860421263791375, + "loss": 1.4605, + "theoretical_loss": 3.944675119586061, + "tokens_seen": 498401280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860260782347042, + "loss": 1.5562, + "theoretical_loss": 3.944619628297704, + "tokens_seen": 498466816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006860100300902708, + "loss": 1.5742, + "theoretical_loss": 3.944564146347089, + "tokens_seen": 498532352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006859939819458376, + "loss": 1.497, + "theoretical_loss": 3.9445086737314186, + "tokens_seen": 498597888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006859779338014042, + "loss": 1.5305, + "theoretical_loss": 3.9444532104478958, + "tokens_seen": 498663424 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685961885656971, + "loss": 1.4614, + "theoretical_loss": 3.9443977564937236, + "tokens_seen": 498728960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006859458375125376, + "loss": 1.5347, + "theoretical_loss": 3.9443423118661074, + "tokens_seen": 498794496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006859297893681044, + "loss": 1.4809, + "theoretical_loss": 3.944286876562254, + "tokens_seen": 498860032 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685913741223671, + "loss": 1.4614, + "theoretical_loss": 3.9442314505793705, + "tokens_seen": 498925568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858976930792378, + "loss": 1.5434, + "theoretical_loss": 3.944176033914666, + "tokens_seen": 498991104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858816449348044, + "loss": 1.4793, + "theoretical_loss": 3.9441206265653497, + "tokens_seen": 499056640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858655967903712, + "loss": 1.5524, + "theoretical_loss": 3.9440652285286335, + "tokens_seen": 499122176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858495486459378, + "loss": 1.4566, + "theoretical_loss": 3.9440098398017294, + "tokens_seen": 499187712 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858335005015046, + "loss": 1.4794, + "theoretical_loss": 3.9439544603818506, + "tokens_seen": 499253248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006858174523570712, + "loss": 1.4982, + "theoretical_loss": 3.9438990902662128, + "tokens_seen": 499318784 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685801404212638, + "loss": 1.5347, + "theoretical_loss": 3.9438437294520305, + "tokens_seen": 499384320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857853560682046, + "loss": 1.5134, + "theoretical_loss": 3.943788377936522, + "tokens_seen": 499449856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857693079237714, + "loss": 1.4345, + "theoretical_loss": 3.943733035716905, + "tokens_seen": 499515392 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685753259779338, + "loss": 1.5914, + "theoretical_loss": 3.9436777027903993, + "tokens_seen": 499580928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857372116349048, + "loss": 1.5062, + "theoretical_loss": 3.9436223791542258, + "tokens_seen": 499646464 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857211634904714, + "loss": 1.3824, + "theoretical_loss": 3.943567064805606, + "tokens_seen": 499712000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006857051153460382, + "loss": 1.4687, + "theoretical_loss": 3.943511759741763, + "tokens_seen": 499777536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856890672016048, + "loss": 1.5585, + "theoretical_loss": 3.9434564639599214, + "tokens_seen": 499843072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856730190571716, + "loss": 1.4283, + "theoretical_loss": 3.9434011774573063, + "tokens_seen": 499908608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856569709127382, + "loss": 1.5084, + "theoretical_loss": 3.9433459002311446, + "tokens_seen": 499974144 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685640922768305, + "loss": 1.5697, + "theoretical_loss": 3.9432906322786643, + "tokens_seen": 500039680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856248746238716, + "loss": 1.5142, + "theoretical_loss": 3.943235373597094, + "tokens_seen": 500105216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006856088264794384, + "loss": 1.4083, + "theoretical_loss": 3.943180124183665, + "tokens_seen": 500170752 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685592778335005, + "loss": 1.518, + "theoretical_loss": 3.9431248840356075, + "tokens_seen": 500236288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855767301905718, + "loss": 1.5122, + "theoretical_loss": 3.943069653150155, + "tokens_seen": 500301824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855606820461384, + "loss": 1.5025, + "theoretical_loss": 3.9430144315245412, + "tokens_seen": 500367360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855446339017052, + "loss": 1.4766, + "theoretical_loss": 3.942959219156, + "tokens_seen": 500432896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855285857572718, + "loss": 1.512, + "theoretical_loss": 3.942904016041769, + "tokens_seen": 500498432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006855125376128386, + "loss": 1.5067, + "theoretical_loss": 3.9428488221790854, + "tokens_seen": 500563968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854964894684052, + "loss": 1.4533, + "theoretical_loss": 3.9427936375651877, + "tokens_seen": 500629504 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685480441323972, + "loss": 1.4543, + "theoretical_loss": 3.9427384621973154, + "tokens_seen": 500695040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854643931795386, + "loss": 1.5801, + "theoretical_loss": 3.9426832960727087, + "tokens_seen": 500760576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854483450351054, + "loss": 1.4239, + "theoretical_loss": 3.9426281391886113, + "tokens_seen": 500826112 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685432296890672, + "loss": 1.5925, + "theoretical_loss": 3.9425729915422654, + "tokens_seen": 500891648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854162487462388, + "loss": 1.5914, + "theoretical_loss": 3.942517853130916, + "tokens_seen": 500957184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006854002006018054, + "loss": 1.5455, + "theoretical_loss": 3.9424627239518086, + "tokens_seen": 501022720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853841524573722, + "loss": 1.4905, + "theoretical_loss": 3.9424076040021903, + "tokens_seen": 501088256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853681043129388, + "loss": 1.4733, + "theoretical_loss": 3.9423524932793086, + "tokens_seen": 501153792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853520561685056, + "loss": 1.5355, + "theoretical_loss": 3.9422973917804134, + "tokens_seen": 501219328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853360080240722, + "loss": 1.4786, + "theoretical_loss": 3.9422422995027544, + "tokens_seen": 501284864 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685319959879639, + "loss": 1.4582, + "theoretical_loss": 3.9421872164435836, + "tokens_seen": 501350400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006853039117352056, + "loss": 1.4059, + "theoretical_loss": 3.942132142600154, + "tokens_seen": 501415936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852878635907724, + "loss": 1.4897, + "theoretical_loss": 3.942077077969719, + "tokens_seen": 501481472 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685271815446339, + "loss": 1.4586, + "theoretical_loss": 3.9420220225495335, + "tokens_seen": 501547008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852557673019058, + "loss": 1.5052, + "theoretical_loss": 3.9419669763368548, + "tokens_seen": 501612544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852397191574724, + "loss": 1.5492, + "theoretical_loss": 3.9419119393289397, + "tokens_seen": 501678080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852236710130392, + "loss": 1.5733, + "theoretical_loss": 3.941856911523047, + "tokens_seen": 501743616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006852076228686058, + "loss": 1.5859, + "theoretical_loss": 3.9418018929164362, + "tokens_seen": 501809152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006851915747241726, + "loss": 1.4799, + "theoretical_loss": 3.9417468835063687, + "tokens_seen": 501874688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006851755265797392, + "loss": 1.5414, + "theoretical_loss": 3.9416918832901064, + "tokens_seen": 501940224 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685159478435306, + "loss": 1.576, + "theoretical_loss": 3.9416368922649134, + "tokens_seen": 502005760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006851434302908726, + "loss": 1.6414, + "theoretical_loss": 3.9415819104280527, + "tokens_seen": 502071296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006851273821464394, + "loss": 1.4958, + "theoretical_loss": 3.941526937776791, + "tokens_seen": 502136832 + }, + { + "epoch": 0.15, + "learning_rate": 0.000685111334002006, + "loss": 1.3764, + "theoretical_loss": 3.941471974308395, + "tokens_seen": 502202368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850952858575728, + "loss": 1.4752, + "theoretical_loss": 3.9414170200201326, + "tokens_seen": 502267904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850792377131394, + "loss": 1.6062, + "theoretical_loss": 3.9413620749092733, + "tokens_seen": 502333440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850631895687062, + "loss": 1.401, + "theoretical_loss": 3.9413071389730874, + "tokens_seen": 502398976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850471414242728, + "loss": 1.539, + "theoretical_loss": 3.941252212208846, + "tokens_seen": 502464512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850310932798396, + "loss": 1.4528, + "theoretical_loss": 3.941197294613822, + "tokens_seen": 502530048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006850150451354062, + "loss": 1.4979, + "theoretical_loss": 3.9411423861852892, + "tokens_seen": 502595584 + }, + { + "epoch": 0.15, + "learning_rate": 0.000684998996990973, + "loss": 1.5176, + "theoretical_loss": 3.941087486920523, + "tokens_seen": 502661120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849829488465396, + "loss": 1.5662, + "theoretical_loss": 3.9410325968167994, + "tokens_seen": 502726656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849669007021064, + "loss": 1.4287, + "theoretical_loss": 3.9409777158713952, + "tokens_seen": 502792192 + }, + { + "epoch": 0.15, + "learning_rate": 0.000684950852557673, + "loss": 1.3871, + "theoretical_loss": 3.9409228440815895, + "tokens_seen": 502857728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849348044132398, + "loss": 1.5545, + "theoretical_loss": 3.9408679814446623, + "tokens_seen": 502923264 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849187562688064, + "loss": 1.3424, + "theoretical_loss": 3.9408131279578935, + "tokens_seen": 502988800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006849027081243732, + "loss": 1.5279, + "theoretical_loss": 3.9407582836185657, + "tokens_seen": 503054336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848866599799398, + "loss": 1.5075, + "theoretical_loss": 3.940703448423962, + "tokens_seen": 503119872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848706118355066, + "loss": 1.5164, + "theoretical_loss": 3.9406486223713664, + "tokens_seen": 503185408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848545636910732, + "loss": 1.4785, + "theoretical_loss": 3.940593805458065, + "tokens_seen": 503250944 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068483851554664, + "loss": 1.5979, + "theoretical_loss": 3.940538997681344, + "tokens_seen": 503316480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848224674022066, + "loss": 1.4346, + "theoretical_loss": 3.9404841990384916, + "tokens_seen": 503382016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006848064192577734, + "loss": 1.5218, + "theoretical_loss": 3.940429409526796, + "tokens_seen": 503447552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068479037111334, + "loss": 1.5219, + "theoretical_loss": 3.940374629143548, + "tokens_seen": 503513088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847743229689068, + "loss": 1.5629, + "theoretical_loss": 3.940319857886039, + "tokens_seen": 503578624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847582748244734, + "loss": 1.5391, + "theoretical_loss": 3.9402650957515606, + "tokens_seen": 503644160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847422266800402, + "loss": 1.4928, + "theoretical_loss": 3.940210342737407, + "tokens_seen": 503709696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847261785356068, + "loss": 1.5532, + "theoretical_loss": 3.940155598840873, + "tokens_seen": 503775232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006847101303911736, + "loss": 1.4604, + "theoretical_loss": 3.9401008640592545, + "tokens_seen": 503840768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006846940822467402, + "loss": 1.4972, + "theoretical_loss": 3.940046138389848, + "tokens_seen": 503906304 + }, + { + "epoch": 0.15, + "learning_rate": 0.000684678034102307, + "loss": 1.5114, + "theoretical_loss": 3.9399914218299523, + "tokens_seen": 503971840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006846619859578737, + "loss": 1.5377, + "theoretical_loss": 3.9399367143768664, + "tokens_seen": 504037376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006846459378134404, + "loss": 1.4433, + "theoretical_loss": 3.939882016027891, + "tokens_seen": 504102912 + }, + { + "epoch": 0.15, + "learning_rate": 0.000684629889669007, + "loss": 1.5481, + "theoretical_loss": 3.939827326780328, + "tokens_seen": 504168448 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006846138415245737, + "loss": 1.4463, + "theoretical_loss": 3.9397726466314795, + "tokens_seen": 504233984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845977933801405, + "loss": 1.368, + "theoretical_loss": 3.93971797557865, + "tokens_seen": 504299520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845817452357071, + "loss": 1.5181, + "theoretical_loss": 3.939663313619145, + "tokens_seen": 504365056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845656970912739, + "loss": 1.4554, + "theoretical_loss": 3.9396086607502694, + "tokens_seen": 504430592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845496489468405, + "loss": 1.5032, + "theoretical_loss": 3.939554016969332, + "tokens_seen": 504496128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845336008024073, + "loss": 1.5561, + "theoretical_loss": 3.939499382273641, + "tokens_seen": 504561664 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 192915, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4771571159362793, + "objective/train/theoretical_loss": 3.939444756660505, + "objective/train/tokens_used": 525087200, + "theoretical_loss": 3.939444756660505, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845175526579739, + "loss": 1.4772, + "theoretical_loss": 3.939444756660505, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006845015045135407, + "loss": 1.3887, + "theoretical_loss": 3.9393901401272364, + "tokens_seen": 504692736 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844854563691073, + "loss": 1.4125, + "theoretical_loss": 3.939335532671146, + "tokens_seen": 504758272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844694082246741, + "loss": 1.4672, + "theoretical_loss": 3.939280934289548, + "tokens_seen": 504823808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844533600802407, + "loss": 1.3927, + "theoretical_loss": 3.9392263449797564, + "tokens_seen": 504889344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844373119358075, + "loss": 1.4563, + "theoretical_loss": 3.939171764739086, + "tokens_seen": 504954880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844212637913741, + "loss": 1.495, + "theoretical_loss": 3.9391171935648535, + "tokens_seen": 505020416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006844052156469409, + "loss": 1.5176, + "theoretical_loss": 3.9390626314543766, + "tokens_seen": 505085952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843891675025075, + "loss": 1.4131, + "theoretical_loss": 3.939008078404975, + "tokens_seen": 505151488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843731193580743, + "loss": 1.5658, + "theoretical_loss": 3.938953534413968, + "tokens_seen": 505217024 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843570712136409, + "loss": 1.5132, + "theoretical_loss": 3.9388989994786767, + "tokens_seen": 505282560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843410230692077, + "loss": 1.5557, + "theoretical_loss": 3.938844473596423, + "tokens_seen": 505348096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843249749247743, + "loss": 1.5351, + "theoretical_loss": 3.9387899567645315, + "tokens_seen": 505413632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006843089267803411, + "loss": 1.4727, + "theoretical_loss": 3.9387354489803252, + "tokens_seen": 505479168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842928786359077, + "loss": 1.4557, + "theoretical_loss": 3.938680950241131, + "tokens_seen": 505544704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842768304914745, + "loss": 1.5463, + "theoretical_loss": 3.938626460544275, + "tokens_seen": 505610240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842607823470411, + "loss": 1.5907, + "theoretical_loss": 3.9385719798870857, + "tokens_seen": 505675776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842447342026079, + "loss": 1.3854, + "theoretical_loss": 3.9385175082668926, + "tokens_seen": 505741312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842286860581745, + "loss": 1.5127, + "theoretical_loss": 3.9384630456810243, + "tokens_seen": 505806848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006842126379137413, + "loss": 1.5117, + "theoretical_loss": 3.9384085921268133, + "tokens_seen": 505872384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841965897693079, + "loss": 1.4626, + "theoretical_loss": 3.9383541476015926, + "tokens_seen": 505937920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841805416248747, + "loss": 1.4934, + "theoretical_loss": 3.9382997121026944, + "tokens_seen": 506003456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841644934804413, + "loss": 1.4904, + "theoretical_loss": 3.938245285627455, + "tokens_seen": 506068992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841484453360081, + "loss": 1.4695, + "theoretical_loss": 3.9381908681732085, + "tokens_seen": 506134528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841323971915747, + "loss": 1.4937, + "theoretical_loss": 3.9381364597372936, + "tokens_seen": 506200064 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841163490471415, + "loss": 1.5277, + "theoretical_loss": 3.938082060317048, + "tokens_seen": 506265600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006841003009027081, + "loss": 1.5658, + "theoretical_loss": 3.9380276699098102, + "tokens_seen": 506331136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840842527582749, + "loss": 1.4609, + "theoretical_loss": 3.9379732885129215, + "tokens_seen": 506396672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840682046138415, + "loss": 1.4391, + "theoretical_loss": 3.9379189161237234, + "tokens_seen": 506462208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840521564694083, + "loss": 1.5199, + "theoretical_loss": 3.9378645527395575, + "tokens_seen": 506527744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840361083249749, + "loss": 1.5124, + "theoretical_loss": 3.937810198357769, + "tokens_seen": 506593280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840200601805417, + "loss": 1.4935, + "theoretical_loss": 3.9377558529757026, + "tokens_seen": 506658816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006840040120361083, + "loss": 1.4662, + "theoretical_loss": 3.937701516590703, + "tokens_seen": 506724352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839879638916751, + "loss": 1.5176, + "theoretical_loss": 3.9376471892001197, + "tokens_seen": 506789888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839719157472417, + "loss": 1.5282, + "theoretical_loss": 3.9375928708012986, + "tokens_seen": 506855424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839558676028085, + "loss": 1.5376, + "theoretical_loss": 3.9375385613915905, + "tokens_seen": 506920960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839398194583751, + "loss": 1.4592, + "theoretical_loss": 3.937484260968346, + "tokens_seen": 506986496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839237713139419, + "loss": 1.5169, + "theoretical_loss": 3.9374299695289157, + "tokens_seen": 507052032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006839077231695085, + "loss": 1.536, + "theoretical_loss": 3.9373756870706536, + "tokens_seen": 507117568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838916750250753, + "loss": 1.6026, + "theoretical_loss": 3.937321413590913, + "tokens_seen": 507183104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838756268806419, + "loss": 1.5024, + "theoretical_loss": 3.9372671490870492, + "tokens_seen": 507248640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838595787362087, + "loss": 1.3873, + "theoretical_loss": 3.937212893556418, + "tokens_seen": 507314176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838435305917753, + "loss": 1.5596, + "theoretical_loss": 3.9371586469963775, + "tokens_seen": 507379712 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838274824473421, + "loss": 1.3812, + "theoretical_loss": 3.937104409404285, + "tokens_seen": 507445248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006838114343029087, + "loss": 1.4625, + "theoretical_loss": 3.9370501807775007, + "tokens_seen": 507510784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837953861584755, + "loss": 1.4738, + "theoretical_loss": 3.936995961113385, + "tokens_seen": 507576320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837793380140421, + "loss": 1.3606, + "theoretical_loss": 3.9369417504093, + "tokens_seen": 507641856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837632898696089, + "loss": 1.404, + "theoretical_loss": 3.936887548662608, + "tokens_seen": 507707392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837472417251755, + "loss": 1.4143, + "theoretical_loss": 3.9368333558706734, + "tokens_seen": 507772928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837311935807423, + "loss": 1.4453, + "theoretical_loss": 3.936779172030861, + "tokens_seen": 507838464 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006837151454363089, + "loss": 1.5579, + "theoretical_loss": 3.936724997140538, + "tokens_seen": 507904000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836990972918757, + "loss": 1.4904, + "theoretical_loss": 3.9366708311970706, + "tokens_seen": 507969536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836830491474423, + "loss": 1.5433, + "theoretical_loss": 3.9366166741978272, + "tokens_seen": 508035072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836670010030091, + "loss": 1.5785, + "theoretical_loss": 3.936562526140178, + "tokens_seen": 508100608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836509528585757, + "loss": 1.5045, + "theoretical_loss": 3.936508387021494, + "tokens_seen": 508166144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836349047141425, + "loss": 1.458, + "theoretical_loss": 3.9364542568391454, + "tokens_seen": 508231680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836188565697091, + "loss": 1.4585, + "theoretical_loss": 3.9364001355905067, + "tokens_seen": 508297216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006836028084252759, + "loss": 1.4411, + "theoretical_loss": 3.936346023272952, + "tokens_seen": 508362752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835867602808425, + "loss": 1.3874, + "theoretical_loss": 3.936291919883855, + "tokens_seen": 508428288 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835707121364093, + "loss": 1.5452, + "theoretical_loss": 3.9362378254205925, + "tokens_seen": 508493824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835546639919759, + "loss": 1.4866, + "theoretical_loss": 3.936183739880543, + "tokens_seen": 508559360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835386158475427, + "loss": 1.5668, + "theoretical_loss": 3.9361296632610827, + "tokens_seen": 508624896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835225677031093, + "loss": 1.5303, + "theoretical_loss": 3.9360755955595934, + "tokens_seen": 508690432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006835065195586761, + "loss": 1.4807, + "theoretical_loss": 3.936021536773455, + "tokens_seen": 508755968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834904714142427, + "loss": 1.4958, + "theoretical_loss": 3.9359674869000485, + "tokens_seen": 508821504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834744232698095, + "loss": 1.4366, + "theoretical_loss": 3.9359134459367575, + "tokens_seen": 508887040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834583751253763, + "loss": 1.4668, + "theoretical_loss": 3.9358594138809657, + "tokens_seen": 508952576 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834423269809429, + "loss": 1.5807, + "theoretical_loss": 3.9358053907300588, + "tokens_seen": 509018112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834262788365097, + "loss": 1.4117, + "theoretical_loss": 3.935751376481422, + "tokens_seen": 509083648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006834102306920763, + "loss": 1.4798, + "theoretical_loss": 3.9356973711324432, + "tokens_seen": 509149184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833941825476431, + "loss": 1.4648, + "theoretical_loss": 3.9356433746805113, + "tokens_seen": 509214720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833781344032097, + "loss": 1.5517, + "theoretical_loss": 3.9355893871230143, + "tokens_seen": 509280256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833620862587765, + "loss": 1.5656, + "theoretical_loss": 3.935535408457344, + "tokens_seen": 509345792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833460381143431, + "loss": 1.5242, + "theoretical_loss": 3.9354814386808923, + "tokens_seen": 509411328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833299899699097, + "loss": 1.4861, + "theoretical_loss": 3.935427477791051, + "tokens_seen": 509476864 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006833139418254765, + "loss": 1.3862, + "theoretical_loss": 3.9353735257852147, + "tokens_seen": 509542400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832978936810431, + "loss": 1.5789, + "theoretical_loss": 3.935319582660778, + "tokens_seen": 509607936 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832818455366099, + "loss": 1.5203, + "theoretical_loss": 3.935265648415138, + "tokens_seen": 509673472 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832657973921766, + "loss": 1.4913, + "theoretical_loss": 3.9352117230456902, + "tokens_seen": 509739008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832497492477433, + "loss": 1.5484, + "theoretical_loss": 3.9351578065498343, + "tokens_seen": 509804544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068323370110331, + "loss": 1.5045, + "theoretical_loss": 3.935103898924969, + "tokens_seen": 509870080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832176529588766, + "loss": 1.4544, + "theoretical_loss": 3.935050000168495, + "tokens_seen": 509935616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006832016048144434, + "loss": 1.4013, + "theoretical_loss": 3.9349961102778135, + "tokens_seen": 510001152 + }, + { + "epoch": 0.15, + "learning_rate": 0.00068318555667001, + "loss": 1.5677, + "theoretical_loss": 3.934942229250328, + "tokens_seen": 510066688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831695085255768, + "loss": 1.4613, + "theoretical_loss": 3.934888357083442, + "tokens_seen": 510132224 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831534603811434, + "loss": 1.6065, + "theoretical_loss": 3.9348344937745594, + "tokens_seen": 510197760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831374122367102, + "loss": 1.4641, + "theoretical_loss": 3.9347806393210876, + "tokens_seen": 510263296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831213640922768, + "loss": 1.559, + "theoretical_loss": 3.934726793720433, + "tokens_seen": 510328832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006831053159478436, + "loss": 1.5181, + "theoretical_loss": 3.9346729569700036, + "tokens_seen": 510394368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006830892678034102, + "loss": 1.5307, + "theoretical_loss": 3.934619129067209, + "tokens_seen": 510459904 + }, + { + "epoch": 0.15, + "learning_rate": 0.000683073219658977, + "loss": 1.5635, + "theoretical_loss": 3.934565310009459, + "tokens_seen": 510525440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006830571715145436, + "loss": 1.6299, + "theoretical_loss": 3.9345114997941657, + "tokens_seen": 510590976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006830411233701104, + "loss": 1.5493, + "theoretical_loss": 3.934457698418741, + "tokens_seen": 510656512 + }, + { + "epoch": 0.15, + "learning_rate": 0.000683025075225677, + "loss": 1.4896, + "theoretical_loss": 3.9344039058805986, + "tokens_seen": 510722048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006830090270812438, + "loss": 1.549, + "theoretical_loss": 3.934350122177154, + "tokens_seen": 510787584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829929789368104, + "loss": 1.535, + "theoretical_loss": 3.9342963473058212, + "tokens_seen": 510853120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829769307923772, + "loss": 1.5197, + "theoretical_loss": 3.9342425812640194, + "tokens_seen": 510918656 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829608826479438, + "loss": 1.5148, + "theoretical_loss": 3.9341888240491647, + "tokens_seen": 510984192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829448345035106, + "loss": 1.5431, + "theoretical_loss": 3.9341350756586766, + "tokens_seen": 511049728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006829287863590772, + "loss": 1.5238, + "theoretical_loss": 3.934081336089976, + "tokens_seen": 511115264 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 195253, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4927643537521362, + "objective/train/theoretical_loss": 3.934027605340483, + "objective/train/tokens_used": 531640800, + "theoretical_loss": 3.934027605340483, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.000682912738214644, + "loss": 1.4928, + "theoretical_loss": 3.934027605340483, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006828966900702106, + "loss": 1.6648, + "theoretical_loss": 3.93397388340762, + "tokens_seen": 511246336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006828806419257774, + "loss": 1.4571, + "theoretical_loss": 3.933920170288811, + "tokens_seen": 511311872 + }, + { + "epoch": 0.15, + "learning_rate": 0.000682864593781344, + "loss": 1.5482, + "theoretical_loss": 3.93386646598148, + "tokens_seen": 511377408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0006828485456369108, + "loss": 1.4836, + "theoretical_loss": 3.9338127704830534, + "tokens_seen": 511442944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006828324974924774, + "loss": 1.571, + "theoretical_loss": 3.933759083790957, + "tokens_seen": 511508480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006828164493480442, + "loss": 1.4634, + "theoretical_loss": 3.933705405902619, + "tokens_seen": 511574016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006828004012036108, + "loss": 1.5006, + "theoretical_loss": 3.9336517368154666, + "tokens_seen": 511639552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006827843530591776, + "loss": 1.3858, + "theoretical_loss": 3.9335980765269323, + "tokens_seen": 511705088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006827683049147442, + "loss": 1.5469, + "theoretical_loss": 3.9335444250344445, + "tokens_seen": 511770624 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682752256770311, + "loss": 1.524, + "theoretical_loss": 3.933490782335437, + "tokens_seen": 511836160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006827362086258776, + "loss": 1.4135, + "theoretical_loss": 3.933437148427342, + "tokens_seen": 511901696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006827201604814444, + "loss": 1.4686, + "theoretical_loss": 3.933383523307594, + "tokens_seen": 511967232 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682704112337011, + "loss": 1.4716, + "theoretical_loss": 3.9333299069736283, + "tokens_seen": 512032768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826880641925778, + "loss": 1.5078, + "theoretical_loss": 3.9332762994228814, + "tokens_seen": 512098304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826720160481444, + "loss": 1.6016, + "theoretical_loss": 3.93322270065279, + "tokens_seen": 512163840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826559679037112, + "loss": 1.4667, + "theoretical_loss": 3.933169110660793, + "tokens_seen": 512229376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826399197592778, + "loss": 1.5683, + "theoretical_loss": 3.93311552944433, + "tokens_seen": 512294912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826238716148446, + "loss": 1.5083, + "theoretical_loss": 3.9330619570008416, + "tokens_seen": 512360448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006826078234704112, + "loss": 1.5222, + "theoretical_loss": 3.9330083933277695, + "tokens_seen": 512425984 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682591775325978, + "loss": 1.5197, + "theoretical_loss": 3.9329548384225568, + "tokens_seen": 512491520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006825757271815446, + "loss": 1.5644, + "theoretical_loss": 3.932901292282647, + "tokens_seen": 512557056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006825596790371114, + "loss": 1.4929, + "theoretical_loss": 3.932847754905485, + "tokens_seen": 512622592 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682543630892678, + "loss": 1.4674, + "theoretical_loss": 3.9327942262885167, + "tokens_seen": 512688128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006825275827482448, + "loss": 1.4877, + "theoretical_loss": 3.9327407064291897, + "tokens_seen": 512753664 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006825115346038114, + "loss": 1.6125, + "theoretical_loss": 3.9326871953249514, + "tokens_seen": 512819200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824954864593782, + "loss": 1.5211, + "theoretical_loss": 3.9326336929732513, + "tokens_seen": 512884736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824794383149448, + "loss": 1.4972, + "theoretical_loss": 3.9325801993715404, + "tokens_seen": 512950272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824633901705116, + "loss": 1.5703, + "theoretical_loss": 3.9325267145172687, + "tokens_seen": 513015808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824473420260782, + "loss": 1.5042, + "theoretical_loss": 3.932473238407889, + "tokens_seen": 513081344 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682431293881645, + "loss": 1.4834, + "theoretical_loss": 3.932419771040856, + "tokens_seen": 513146880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006824152457372116, + "loss": 1.5961, + "theoretical_loss": 3.932366312413623, + "tokens_seen": 513212416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823991975927784, + "loss": 1.4819, + "theoretical_loss": 3.932312862523646, + "tokens_seen": 513277952 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682383149448345, + "loss": 1.4637, + "theoretical_loss": 3.9322594213683812, + "tokens_seen": 513343488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823671013039118, + "loss": 1.4682, + "theoretical_loss": 3.9322059889452867, + "tokens_seen": 513409024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823510531594784, + "loss": 1.6445, + "theoretical_loss": 3.9321525652518217, + "tokens_seen": 513474560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823350050150452, + "loss": 1.5195, + "theoretical_loss": 3.9320991502854454, + "tokens_seen": 513540096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823189568706118, + "loss": 1.4613, + "theoretical_loss": 3.932045744043619, + "tokens_seen": 513605632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006823029087261786, + "loss": 1.4843, + "theoretical_loss": 3.9319923465238054, + "tokens_seen": 513671168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822868605817452, + "loss": 1.4426, + "theoretical_loss": 3.9319389577234665, + "tokens_seen": 513736704 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682270812437312, + "loss": 1.5403, + "theoretical_loss": 3.931885577640066, + "tokens_seen": 513802240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822547642928787, + "loss": 1.4328, + "theoretical_loss": 3.9318322062710704, + "tokens_seen": 513867776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822387161484454, + "loss": 1.5424, + "theoretical_loss": 3.931778843613945, + "tokens_seen": 513933312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822226680040121, + "loss": 1.4345, + "theoretical_loss": 3.9317254896661575, + "tokens_seen": 513998848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006822066198595788, + "loss": 1.5545, + "theoretical_loss": 3.9316721444251765, + "tokens_seen": 514064384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006821905717151455, + "loss": 1.4534, + "theoretical_loss": 3.9316188078884706, + "tokens_seen": 514129920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006821745235707122, + "loss": 1.4149, + "theoretical_loss": 3.931565480053511, + "tokens_seen": 514195456 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682158475426279, + "loss": 1.459, + "theoretical_loss": 3.9315121609177694, + "tokens_seen": 514260992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006821424272818456, + "loss": 1.5464, + "theoretical_loss": 3.931458850478718, + "tokens_seen": 514326528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006821263791374124, + "loss": 1.4688, + "theoretical_loss": 3.9314055487338297, + "tokens_seen": 514392064 + }, + { + "epoch": 0.16, + "learning_rate": 0.000682110330992979, + "loss": 1.49, + "theoretical_loss": 3.93135225568058, + "tokens_seen": 514457600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820942828485458, + "loss": 1.5361, + "theoretical_loss": 3.9312989713164455, + "tokens_seen": 514523136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820782347041124, + "loss": 1.4578, + "theoretical_loss": 3.9312456956389017, + "tokens_seen": 514588672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820621865596792, + "loss": 1.4721, + "theoretical_loss": 3.931192428645427, + "tokens_seen": 514654208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820461384152458, + "loss": 1.4551, + "theoretical_loss": 3.9311391703334997, + "tokens_seen": 514719744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820300902708126, + "loss": 1.5363, + "theoretical_loss": 3.9310859207006006, + "tokens_seen": 514785280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006820140421263792, + "loss": 1.4793, + "theoretical_loss": 3.9310326797442103, + "tokens_seen": 514850816 + }, + { + "epoch": 0.16, + "learning_rate": 0.000681997993981946, + "loss": 1.6466, + "theoretical_loss": 3.930979447461811, + "tokens_seen": 514916352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006819819458375126, + "loss": 1.5345, + "theoretical_loss": 3.9309262238508857, + "tokens_seen": 514981888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006819658976930794, + "loss": 1.5342, + "theoretical_loss": 3.930873008908919, + "tokens_seen": 515047424 + }, + { + "epoch": 0.16, + "learning_rate": 0.000681949849548646, + "loss": 1.4533, + "theoretical_loss": 3.9308198026333954, + "tokens_seen": 515112960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006819338014042126, + "loss": 1.5149, + "theoretical_loss": 3.9307666050218018, + "tokens_seen": 515178496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006819177532597794, + "loss": 1.4271, + "theoretical_loss": 3.930713416071625, + "tokens_seen": 515244032 + }, + { + "epoch": 0.16, + "learning_rate": 0.000681901705115346, + "loss": 1.4856, + "theoretical_loss": 3.930660235780354, + "tokens_seen": 515309568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818856569709128, + "loss": 1.4503, + "theoretical_loss": 3.9306070641454776, + "tokens_seen": 515375104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818696088264795, + "loss": 1.5552, + "theoretical_loss": 3.9305539011644863, + "tokens_seen": 515440640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818535606820462, + "loss": 1.49, + "theoretical_loss": 3.9305007468348725, + "tokens_seen": 515506176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818375125376129, + "loss": 1.4328, + "theoretical_loss": 3.9304476011541274, + "tokens_seen": 515571712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818214643931796, + "loss": 1.4305, + "theoretical_loss": 3.930394464119746, + "tokens_seen": 515637248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006818054162487463, + "loss": 1.5374, + "theoretical_loss": 3.9303413357292216, + "tokens_seen": 515702784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817893681043129, + "loss": 1.4602, + "theoretical_loss": 3.930288215980051, + "tokens_seen": 515768320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817733199598797, + "loss": 1.4747, + "theoretical_loss": 3.9302351048697304, + "tokens_seen": 515833856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817572718154463, + "loss": 1.4172, + "theoretical_loss": 3.9301820023957577, + "tokens_seen": 515899392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817412236710131, + "loss": 1.4934, + "theoretical_loss": 3.930128908555632, + "tokens_seen": 515964928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817251755265797, + "loss": 1.5599, + "theoretical_loss": 3.9300758233468525, + "tokens_seen": 516030464 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006817091273821465, + "loss": 1.4585, + "theoretical_loss": 3.9300227467669204, + "tokens_seen": 516096000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816930792377131, + "loss": 1.44, + "theoretical_loss": 3.929969678813338, + "tokens_seen": 516161536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816770310932799, + "loss": 1.5235, + "theoretical_loss": 3.929916619483608, + "tokens_seen": 516227072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816609829488465, + "loss": 1.4747, + "theoretical_loss": 3.9298635687752337, + "tokens_seen": 516292608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816449348044133, + "loss": 1.5545, + "theoretical_loss": 3.9298105266857215, + "tokens_seen": 516358144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816288866599799, + "loss": 1.5144, + "theoretical_loss": 3.929757493212577, + "tokens_seen": 516423680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006816128385155467, + "loss": 1.5349, + "theoretical_loss": 3.929704468353307, + "tokens_seen": 516489216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815967903711133, + "loss": 1.4878, + "theoretical_loss": 3.9296514521054196, + "tokens_seen": 516554752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815807422266801, + "loss": 1.4512, + "theoretical_loss": 3.929598444466425, + "tokens_seen": 516620288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815646940822467, + "loss": 1.4506, + "theoretical_loss": 3.9295454454338326, + "tokens_seen": 516685824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815486459378135, + "loss": 1.4666, + "theoretical_loss": 3.9294924550051533, + "tokens_seen": 516751360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815325977933801, + "loss": 1.563, + "theoretical_loss": 3.9294394731779008, + "tokens_seen": 516816896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815165496489469, + "loss": 1.4868, + "theoretical_loss": 3.929386499949587, + "tokens_seen": 516882432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006815005015045135, + "loss": 1.5435, + "theoretical_loss": 3.929333535317727, + "tokens_seen": 516947968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814844533600803, + "loss": 1.5296, + "theoretical_loss": 3.929280579279837, + "tokens_seen": 517013504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814684052156469, + "loss": 1.4191, + "theoretical_loss": 3.929227631833432, + "tokens_seen": 517079040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814523570712137, + "loss": 1.4075, + "theoretical_loss": 3.92917469297603, + "tokens_seen": 517144576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814363089267803, + "loss": 1.4132, + "theoretical_loss": 3.9291217627051496, + "tokens_seen": 517210112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814202607823471, + "loss": 1.4545, + "theoretical_loss": 3.9290688410183106, + "tokens_seen": 517275648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006814042126379137, + "loss": 1.5751, + "theoretical_loss": 3.9290159279130332, + "tokens_seen": 517341184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813881644934805, + "loss": 1.4935, + "theoretical_loss": 3.928963023386839, + "tokens_seen": 517406720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813721163490471, + "loss": 1.4569, + "theoretical_loss": 3.9289101274372515, + "tokens_seen": 517472256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813560682046139, + "loss": 1.5109, + "theoretical_loss": 3.9288572400617934, + "tokens_seen": 517537792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813400200601805, + "loss": 1.5265, + "theoretical_loss": 3.9288043612579893, + "tokens_seen": 517603328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813239719157473, + "loss": 1.5922, + "theoretical_loss": 3.928751491023366, + "tokens_seen": 517668864 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 197659, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3977913856506348, + "objective/train/theoretical_loss": 3.9286986293554493, + "objective/train/tokens_used": 538194400, + "theoretical_loss": 3.9286986293554493, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006813079237713139, + "loss": 1.3978, + "theoretical_loss": 3.9286986293554493, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812918756268807, + "loss": 1.4508, + "theoretical_loss": 3.9286457762517673, + "tokens_seen": 517799936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812758274824473, + "loss": 1.384, + "theoretical_loss": 3.928592931709849, + "tokens_seen": 517865472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812597793380141, + "loss": 1.4557, + "theoretical_loss": 3.9285400957272234, + "tokens_seen": 517931008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812437311935807, + "loss": 1.5916, + "theoretical_loss": 3.9284872683014225, + "tokens_seen": 517996544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812276830491475, + "loss": 1.4564, + "theoretical_loss": 3.9284344494299774, + "tokens_seen": 518062080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006812116349047141, + "loss": 1.4545, + "theoretical_loss": 3.9283816391104214, + "tokens_seen": 518127616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811955867602809, + "loss": 1.4759, + "theoretical_loss": 3.9283288373402883, + "tokens_seen": 518193152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811795386158475, + "loss": 1.6175, + "theoretical_loss": 3.9282760441171134, + "tokens_seen": 518258688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811634904714143, + "loss": 1.4287, + "theoretical_loss": 3.9282232594384316, + "tokens_seen": 518324224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811474423269809, + "loss": 1.4837, + "theoretical_loss": 3.9281704833017814, + "tokens_seen": 518389760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811313941825477, + "loss": 1.4705, + "theoretical_loss": 3.9281177157047003, + "tokens_seen": 518455296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006811153460381143, + "loss": 1.5372, + "theoretical_loss": 3.928064956644726, + "tokens_seen": 518520832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810992978936811, + "loss": 1.4273, + "theoretical_loss": 3.9280122061194005, + "tokens_seen": 518586368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810832497492477, + "loss": 1.5432, + "theoretical_loss": 3.927959464126264, + "tokens_seen": 518651904 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810672016048145, + "loss": 1.5374, + "theoretical_loss": 3.9279067306628583, + "tokens_seen": 518717440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810511534603812, + "loss": 1.442, + "theoretical_loss": 3.9278540057267266, + "tokens_seen": 518782976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810351053159479, + "loss": 1.5838, + "theoretical_loss": 3.927801289315414, + "tokens_seen": 518848512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810190571715146, + "loss": 1.5246, + "theoretical_loss": 3.9277485814264645, + "tokens_seen": 518914048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006810030090270813, + "loss": 1.4806, + "theoretical_loss": 3.9276958820574253, + "tokens_seen": 518979584 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680986960882648, + "loss": 1.4541, + "theoretical_loss": 3.9276431912058425, + "tokens_seen": 519045120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809709127382147, + "loss": 1.6333, + "theoretical_loss": 3.9275905088692653, + "tokens_seen": 519110656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809548645937814, + "loss": 1.5028, + "theoretical_loss": 3.927537835045242, + "tokens_seen": 519176192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809388164493481, + "loss": 1.4962, + "theoretical_loss": 3.927485169731323, + "tokens_seen": 519241728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809227683049148, + "loss": 1.5856, + "theoretical_loss": 3.9274325129250602, + "tokens_seen": 519307264 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006809067201604815, + "loss": 1.5406, + "theoretical_loss": 3.9273798646240055, + "tokens_seen": 519372800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808906720160482, + "loss": 1.5551, + "theoretical_loss": 3.9273272248257123, + "tokens_seen": 519438336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808746238716149, + "loss": 1.5505, + "theoretical_loss": 3.9272745935277347, + "tokens_seen": 519503872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808585757271816, + "loss": 1.4709, + "theoretical_loss": 3.927221970727628, + "tokens_seen": 519569408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808425275827483, + "loss": 1.5524, + "theoretical_loss": 3.927169356422948, + "tokens_seen": 519634944 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680826479438315, + "loss": 1.4632, + "theoretical_loss": 3.9271167506112534, + "tokens_seen": 519700480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006808104312938817, + "loss": 1.5375, + "theoretical_loss": 3.927064153290101, + "tokens_seen": 519766016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807943831494484, + "loss": 1.4494, + "theoretical_loss": 3.9270115644570516, + "tokens_seen": 519831552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807783350050151, + "loss": 1.5576, + "theoretical_loss": 3.9269589841096644, + "tokens_seen": 519897088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807622868605819, + "loss": 1.5529, + "theoretical_loss": 3.9269064122455006, + "tokens_seen": 519962624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807462387161485, + "loss": 1.5, + "theoretical_loss": 3.926853848862124, + "tokens_seen": 520028160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807301905717153, + "loss": 1.4727, + "theoretical_loss": 3.9268012939570967, + "tokens_seen": 520093696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006807141424272819, + "loss": 1.4417, + "theoretical_loss": 3.9267487475279834, + "tokens_seen": 520159232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806980942828487, + "loss": 1.5048, + "theoretical_loss": 3.9266962095723494, + "tokens_seen": 520224768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806820461384153, + "loss": 1.3845, + "theoretical_loss": 3.9266436800877615, + "tokens_seen": 520290304 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680665997993982, + "loss": 1.5835, + "theoretical_loss": 3.926591159071787, + "tokens_seen": 520355840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806499498495487, + "loss": 1.4934, + "theoretical_loss": 3.9265386465219936, + "tokens_seen": 520421376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806339017051155, + "loss": 1.5652, + "theoretical_loss": 3.9264861424359516, + "tokens_seen": 520486912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806178535606821, + "loss": 1.4982, + "theoretical_loss": 3.9264336468112315, + "tokens_seen": 520552448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006806018054162489, + "loss": 1.567, + "theoretical_loss": 3.9263811596454037, + "tokens_seen": 520617984 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805857572718155, + "loss": 1.497, + "theoretical_loss": 3.9263286809360407, + "tokens_seen": 520683520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805697091273823, + "loss": 1.5281, + "theoretical_loss": 3.9262762106807174, + "tokens_seen": 520749056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805536609829489, + "loss": 1.5089, + "theoretical_loss": 3.926223748877007, + "tokens_seen": 520814592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805376128385156, + "loss": 1.6058, + "theoretical_loss": 3.926171295522485, + "tokens_seen": 520880128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006805215646940823, + "loss": 1.5989, + "theoretical_loss": 3.9261188506147278, + "tokens_seen": 520945664 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680505516549649, + "loss": 1.4645, + "theoretical_loss": 3.926066414151313, + "tokens_seen": 521011200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804894684052157, + "loss": 1.5599, + "theoretical_loss": 3.926013986129819, + "tokens_seen": 521076736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804734202607824, + "loss": 1.5434, + "theoretical_loss": 3.9259615665478256, + "tokens_seen": 521142272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804573721163491, + "loss": 1.457, + "theoretical_loss": 3.9259091554029126, + "tokens_seen": 521207808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804413239719158, + "loss": 1.5803, + "theoretical_loss": 3.9258567526926615, + "tokens_seen": 521273344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804252758274825, + "loss": 1.6173, + "theoretical_loss": 3.9258043584146556, + "tokens_seen": 521338880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006804092276830492, + "loss": 1.4959, + "theoretical_loss": 3.925751972566477, + "tokens_seen": 521404416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803931795386158, + "loss": 1.5505, + "theoretical_loss": 3.9256995951457103, + "tokens_seen": 521469952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803771313941826, + "loss": 1.3809, + "theoretical_loss": 3.925647226149942, + "tokens_seen": 521535488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803610832497492, + "loss": 1.5001, + "theoretical_loss": 3.925594865576758, + "tokens_seen": 521601024 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680345035105316, + "loss": 1.4417, + "theoretical_loss": 3.9255425134237445, + "tokens_seen": 521666560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803289869608826, + "loss": 1.5312, + "theoretical_loss": 3.9254901696884916, + "tokens_seen": 521732096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006803129388164494, + "loss": 1.4324, + "theoretical_loss": 3.925437834368588, + "tokens_seen": 521797632 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680296890672016, + "loss": 1.5199, + "theoretical_loss": 3.925385507461624, + "tokens_seen": 521863168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802808425275828, + "loss": 1.5288, + "theoretical_loss": 3.9253331889651912, + "tokens_seen": 521928704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802647943831494, + "loss": 1.5455, + "theoretical_loss": 3.9252808788768814, + "tokens_seen": 521994240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802487462387162, + "loss": 1.5087, + "theoretical_loss": 3.9252285771942876, + "tokens_seen": 522059776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802326980942828, + "loss": 1.519, + "theoretical_loss": 3.9251762839150057, + "tokens_seen": 522125312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802166499498496, + "loss": 1.5797, + "theoretical_loss": 3.9251239990366305, + "tokens_seen": 522190848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006802006018054162, + "loss": 1.6343, + "theoretical_loss": 3.925071722556757, + "tokens_seen": 522256384 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680184553660983, + "loss": 1.447, + "theoretical_loss": 3.9250194544729844, + "tokens_seen": 522321920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006801685055165496, + "loss": 1.4694, + "theoretical_loss": 3.9249671947829095, + "tokens_seen": 522387456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006801524573721164, + "loss": 1.53, + "theoretical_loss": 3.924914943484133, + "tokens_seen": 522452992 + }, + { + "epoch": 0.16, + "learning_rate": 0.000680136409227683, + "loss": 1.6372, + "theoretical_loss": 3.9248627005742533, + "tokens_seen": 522518528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006801203610832498, + "loss": 1.5316, + "theoretical_loss": 3.9248104660508734, + "tokens_seen": 522584064 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006801043129388164, + "loss": 1.433, + "theoretical_loss": 3.9247582399115943, + "tokens_seen": 522649600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800882647943832, + "loss": 1.4872, + "theoretical_loss": 3.92470602215402, + "tokens_seen": 522715136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800722166499498, + "loss": 1.4878, + "theoretical_loss": 3.9246538127757544, + "tokens_seen": 522780672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800561685055166, + "loss": 1.5593, + "theoretical_loss": 3.9246016117744036, + "tokens_seen": 522846208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800401203610832, + "loss": 1.3854, + "theoretical_loss": 3.924549419147572, + "tokens_seen": 522911744 + }, + { + "epoch": 0.16, + "learning_rate": 0.00068002407221665, + "loss": 1.4211, + "theoretical_loss": 3.9244972348928684, + "tokens_seen": 522977280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006800080240722166, + "loss": 1.4798, + "theoretical_loss": 3.9244450590079003, + "tokens_seen": 523042816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799919759277834, + "loss": 1.5415, + "theoretical_loss": 3.9243928914902764, + "tokens_seen": 523108352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00067997592778335, + "loss": 1.3798, + "theoretical_loss": 3.924340732337608, + "tokens_seen": 523173888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799598796389168, + "loss": 1.4309, + "theoretical_loss": 3.924288581547505, + "tokens_seen": 523239424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799438314944834, + "loss": 1.5864, + "theoretical_loss": 3.9242364391175797, + "tokens_seen": 523304960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799277833500502, + "loss": 1.5678, + "theoretical_loss": 3.924184305045446, + "tokens_seen": 523370496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006799117352056168, + "loss": 1.4642, + "theoretical_loss": 3.9241321793287174, + "tokens_seen": 523436032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798956870611836, + "loss": 1.5081, + "theoretical_loss": 3.9240800619650082, + "tokens_seen": 523501568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798796389167503, + "loss": 1.5093, + "theoretical_loss": 3.9240279529519357, + "tokens_seen": 523567104 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679863590772317, + "loss": 1.5531, + "theoretical_loss": 3.9239758522871164, + "tokens_seen": 523632640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798475426278837, + "loss": 1.5214, + "theoretical_loss": 3.923923759968168, + "tokens_seen": 523698176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798314944834504, + "loss": 1.5054, + "theoretical_loss": 3.9238716759927095, + "tokens_seen": 523763712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006798154463390171, + "loss": 1.5177, + "theoretical_loss": 3.923819600358361, + "tokens_seen": 523829248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797993981945838, + "loss": 1.5365, + "theoretical_loss": 3.923767533062743, + "tokens_seen": 523894784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797833500501505, + "loss": 1.5573, + "theoretical_loss": 3.9237154741034783, + "tokens_seen": 523960320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797673019057172, + "loss": 1.4597, + "theoretical_loss": 3.923663423478189, + "tokens_seen": 524025856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797512537612839, + "loss": 1.514, + "theoretical_loss": 3.9236113811844984, + "tokens_seen": 524091392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797352056168506, + "loss": 1.4305, + "theoretical_loss": 3.9235593472200323, + "tokens_seen": 524156928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006797191574724173, + "loss": 1.4422, + "theoretical_loss": 3.9235073215824166, + "tokens_seen": 524222464 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 199786, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4710164070129395, + "objective/train/theoretical_loss": 3.9234553042692766, + "objective/train/tokens_used": 544748000, + "theoretical_loss": 3.9234553042692766, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679703109327984, + "loss": 1.471, + "theoretical_loss": 3.9234553042692766, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796870611835507, + "loss": 1.4901, + "theoretical_loss": 3.9234032952782414, + "tokens_seen": 524353536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796710130391174, + "loss": 1.5114, + "theoretical_loss": 3.923351294606939, + "tokens_seen": 524419072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796549648946841, + "loss": 1.4697, + "theoretical_loss": 3.923299302253, + "tokens_seen": 524484608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796389167502508, + "loss": 1.4857, + "theoretical_loss": 3.923247318214054, + "tokens_seen": 524550144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796228686058175, + "loss": 1.5773, + "theoretical_loss": 3.923195342487733, + "tokens_seen": 524615680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006796068204613842, + "loss": 1.5282, + "theoretical_loss": 3.9231433750716693, + "tokens_seen": 524681216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795907723169509, + "loss": 1.4995, + "theoretical_loss": 3.9230914159634964, + "tokens_seen": 524746752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795747241725176, + "loss": 1.5593, + "theoretical_loss": 3.9230394651608496, + "tokens_seen": 524812288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795586760280843, + "loss": 1.4765, + "theoretical_loss": 3.922987522661363, + "tokens_seen": 524877824 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679542627883651, + "loss": 1.5215, + "theoretical_loss": 3.922935588462675, + "tokens_seen": 524943360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795265797392177, + "loss": 1.5005, + "theoretical_loss": 3.9228836625624206, + "tokens_seen": 525008896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006795105315947844, + "loss": 1.5552, + "theoretical_loss": 3.9228317449582404, + "tokens_seen": 525074432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794944834503511, + "loss": 1.4967, + "theoretical_loss": 3.9227798356477726, + "tokens_seen": 525139968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794784353059178, + "loss": 1.52, + "theoretical_loss": 3.922727934628658, + "tokens_seen": 525205504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794623871614845, + "loss": 1.5624, + "theoretical_loss": 3.922676041898537, + "tokens_seen": 525271040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794463390170512, + "loss": 1.4973, + "theoretical_loss": 3.9226241574550533, + "tokens_seen": 525336576 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679430290872618, + "loss": 1.4885, + "theoretical_loss": 3.9225722812958486, + "tokens_seen": 525402112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006794142427281846, + "loss": 1.5818, + "theoretical_loss": 3.922520413418568, + "tokens_seen": 525467648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793981945837513, + "loss": 1.51, + "theoretical_loss": 3.922468553820856, + "tokens_seen": 525533184 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679382146439318, + "loss": 1.5027, + "theoretical_loss": 3.9224167025003593, + "tokens_seen": 525598720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793660982948848, + "loss": 1.5383, + "theoretical_loss": 3.922364859454725, + "tokens_seen": 525664256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793500501504514, + "loss": 1.6172, + "theoretical_loss": 3.9223130246816007, + "tokens_seen": 525729792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793340020060182, + "loss": 1.5414, + "theoretical_loss": 3.9222611981786355, + "tokens_seen": 525795328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793179538615848, + "loss": 1.4279, + "theoretical_loss": 3.92220937994348, + "tokens_seen": 525860864 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006793019057171516, + "loss": 1.4529, + "theoretical_loss": 3.9221575699737836, + "tokens_seen": 525926400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006792858575727182, + "loss": 1.4595, + "theoretical_loss": 3.9221057682672, + "tokens_seen": 525991936 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679269809428285, + "loss": 1.5844, + "theoretical_loss": 3.92205397482138, + "tokens_seen": 526057472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006792537612838516, + "loss": 1.5545, + "theoretical_loss": 3.9220021896339796, + "tokens_seen": 526123008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006792377131394184, + "loss": 1.5428, + "theoretical_loss": 3.921950412702652, + "tokens_seen": 526188544 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679221664994985, + "loss": 1.4916, + "theoretical_loss": 3.9218986440250534, + "tokens_seen": 526254080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006792056168505518, + "loss": 1.4157, + "theoretical_loss": 3.9218468835988403, + "tokens_seen": 526319616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791895687061184, + "loss": 1.5477, + "theoretical_loss": 3.9217951314216704, + "tokens_seen": 526385152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791735205616852, + "loss": 1.5625, + "theoretical_loss": 3.9217433874912024, + "tokens_seen": 526450688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791574724172518, + "loss": 1.5584, + "theoretical_loss": 3.921691651805096, + "tokens_seen": 526516224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791414242728185, + "loss": 1.5538, + "theoretical_loss": 3.9216399243610116, + "tokens_seen": 526581760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791253761283852, + "loss": 1.5673, + "theoretical_loss": 3.9215882051566098, + "tokens_seen": 526647296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006791093279839519, + "loss": 1.5533, + "theoretical_loss": 3.921536494189554, + "tokens_seen": 526712832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790932798395186, + "loss": 1.4866, + "theoretical_loss": 3.921484791457507, + "tokens_seen": 526778368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790772316950853, + "loss": 1.5309, + "theoretical_loss": 3.921433096958134, + "tokens_seen": 526843904 + }, + { + "epoch": 0.16, + "learning_rate": 0.000679061183550652, + "loss": 1.4798, + "theoretical_loss": 3.921381410689099, + "tokens_seen": 526909440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790451354062187, + "loss": 1.5223, + "theoretical_loss": 3.9213297326480694, + "tokens_seen": 526974976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790290872617854, + "loss": 1.5074, + "theoretical_loss": 3.9212780628327115, + "tokens_seen": 527040512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006790130391173521, + "loss": 1.4828, + "theoretical_loss": 3.921226401240693, + "tokens_seen": 527106048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789969909729187, + "loss": 1.6028, + "theoretical_loss": 3.9211747478696846, + "tokens_seen": 527171584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789809428284855, + "loss": 1.5113, + "theoretical_loss": 3.9211231027173548, + "tokens_seen": 527237120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789648946840521, + "loss": 1.4836, + "theoretical_loss": 3.921071465781375, + "tokens_seen": 527302656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789488465396189, + "loss": 1.5652, + "theoretical_loss": 3.9210198370594176, + "tokens_seen": 527368192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789327983951855, + "loss": 1.4973, + "theoretical_loss": 3.920968216549155, + "tokens_seen": 527433728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789167502507523, + "loss": 1.591, + "theoretical_loss": 3.9209166042482613, + "tokens_seen": 527499264 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006789007021063189, + "loss": 1.425, + "theoretical_loss": 3.9208650001544108, + "tokens_seen": 527564800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788846539618857, + "loss": 1.4892, + "theoretical_loss": 3.920813404265279, + "tokens_seen": 527630336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788686058174523, + "loss": 1.6404, + "theoretical_loss": 3.920761816578543, + "tokens_seen": 527695872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788525576730191, + "loss": 1.4914, + "theoretical_loss": 3.920710237091881, + "tokens_seen": 527761408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788365095285857, + "loss": 1.4574, + "theoretical_loss": 3.9206586658029714, + "tokens_seen": 527826944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788204613841525, + "loss": 1.4975, + "theoretical_loss": 3.920607102709493, + "tokens_seen": 527892480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006788044132397191, + "loss": 1.5272, + "theoretical_loss": 3.9205555478091254, + "tokens_seen": 527958016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787883650952859, + "loss": 1.4515, + "theoretical_loss": 3.920504001099552, + "tokens_seen": 528023552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787723169508525, + "loss": 1.5712, + "theoretical_loss": 3.920452462578454, + "tokens_seen": 528089088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787562688064193, + "loss": 1.4757, + "theoretical_loss": 3.920400932243515, + "tokens_seen": 528154624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787402206619859, + "loss": 1.6068, + "theoretical_loss": 3.920349410092419, + "tokens_seen": 528220160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787241725175527, + "loss": 1.4963, + "theoretical_loss": 3.920297896122851, + "tokens_seen": 528285696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006787081243731193, + "loss": 1.5176, + "theoretical_loss": 3.9202463903324976, + "tokens_seen": 528351232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786920762286861, + "loss": 1.4593, + "theoretical_loss": 3.920194892719045, + "tokens_seen": 528416768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786760280842528, + "loss": 1.4324, + "theoretical_loss": 3.920143403280182, + "tokens_seen": 528482304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786599799398195, + "loss": 1.4706, + "theoretical_loss": 3.9200919220135972, + "tokens_seen": 528547840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786439317953862, + "loss": 1.5796, + "theoretical_loss": 3.920040448916981, + "tokens_seen": 528613376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786278836509529, + "loss": 1.5048, + "theoretical_loss": 3.919988983988023, + "tokens_seen": 528678912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006786118355065196, + "loss": 1.5156, + "theoretical_loss": 3.919937527224416, + "tokens_seen": 528744448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785957873620863, + "loss": 1.5273, + "theoretical_loss": 3.9198860786238523, + "tokens_seen": 528809984 + }, + { + "epoch": 0.16, + "learning_rate": 0.000678579739217653, + "loss": 1.489, + "theoretical_loss": 3.919834638184025, + "tokens_seen": 528875520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785636910732197, + "loss": 1.4565, + "theoretical_loss": 3.91978320590263, + "tokens_seen": 528941056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785476429287864, + "loss": 1.4633, + "theoretical_loss": 3.919731781777361, + "tokens_seen": 529006592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785315947843531, + "loss": 1.5683, + "theoretical_loss": 3.919680365805916, + "tokens_seen": 529072128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006785155466399198, + "loss": 1.4105, + "theoretical_loss": 3.9196289579859913, + "tokens_seen": 529137664 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784994984954865, + "loss": 1.4141, + "theoretical_loss": 3.919577558315286, + "tokens_seen": 529203200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784834503510532, + "loss": 1.4611, + "theoretical_loss": 3.9195261667914982, + "tokens_seen": 529268736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784674022066199, + "loss": 1.557, + "theoretical_loss": 3.91947478341233, + "tokens_seen": 529334272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784513540621866, + "loss": 1.5616, + "theoretical_loss": 3.9194234081754797, + "tokens_seen": 529399808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784353059177533, + "loss": 1.4557, + "theoretical_loss": 3.9193720410786517, + "tokens_seen": 529465344 + }, + { + "epoch": 0.16, + "learning_rate": 0.00067841925777332, + "loss": 1.5214, + "theoretical_loss": 3.9193206821195483, + "tokens_seen": 529530880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006784032096288867, + "loss": 1.5456, + "theoretical_loss": 3.9192693312958733, + "tokens_seen": 529596416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783871614844534, + "loss": 1.5274, + "theoretical_loss": 3.919217988605331, + "tokens_seen": 529661952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783711133400201, + "loss": 1.4742, + "theoretical_loss": 3.919166654045628, + "tokens_seen": 529727488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783550651955868, + "loss": 1.3755, + "theoretical_loss": 3.919115327614471, + "tokens_seen": 529793024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783390170511535, + "loss": 1.5839, + "theoretical_loss": 3.9190640093095666, + "tokens_seen": 529858560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783229689067202, + "loss": 1.5353, + "theoretical_loss": 3.9190126991286243, + "tokens_seen": 529924096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006783069207622869, + "loss": 1.5977, + "theoretical_loss": 3.918961397069353, + "tokens_seen": 529989632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782908726178536, + "loss": 1.5216, + "theoretical_loss": 3.9189101031294635, + "tokens_seen": 530055168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782748244734203, + "loss": 1.4668, + "theoretical_loss": 3.918858817306667, + "tokens_seen": 530120704 + }, + { + "epoch": 0.16, + "learning_rate": 0.000678258776328987, + "loss": 1.4352, + "theoretical_loss": 3.918807539598676, + "tokens_seen": 530186240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782427281845537, + "loss": 1.4745, + "theoretical_loss": 3.918756270003203, + "tokens_seen": 530251776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782266800401204, + "loss": 1.5607, + "theoretical_loss": 3.9187050085179633, + "tokens_seen": 530317312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006782106318956871, + "loss": 1.6128, + "theoretical_loss": 3.918653755140671, + "tokens_seen": 530382848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781945837512538, + "loss": 1.425, + "theoretical_loss": 3.918602509869042, + "tokens_seen": 530448384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781785356068205, + "loss": 1.5345, + "theoretical_loss": 3.9185512727007934, + "tokens_seen": 530513920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781624874623872, + "loss": 1.5292, + "theoretical_loss": 3.9185000436336432, + "tokens_seen": 530579456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781464393179539, + "loss": 1.5637, + "theoretical_loss": 3.9184488226653102, + "tokens_seen": 530644992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781303911735206, + "loss": 1.6626, + "theoretical_loss": 3.918397609793514, + "tokens_seen": 530710528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006781143430290873, + "loss": 1.573, + "theoretical_loss": 3.9183464050159746, + "tokens_seen": 530776064 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 202413, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5111300945281982, + "objective/train/theoretical_loss": 3.918295208330415, + "objective/train/tokens_used": 551301600, + "theoretical_loss": 3.918295208330415, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.000678098294884654, + "loss": 1.5111, + "theoretical_loss": 3.918295208330415, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780822467402207, + "loss": 1.4422, + "theoretical_loss": 3.918244019734556, + "tokens_seen": 530907136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780661985957874, + "loss": 1.5778, + "theoretical_loss": 3.9181928392261214, + "tokens_seen": 530972672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780501504513541, + "loss": 1.564, + "theoretical_loss": 3.9181416668028355, + "tokens_seen": 531038208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780341023069208, + "loss": 1.4484, + "theoretical_loss": 3.918090502462425, + "tokens_seen": 531103744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780180541624875, + "loss": 1.5584, + "theoretical_loss": 3.9180393462026135, + "tokens_seen": 531169280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006780020060180543, + "loss": 1.5352, + "theoretical_loss": 3.9179881980211295, + "tokens_seen": 531234816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779859578736209, + "loss": 1.466, + "theoretical_loss": 3.917937057915701, + "tokens_seen": 531300352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779699097291877, + "loss": 1.5966, + "theoretical_loss": 3.9178859258840566, + "tokens_seen": 531365888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779538615847543, + "loss": 1.4532, + "theoretical_loss": 3.9178348019239264, + "tokens_seen": 531431424 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677937813440321, + "loss": 1.5094, + "theoretical_loss": 3.91778368603304, + "tokens_seen": 531496960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779217652958877, + "loss": 1.4795, + "theoretical_loss": 3.9177325782091303, + "tokens_seen": 531562496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006779057171514545, + "loss": 1.5152, + "theoretical_loss": 3.9176814784499303, + "tokens_seen": 531628032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778896690070211, + "loss": 1.4845, + "theoretical_loss": 3.9176303867531717, + "tokens_seen": 531693568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778736208625879, + "loss": 1.4352, + "theoretical_loss": 3.9175793031165895, + "tokens_seen": 531759104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778575727181545, + "loss": 1.4657, + "theoretical_loss": 3.91752822753792, + "tokens_seen": 531824640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778415245737213, + "loss": 1.5559, + "theoretical_loss": 3.917477160014898, + "tokens_seen": 531890176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778254764292879, + "loss": 1.4674, + "theoretical_loss": 3.9174261005452617, + "tokens_seen": 531955712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006778094282848547, + "loss": 1.5045, + "theoretical_loss": 3.917375049126749, + "tokens_seen": 532021248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777933801404213, + "loss": 1.4319, + "theoretical_loss": 3.9173240057570986, + "tokens_seen": 532086784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777773319959881, + "loss": 1.5927, + "theoretical_loss": 3.91727297043405, + "tokens_seen": 532152320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777612838515547, + "loss": 1.4918, + "theoretical_loss": 3.917221943155345, + "tokens_seen": 532217856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777452357071214, + "loss": 1.3895, + "theoretical_loss": 3.9171709239187247, + "tokens_seen": 532283392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777291875626881, + "loss": 1.5161, + "theoretical_loss": 3.9171199127219314, + "tokens_seen": 532348928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006777131394182548, + "loss": 1.3735, + "theoretical_loss": 3.917068909562709, + "tokens_seen": 532414464 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776970912738215, + "loss": 1.4222, + "theoretical_loss": 3.917017914438802, + "tokens_seen": 532480000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776810431293882, + "loss": 1.5352, + "theoretical_loss": 3.9169669273479553, + "tokens_seen": 532545536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776649949849549, + "loss": 1.5269, + "theoretical_loss": 3.916915948287916, + "tokens_seen": 532611072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776489468405216, + "loss": 1.4565, + "theoretical_loss": 3.91686497725643, + "tokens_seen": 532676608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776328986960883, + "loss": 1.5272, + "theoretical_loss": 3.916814014251247, + "tokens_seen": 532742144 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677616850551655, + "loss": 1.4956, + "theoretical_loss": 3.9167630592701146, + "tokens_seen": 532807680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006776008024072216, + "loss": 1.5153, + "theoretical_loss": 3.9167121123107833, + "tokens_seen": 532873216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775847542627884, + "loss": 1.4216, + "theoretical_loss": 3.9166611733710037, + "tokens_seen": 532938752 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677568706118355, + "loss": 1.563, + "theoretical_loss": 3.9166102424485274, + "tokens_seen": 533004288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775526579739218, + "loss": 1.429, + "theoretical_loss": 3.9165593195411077, + "tokens_seen": 533069824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775366098294884, + "loss": 1.5391, + "theoretical_loss": 3.916508404646497, + "tokens_seen": 533135360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775205616850552, + "loss": 1.6115, + "theoretical_loss": 3.9164574977624507, + "tokens_seen": 533200896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006775045135406218, + "loss": 1.6296, + "theoretical_loss": 3.916406598886724, + "tokens_seen": 533266432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774884653961886, + "loss": 1.5436, + "theoretical_loss": 3.916355708017073, + "tokens_seen": 533331968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774724172517553, + "loss": 1.5042, + "theoretical_loss": 3.916304825151254, + "tokens_seen": 533397504 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677456369107322, + "loss": 1.492, + "theoretical_loss": 3.916253950287026, + "tokens_seen": 533463040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774403209628887, + "loss": 1.5206, + "theoretical_loss": 3.916203083422148, + "tokens_seen": 533528576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774242728184554, + "loss": 1.4789, + "theoretical_loss": 3.91615222455438, + "tokens_seen": 533594112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006774082246740221, + "loss": 1.622, + "theoretical_loss": 3.9161013736814816, + "tokens_seen": 533659648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773921765295888, + "loss": 1.5202, + "theoretical_loss": 3.916050530801215, + "tokens_seen": 533725184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773761283851555, + "loss": 1.5806, + "theoretical_loss": 3.915999695911343, + "tokens_seen": 533790720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773600802407222, + "loss": 1.6198, + "theoretical_loss": 3.91594886900963, + "tokens_seen": 533856256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773440320962889, + "loss": 1.5917, + "theoretical_loss": 3.9158980500938383, + "tokens_seen": 533921792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773279839518556, + "loss": 1.5048, + "theoretical_loss": 3.9158472391617343, + "tokens_seen": 533987328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006773119358074223, + "loss": 1.4942, + "theoretical_loss": 3.9157964362110844, + "tokens_seen": 534052864 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677295887662989, + "loss": 1.4901, + "theoretical_loss": 3.9157456412396554, + "tokens_seen": 534118400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772798395185557, + "loss": 1.5294, + "theoretical_loss": 3.9156948542452144, + "tokens_seen": 534183936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772637913741224, + "loss": 1.5677, + "theoretical_loss": 3.915644075225532, + "tokens_seen": 534249472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772477432296891, + "loss": 1.6026, + "theoretical_loss": 3.915593304178376, + "tokens_seen": 534315008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772316950852558, + "loss": 1.5584, + "theoretical_loss": 3.9155425411015186, + "tokens_seen": 534380544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006772156469408225, + "loss": 1.5201, + "theoretical_loss": 3.9154917859927307, + "tokens_seen": 534446080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771995987963892, + "loss": 1.5186, + "theoretical_loss": 3.9154410388497842, + "tokens_seen": 534511616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771835506519559, + "loss": 1.4542, + "theoretical_loss": 3.9153902996704533, + "tokens_seen": 534577152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771675025075226, + "loss": 1.5519, + "theoretical_loss": 3.915339568452512, + "tokens_seen": 534642688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771514543630893, + "loss": 1.6118, + "theoretical_loss": 3.9152888451937358, + "tokens_seen": 534708224 + }, + { + "epoch": 0.16, + "learning_rate": 0.000677135406218656, + "loss": 1.5252, + "theoretical_loss": 3.9152381298919, + "tokens_seen": 534773760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771193580742227, + "loss": 1.4937, + "theoretical_loss": 3.915187422544782, + "tokens_seen": 534839296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006771033099297894, + "loss": 1.3919, + "theoretical_loss": 3.915136723150159, + "tokens_seen": 534904832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770872617853561, + "loss": 1.549, + "theoretical_loss": 3.91508603170581, + "tokens_seen": 534970368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770712136409228, + "loss": 1.6436, + "theoretical_loss": 3.915035348209514, + "tokens_seen": 535035904 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770551654964895, + "loss": 1.5406, + "theoretical_loss": 3.9149846726590534, + "tokens_seen": 535101440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770391173520562, + "loss": 1.5235, + "theoretical_loss": 3.914934005052208, + "tokens_seen": 535166976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770230692076229, + "loss": 1.4897, + "theoretical_loss": 3.91488334538676, + "tokens_seen": 535232512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006770070210631896, + "loss": 1.4307, + "theoretical_loss": 3.914832693660493, + "tokens_seen": 535298048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769909729187563, + "loss": 1.5801, + "theoretical_loss": 3.9147820498711914, + "tokens_seen": 535363584 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676974924774323, + "loss": 1.5116, + "theoretical_loss": 3.91473141401664, + "tokens_seen": 535429120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769588766298897, + "loss": 1.423, + "theoretical_loss": 3.9146807860946238, + "tokens_seen": 535494656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769428284854564, + "loss": 1.4808, + "theoretical_loss": 3.91463016610293, + "tokens_seen": 535560192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769267803410231, + "loss": 1.5459, + "theoretical_loss": 3.9145795540393467, + "tokens_seen": 535625728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006769107321965898, + "loss": 1.4402, + "theoretical_loss": 3.9145289499016616, + "tokens_seen": 535691264 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768946840521565, + "loss": 1.5455, + "theoretical_loss": 3.9144783536876644, + "tokens_seen": 535756800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768786359077232, + "loss": 1.5371, + "theoretical_loss": 3.9144277653951454, + "tokens_seen": 535822336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768625877632899, + "loss": 1.4001, + "theoretical_loss": 3.914377185021896, + "tokens_seen": 535887872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768465396188566, + "loss": 1.6264, + "theoretical_loss": 3.914326612565708, + "tokens_seen": 535953408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006768304914744233, + "loss": 1.5014, + "theoretical_loss": 3.914276048024374, + "tokens_seen": 536018944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00067681444332999, + "loss": 1.4535, + "theoretical_loss": 3.9142254913956886, + "tokens_seen": 536084480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767983951855567, + "loss": 1.5008, + "theoretical_loss": 3.914174942677446, + "tokens_seen": 536150016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767823470411234, + "loss": 1.5274, + "theoretical_loss": 3.9141244018674413, + "tokens_seen": 536215552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767662988966901, + "loss": 1.4813, + "theoretical_loss": 3.9140738689634715, + "tokens_seen": 536281088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767502507522568, + "loss": 1.5395, + "theoretical_loss": 3.914023343963334, + "tokens_seen": 536346624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767342026078235, + "loss": 1.4882, + "theoretical_loss": 3.9139728268648266, + "tokens_seen": 536412160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006767181544633902, + "loss": 1.496, + "theoretical_loss": 3.913922317665749, + "tokens_seen": 536477696 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676702106318957, + "loss": 1.4456, + "theoretical_loss": 3.9138718163639012, + "tokens_seen": 536543232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766860581745236, + "loss": 1.5603, + "theoretical_loss": 3.9138213229570837, + "tokens_seen": 536608768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766700100300903, + "loss": 1.6238, + "theoretical_loss": 3.9137708374430984, + "tokens_seen": 536674304 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676653961885657, + "loss": 1.6375, + "theoretical_loss": 3.913720359819748, + "tokens_seen": 536739840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766379137412238, + "loss": 1.4108, + "theoretical_loss": 3.913669890084835, + "tokens_seen": 536805376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766218655967904, + "loss": 1.4363, + "theoretical_loss": 3.9136194282361654, + "tokens_seen": 536870912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006766058174523572, + "loss": 1.4601, + "theoretical_loss": 3.913568974271543, + "tokens_seen": 536936448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765897693079238, + "loss": 1.4873, + "theoretical_loss": 3.913518528188775, + "tokens_seen": 537001984 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765737211634906, + "loss": 1.5317, + "theoretical_loss": 3.9134680899856686, + "tokens_seen": 537067520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765576730190572, + "loss": 1.5006, + "theoretical_loss": 3.913417659660031, + "tokens_seen": 537133056 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676541624874624, + "loss": 1.6038, + "theoretical_loss": 3.913367237209671, + "tokens_seen": 537198592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765255767301906, + "loss": 1.53, + "theoretical_loss": 3.9133168226323987, + "tokens_seen": 537264128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006765095285857574, + "loss": 1.4758, + "theoretical_loss": 3.913266415926024, + "tokens_seen": 537329664 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 204661, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5203614234924316, + "objective/train/theoretical_loss": 3.9132160170883585, + "objective/train/tokens_used": 557855200, + "theoretical_loss": 3.9132160170883585, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676493480441324, + "loss": 1.5204, + "theoretical_loss": 3.9132160170883585, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764774322968908, + "loss": 1.5095, + "theoretical_loss": 3.913165626117215, + "tokens_seen": 537460736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764613841524574, + "loss": 1.4767, + "theoretical_loss": 3.913115243010407, + "tokens_seen": 537526272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764453360080242, + "loss": 1.4376, + "theoretical_loss": 3.9130648677657476, + "tokens_seen": 537591808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764292878635908, + "loss": 1.557, + "theoretical_loss": 3.9130145003810517, + "tokens_seen": 537657344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006764132397191576, + "loss": 1.5623, + "theoretical_loss": 3.912964140854135, + "tokens_seen": 537722880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763971915747242, + "loss": 1.5442, + "theoretical_loss": 3.9129137891828147, + "tokens_seen": 537788416 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676381143430291, + "loss": 1.6309, + "theoretical_loss": 3.9128634453649083, + "tokens_seen": 537853952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763650952858576, + "loss": 1.3966, + "theoretical_loss": 3.9128131093982343, + "tokens_seen": 537919488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763490471414244, + "loss": 1.5928, + "theoretical_loss": 3.9127627812806116, + "tokens_seen": 537985024 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676332998996991, + "loss": 1.4628, + "theoretical_loss": 3.91271246100986, + "tokens_seen": 538050560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763169508525577, + "loss": 1.4975, + "theoretical_loss": 3.9126621485838013, + "tokens_seen": 538116096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006763009027081244, + "loss": 1.4841, + "theoretical_loss": 3.9126118440002573, + "tokens_seen": 538181632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762848545636911, + "loss": 1.5224, + "theoretical_loss": 3.9125615472570505, + "tokens_seen": 538247168 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762688064192578, + "loss": 1.4918, + "theoretical_loss": 3.912511258352004, + "tokens_seen": 538312704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762527582748245, + "loss": 1.4796, + "theoretical_loss": 3.9124609772829437, + "tokens_seen": 538378240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762367101303912, + "loss": 1.4707, + "theoretical_loss": 3.9124107040476934, + "tokens_seen": 538443776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762206619859579, + "loss": 1.5656, + "theoretical_loss": 3.91236043864408, + "tokens_seen": 538509312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006762046138415246, + "loss": 1.5513, + "theoretical_loss": 3.912310181069931, + "tokens_seen": 538574848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761885656970913, + "loss": 1.5237, + "theoretical_loss": 3.9122599313230735, + "tokens_seen": 538640384 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676172517552658, + "loss": 1.5069, + "theoretical_loss": 3.9122096894013376, + "tokens_seen": 538705920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761564694082247, + "loss": 1.4998, + "theoretical_loss": 3.9121594553025516, + "tokens_seen": 538771456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761404212637914, + "loss": 1.5388, + "theoretical_loss": 3.912109229024547, + "tokens_seen": 538836992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761243731193581, + "loss": 1.4984, + "theoretical_loss": 3.9120590105651547, + "tokens_seen": 538902528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006761083249749248, + "loss": 1.4839, + "theoretical_loss": 3.9120087999222073, + "tokens_seen": 538968064 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760922768304915, + "loss": 1.5011, + "theoretical_loss": 3.9119585970935375, + "tokens_seen": 539033600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760762286860582, + "loss": 1.5182, + "theoretical_loss": 3.91190840207698, + "tokens_seen": 539099136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760601805416249, + "loss": 1.4834, + "theoretical_loss": 3.9118582148703696, + "tokens_seen": 539164672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760441323971916, + "loss": 1.4267, + "theoretical_loss": 3.9118080354715414, + "tokens_seen": 539230208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006760280842527583, + "loss": 1.5204, + "theoretical_loss": 3.911757863878333, + "tokens_seen": 539295744 + }, + { + "epoch": 0.16, + "learning_rate": 0.000676012036108325, + "loss": 1.3502, + "theoretical_loss": 3.9117077000885807, + "tokens_seen": 539361280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759959879638917, + "loss": 1.5773, + "theoretical_loss": 3.911657544100124, + "tokens_seen": 539426816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759799398194584, + "loss": 1.5165, + "theoretical_loss": 3.911607395910801, + "tokens_seen": 539492352 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759638916750251, + "loss": 1.6352, + "theoretical_loss": 3.911557255518453, + "tokens_seen": 539557888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759478435305918, + "loss": 1.492, + "theoretical_loss": 3.9115071229209195, + "tokens_seen": 539623424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759317953861585, + "loss": 1.4737, + "theoretical_loss": 3.9114569981160434, + "tokens_seen": 539688960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006759157472417252, + "loss": 1.4367, + "theoretical_loss": 3.9114068811016662, + "tokens_seen": 539754496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758996990972919, + "loss": 1.5374, + "theoretical_loss": 3.9113567718756332, + "tokens_seen": 539820032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758836509528586, + "loss": 1.5316, + "theoretical_loss": 3.9113066704357866, + "tokens_seen": 539885568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758676028084253, + "loss": 1.5462, + "theoretical_loss": 3.9112565767799734, + "tokens_seen": 539951104 + }, + { + "epoch": 0.16, + "learning_rate": 0.000675851554663992, + "loss": 1.5831, + "theoretical_loss": 3.9112064909060393, + "tokens_seen": 540016640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758355065195587, + "loss": 1.5283, + "theoretical_loss": 3.91115641281183, + "tokens_seen": 540082176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758194583751254, + "loss": 1.487, + "theoretical_loss": 3.911106342495195, + "tokens_seen": 540147712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006758034102306921, + "loss": 1.4563, + "theoretical_loss": 3.911056279953981, + "tokens_seen": 540213248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757873620862588, + "loss": 1.5454, + "theoretical_loss": 3.91100622518604, + "tokens_seen": 540278784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757713139418255, + "loss": 1.4832, + "theoretical_loss": 3.9109561781892204, + "tokens_seen": 540344320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757552657973922, + "loss": 1.5616, + "theoretical_loss": 3.9109061389613737, + "tokens_seen": 540409856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757392176529589, + "loss": 1.5055, + "theoretical_loss": 3.9108561075003525, + "tokens_seen": 540475392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757231695085256, + "loss": 1.5407, + "theoretical_loss": 3.9108060838040095, + "tokens_seen": 540540928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006757071213640923, + "loss": 1.4842, + "theoretical_loss": 3.9107560678701985, + "tokens_seen": 540606464 + }, + { + "epoch": 0.16, + "learning_rate": 0.000675691073219659, + "loss": 1.543, + "theoretical_loss": 3.910706059696774, + "tokens_seen": 540672000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756750250752257, + "loss": 1.4772, + "theoretical_loss": 3.910656059281591, + "tokens_seen": 540737536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756589769307924, + "loss": 1.4419, + "theoretical_loss": 3.910606066622507, + "tokens_seen": 540803072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756429287863591, + "loss": 1.6733, + "theoretical_loss": 3.9105560817173783, + "tokens_seen": 540868608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756268806419258, + "loss": 1.4726, + "theoretical_loss": 3.910506104564063, + "tokens_seen": 540934144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006756108324974925, + "loss": 1.5917, + "theoretical_loss": 3.9104561351604206, + "tokens_seen": 540999680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755947843530592, + "loss": 1.5411, + "theoretical_loss": 3.9104061735043105, + "tokens_seen": 541065216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755787362086259, + "loss": 1.5808, + "theoretical_loss": 3.9103562195935933, + "tokens_seen": 541130752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755626880641926, + "loss": 1.5569, + "theoretical_loss": 3.91030627342613, + "tokens_seen": 541196288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755466399197593, + "loss": 1.4738, + "theoretical_loss": 3.910256334999783, + "tokens_seen": 541261824 + }, + { + "epoch": 0.16, + "learning_rate": 0.000675530591775326, + "loss": 1.4502, + "theoretical_loss": 3.910206404312416, + "tokens_seen": 541327360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006755145436308927, + "loss": 1.4764, + "theoretical_loss": 3.9101564813618923, + "tokens_seen": 541392896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754984954864594, + "loss": 1.461, + "theoretical_loss": 3.9101065661460774, + "tokens_seen": 541458432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754824473420261, + "loss": 1.46, + "theoretical_loss": 3.9100566586628362, + "tokens_seen": 541523968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754663991975928, + "loss": 1.5397, + "theoretical_loss": 3.9100067589100362, + "tokens_seen": 541589504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754503510531595, + "loss": 1.6247, + "theoretical_loss": 3.909956866885544, + "tokens_seen": 541655040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754343029087262, + "loss": 1.4387, + "theoretical_loss": 3.909906982587228, + "tokens_seen": 541720576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754182547642929, + "loss": 1.4676, + "theoretical_loss": 3.909857106012957, + "tokens_seen": 541786112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006754022066198596, + "loss": 1.4333, + "theoretical_loss": 3.9098072371606016, + "tokens_seen": 541851648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753861584754263, + "loss": 1.4643, + "theoretical_loss": 3.9097573760280326, + "tokens_seen": 541917184 + }, + { + "epoch": 0.16, + "learning_rate": 0.000675370110330993, + "loss": 1.4891, + "theoretical_loss": 3.90970752261312, + "tokens_seen": 541982720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753540621865597, + "loss": 1.5667, + "theoretical_loss": 3.9096576769137386, + "tokens_seen": 542048256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753380140421264, + "loss": 1.5635, + "theoretical_loss": 3.90960783892776, + "tokens_seen": 542113792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753219658976931, + "loss": 1.4218, + "theoretical_loss": 3.9095580086530592, + "tokens_seen": 542179328 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006753059177532598, + "loss": 1.5102, + "theoretical_loss": 3.90950818608751, + "tokens_seen": 542244864 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752898696088265, + "loss": 1.4862, + "theoretical_loss": 3.9094583712289896, + "tokens_seen": 542310400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752738214643933, + "loss": 1.4944, + "theoretical_loss": 3.909408564075374, + "tokens_seen": 542375936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752577733199599, + "loss": 1.5736, + "theoretical_loss": 3.9093587646245407, + "tokens_seen": 542441472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752417251755267, + "loss": 1.4892, + "theoretical_loss": 3.9093089728743684, + "tokens_seen": 542507008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006752256770310933, + "loss": 1.5309, + "theoretical_loss": 3.9092591888227357, + "tokens_seen": 542572544 + }, + { + "epoch": 0.16, + "learning_rate": 0.00067520962888666, + "loss": 1.5004, + "theoretical_loss": 3.909209412467523, + "tokens_seen": 542638080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751935807422267, + "loss": 1.4304, + "theoretical_loss": 3.9091596438066114, + "tokens_seen": 542703616 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751775325977935, + "loss": 1.565, + "theoretical_loss": 3.909109882837882, + "tokens_seen": 542769152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751614844533601, + "loss": 1.5355, + "theoretical_loss": 3.9090601295592178, + "tokens_seen": 542834688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751454363089269, + "loss": 1.5311, + "theoretical_loss": 3.9090103839685013, + "tokens_seen": 542900224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751293881644935, + "loss": 1.4342, + "theoretical_loss": 3.908960646063618, + "tokens_seen": 542965760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006751133400200603, + "loss": 1.4602, + "theoretical_loss": 3.908910915842452, + "tokens_seen": 543031296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750972918756269, + "loss": 1.5068, + "theoretical_loss": 3.90886119330289, + "tokens_seen": 543096832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750812437311937, + "loss": 1.6493, + "theoretical_loss": 3.908811478442818, + "tokens_seen": 543162368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750651955867603, + "loss": 1.6192, + "theoretical_loss": 3.908761771260124, + "tokens_seen": 543227904 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750491474423271, + "loss": 1.4698, + "theoretical_loss": 3.908712071752696, + "tokens_seen": 543293440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750330992978937, + "loss": 1.5035, + "theoretical_loss": 3.908662379918424, + "tokens_seen": 543358976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750170511534605, + "loss": 1.6014, + "theoretical_loss": 3.908612695755197, + "tokens_seen": 543424512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006750010030090271, + "loss": 1.4909, + "theoretical_loss": 3.9085630192609067, + "tokens_seen": 543490048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749849548645939, + "loss": 1.5095, + "theoretical_loss": 3.908513350433444, + "tokens_seen": 543555584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749689067201605, + "loss": 1.5749, + "theoretical_loss": 3.908463689270703, + "tokens_seen": 543621120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749528585757273, + "loss": 1.5513, + "theoretical_loss": 3.9084140357705754, + "tokens_seen": 543686656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749368104312939, + "loss": 1.5023, + "theoretical_loss": 3.9083643899309566, + "tokens_seen": 543752192 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749207622868606, + "loss": 1.6167, + "theoretical_loss": 3.9083147517497414, + "tokens_seen": 543817728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006749047141424273, + "loss": 1.4988, + "theoretical_loss": 3.908265121224825, + "tokens_seen": 543883264 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 207102, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5208964347839355, + "objective/train/theoretical_loss": 3.908215498354105, + "objective/train/tokens_used": 564408800, + "theoretical_loss": 3.908215498354105, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.000674888665997994, + "loss": 1.5209, + "theoretical_loss": 3.908215498354105, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748726178535607, + "loss": 1.6534, + "theoretical_loss": 3.9081658831354784, + "tokens_seen": 544014336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748565697091274, + "loss": 1.5042, + "theoretical_loss": 3.908116275566844, + "tokens_seen": 544079872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748405215646941, + "loss": 1.501, + "theoretical_loss": 3.9080666756461007, + "tokens_seen": 544145408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748244734202608, + "loss": 1.4516, + "theoretical_loss": 3.908017083371149, + "tokens_seen": 544210944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006748084252758275, + "loss": 1.5368, + "theoretical_loss": 3.9079674987398896, + "tokens_seen": 544276480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006747923771313942, + "loss": 1.4498, + "theoretical_loss": 3.907917921750224, + "tokens_seen": 544342016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006747763289869609, + "loss": 1.5689, + "theoretical_loss": 3.9078683524000546, + "tokens_seen": 544407552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0006747602808425276, + "loss": 1.507, + "theoretical_loss": 3.907818790687285, + "tokens_seen": 544473088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006747442326980943, + "loss": 1.4899, + "theoretical_loss": 3.90776923660982, + "tokens_seen": 544538624 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674728184553661, + "loss": 1.4558, + "theoretical_loss": 3.907719690165564, + "tokens_seen": 544604160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006747121364092277, + "loss": 1.4594, + "theoretical_loss": 3.9076701513524226, + "tokens_seen": 544669696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746960882647944, + "loss": 1.5122, + "theoretical_loss": 3.9076206201683026, + "tokens_seen": 544735232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746800401203611, + "loss": 1.5333, + "theoretical_loss": 3.907571096611112, + "tokens_seen": 544800768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746639919759278, + "loss": 1.5176, + "theoretical_loss": 3.9075215806787584, + "tokens_seen": 544866304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746479438314945, + "loss": 1.4808, + "theoretical_loss": 3.907472072369152, + "tokens_seen": 544931840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746318956870612, + "loss": 1.5216, + "theoretical_loss": 3.907422571680202, + "tokens_seen": 544997376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006746158475426279, + "loss": 1.6039, + "theoretical_loss": 3.907373078609819, + "tokens_seen": 545062912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745997993981946, + "loss": 1.524, + "theoretical_loss": 3.9073235931559154, + "tokens_seen": 545128448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745837512537613, + "loss": 1.5329, + "theoretical_loss": 3.907274115316403, + "tokens_seen": 545193984 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674567703109328, + "loss": 1.4872, + "theoretical_loss": 3.907224645089195, + "tokens_seen": 545259520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745516549648947, + "loss": 1.4412, + "theoretical_loss": 3.907175182472206, + "tokens_seen": 545325056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745356068204614, + "loss": 1.4947, + "theoretical_loss": 3.9071257274633506, + "tokens_seen": 545390592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745195586760281, + "loss": 1.5135, + "theoretical_loss": 3.9070762800605454, + "tokens_seen": 545456128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006745035105315948, + "loss": 1.5662, + "theoretical_loss": 3.9070268402617057, + "tokens_seen": 545521664 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744874623871615, + "loss": 1.5566, + "theoretical_loss": 3.9069774080647495, + "tokens_seen": 545587200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744714142427282, + "loss": 1.5257, + "theoretical_loss": 3.9069279834675945, + "tokens_seen": 545652736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744553660982949, + "loss": 1.4452, + "theoretical_loss": 3.9068785664681602, + "tokens_seen": 545718272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744393179538616, + "loss": 1.4574, + "theoretical_loss": 3.906829157064367, + "tokens_seen": 545783808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006744232698094283, + "loss": 1.5707, + "theoretical_loss": 3.906779755254134, + "tokens_seen": 545849344 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674407221664995, + "loss": 1.5503, + "theoretical_loss": 3.9067303610353847, + "tokens_seen": 545914880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743911735205617, + "loss": 1.3868, + "theoretical_loss": 3.9066809744060396, + "tokens_seen": 545980416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743751253761284, + "loss": 1.4874, + "theoretical_loss": 3.9066315953640225, + "tokens_seen": 546045952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743590772316951, + "loss": 1.5051, + "theoretical_loss": 3.9065822239072574, + "tokens_seen": 546111488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743430290872618, + "loss": 1.5636, + "theoretical_loss": 3.9065328600336695, + "tokens_seen": 546177024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743269809428285, + "loss": 1.5163, + "theoretical_loss": 3.9064835037411836, + "tokens_seen": 546242560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006743109327983952, + "loss": 1.5592, + "theoretical_loss": 3.9064341550277266, + "tokens_seen": 546308096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742948846539619, + "loss": 1.5235, + "theoretical_loss": 3.9063848138912247, + "tokens_seen": 546373632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742788365095286, + "loss": 1.4624, + "theoretical_loss": 3.9063354803296075, + "tokens_seen": 546439168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742627883650953, + "loss": 1.5937, + "theoretical_loss": 3.906286154340802, + "tokens_seen": 546504704 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674246740220662, + "loss": 1.5089, + "theoretical_loss": 3.9062368359227397, + "tokens_seen": 546570240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742306920762287, + "loss": 1.5635, + "theoretical_loss": 3.90618752507335, + "tokens_seen": 546635776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006742146439317954, + "loss": 1.6191, + "theoretical_loss": 3.906138221790565, + "tokens_seen": 546701312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741985957873621, + "loss": 1.5807, + "theoretical_loss": 3.9060889260723153, + "tokens_seen": 546766848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741825476429288, + "loss": 1.4968, + "theoretical_loss": 3.906039637916535, + "tokens_seen": 546832384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741664994984955, + "loss": 1.4997, + "theoretical_loss": 3.9059903573211576, + "tokens_seen": 546897920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741504513540622, + "loss": 1.5509, + "theoretical_loss": 3.905941084284117, + "tokens_seen": 546963456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741344032096289, + "loss": 1.5276, + "theoretical_loss": 3.9058918188033496, + "tokens_seen": 547028992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741183550651956, + "loss": 1.4481, + "theoretical_loss": 3.905842560876791, + "tokens_seen": 547094528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006741023069207623, + "loss": 1.3893, + "theoretical_loss": 3.9057933105023777, + "tokens_seen": 547160064 + }, + { + "epoch": 0.17, + "learning_rate": 0.000674086258776329, + "loss": 1.4598, + "theoretical_loss": 3.9057440676780484, + "tokens_seen": 547225600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740702106318957, + "loss": 1.5256, + "theoretical_loss": 3.905694832401741, + "tokens_seen": 547291136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740541624874624, + "loss": 1.5249, + "theoretical_loss": 3.905645604671395, + "tokens_seen": 547356672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740381143430291, + "loss": 1.6073, + "theoretical_loss": 3.9055963844849506, + "tokens_seen": 547422208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740220661985958, + "loss": 1.545, + "theoretical_loss": 3.9055471718403494, + "tokens_seen": 547487744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006740060180541625, + "loss": 1.4752, + "theoretical_loss": 3.905497966735532, + "tokens_seen": 547553280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006739899699097292, + "loss": 1.4941, + "theoretical_loss": 3.9054487691684425, + "tokens_seen": 547618816 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673973921765296, + "loss": 1.3962, + "theoretical_loss": 3.905399579137023, + "tokens_seen": 547684352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006739578736208626, + "loss": 1.4772, + "theoretical_loss": 3.9053503966392187, + "tokens_seen": 547749888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006739418254764293, + "loss": 1.563, + "theoretical_loss": 3.905301221672974, + "tokens_seen": 547815424 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673925777331996, + "loss": 1.4325, + "theoretical_loss": 3.9052520542362346, + "tokens_seen": 547880960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006739097291875627, + "loss": 1.4141, + "theoretical_loss": 3.9052028943269477, + "tokens_seen": 547946496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738936810431294, + "loss": 1.5465, + "theoretical_loss": 3.905153741943061, + "tokens_seen": 548012032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738776328986962, + "loss": 1.4706, + "theoretical_loss": 3.905104597082522, + "tokens_seen": 548077568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738615847542629, + "loss": 1.484, + "theoretical_loss": 3.9050554597432807, + "tokens_seen": 548143104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738455366098296, + "loss": 1.5229, + "theoretical_loss": 3.905006329923286, + "tokens_seen": 548208640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006738294884653963, + "loss": 1.4753, + "theoretical_loss": 3.904957207620489, + "tokens_seen": 548274176 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673813440320963, + "loss": 1.573, + "theoretical_loss": 3.904908092832841, + "tokens_seen": 548339712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737973921765297, + "loss": 1.5113, + "theoretical_loss": 3.904858985558295, + "tokens_seen": 548405248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737813440320964, + "loss": 1.4792, + "theoretical_loss": 3.904809885794803, + "tokens_seen": 548470784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737652958876631, + "loss": 1.5143, + "theoretical_loss": 3.9047607935403197, + "tokens_seen": 548536320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737492477432298, + "loss": 1.4158, + "theoretical_loss": 3.9047117087927994, + "tokens_seen": 548601856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737331995987965, + "loss": 1.6346, + "theoretical_loss": 3.904662631550198, + "tokens_seen": 548667392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737171514543632, + "loss": 1.5707, + "theoretical_loss": 3.904613561810471, + "tokens_seen": 548732928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006737011033099299, + "loss": 1.5596, + "theoretical_loss": 3.9045644995715767, + "tokens_seen": 548798464 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006736850551654966, + "loss": 1.5147, + "theoretical_loss": 3.9045154448314716, + "tokens_seen": 548864000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006736690070210632, + "loss": 1.4624, + "theoretical_loss": 3.9044663975881155, + "tokens_seen": 548929536 + }, + { + "epoch": 0.17, + "learning_rate": 0.00067365295887663, + "loss": 1.5437, + "theoretical_loss": 3.9044173578394674, + "tokens_seen": 548995072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006736369107321966, + "loss": 1.4338, + "theoretical_loss": 3.904368325583488, + "tokens_seen": 549060608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006736208625877634, + "loss": 1.4624, + "theoretical_loss": 3.9043193008181376, + "tokens_seen": 549126144 + }, + { + "epoch": 0.17, + "learning_rate": 0.00067360481444333, + "loss": 1.5098, + "theoretical_loss": 3.9042702835413787, + "tokens_seen": 549191680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735887662988968, + "loss": 1.5239, + "theoretical_loss": 3.9042212737511743, + "tokens_seen": 549257216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735727181544634, + "loss": 1.4748, + "theoretical_loss": 3.904172271445487, + "tokens_seen": 549322752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735566700100302, + "loss": 1.4697, + "theoretical_loss": 3.9041232766222813, + "tokens_seen": 549388288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735406218655968, + "loss": 1.539, + "theoretical_loss": 3.9040742892795226, + "tokens_seen": 549453824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735245737211635, + "loss": 1.5083, + "theoretical_loss": 3.9040253094151773, + "tokens_seen": 549519360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006735085255767302, + "loss": 1.4952, + "theoretical_loss": 3.9039763370272107, + "tokens_seen": 549584896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734924774322969, + "loss": 1.4248, + "theoretical_loss": 3.903927372113592, + "tokens_seen": 549650432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734764292878636, + "loss": 1.5336, + "theoretical_loss": 3.9038784146722874, + "tokens_seen": 549715968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734603811434303, + "loss": 1.5178, + "theoretical_loss": 3.9038294647012672, + "tokens_seen": 549781504 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673444332998997, + "loss": 1.4301, + "theoretical_loss": 3.9037805221985016, + "tokens_seen": 549847040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734282848545637, + "loss": 1.5305, + "theoretical_loss": 3.90373158716196, + "tokens_seen": 549912576 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006734122367101304, + "loss": 1.6015, + "theoretical_loss": 3.903682659589615, + "tokens_seen": 549978112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733961885656971, + "loss": 1.467, + "theoretical_loss": 3.903633739479438, + "tokens_seen": 550043648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733801404212638, + "loss": 1.4949, + "theoretical_loss": 3.9035848268294027, + "tokens_seen": 550109184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733640922768305, + "loss": 1.4436, + "theoretical_loss": 3.9035359216374825, + "tokens_seen": 550174720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733480441323972, + "loss": 1.3601, + "theoretical_loss": 3.9034870239016524, + "tokens_seen": 550240256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733319959879639, + "loss": 1.3849, + "theoretical_loss": 3.903438133619887, + "tokens_seen": 550305792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006733159478435306, + "loss": 1.5425, + "theoretical_loss": 3.9033892507901635, + "tokens_seen": 550371328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732998996990973, + "loss": 1.4299, + "theoretical_loss": 3.9033403754104583, + "tokens_seen": 550436864 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 209442, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5209815502166748, + "objective/train/theoretical_loss": 3.9032915074787486, + "objective/train/tokens_used": 570962400, + "theoretical_loss": 3.9032915074787486, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673283851554664, + "loss": 1.521, + "theoretical_loss": 3.9032915074787486, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732678034102307, + "loss": 1.5382, + "theoretical_loss": 3.903242646993014, + "tokens_seen": 550567936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732517552657974, + "loss": 1.4649, + "theoretical_loss": 3.9031937939512336, + "tokens_seen": 550633472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732357071213641, + "loss": 1.5026, + "theoretical_loss": 3.903144948351387, + "tokens_seen": 550699008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732196589769308, + "loss": 1.5113, + "theoretical_loss": 3.9030961101914556, + "tokens_seen": 550764544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006732036108324975, + "loss": 1.4883, + "theoretical_loss": 3.903047279469421, + "tokens_seen": 550830080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731875626880642, + "loss": 1.5859, + "theoretical_loss": 3.9029984561832656, + "tokens_seen": 550895616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731715145436309, + "loss": 1.6847, + "theoretical_loss": 3.9029496403309727, + "tokens_seen": 550961152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731554663991976, + "loss": 1.3976, + "theoretical_loss": 3.902900831910527, + "tokens_seen": 551026688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731394182547643, + "loss": 1.5753, + "theoretical_loss": 3.902852030919912, + "tokens_seen": 551092224 + }, + { + "epoch": 0.17, + "learning_rate": 0.000673123370110331, + "loss": 1.5123, + "theoretical_loss": 3.902803237357115, + "tokens_seen": 551157760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006731073219658977, + "loss": 1.3881, + "theoretical_loss": 3.9027544512201207, + "tokens_seen": 551223296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730912738214644, + "loss": 1.5475, + "theoretical_loss": 3.902705672506918, + "tokens_seen": 551288832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730752256770311, + "loss": 1.5725, + "theoretical_loss": 3.9026569012154932, + "tokens_seen": 551354368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730591775325978, + "loss": 1.4083, + "theoretical_loss": 3.9026081373438366, + "tokens_seen": 551419904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730431293881645, + "loss": 1.5051, + "theoretical_loss": 3.9025593808899366, + "tokens_seen": 551485440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730270812437312, + "loss": 1.4575, + "theoretical_loss": 3.9025106318517846, + "tokens_seen": 551550976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006730110330992979, + "loss": 1.5961, + "theoretical_loss": 3.902461890227371, + "tokens_seen": 551616512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729949849548646, + "loss": 1.5791, + "theoretical_loss": 3.902413156014688, + "tokens_seen": 551682048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729789368104313, + "loss": 1.5104, + "theoretical_loss": 3.902364429211728, + "tokens_seen": 551747584 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672962888665998, + "loss": 1.5209, + "theoretical_loss": 3.9023157098164853, + "tokens_seen": 551813120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729468405215647, + "loss": 1.5507, + "theoretical_loss": 3.902266997826953, + "tokens_seen": 551878656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729307923771314, + "loss": 1.4593, + "theoretical_loss": 3.902218293241127, + "tokens_seen": 551944192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006729147442326981, + "loss": 1.4701, + "theoretical_loss": 3.9021695960570026, + "tokens_seen": 552009728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728986960882648, + "loss": 1.5732, + "theoretical_loss": 3.902120906272577, + "tokens_seen": 552075264 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728826479438315, + "loss": 1.4265, + "theoretical_loss": 3.9020722238858467, + "tokens_seen": 552140800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728665997993982, + "loss": 1.4971, + "theoretical_loss": 3.902023548894811, + "tokens_seen": 552206336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728505516549649, + "loss": 1.5611, + "theoretical_loss": 3.901974881297468, + "tokens_seen": 552271872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728345035105316, + "loss": 1.4395, + "theoretical_loss": 3.9019262210918173, + "tokens_seen": 552337408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006728184553660983, + "loss": 1.5643, + "theoretical_loss": 3.90187756827586, + "tokens_seen": 552402944 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672802407221665, + "loss": 1.4878, + "theoretical_loss": 3.9018289228475975, + "tokens_seen": 552468480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727863590772317, + "loss": 1.515, + "theoretical_loss": 3.9017802848050307, + "tokens_seen": 552534016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727703109327984, + "loss": 1.4369, + "theoretical_loss": 3.9017316541461637, + "tokens_seen": 552599552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727542627883651, + "loss": 1.4524, + "theoretical_loss": 3.9016830308689996, + "tokens_seen": 552665088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727382146439318, + "loss": 1.4585, + "theoretical_loss": 3.901634414971543, + "tokens_seen": 552730624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727221664994985, + "loss": 1.5475, + "theoretical_loss": 3.901585806451799, + "tokens_seen": 552796160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006727061183550652, + "loss": 1.4747, + "theoretical_loss": 3.901537205307773, + "tokens_seen": 552861696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726900702106319, + "loss": 1.3849, + "theoretical_loss": 3.9014886115374723, + "tokens_seen": 552927232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726740220661986, + "loss": 1.5381, + "theoretical_loss": 3.9014400251389043, + "tokens_seen": 552992768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726579739217654, + "loss": 1.6513, + "theoretical_loss": 3.901391446110077, + "tokens_seen": 553058304 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672641925777332, + "loss": 1.4904, + "theoretical_loss": 3.901342874449, + "tokens_seen": 553123840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726258776328988, + "loss": 1.477, + "theoretical_loss": 3.9012943101536823, + "tokens_seen": 553189376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006726098294884654, + "loss": 1.4874, + "theoretical_loss": 3.901245753222135, + "tokens_seen": 553254912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725937813440322, + "loss": 1.524, + "theoretical_loss": 3.90119720365237, + "tokens_seen": 553320448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725777331995988, + "loss": 1.5648, + "theoretical_loss": 3.9011486614423982, + "tokens_seen": 553385984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725616850551656, + "loss": 1.5664, + "theoretical_loss": 3.901100126590233, + "tokens_seen": 553451520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725456369107322, + "loss": 1.4986, + "theoretical_loss": 3.901051599093889, + "tokens_seen": 553517056 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672529588766299, + "loss": 1.5321, + "theoretical_loss": 3.901003078951379, + "tokens_seen": 553582592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006725135406218657, + "loss": 1.5565, + "theoretical_loss": 3.9009545661607192, + "tokens_seen": 553648128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724974924774324, + "loss": 1.5124, + "theoretical_loss": 3.900906060719925, + "tokens_seen": 553713664 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672481444332999, + "loss": 1.5717, + "theoretical_loss": 3.9008575626270146, + "tokens_seen": 553779200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724653961885658, + "loss": 1.5561, + "theoretical_loss": 3.900809071880004, + "tokens_seen": 553844736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724493480441325, + "loss": 1.5441, + "theoretical_loss": 3.900760588476912, + "tokens_seen": 553910272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724332998996992, + "loss": 1.5754, + "theoretical_loss": 3.900712112415758, + "tokens_seen": 553975808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724172517552659, + "loss": 1.4774, + "theoretical_loss": 3.900663643694561, + "tokens_seen": 554041344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006724012036108326, + "loss": 1.6281, + "theoretical_loss": 3.900615182311342, + "tokens_seen": 554106880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723851554663993, + "loss": 1.5086, + "theoretical_loss": 3.900566728264123, + "tokens_seen": 554172416 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672369107321966, + "loss": 1.5087, + "theoretical_loss": 3.9005182815509256, + "tokens_seen": 554237952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723530591775327, + "loss": 1.4745, + "theoretical_loss": 3.9004698421697723, + "tokens_seen": 554303488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723370110330994, + "loss": 1.5214, + "theoretical_loss": 3.900421410118688, + "tokens_seen": 554369024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723209628886661, + "loss": 1.4856, + "theoretical_loss": 3.9003729853956957, + "tokens_seen": 554434560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006723049147442328, + "loss": 1.4927, + "theoretical_loss": 3.9003245679988217, + "tokens_seen": 554500096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722888665997995, + "loss": 1.4622, + "theoretical_loss": 3.9002761579260907, + "tokens_seen": 554565632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722728184553661, + "loss": 1.5582, + "theoretical_loss": 3.9002277551755307, + "tokens_seen": 554631168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722567703109329, + "loss": 1.567, + "theoretical_loss": 3.9001793597451684, + "tokens_seen": 554696704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722407221664995, + "loss": 1.2988, + "theoretical_loss": 3.900130971633033, + "tokens_seen": 554762240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722246740220663, + "loss": 1.3807, + "theoretical_loss": 3.9000825908371524, + "tokens_seen": 554827776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006722086258776329, + "loss": 1.5109, + "theoretical_loss": 3.9000342173555573, + "tokens_seen": 554893312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721925777331997, + "loss": 1.4393, + "theoretical_loss": 3.899985851186277, + "tokens_seen": 554958848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721765295887663, + "loss": 1.5393, + "theoretical_loss": 3.8999374923273447, + "tokens_seen": 555024384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721604814443331, + "loss": 1.4382, + "theoretical_loss": 3.899889140776791, + "tokens_seen": 555089920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721444332998997, + "loss": 1.4461, + "theoretical_loss": 3.8998407965326494, + "tokens_seen": 555155456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721283851554664, + "loss": 1.3795, + "theoretical_loss": 3.899792459592953, + "tokens_seen": 555220992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006721123370110331, + "loss": 1.5029, + "theoretical_loss": 3.899744129955737, + "tokens_seen": 555286528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720962888665998, + "loss": 1.4414, + "theoretical_loss": 3.899695807619035, + "tokens_seen": 555352064 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720802407221665, + "loss": 1.4379, + "theoretical_loss": 3.899647492580885, + "tokens_seen": 555417600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720641925777332, + "loss": 1.4131, + "theoretical_loss": 3.899599184839322, + "tokens_seen": 555483136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720481444332999, + "loss": 1.4832, + "theoretical_loss": 3.899550884392384, + "tokens_seen": 555548672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720320962888666, + "loss": 1.4517, + "theoretical_loss": 3.8995025912381087, + "tokens_seen": 555614208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006720160481444333, + "loss": 1.5545, + "theoretical_loss": 3.899454305374536, + "tokens_seen": 555679744 + }, + { + "epoch": 0.17, + "learning_rate": 0.000672, + "loss": 1.6385, + "theoretical_loss": 3.899406026799704, + "tokens_seen": 555745280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719839518555667, + "loss": 1.4285, + "theoretical_loss": 3.899357755511655, + "tokens_seen": 555810816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719679037111334, + "loss": 1.4987, + "theoretical_loss": 3.899309491508429, + "tokens_seen": 555876352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719518555667001, + "loss": 1.5927, + "theoretical_loss": 3.899261234788068, + "tokens_seen": 555941888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719358074222668, + "loss": 1.398, + "theoretical_loss": 3.8992129853486155, + "tokens_seen": 556007424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719197592778335, + "loss": 1.4179, + "theoretical_loss": 3.8991647431881145, + "tokens_seen": 556072960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006719037111334002, + "loss": 1.4673, + "theoretical_loss": 3.899116508304609, + "tokens_seen": 556138496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718876629889669, + "loss": 1.5934, + "theoretical_loss": 3.899068280696144, + "tokens_seen": 556204032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718716148445336, + "loss": 1.4927, + "theoretical_loss": 3.8990200603607654, + "tokens_seen": 556269568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718555667001003, + "loss": 1.5694, + "theoretical_loss": 3.8989718472965196, + "tokens_seen": 556335104 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671839518555667, + "loss": 1.5221, + "theoretical_loss": 3.898923641501454, + "tokens_seen": 556400640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718234704112337, + "loss": 1.5108, + "theoretical_loss": 3.898875442973617, + "tokens_seen": 556466176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006718074222668004, + "loss": 1.5048, + "theoretical_loss": 3.8988272517110563, + "tokens_seen": 556531712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717913741223671, + "loss": 1.4982, + "theoretical_loss": 3.898779067711822, + "tokens_seen": 556597248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717753259779338, + "loss": 1.5628, + "theoretical_loss": 3.8987308909739644, + "tokens_seen": 556662784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717592778335005, + "loss": 1.4499, + "theoretical_loss": 3.898682721495535, + "tokens_seen": 556728320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717432296890672, + "loss": 1.5246, + "theoretical_loss": 3.8986345592745844, + "tokens_seen": 556793856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717271815446339, + "loss": 1.4517, + "theoretical_loss": 3.898586404309166, + "tokens_seen": 556859392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006717111334002006, + "loss": 1.2756, + "theoretical_loss": 3.8985382565973326, + "tokens_seen": 556924928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716950852557673, + "loss": 1.5299, + "theoretical_loss": 3.898490116137139, + "tokens_seen": 556990464 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 211950, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5476175546646118, + "objective/train/theoretical_loss": 3.898441982926639, + "objective/train/tokens_used": 577516000, + "theoretical_loss": 3.898441982926639, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671679037111334, + "loss": 1.5476, + "theoretical_loss": 3.898441982926639, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716629889669007, + "loss": 1.4725, + "theoretical_loss": 3.8983938569638883, + "tokens_seen": 557121536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716469408224674, + "loss": 1.5025, + "theoretical_loss": 3.8983457382469435, + "tokens_seen": 557187072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716308926780341, + "loss": 1.5505, + "theoretical_loss": 3.8982976267738616, + "tokens_seen": 557252608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006716148445336008, + "loss": 1.4456, + "theoretical_loss": 3.8982495225427005, + "tokens_seen": 557318144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715987963891675, + "loss": 1.515, + "theoretical_loss": 3.8982014255515183, + "tokens_seen": 557383680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715827482447342, + "loss": 1.4586, + "theoretical_loss": 3.8981533357983746, + "tokens_seen": 557449216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715667001003009, + "loss": 1.4789, + "theoretical_loss": 3.8981052532813294, + "tokens_seen": 557514752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715506519558676, + "loss": 1.4863, + "theoretical_loss": 3.898057177998443, + "tokens_seen": 557580288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715346038114343, + "loss": 1.5423, + "theoretical_loss": 3.898009109947777, + "tokens_seen": 557645824 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671518555667001, + "loss": 1.5739, + "theoretical_loss": 3.897961049127395, + "tokens_seen": 557711360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006715025075225677, + "loss": 1.5276, + "theoretical_loss": 3.8979129955353584, + "tokens_seen": 557776896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714864593781344, + "loss": 1.6173, + "theoretical_loss": 3.8978649491697315, + "tokens_seen": 557842432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714704112337011, + "loss": 1.4682, + "theoretical_loss": 3.8978169100285784, + "tokens_seen": 557907968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714543630892679, + "loss": 1.4947, + "theoretical_loss": 3.8977688781099653, + "tokens_seen": 557973504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714383149448345, + "loss": 1.4742, + "theoretical_loss": 3.8977208534119576, + "tokens_seen": 558039040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714222668004013, + "loss": 1.4515, + "theoretical_loss": 3.8976728359326223, + "tokens_seen": 558104576 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006714062186559679, + "loss": 1.5246, + "theoretical_loss": 3.897624825670026, + "tokens_seen": 558170112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713901705115347, + "loss": 1.59, + "theoretical_loss": 3.8975768226222383, + "tokens_seen": 558235648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713741223671013, + "loss": 1.4687, + "theoretical_loss": 3.897528826787327, + "tokens_seen": 558301184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713580742226681, + "loss": 1.4938, + "theoretical_loss": 3.8974808381633625, + "tokens_seen": 558366720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713420260782347, + "loss": 1.4449, + "theoretical_loss": 3.8974328567484147, + "tokens_seen": 558432256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713259779338015, + "loss": 1.536, + "theoretical_loss": 3.8973848825405555, + "tokens_seen": 558497792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006713099297893681, + "loss": 1.4851, + "theoretical_loss": 3.8973369155378563, + "tokens_seen": 558563328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712938816449349, + "loss": 1.4965, + "theoretical_loss": 3.89728895573839, + "tokens_seen": 558628864 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712778335005015, + "loss": 1.6116, + "theoretical_loss": 3.8972410031402296, + "tokens_seen": 558694400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712617853560683, + "loss": 1.5205, + "theoretical_loss": 3.8971930577414495, + "tokens_seen": 558759936 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671245737211635, + "loss": 1.4946, + "theoretical_loss": 3.8971451195401254, + "tokens_seen": 558825472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712296890672017, + "loss": 1.552, + "theoretical_loss": 3.897097188534332, + "tokens_seen": 558891008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006712136409227683, + "loss": 1.5321, + "theoretical_loss": 3.8970492647221464, + "tokens_seen": 558956544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711975927783351, + "loss": 1.4608, + "theoretical_loss": 3.897001348101645, + "tokens_seen": 559022080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711815446339017, + "loss": 1.5873, + "theoretical_loss": 3.8969534386709053, + "tokens_seen": 559087616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711654964894685, + "loss": 1.5346, + "theoretical_loss": 3.896905536428007, + "tokens_seen": 559153152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711494483450352, + "loss": 1.4718, + "theoretical_loss": 3.8968576413710294, + "tokens_seen": 559218688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711334002006019, + "loss": 1.4614, + "theoretical_loss": 3.8968097534980517, + "tokens_seen": 559284224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711173520561686, + "loss": 1.555, + "theoretical_loss": 3.8967618728071556, + "tokens_seen": 559349760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006711013039117353, + "loss": 1.5364, + "theoretical_loss": 3.896713999296422, + "tokens_seen": 559415296 + }, + { + "epoch": 0.17, + "learning_rate": 0.000671085255767302, + "loss": 1.4636, + "theoretical_loss": 3.8966661329639334, + "tokens_seen": 559480832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710692076228687, + "loss": 1.6023, + "theoretical_loss": 3.896618273807773, + "tokens_seen": 559546368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710531594784354, + "loss": 1.528, + "theoretical_loss": 3.8965704218260244, + "tokens_seen": 559611904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710371113340021, + "loss": 1.5075, + "theoretical_loss": 3.896522577016772, + "tokens_seen": 559677440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710210631895688, + "loss": 1.4652, + "theoretical_loss": 3.896474739378102, + "tokens_seen": 559742976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006710050150451355, + "loss": 1.5473, + "theoretical_loss": 3.896426908908099, + "tokens_seen": 559808512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709889669007022, + "loss": 1.5271, + "theoretical_loss": 3.89637908560485, + "tokens_seen": 559874048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709729187562689, + "loss": 1.4425, + "theoretical_loss": 3.896331269466443, + "tokens_seen": 559939584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709568706118356, + "loss": 1.4605, + "theoretical_loss": 3.8962834604909657, + "tokens_seen": 560005120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709408224674023, + "loss": 1.5405, + "theoretical_loss": 3.8962356586765075, + "tokens_seen": 560070656 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670924774322969, + "loss": 1.4699, + "theoretical_loss": 3.896187864021157, + "tokens_seen": 560136192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006709087261785357, + "loss": 1.5171, + "theoretical_loss": 3.8961400765230065, + "tokens_seen": 560201728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708926780341024, + "loss": 1.512, + "theoretical_loss": 3.8960922961801447, + "tokens_seen": 560267264 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708766298896691, + "loss": 1.4642, + "theoretical_loss": 3.8960445229906657, + "tokens_seen": 560332800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708605817452358, + "loss": 1.3898, + "theoretical_loss": 3.895996756952661, + "tokens_seen": 560398336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708445336008024, + "loss": 1.5515, + "theoretical_loss": 3.895948998064223, + "tokens_seen": 560463872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708284854563692, + "loss": 1.4898, + "theoretical_loss": 3.8959012463234473, + "tokens_seen": 560529408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006708124373119358, + "loss": 1.4294, + "theoretical_loss": 3.8958535017284284, + "tokens_seen": 560594944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707963891675026, + "loss": 1.4789, + "theoretical_loss": 3.895805764277261, + "tokens_seen": 560660480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707803410230692, + "loss": 1.5495, + "theoretical_loss": 3.895758033968042, + "tokens_seen": 560726016 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670764292878636, + "loss": 1.5561, + "theoretical_loss": 3.895710310798868, + "tokens_seen": 560791552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707482447342026, + "loss": 1.5342, + "theoretical_loss": 3.895662594767837, + "tokens_seen": 560857088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707321965897693, + "loss": 1.4923, + "theoretical_loss": 3.895614885873047, + "tokens_seen": 560922624 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670716148445336, + "loss": 1.5322, + "theoretical_loss": 3.895567184112598, + "tokens_seen": 560988160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006707001003009027, + "loss": 1.5897, + "theoretical_loss": 3.8955194894845886, + "tokens_seen": 561053696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706840521564694, + "loss": 1.4856, + "theoretical_loss": 3.89547180198712, + "tokens_seen": 561119232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706680040120361, + "loss": 1.489, + "theoretical_loss": 3.8954241216182943, + "tokens_seen": 561184768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706519558676028, + "loss": 1.5728, + "theoretical_loss": 3.8953764483762123, + "tokens_seen": 561250304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706359077231695, + "loss": 1.4674, + "theoretical_loss": 3.8953287822589777, + "tokens_seen": 561315840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706198595787362, + "loss": 1.5919, + "theoretical_loss": 3.8952811232646933, + "tokens_seen": 561381376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006706038114343029, + "loss": 1.4547, + "theoretical_loss": 3.895233471391464, + "tokens_seen": 561446912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705877632898696, + "loss": 1.5291, + "theoretical_loss": 3.895185826637394, + "tokens_seen": 561512448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705717151454363, + "loss": 1.6235, + "theoretical_loss": 3.89513818900059, + "tokens_seen": 561577984 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670555667001003, + "loss": 1.5601, + "theoretical_loss": 3.8950905584791577, + "tokens_seen": 561643520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705396188565697, + "loss": 1.4593, + "theoretical_loss": 3.8950429350712037, + "tokens_seen": 561709056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705235707121364, + "loss": 1.5017, + "theoretical_loss": 3.8949953187748374, + "tokens_seen": 561774592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006705075225677031, + "loss": 1.4517, + "theoretical_loss": 3.894947709588166, + "tokens_seen": 561840128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704914744232698, + "loss": 1.6024, + "theoretical_loss": 3.8949001075093, + "tokens_seen": 561905664 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704754262788365, + "loss": 1.5709, + "theoretical_loss": 3.8948525125363487, + "tokens_seen": 561971200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704593781344032, + "loss": 1.5276, + "theoretical_loss": 3.8948049246674223, + "tokens_seen": 562036736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704433299899699, + "loss": 1.4933, + "theoretical_loss": 3.894757343900633, + "tokens_seen": 562102272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704272818455366, + "loss": 1.4626, + "theoretical_loss": 3.894709770234093, + "tokens_seen": 562167808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006704112337011033, + "loss": 1.4766, + "theoretical_loss": 3.8946622036659155, + "tokens_seen": 562233344 + }, + { + "epoch": 0.17, + "learning_rate": 0.00067039518555667, + "loss": 1.4908, + "theoretical_loss": 3.8946146441942138, + "tokens_seen": 562298880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703791374122367, + "loss": 1.5405, + "theoretical_loss": 3.894567091817102, + "tokens_seen": 562364416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703630892678034, + "loss": 1.4678, + "theoretical_loss": 3.8945195465326954, + "tokens_seen": 562429952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703470411233701, + "loss": 1.4728, + "theoretical_loss": 3.8944720083391093, + "tokens_seen": 562495488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703309929789368, + "loss": 1.4172, + "theoretical_loss": 3.8944244772344616, + "tokens_seen": 562561024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006703149448345035, + "loss": 1.5571, + "theoretical_loss": 3.894376953216868, + "tokens_seen": 562626560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702988966900702, + "loss": 1.4354, + "theoretical_loss": 3.8943294362844476, + "tokens_seen": 562692096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702828485456369, + "loss": 1.4727, + "theoretical_loss": 3.8942819264353186, + "tokens_seen": 562757632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702668004012036, + "loss": 1.5132, + "theoretical_loss": 3.8942344236676005, + "tokens_seen": 562823168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702507522567704, + "loss": 1.5421, + "theoretical_loss": 3.894186927979413, + "tokens_seen": 562888704 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670234704112337, + "loss": 1.6154, + "theoretical_loss": 3.8941394393688777, + "tokens_seen": 562954240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702186559679038, + "loss": 1.4854, + "theoretical_loss": 3.8940919578341155, + "tokens_seen": 563019776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006702026078234704, + "loss": 1.5619, + "theoretical_loss": 3.894044483373249, + "tokens_seen": 563085312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701865596790372, + "loss": 1.5895, + "theoretical_loss": 3.893997015984401, + "tokens_seen": 563150848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701705115346038, + "loss": 1.4367, + "theoretical_loss": 3.893949555665696, + "tokens_seen": 563216384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701544633901706, + "loss": 1.5503, + "theoretical_loss": 3.893902102415257, + "tokens_seen": 563281920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701384152457372, + "loss": 1.4232, + "theoretical_loss": 3.89385465623121, + "tokens_seen": 563347456 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670122367101304, + "loss": 1.5053, + "theoretical_loss": 3.8938072171116804, + "tokens_seen": 563412992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006701063189568706, + "loss": 1.3943, + "theoretical_loss": 3.893759785054796, + "tokens_seen": 563478528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700902708124374, + "loss": 1.6669, + "theoretical_loss": 3.8937123600586823, + "tokens_seen": 563544064 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 214661, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4646552801132202, + "objective/train/theoretical_loss": 3.8936649421214686, + "objective/train/tokens_used": 584069600, + "theoretical_loss": 3.8936649421214686, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.000670074222668004, + "loss": 1.4647, + "theoretical_loss": 3.8936649421214686, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700581745235708, + "loss": 1.4413, + "theoretical_loss": 3.8936175312412833, + "tokens_seen": 563675136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700421263791374, + "loss": 1.5119, + "theoretical_loss": 3.8935701274162553, + "tokens_seen": 563740672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700260782347042, + "loss": 1.3737, + "theoretical_loss": 3.8935227306445155, + "tokens_seen": 563806208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006700100300902708, + "loss": 1.5451, + "theoretical_loss": 3.8934753409241942, + "tokens_seen": 563871744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006699939819458376, + "loss": 1.4915, + "theoretical_loss": 3.8934279582534232, + "tokens_seen": 563937280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006699779338014042, + "loss": 1.4854, + "theoretical_loss": 3.8933805826303347, + "tokens_seen": 564002816 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669961885656971, + "loss": 1.4023, + "theoretical_loss": 3.893333214053062, + "tokens_seen": 564068352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006699458375125376, + "loss": 1.5233, + "theoretical_loss": 3.8932858525197385, + "tokens_seen": 564133888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006699297893681044, + "loss": 1.5345, + "theoretical_loss": 3.8932384980284986, + "tokens_seen": 564199424 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669913741223671, + "loss": 1.5521, + "theoretical_loss": 3.8931911505774766, + "tokens_seen": 564264960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698976930792378, + "loss": 1.4262, + "theoretical_loss": 3.89314381016481, + "tokens_seen": 564330496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698816449348044, + "loss": 1.5299, + "theoretical_loss": 3.8930964767886347, + "tokens_seen": 564396032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698655967903712, + "loss": 1.4541, + "theoretical_loss": 3.893049150447087, + "tokens_seen": 564461568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698495486459378, + "loss": 1.4656, + "theoretical_loss": 3.8930018311383066, + "tokens_seen": 564527104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698335005015046, + "loss": 1.4944, + "theoretical_loss": 3.8929545188604306, + "tokens_seen": 564592640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006698174523570712, + "loss": 1.5333, + "theoretical_loss": 3.8929072136115987, + "tokens_seen": 564658176 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669801404212638, + "loss": 1.5386, + "theoretical_loss": 3.8928599153899515, + "tokens_seen": 564723712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697853560682047, + "loss": 1.4559, + "theoretical_loss": 3.8928126241936294, + "tokens_seen": 564789248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697693079237714, + "loss": 1.4816, + "theoretical_loss": 3.8927653400207745, + "tokens_seen": 564854784 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669753259779338, + "loss": 1.6081, + "theoretical_loss": 3.892718062869528, + "tokens_seen": 564920320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697372116349048, + "loss": 1.484, + "theoretical_loss": 3.8926707927380337, + "tokens_seen": 564985856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697211634904715, + "loss": 1.4528, + "theoretical_loss": 3.892623529624435, + "tokens_seen": 565051392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006697051153460382, + "loss": 1.453, + "theoretical_loss": 3.892576273526876, + "tokens_seen": 565116928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696890672016049, + "loss": 1.524, + "theoretical_loss": 3.892529024443502, + "tokens_seen": 565182464 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696730190571716, + "loss": 1.3675, + "theoretical_loss": 3.892481782372458, + "tokens_seen": 565248000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696569709127383, + "loss": 1.5243, + "theoretical_loss": 3.8924345473118915, + "tokens_seen": 565313536 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669640922768305, + "loss": 1.4052, + "theoretical_loss": 3.892387319259949, + "tokens_seen": 565379072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696248746238717, + "loss": 1.549, + "theoretical_loss": 3.8923400982147784, + "tokens_seen": 565444608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006696088264794384, + "loss": 1.509, + "theoretical_loss": 3.8922928841745286, + "tokens_seen": 565510144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695927783350051, + "loss": 1.4921, + "theoretical_loss": 3.892245677137349, + "tokens_seen": 565575680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695767301905718, + "loss": 1.4968, + "theoretical_loss": 3.8921984771013887, + "tokens_seen": 565641216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695606820461385, + "loss": 1.3302, + "theoretical_loss": 3.8921512840647985, + "tokens_seen": 565706752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695446339017052, + "loss": 1.5419, + "theoretical_loss": 3.892104098025731, + "tokens_seen": 565772288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695285857572719, + "loss": 1.4738, + "theoretical_loss": 3.892056918982337, + "tokens_seen": 565837824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006695125376128386, + "loss": 1.469, + "theoretical_loss": 3.8920097469327697, + "tokens_seen": 565903360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694964894684053, + "loss": 1.4974, + "theoretical_loss": 3.8919625818751826, + "tokens_seen": 565968896 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669480441323972, + "loss": 1.4961, + "theoretical_loss": 3.8919154238077294, + "tokens_seen": 566034432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694643931795387, + "loss": 1.4899, + "theoretical_loss": 3.8918682727285665, + "tokens_seen": 566099968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694483450351053, + "loss": 1.4481, + "theoretical_loss": 3.8918211286358475, + "tokens_seen": 566165504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694322968906721, + "loss": 1.4872, + "theoretical_loss": 3.89177399152773, + "tokens_seen": 566231040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694162487462387, + "loss": 1.3595, + "theoretical_loss": 3.89172686140237, + "tokens_seen": 566296576 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006694002006018055, + "loss": 1.5408, + "theoretical_loss": 3.8916797382579262, + "tokens_seen": 566362112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693841524573721, + "loss": 1.5383, + "theoretical_loss": 3.8916326220925566, + "tokens_seen": 566427648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693681043129389, + "loss": 1.5041, + "theoretical_loss": 3.8915855129044195, + "tokens_seen": 566493184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693520561685055, + "loss": 1.4612, + "theoretical_loss": 3.891538410691676, + "tokens_seen": 566558720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693360080240723, + "loss": 1.4447, + "theoretical_loss": 3.8914913154524857, + "tokens_seen": 566624256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693199598796389, + "loss": 1.4394, + "theoretical_loss": 3.89144422718501, + "tokens_seen": 566689792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006693039117352056, + "loss": 1.431, + "theoretical_loss": 3.891397145887411, + "tokens_seen": 566755328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692878635907723, + "loss": 1.4896, + "theoretical_loss": 3.8913500715578504, + "tokens_seen": 566820864 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669271815446339, + "loss": 1.538, + "theoretical_loss": 3.8913030041944925, + "tokens_seen": 566886400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692557673019057, + "loss": 1.5032, + "theoretical_loss": 3.891255943795501, + "tokens_seen": 566951936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692397191574724, + "loss": 1.4688, + "theoretical_loss": 3.8912088903590405, + "tokens_seen": 567017472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692236710130391, + "loss": 1.484, + "theoretical_loss": 3.8911618438832756, + "tokens_seen": 567083008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006692076228686058, + "loss": 1.4003, + "theoretical_loss": 3.8911148043663735, + "tokens_seen": 567148544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691915747241725, + "loss": 1.5101, + "theoretical_loss": 3.8910677718065, + "tokens_seen": 567214080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691755265797392, + "loss": 1.5042, + "theoretical_loss": 3.8910207462018236, + "tokens_seen": 567279616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691594784353059, + "loss": 1.4769, + "theoretical_loss": 3.890973727550511, + "tokens_seen": 567345152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691434302908726, + "loss": 1.5047, + "theoretical_loss": 3.890926715850733, + "tokens_seen": 567410688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006691273821464393, + "loss": 1.4226, + "theoretical_loss": 3.890879711100657, + "tokens_seen": 567476224 + }, + { + "epoch": 0.17, + "learning_rate": 0.000669111334002006, + "loss": 1.5244, + "theoretical_loss": 3.8908327132984546, + "tokens_seen": 567541760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690952858575727, + "loss": 1.5131, + "theoretical_loss": 3.890785722442296, + "tokens_seen": 567607296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690792377131394, + "loss": 1.4516, + "theoretical_loss": 3.890738738530353, + "tokens_seen": 567672832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690631895687061, + "loss": 1.5164, + "theoretical_loss": 3.890691761560798, + "tokens_seen": 567738368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690471414242729, + "loss": 1.5196, + "theoretical_loss": 3.8906447915318036, + "tokens_seen": 567803904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690310932798395, + "loss": 1.4555, + "theoretical_loss": 3.8905978284415443, + "tokens_seen": 567869440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006690150451354063, + "loss": 1.5196, + "theoretical_loss": 3.890550872288194, + "tokens_seen": 567934976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689989969909729, + "loss": 1.4135, + "theoretical_loss": 3.890503923069927, + "tokens_seen": 568000512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689829488465397, + "loss": 1.6256, + "theoretical_loss": 3.8904569807849203, + "tokens_seen": 568066048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689669007021063, + "loss": 1.4741, + "theoretical_loss": 3.8904100454313495, + "tokens_seen": 568131584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689508525576731, + "loss": 1.5054, + "theoretical_loss": 3.890363117007392, + "tokens_seen": 568197120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689348044132397, + "loss": 1.4886, + "theoretical_loss": 3.890316195511226, + "tokens_seen": 568262656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689187562688065, + "loss": 1.5245, + "theoretical_loss": 3.8902692809410295, + "tokens_seen": 568328192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006689027081243731, + "loss": 1.4749, + "theoretical_loss": 3.8902223732949817, + "tokens_seen": 568393728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688866599799399, + "loss": 1.5976, + "theoretical_loss": 3.8901754725712627, + "tokens_seen": 568459264 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688706118355065, + "loss": 1.5261, + "theoretical_loss": 3.890128578768053, + "tokens_seen": 568524800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688545636910733, + "loss": 1.4555, + "theoretical_loss": 3.8900816918835335, + "tokens_seen": 568590336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688385155466399, + "loss": 1.5039, + "theoretical_loss": 3.8900348119158865, + "tokens_seen": 568655872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688224674022067, + "loss": 1.5373, + "theoretical_loss": 3.8899879388632947, + "tokens_seen": 568721408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006688064192577733, + "loss": 1.5355, + "theoretical_loss": 3.889941072723942, + "tokens_seen": 568786944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687903711133401, + "loss": 1.4875, + "theoretical_loss": 3.8898942134960106, + "tokens_seen": 568852480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687743229689067, + "loss": 1.4132, + "theoretical_loss": 3.8898473611776865, + "tokens_seen": 568918016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687582748244735, + "loss": 1.571, + "theoretical_loss": 3.8898005157671554, + "tokens_seen": 568983552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687422266800401, + "loss": 1.4483, + "theoretical_loss": 3.889753677262603, + "tokens_seen": 569049088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687261785356069, + "loss": 1.5358, + "theoretical_loss": 3.8897068456622153, + "tokens_seen": 569114624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006687101303911735, + "loss": 1.5359, + "theoretical_loss": 3.8896600209641807, + "tokens_seen": 569180160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686940822467403, + "loss": 1.4719, + "theoretical_loss": 3.889613203166687, + "tokens_seen": 569245696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686780341023069, + "loss": 1.4899, + "theoretical_loss": 3.8895663922679224, + "tokens_seen": 569311232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686619859578737, + "loss": 1.5, + "theoretical_loss": 3.8895195882660776, + "tokens_seen": 569376768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686459378134403, + "loss": 1.4583, + "theoretical_loss": 3.8894727911593416, + "tokens_seen": 569442304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686298896690071, + "loss": 1.3883, + "theoretical_loss": 3.889426000945906, + "tokens_seen": 569507840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006686138415245737, + "loss": 1.4802, + "theoretical_loss": 3.889379217623962, + "tokens_seen": 569573376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685977933801405, + "loss": 1.5759, + "theoretical_loss": 3.8893324411917014, + "tokens_seen": 569638912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685817452357071, + "loss": 1.4144, + "theoretical_loss": 3.8892856716473183, + "tokens_seen": 569704448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685656970912739, + "loss": 1.4532, + "theoretical_loss": 3.8892389089890056, + "tokens_seen": 569769984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685496489468405, + "loss": 1.4487, + "theoretical_loss": 3.8891921532149576, + "tokens_seen": 569835520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685336008024073, + "loss": 1.4613, + "theoretical_loss": 3.8891454043233686, + "tokens_seen": 569901056 + }, + { + "epoch": 0.17, + "learning_rate": 0.000668517552657974, + "loss": 1.4924, + "theoretical_loss": 3.8890986623124353, + "tokens_seen": 569966592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006685015045135407, + "loss": 1.4606, + "theoretical_loss": 3.889051927180353, + "tokens_seen": 570032128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684854563691073, + "loss": 1.5533, + "theoretical_loss": 3.88900519892532, + "tokens_seen": 570097664 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 217201, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4288733005523682, + "objective/train/theoretical_loss": 3.8889584775455326, + "objective/train/tokens_used": 590623200, + "theoretical_loss": 3.8889584775455326, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684694082246741, + "loss": 1.4289, + "theoretical_loss": 3.8889584775455326, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684533600802407, + "loss": 1.5173, + "theoretical_loss": 3.8889117630391894, + "tokens_seen": 570228736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684373119358075, + "loss": 1.3628, + "theoretical_loss": 3.888865055404491, + "tokens_seen": 570294272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684212637913742, + "loss": 1.4813, + "theoretical_loss": 3.8888183546396347, + "tokens_seen": 570359808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006684052156469409, + "loss": 1.5031, + "theoretical_loss": 3.8887716607428224, + "tokens_seen": 570425344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683891675025076, + "loss": 1.4494, + "theoretical_loss": 3.8887249737122547, + "tokens_seen": 570490880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683731193580743, + "loss": 1.4542, + "theoretical_loss": 3.8886782935461333, + "tokens_seen": 570556416 + }, + { + "epoch": 0.17, + "learning_rate": 0.000668357071213641, + "loss": 1.5168, + "theoretical_loss": 3.888631620242661, + "tokens_seen": 570621952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683410230692077, + "loss": 1.4891, + "theoretical_loss": 3.8885849538000397, + "tokens_seen": 570687488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683249749247744, + "loss": 1.4326, + "theoretical_loss": 3.8885382942164752, + "tokens_seen": 570753024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006683089267803411, + "loss": 1.4861, + "theoretical_loss": 3.8884916414901705, + "tokens_seen": 570818560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682928786359078, + "loss": 1.4437, + "theoretical_loss": 3.8884449956193308, + "tokens_seen": 570884096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682768304914745, + "loss": 1.4803, + "theoretical_loss": 3.8883983566021625, + "tokens_seen": 570949632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682607823470412, + "loss": 1.4795, + "theoretical_loss": 3.8883517244368715, + "tokens_seen": 571015168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682447342026079, + "loss": 1.5267, + "theoretical_loss": 3.8883050991216654, + "tokens_seen": 571080704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682286860581746, + "loss": 1.4385, + "theoretical_loss": 3.888258480654751, + "tokens_seen": 571146240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006682126379137413, + "loss": 1.4541, + "theoretical_loss": 3.888211869034339, + "tokens_seen": 571211776 + }, + { + "epoch": 0.17, + "learning_rate": 0.000668196589769308, + "loss": 1.4689, + "theoretical_loss": 3.888165264258636, + "tokens_seen": 571277312 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681805416248747, + "loss": 1.4344, + "theoretical_loss": 3.888118666325853, + "tokens_seen": 571342848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681644934804414, + "loss": 1.4824, + "theoretical_loss": 3.888072075234201, + "tokens_seen": 571408384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681484453360081, + "loss": 1.403, + "theoretical_loss": 3.8880254909818905, + "tokens_seen": 571473920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681323971915748, + "loss": 1.5724, + "theoretical_loss": 3.8879789135671334, + "tokens_seen": 571539456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681163490471415, + "loss": 1.5387, + "theoretical_loss": 3.8879323429881425, + "tokens_seen": 571604992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006681003009027082, + "loss": 1.5681, + "theoretical_loss": 3.887885779243131, + "tokens_seen": 571670528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680842527582749, + "loss": 1.3829, + "theoretical_loss": 3.887839222330313, + "tokens_seen": 571736064 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680682046138416, + "loss": 1.3526, + "theoretical_loss": 3.887792672247902, + "tokens_seen": 571801600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680521564694082, + "loss": 1.4094, + "theoretical_loss": 3.887746128994114, + "tokens_seen": 571867136 + }, + { + "epoch": 0.17, + "learning_rate": 0.000668036108324975, + "loss": 1.4718, + "theoretical_loss": 3.8876995925671656, + "tokens_seen": 571932672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680200601805416, + "loss": 1.5132, + "theoretical_loss": 3.8876530629652724, + "tokens_seen": 571998208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006680040120361084, + "loss": 1.5718, + "theoretical_loss": 3.8876065401866513, + "tokens_seen": 572063744 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667987963891675, + "loss": 1.4837, + "theoretical_loss": 3.887560024229521, + "tokens_seen": 572129280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679719157472418, + "loss": 1.4954, + "theoretical_loss": 3.8875135150921, + "tokens_seen": 572194816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679558676028084, + "loss": 1.4533, + "theoretical_loss": 3.887467012772607, + "tokens_seen": 572260352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679398194583752, + "loss": 1.5866, + "theoretical_loss": 3.8874205172692626, + "tokens_seen": 572325888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679237713139418, + "loss": 1.5332, + "theoretical_loss": 3.887374028580287, + "tokens_seen": 572391424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006679077231695085, + "loss": 1.4713, + "theoretical_loss": 3.8873275467039017, + "tokens_seen": 572456960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678916750250752, + "loss": 1.5634, + "theoretical_loss": 3.887281071638328, + "tokens_seen": 572522496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678756268806419, + "loss": 1.65, + "theoretical_loss": 3.8872346033817893, + "tokens_seen": 572588032 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678595787362086, + "loss": 1.5073, + "theoretical_loss": 3.887188141932508, + "tokens_seen": 572653568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678435305917754, + "loss": 1.4923, + "theoretical_loss": 3.8871416872887083, + "tokens_seen": 572719104 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667827482447342, + "loss": 1.5372, + "theoretical_loss": 3.8870952394486156, + "tokens_seen": 572784640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006678114343029088, + "loss": 1.4996, + "theoretical_loss": 3.8870487984104543, + "tokens_seen": 572850176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677953861584754, + "loss": 1.5229, + "theoretical_loss": 3.8870023641724503, + "tokens_seen": 572915712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677793380140422, + "loss": 1.5847, + "theoretical_loss": 3.886955936732831, + "tokens_seen": 572981248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677632898696088, + "loss": 1.5271, + "theoretical_loss": 3.886909516089822, + "tokens_seen": 573046784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677472417251756, + "loss": 1.5696, + "theoretical_loss": 3.8868631022416533, + "tokens_seen": 573112320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006677311935807422, + "loss": 1.6653, + "theoretical_loss": 3.886816695186552, + "tokens_seen": 573177856 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667715145436309, + "loss": 1.4621, + "theoretical_loss": 3.886770294922748, + "tokens_seen": 573243392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676990972918756, + "loss": 1.4493, + "theoretical_loss": 3.8867239014484705, + "tokens_seen": 573308928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676830491474424, + "loss": 1.571, + "theoretical_loss": 3.886677514761951, + "tokens_seen": 573374464 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667667001003009, + "loss": 1.5763, + "theoretical_loss": 3.8866311348614198, + "tokens_seen": 573440000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676509528585758, + "loss": 1.498, + "theoretical_loss": 3.8865847617451097, + "tokens_seen": 573505536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676349047141424, + "loss": 1.5396, + "theoretical_loss": 3.8865383954112525, + "tokens_seen": 573571072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676188565697092, + "loss": 1.4449, + "theoretical_loss": 3.8864920358580823, + "tokens_seen": 573636608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006676028084252758, + "loss": 1.5381, + "theoretical_loss": 3.8864456830838314, + "tokens_seen": 573702144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006675867602808426, + "loss": 1.5115, + "theoretical_loss": 3.8863993370867362, + "tokens_seen": 573767680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006675707121364092, + "loss": 1.4637, + "theoretical_loss": 3.886352997865031, + "tokens_seen": 573833216 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667554663991976, + "loss": 1.5585, + "theoretical_loss": 3.8863066654169516, + "tokens_seen": 573898752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006675386158475426, + "loss": 1.6178, + "theoretical_loss": 3.886260339740735, + "tokens_seen": 573964288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006675225677031094, + "loss": 1.4068, + "theoretical_loss": 3.8862140208346174, + "tokens_seen": 574029824 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667506519558676, + "loss": 1.4945, + "theoretical_loss": 3.8861677086968376, + "tokens_seen": 574095360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674904714142428, + "loss": 1.6049, + "theoretical_loss": 3.8861214033256344, + "tokens_seen": 574160896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674744232698094, + "loss": 1.4864, + "theoretical_loss": 3.8860751047192457, + "tokens_seen": 574226432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674583751253762, + "loss": 1.4481, + "theoretical_loss": 3.886028812875912, + "tokens_seen": 574291968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674423269809428, + "loss": 1.4883, + "theoretical_loss": 3.885982527793874, + "tokens_seen": 574357504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674262788365096, + "loss": 1.4112, + "theoretical_loss": 3.8859362494713725, + "tokens_seen": 574423040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006674102306920762, + "loss": 1.5699, + "theoretical_loss": 3.8858899779066496, + "tokens_seen": 574488576 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667394182547643, + "loss": 1.5145, + "theoretical_loss": 3.8858437130979473, + "tokens_seen": 574554112 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006673781344032096, + "loss": 1.5351, + "theoretical_loss": 3.8857974550435097, + "tokens_seen": 574619648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006673620862587764, + "loss": 1.4705, + "theoretical_loss": 3.885751203741579, + "tokens_seen": 574685184 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667346038114343, + "loss": 1.4656, + "theoretical_loss": 3.8857049591904014, + "tokens_seen": 574750720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006673299899699098, + "loss": 1.4563, + "theoretical_loss": 3.8856587213882205, + "tokens_seen": 574816256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006673139418254764, + "loss": 1.4182, + "theoretical_loss": 3.8856124903332825, + "tokens_seen": 574881792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672978936810432, + "loss": 1.3568, + "theoretical_loss": 3.885566266023834, + "tokens_seen": 574947328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672818455366098, + "loss": 1.5186, + "theoretical_loss": 3.8855200484581225, + "tokens_seen": 575012864 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672657973921766, + "loss": 1.4667, + "theoretical_loss": 3.885473837634395, + "tokens_seen": 575078400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672497492477432, + "loss": 1.4794, + "theoretical_loss": 3.8854276335508997, + "tokens_seen": 575143936 + }, + { + "epoch": 0.17, + "learning_rate": 0.00066723370110331, + "loss": 1.6099, + "theoretical_loss": 3.885381436205886, + "tokens_seen": 575209472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672176529588766, + "loss": 1.4558, + "theoretical_loss": 3.885335245597604, + "tokens_seen": 575275008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006672016048144434, + "loss": 1.4809, + "theoretical_loss": 3.885289061724303, + "tokens_seen": 575340544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00066718555667001, + "loss": 1.4496, + "theoretical_loss": 3.885242884584235, + "tokens_seen": 575406080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671695085255768, + "loss": 1.4464, + "theoretical_loss": 3.8851967141756507, + "tokens_seen": 575471616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671534603811434, + "loss": 1.5502, + "theoretical_loss": 3.885150550496803, + "tokens_seen": 575537152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671374122367102, + "loss": 1.5008, + "theoretical_loss": 3.8851043935459453, + "tokens_seen": 575602688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671213640922768, + "loss": 1.4732, + "theoretical_loss": 3.88505824332133, + "tokens_seen": 575668224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006671053159478436, + "loss": 1.5417, + "theoretical_loss": 3.8850120998212123, + "tokens_seen": 575733760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006670892678034102, + "loss": 1.5514, + "theoretical_loss": 3.884965963043847, + "tokens_seen": 575799296 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667073219658977, + "loss": 1.5767, + "theoretical_loss": 3.884919832987489, + "tokens_seen": 575864832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006670571715145436, + "loss": 1.3884, + "theoretical_loss": 3.884873709650395, + "tokens_seen": 575930368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006670411233701104, + "loss": 1.4891, + "theoretical_loss": 3.8848275930308223, + "tokens_seen": 575995904 + }, + { + "epoch": 0.17, + "learning_rate": 0.000667025075225677, + "loss": 1.5209, + "theoretical_loss": 3.884781483127027, + "tokens_seen": 576061440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006670090270812438, + "loss": 1.5727, + "theoretical_loss": 3.8847353799372684, + "tokens_seen": 576126976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669929789368105, + "loss": 1.4639, + "theoretical_loss": 3.8846892834598052, + "tokens_seen": 576192512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669769307923772, + "loss": 1.5604, + "theoretical_loss": 3.884643193692896, + "tokens_seen": 576258048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669608826479439, + "loss": 1.4891, + "theoretical_loss": 3.884597110634802, + "tokens_seen": 576323584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669448345035106, + "loss": 1.4047, + "theoretical_loss": 3.884551034283784, + "tokens_seen": 576389120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006669287863590773, + "loss": 1.4425, + "theoretical_loss": 3.8845049646381025, + "tokens_seen": 576454656 + }, + { + "epoch": 0.17, + "learning_rate": 0.000666912738214644, + "loss": 1.4263, + "theoretical_loss": 3.8844589016960196, + "tokens_seen": 576520192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668966900702107, + "loss": 1.4342, + "theoretical_loss": 3.884412845455799, + "tokens_seen": 576585728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668806419257774, + "loss": 1.4769, + "theoretical_loss": 3.8843667959157027, + "tokens_seen": 576651264 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 219460, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5097222328186035, + "objective/train/theoretical_loss": 3.884320753073996, + "objective/train/tokens_used": 597176800, + "theoretical_loss": 3.884320753073996, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668645937813441, + "loss": 1.5097, + "theoretical_loss": 3.884320753073996, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668485456369108, + "loss": 1.4855, + "theoretical_loss": 3.884274716928942, + "tokens_seen": 576782336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668324974924775, + "loss": 1.4264, + "theoretical_loss": 3.8842286874788075, + "tokens_seen": 576847872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668164493480442, + "loss": 1.5033, + "theoretical_loss": 3.8841826647218576, + "tokens_seen": 576913408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006668004012036109, + "loss": 1.3852, + "theoretical_loss": 3.8841366486563595, + "tokens_seen": 576978944 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006667843530591776, + "loss": 1.5038, + "theoretical_loss": 3.8840906392805796, + "tokens_seen": 577044480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006667683049147443, + "loss": 1.4043, + "theoretical_loss": 3.8840446365927863, + "tokens_seen": 577110016 + }, + { + "epoch": 0.17, + "learning_rate": 0.000666752256770311, + "loss": 1.4962, + "theoretical_loss": 3.8839986405912477, + "tokens_seen": 577175552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006667362086258777, + "loss": 1.4711, + "theoretical_loss": 3.8839526512742335, + "tokens_seen": 577241088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006667201604814444, + "loss": 1.4229, + "theoretical_loss": 3.883906668640013, + "tokens_seen": 577306624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006667041123370111, + "loss": 1.6302, + "theoretical_loss": 3.8838606926868575, + "tokens_seen": 577372160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0006666880641925778, + "loss": 1.4112, + "theoretical_loss": 3.8838147234130367, + "tokens_seen": 577437696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666720160481445, + "loss": 1.4938, + "theoretical_loss": 3.8837687608168237, + "tokens_seen": 577503232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666559679037111, + "loss": 1.5936, + "theoretical_loss": 3.8837228048964896, + "tokens_seen": 577568768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666399197592779, + "loss": 1.3643, + "theoretical_loss": 3.8836768556503083, + "tokens_seen": 577634304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666238716148445, + "loss": 1.5344, + "theoretical_loss": 3.883630913076553, + "tokens_seen": 577699840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006666078234704113, + "loss": 1.4971, + "theoretical_loss": 3.883584977173499, + "tokens_seen": 577765376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665917753259779, + "loss": 1.5703, + "theoretical_loss": 3.8835390479394203, + "tokens_seen": 577830912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665757271815447, + "loss": 1.4558, + "theoretical_loss": 3.883493125372592, + "tokens_seen": 577896448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665596790371113, + "loss": 1.4916, + "theoretical_loss": 3.8834472094712913, + "tokens_seen": 577961984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665436308926781, + "loss": 1.5021, + "theoretical_loss": 3.8834013002337953, + "tokens_seen": 578027520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665275827482447, + "loss": 1.5162, + "theoretical_loss": 3.883355397658381, + "tokens_seen": 578093056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006665115346038115, + "loss": 1.5446, + "theoretical_loss": 3.8833095017433257, + "tokens_seen": 578158592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664954864593781, + "loss": 1.5309, + "theoretical_loss": 3.8832636124869095, + "tokens_seen": 578224128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664794383149449, + "loss": 1.5441, + "theoretical_loss": 3.883217729887411, + "tokens_seen": 578289664 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664633901705115, + "loss": 1.5071, + "theoretical_loss": 3.883171853943111, + "tokens_seen": 578355200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664473420260783, + "loss": 1.4474, + "theoretical_loss": 3.8831259846522896, + "tokens_seen": 578420736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664312938816449, + "loss": 1.5088, + "theoretical_loss": 3.8830801220132285, + "tokens_seen": 578486272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006664152457372117, + "loss": 1.4753, + "theoretical_loss": 3.883034266024209, + "tokens_seen": 578551808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663991975927783, + "loss": 1.4132, + "theoretical_loss": 3.8829884166835154, + "tokens_seen": 578617344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663831494483451, + "loss": 1.4475, + "theoretical_loss": 3.882942573989429, + "tokens_seen": 578682880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663671013039117, + "loss": 1.4822, + "theoretical_loss": 3.882896737940235, + "tokens_seen": 578748416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663510531594785, + "loss": 1.4978, + "theoretical_loss": 3.882850908534217, + "tokens_seen": 578813952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663350050150451, + "loss": 1.4942, + "theoretical_loss": 3.882805085769661, + "tokens_seen": 578879488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663189568706119, + "loss": 1.4978, + "theoretical_loss": 3.8827592696448523, + "tokens_seen": 578945024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006663029087261785, + "loss": 1.5268, + "theoretical_loss": 3.8827134601580777, + "tokens_seen": 579010560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662868605817453, + "loss": 1.3868, + "theoretical_loss": 3.8826676573076235, + "tokens_seen": 579076096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662708124373119, + "loss": 1.6254, + "theoretical_loss": 3.8826218610917786, + "tokens_seen": 579141632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662547642928787, + "loss": 1.5165, + "theoretical_loss": 3.8825760715088307, + "tokens_seen": 579207168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662387161484453, + "loss": 1.5266, + "theoretical_loss": 3.882530288557069, + "tokens_seen": 579272704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662226680040121, + "loss": 1.4868, + "theoretical_loss": 3.8824845122347824, + "tokens_seen": 579338240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006662066198595787, + "loss": 1.4787, + "theoretical_loss": 3.8824387425402618, + "tokens_seen": 579403776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661905717151455, + "loss": 1.5221, + "theoretical_loss": 3.8823929794717986, + "tokens_seen": 579469312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661745235707121, + "loss": 1.5738, + "theoretical_loss": 3.882347223027683, + "tokens_seen": 579534848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661584754262789, + "loss": 1.4122, + "theoretical_loss": 3.882301473206209, + "tokens_seen": 579600384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661424272818455, + "loss": 1.5764, + "theoretical_loss": 3.8822557300056673, + "tokens_seen": 579665920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661263791374123, + "loss": 1.445, + "theoretical_loss": 3.882209993424353, + "tokens_seen": 579731456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006661103309929789, + "loss": 1.5172, + "theoretical_loss": 3.8821642634605587, + "tokens_seen": 579796992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660942828485457, + "loss": 1.433, + "theoretical_loss": 3.8821185401125806, + "tokens_seen": 579862528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660782347041123, + "loss": 1.4529, + "theoretical_loss": 3.8820728233787136, + "tokens_seen": 579928064 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660621865596791, + "loss": 1.4999, + "theoretical_loss": 3.8820271132572524, + "tokens_seen": 579993600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660461384152457, + "loss": 1.4728, + "theoretical_loss": 3.881981409746495, + "tokens_seen": 580059136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660300902708125, + "loss": 1.3974, + "theoretical_loss": 3.8819357128447383, + "tokens_seen": 580124672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006660140421263791, + "loss": 1.4184, + "theoretical_loss": 3.88189002255028, + "tokens_seen": 580190208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659979939819459, + "loss": 1.5259, + "theoretical_loss": 3.8818443388614186, + "tokens_seen": 580255744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659819458375125, + "loss": 1.3845, + "theoretical_loss": 3.881798661776453, + "tokens_seen": 580321280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659658976930793, + "loss": 1.4292, + "theoretical_loss": 3.8817529912936832, + "tokens_seen": 580386816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659498495486459, + "loss": 1.4712, + "theoretical_loss": 3.88170732741141, + "tokens_seen": 580452352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659338014042127, + "loss": 1.5801, + "theoretical_loss": 3.8816616701279334, + "tokens_seen": 580517888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659177532597793, + "loss": 1.5285, + "theoretical_loss": 3.881616019441556, + "tokens_seen": 580583424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006659017051153461, + "loss": 1.5466, + "theoretical_loss": 3.881570375350579, + "tokens_seen": 580648960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658856569709127, + "loss": 1.5584, + "theoretical_loss": 3.881524737853306, + "tokens_seen": 580714496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658696088264795, + "loss": 1.6086, + "theoretical_loss": 3.881479106948041, + "tokens_seen": 580780032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658535606820461, + "loss": 1.5561, + "theoretical_loss": 3.8814334826330876, + "tokens_seen": 580845568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658375125376129, + "loss": 1.4337, + "theoretical_loss": 3.88138786490675, + "tokens_seen": 580911104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658214643931795, + "loss": 1.4006, + "theoretical_loss": 3.8813422537673343, + "tokens_seen": 580976640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006658054162487463, + "loss": 1.5593, + "theoretical_loss": 3.881296649213146, + "tokens_seen": 581042176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657893681043129, + "loss": 1.5677, + "theoretical_loss": 3.8812510512424927, + "tokens_seen": 581107712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657733199598797, + "loss": 1.4271, + "theoretical_loss": 3.881205459853681, + "tokens_seen": 581173248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657572718154463, + "loss": 1.4254, + "theoretical_loss": 3.8811598750450185, + "tokens_seen": 581238784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657412236710131, + "loss": 1.5233, + "theoretical_loss": 3.8811142968148142, + "tokens_seen": 581304320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657251755265797, + "loss": 1.561, + "theoretical_loss": 3.881068725161377, + "tokens_seen": 581369856 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006657091273821465, + "loss": 1.5094, + "theoretical_loss": 3.881023160083018, + "tokens_seen": 581435392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656930792377131, + "loss": 1.537, + "theoretical_loss": 3.8809776015780453, + "tokens_seen": 581500928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656770310932799, + "loss": 1.4653, + "theoretical_loss": 3.880932049644771, + "tokens_seen": 581566464 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656609829488466, + "loss": 1.4386, + "theoretical_loss": 3.8808865042815075, + "tokens_seen": 581632000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656449348044133, + "loss": 1.4296, + "theoretical_loss": 3.880840965486566, + "tokens_seen": 581697536 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066562888665998, + "loss": 1.5901, + "theoretical_loss": 3.8807954332582595, + "tokens_seen": 581763072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006656128385155467, + "loss": 1.4501, + "theoretical_loss": 3.880749907594902, + "tokens_seen": 581828608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655967903711134, + "loss": 1.4349, + "theoretical_loss": 3.8807043884948085, + "tokens_seen": 581894144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655807422266801, + "loss": 1.5229, + "theoretical_loss": 3.880658875956292, + "tokens_seen": 581959680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655646940822468, + "loss": 1.5032, + "theoretical_loss": 3.8806133699776684, + "tokens_seen": 582025216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655486459378135, + "loss": 1.4514, + "theoretical_loss": 3.880567870557254, + "tokens_seen": 582090752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655325977933802, + "loss": 1.519, + "theoretical_loss": 3.8805223776933655, + "tokens_seen": 582156288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655165496489469, + "loss": 1.5191, + "theoretical_loss": 3.88047689138432, + "tokens_seen": 582221824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006655005015045136, + "loss": 1.4464, + "theoretical_loss": 3.880431411628436, + "tokens_seen": 582287360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654844533600803, + "loss": 1.5692, + "theoretical_loss": 3.8803859384240305, + "tokens_seen": 582352896 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665468405215647, + "loss": 1.5313, + "theoretical_loss": 3.880340471769424, + "tokens_seen": 582418432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654523570712137, + "loss": 1.4007, + "theoretical_loss": 3.880295011662936, + "tokens_seen": 582483968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654363089267805, + "loss": 1.488, + "theoretical_loss": 3.8802495581028857, + "tokens_seen": 582549504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654202607823471, + "loss": 1.4464, + "theoretical_loss": 3.880204111087596, + "tokens_seen": 582615040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006654042126379139, + "loss": 1.4906, + "theoretical_loss": 3.880158670615387, + "tokens_seen": 582680576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653881644934805, + "loss": 1.5665, + "theoretical_loss": 3.880113236684582, + "tokens_seen": 582746112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653721163490472, + "loss": 1.4177, + "theoretical_loss": 3.880067809293503, + "tokens_seen": 582811648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653560682046139, + "loss": 1.3977, + "theoretical_loss": 3.880022388440474, + "tokens_seen": 582877184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653400200601806, + "loss": 1.5624, + "theoretical_loss": 3.879976974123818, + "tokens_seen": 582942720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006653239719157473, + "loss": 1.4499, + "theoretical_loss": 3.8799315663418614, + "tokens_seen": 583008256 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665307923771314, + "loss": 1.4931, + "theoretical_loss": 3.8798861650929277, + "tokens_seen": 583073792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652918756268807, + "loss": 1.5528, + "theoretical_loss": 3.8798407703753446, + "tokens_seen": 583139328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652758274824474, + "loss": 1.4739, + "theoretical_loss": 3.879795382187438, + "tokens_seen": 583204864 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 221910, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4041215181350708, + "objective/train/theoretical_loss": 3.879750000527534, + "objective/train/tokens_used": 603730400, + "theoretical_loss": 3.879750000527534, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665259779338014, + "loss": 1.4041, + "theoretical_loss": 3.879750000527534, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652437311935808, + "loss": 1.473, + "theoretical_loss": 3.8797046253939618, + "tokens_seen": 583335936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652276830491474, + "loss": 1.5725, + "theoretical_loss": 3.8796592567850485, + "tokens_seen": 583401472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006652116349047142, + "loss": 1.4553, + "theoretical_loss": 3.8796138946991245, + "tokens_seen": 583467008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651955867602808, + "loss": 1.4174, + "theoretical_loss": 3.8795685391345183, + "tokens_seen": 583532544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651795386158476, + "loss": 1.4912, + "theoretical_loss": 3.879523190089561, + "tokens_seen": 583598080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651634904714142, + "loss": 1.5175, + "theoretical_loss": 3.8794778475625833, + "tokens_seen": 583663616 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665147442326981, + "loss": 1.4781, + "theoretical_loss": 3.879432511551916, + "tokens_seen": 583729152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651313941825476, + "loss": 1.521, + "theoretical_loss": 3.8793871820558916, + "tokens_seen": 583794688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006651153460381144, + "loss": 1.5027, + "theoretical_loss": 3.8793418590728423, + "tokens_seen": 583860224 + }, + { + "epoch": 0.18, + "learning_rate": 0.000665099297893681, + "loss": 1.5852, + "theoretical_loss": 3.879296542601103, + "tokens_seen": 583925760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650832497492478, + "loss": 1.4657, + "theoretical_loss": 3.879251232639006, + "tokens_seen": 583991296 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650672016048144, + "loss": 1.3985, + "theoretical_loss": 3.879205929184886, + "tokens_seen": 584056832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650511534603812, + "loss": 1.5415, + "theoretical_loss": 3.8791606322370793, + "tokens_seen": 584122368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650351053159478, + "loss": 1.3816, + "theoretical_loss": 3.8791153417939204, + "tokens_seen": 584187904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650190571715146, + "loss": 1.3885, + "theoretical_loss": 3.8790700578537463, + "tokens_seen": 584253440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006650030090270812, + "loss": 1.574, + "theoretical_loss": 3.879024780414894, + "tokens_seen": 584318976 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664986960882648, + "loss": 1.4758, + "theoretical_loss": 3.878979509475701, + "tokens_seen": 584384512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006649709127382146, + "loss": 1.5093, + "theoretical_loss": 3.8789342450345052, + "tokens_seen": 584450048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006649548645937814, + "loss": 1.3754, + "theoretical_loss": 3.8788889870896455, + "tokens_seen": 584515584 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664938816449348, + "loss": 1.5123, + "theoretical_loss": 3.878843735639462, + "tokens_seen": 584581120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006649227683049148, + "loss": 1.4916, + "theoretical_loss": 3.8787984906822945, + "tokens_seen": 584646656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006649067201604814, + "loss": 1.5501, + "theoretical_loss": 3.878753252216483, + "tokens_seen": 584712192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648906720160482, + "loss": 1.426, + "theoretical_loss": 3.8787080202403694, + "tokens_seen": 584777728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648746238716148, + "loss": 1.6486, + "theoretical_loss": 3.8786627947522954, + "tokens_seen": 584843264 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648585757271816, + "loss": 1.4119, + "theoretical_loss": 3.878617575750603, + "tokens_seen": 584908800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648425275827482, + "loss": 1.4517, + "theoretical_loss": 3.8785723632336366, + "tokens_seen": 584974336 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664826479438315, + "loss": 1.4637, + "theoretical_loss": 3.8785271571997386, + "tokens_seen": 585039872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006648104312938816, + "loss": 1.4754, + "theoretical_loss": 3.878481957647254, + "tokens_seen": 585105408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647943831494484, + "loss": 1.4334, + "theoretical_loss": 3.878436764574527, + "tokens_seen": 585170944 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664778335005015, + "loss": 1.5349, + "theoretical_loss": 3.8783915779799036, + "tokens_seen": 585236480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647622868605818, + "loss": 1.5239, + "theoretical_loss": 3.8783463978617303, + "tokens_seen": 585302016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647462387161484, + "loss": 1.5186, + "theoretical_loss": 3.878301224218353, + "tokens_seen": 585367552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647301905717152, + "loss": 1.514, + "theoretical_loss": 3.8782560570481204, + "tokens_seen": 585433088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006647141424272818, + "loss": 1.5859, + "theoretical_loss": 3.8782108963493793, + "tokens_seen": 585498624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646980942828486, + "loss": 1.4746, + "theoretical_loss": 3.878165742120478, + "tokens_seen": 585564160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646820461384152, + "loss": 1.5492, + "theoretical_loss": 3.8781205943597667, + "tokens_seen": 585629696 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664665997993982, + "loss": 1.4738, + "theoretical_loss": 3.8780754530655948, + "tokens_seen": 585695232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646499498495486, + "loss": 1.4957, + "theoretical_loss": 3.8780303182363123, + "tokens_seen": 585760768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646339017051154, + "loss": 1.4051, + "theoretical_loss": 3.877985189870271, + "tokens_seen": 585826304 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664617853560682, + "loss": 1.4884, + "theoretical_loss": 3.877940067965821, + "tokens_seen": 585891840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006646018054162488, + "loss": 1.4547, + "theoretical_loss": 3.8778949525213164, + "tokens_seen": 585957376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645857572718154, + "loss": 1.3488, + "theoretical_loss": 3.8778498435351088, + "tokens_seen": 586022912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645697091273822, + "loss": 1.4429, + "theoretical_loss": 3.8778047410055514, + "tokens_seen": 586088448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645536609829488, + "loss": 1.5628, + "theoretical_loss": 3.8777596449309994, + "tokens_seen": 586153984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645376128385156, + "loss": 1.5447, + "theoretical_loss": 3.8777145553098062, + "tokens_seen": 586219520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006645215646940822, + "loss": 1.4109, + "theoretical_loss": 3.8776694721403278, + "tokens_seen": 586285056 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664505516549649, + "loss": 1.5376, + "theoretical_loss": 3.877624395420919, + "tokens_seen": 586350592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644894684052156, + "loss": 1.4292, + "theoretical_loss": 3.8775793251499375, + "tokens_seen": 586416128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644734202607824, + "loss": 1.6319, + "theoretical_loss": 3.8775342613257395, + "tokens_seen": 586481664 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664457372116349, + "loss": 1.447, + "theoretical_loss": 3.8774892039466833, + "tokens_seen": 586547200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644413239719158, + "loss": 1.4571, + "theoretical_loss": 3.877444153011126, + "tokens_seen": 586612736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644252758274824, + "loss": 1.4108, + "theoretical_loss": 3.8773991085174284, + "tokens_seen": 586678272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006644092276830492, + "loss": 1.4757, + "theoretical_loss": 3.8773540704639475, + "tokens_seen": 586743808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643931795386158, + "loss": 1.4353, + "theoretical_loss": 3.8773090388490448, + "tokens_seen": 586809344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643771313941826, + "loss": 1.5196, + "theoretical_loss": 3.8772640136710805, + "tokens_seen": 586874880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643610832497492, + "loss": 1.5863, + "theoretical_loss": 3.877218994928416, + "tokens_seen": 586940416 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664345035105316, + "loss": 1.4168, + "theoretical_loss": 3.8771739826194134, + "tokens_seen": 587005952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643289869608826, + "loss": 1.4554, + "theoretical_loss": 3.8771289767424344, + "tokens_seen": 587071488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006643129388164494, + "loss": 1.562, + "theoretical_loss": 3.8770839772958428, + "tokens_seen": 587137024 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664296890672016, + "loss": 1.4093, + "theoretical_loss": 3.877038984278002, + "tokens_seen": 587202560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642808425275828, + "loss": 1.547, + "theoretical_loss": 3.8769939976872756, + "tokens_seen": 587268096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642647943831495, + "loss": 1.4914, + "theoretical_loss": 3.8769490175220294, + "tokens_seen": 587333632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642487462387162, + "loss": 1.5115, + "theoretical_loss": 3.876904043780628, + "tokens_seen": 587399168 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664232698094283, + "loss": 1.5733, + "theoretical_loss": 3.8768590764614377, + "tokens_seen": 587464704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642166499498496, + "loss": 1.4817, + "theoretical_loss": 3.8768141155628255, + "tokens_seen": 587530240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006642006018054164, + "loss": 1.4335, + "theoretical_loss": 3.8767691610831587, + "tokens_seen": 587595776 + }, + { + "epoch": 0.18, + "learning_rate": 0.000664184553660983, + "loss": 1.5439, + "theoretical_loss": 3.876724213020804, + "tokens_seen": 587661312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641685055165498, + "loss": 1.6219, + "theoretical_loss": 3.876679271374131, + "tokens_seen": 587726848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641524573721164, + "loss": 1.5227, + "theoretical_loss": 3.8766343361415085, + "tokens_seen": 587792384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641364092276832, + "loss": 1.3738, + "theoretical_loss": 3.876589407321305, + "tokens_seen": 587857920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641203610832498, + "loss": 1.409, + "theoretical_loss": 3.8765444849118924, + "tokens_seen": 587923456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006641043129388166, + "loss": 1.5243, + "theoretical_loss": 3.8764995689116404, + "tokens_seen": 587988992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006640882647943832, + "loss": 1.468, + "theoretical_loss": 3.8764546593189206, + "tokens_seen": 588054528 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066407221664995, + "loss": 1.6056, + "theoretical_loss": 3.876409756132105, + "tokens_seen": 588120064 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006640561685055166, + "loss": 1.564, + "theoretical_loss": 3.8763648593495663, + "tokens_seen": 588185600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006640401203610834, + "loss": 1.491, + "theoretical_loss": 3.8763199689696783, + "tokens_seen": 588251136 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066402407221665, + "loss": 1.4604, + "theoretical_loss": 3.876275084990813, + "tokens_seen": 588316672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006640080240722168, + "loss": 1.5556, + "theoretical_loss": 3.876230207411347, + "tokens_seen": 588382208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639919759277834, + "loss": 1.5533, + "theoretical_loss": 3.8761853362296534, + "tokens_seen": 588447744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639759277833501, + "loss": 1.4383, + "theoretical_loss": 3.8761404714441086, + "tokens_seen": 588513280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639598796389168, + "loss": 1.5137, + "theoretical_loss": 3.8760956130530886, + "tokens_seen": 588578816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639438314944835, + "loss": 1.4285, + "theoretical_loss": 3.8760507610549704, + "tokens_seen": 588644352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639277833500502, + "loss": 1.578, + "theoretical_loss": 3.876005915448131, + "tokens_seen": 588709888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006639117352056169, + "loss": 1.5701, + "theoretical_loss": 3.8759610762309484, + "tokens_seen": 588775424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638956870611836, + "loss": 1.5023, + "theoretical_loss": 3.875916243401801, + "tokens_seen": 588840960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638796389167503, + "loss": 1.4444, + "theoretical_loss": 3.8758714169590682, + "tokens_seen": 588906496 + }, + { + "epoch": 0.18, + "learning_rate": 0.000663863590772317, + "loss": 1.4576, + "theoretical_loss": 3.87582659690113, + "tokens_seen": 588972032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638475426278837, + "loss": 1.5451, + "theoretical_loss": 3.875781783226366, + "tokens_seen": 589037568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638314944834503, + "loss": 1.4894, + "theoretical_loss": 3.8757369759331572, + "tokens_seen": 589103104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006638154463390171, + "loss": 1.471, + "theoretical_loss": 3.8756921750198856, + "tokens_seen": 589168640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637993981945837, + "loss": 1.5179, + "theoretical_loss": 3.8756473804849323, + "tokens_seen": 589234176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637833500501505, + "loss": 1.4629, + "theoretical_loss": 3.8756025923266813, + "tokens_seen": 589299712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637673019057171, + "loss": 1.4521, + "theoretical_loss": 3.8755578105435147, + "tokens_seen": 589365248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637512537612839, + "loss": 1.5416, + "theoretical_loss": 3.8755130351338165, + "tokens_seen": 589430784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637352056168505, + "loss": 1.4798, + "theoretical_loss": 3.8754682660959716, + "tokens_seen": 589496320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637191574724173, + "loss": 1.4761, + "theoretical_loss": 3.8754235034283653, + "tokens_seen": 589561856 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006637031093279839, + "loss": 1.4506, + "theoretical_loss": 3.875378747129382, + "tokens_seen": 589627392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636870611835507, + "loss": 1.5015, + "theoretical_loss": 3.8753339971974086, + "tokens_seen": 589692928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636710130391173, + "loss": 1.492, + "theoretical_loss": 3.8752892536308323, + "tokens_seen": 589758464 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 224258, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.424971342086792, + "objective/train/theoretical_loss": 3.87524451642804, + "objective/train/tokens_used": 610284000, + "theoretical_loss": 3.87524451642804, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636549648946841, + "loss": 1.425, + "theoretical_loss": 3.87524451642804, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636389167502507, + "loss": 1.4776, + "theoretical_loss": 3.8751997855874194, + "tokens_seen": 589889536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636228686058175, + "loss": 1.5262, + "theoretical_loss": 3.8751550611073595, + "tokens_seen": 589955072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006636068204613841, + "loss": 1.5614, + "theoretical_loss": 3.875110342986249, + "tokens_seen": 590020608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635907723169509, + "loss": 1.57, + "theoretical_loss": 3.875065631222478, + "tokens_seen": 590086144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635747241725175, + "loss": 1.4816, + "theoretical_loss": 3.875020925814437, + "tokens_seen": 590151680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635586760280843, + "loss": 1.5331, + "theoretical_loss": 3.8749762267605163, + "tokens_seen": 590217216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635426278836509, + "loss": 1.6611, + "theoretical_loss": 3.8749315340591073, + "tokens_seen": 590282752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635265797392177, + "loss": 1.5631, + "theoretical_loss": 3.874886847708603, + "tokens_seen": 590348288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006635105315947843, + "loss": 1.5863, + "theoretical_loss": 3.8748421677073956, + "tokens_seen": 590413824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634944834503511, + "loss": 1.4512, + "theoretical_loss": 3.874797494053878, + "tokens_seen": 590479360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634784353059177, + "loss": 1.5475, + "theoretical_loss": 3.8747528267464437, + "tokens_seen": 590544896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634623871614845, + "loss": 1.5883, + "theoretical_loss": 3.8747081657834874, + "tokens_seen": 590610432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634463390170511, + "loss": 1.5669, + "theoretical_loss": 3.874663511163405, + "tokens_seen": 590675968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634302908726179, + "loss": 1.5869, + "theoretical_loss": 3.8746188628845912, + "tokens_seen": 590741504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006634142427281845, + "loss": 1.4681, + "theoretical_loss": 3.8745742209454423, + "tokens_seen": 590807040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633981945837513, + "loss": 1.4747, + "theoretical_loss": 3.874529585344355, + "tokens_seen": 590872576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633821464393179, + "loss": 1.5044, + "theoretical_loss": 3.874484956079727, + "tokens_seen": 590938112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633660982948847, + "loss": 1.4848, + "theoretical_loss": 3.8744403331499555, + "tokens_seen": 591003648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633500501504513, + "loss": 1.4478, + "theoretical_loss": 3.8743957165534395, + "tokens_seen": 591069184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633340020060181, + "loss": 1.4649, + "theoretical_loss": 3.8743511062885783, + "tokens_seen": 591134720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633179538615847, + "loss": 1.4106, + "theoretical_loss": 3.874306502353771, + "tokens_seen": 591200256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006633019057171515, + "loss": 1.4933, + "theoretical_loss": 3.8742619047474176, + "tokens_seen": 591265792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632858575727181, + "loss": 1.4814, + "theoretical_loss": 3.87421731346792, + "tokens_seen": 591331328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632698094282849, + "loss": 1.3148, + "theoretical_loss": 3.874172728513678, + "tokens_seen": 591396864 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632537612838515, + "loss": 1.4518, + "theoretical_loss": 3.874128149883095, + "tokens_seen": 591462400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632377131394183, + "loss": 1.4433, + "theoretical_loss": 3.8740835775745732, + "tokens_seen": 591527936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632216649949849, + "loss": 1.5372, + "theoretical_loss": 3.8740390115865155, + "tokens_seen": 591593472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006632056168505517, + "loss": 1.5358, + "theoretical_loss": 3.873994451917326, + "tokens_seen": 591659008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631895687061183, + "loss": 1.4701, + "theoretical_loss": 3.8739498985654084, + "tokens_seen": 591724544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631735205616851, + "loss": 1.4587, + "theoretical_loss": 3.873905351529168, + "tokens_seen": 591790080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631574724172517, + "loss": 1.4701, + "theoretical_loss": 3.8738608108070096, + "tokens_seen": 591855616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631414242728185, + "loss": 1.5064, + "theoretical_loss": 3.8738162763973407, + "tokens_seen": 591921152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631253761283851, + "loss": 1.4086, + "theoretical_loss": 3.8737717482985667, + "tokens_seen": 591986688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006631093279839519, + "loss": 1.4271, + "theoretical_loss": 3.8737272265090947, + "tokens_seen": 592052224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630932798395185, + "loss": 1.4557, + "theoretical_loss": 3.8736827110273335, + "tokens_seen": 592117760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630772316950853, + "loss": 1.57, + "theoretical_loss": 3.8736382018516906, + "tokens_seen": 592183296 + }, + { + "epoch": 0.18, + "learning_rate": 0.000663061183550652, + "loss": 1.5244, + "theoretical_loss": 3.8735936989805744, + "tokens_seen": 592248832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630451354062187, + "loss": 1.5087, + "theoretical_loss": 3.873549202412396, + "tokens_seen": 592314368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630290872617855, + "loss": 1.4189, + "theoretical_loss": 3.8735047121455644, + "tokens_seen": 592379904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006630130391173521, + "loss": 1.4898, + "theoretical_loss": 3.873460228178491, + "tokens_seen": 592445440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629969909729189, + "loss": 1.5874, + "theoretical_loss": 3.8734157505095856, + "tokens_seen": 592510976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629809428284855, + "loss": 1.4631, + "theoretical_loss": 3.8733712791372614, + "tokens_seen": 592576512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629648946840523, + "loss": 1.3828, + "theoretical_loss": 3.8733268140599306, + "tokens_seen": 592642048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629488465396189, + "loss": 1.5233, + "theoretical_loss": 3.8732823552760056, + "tokens_seen": 592707584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629327983951857, + "loss": 1.4799, + "theoretical_loss": 3.8732379027839, + "tokens_seen": 592773120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629167502507523, + "loss": 1.5156, + "theoretical_loss": 3.8731934565820287, + "tokens_seen": 592838656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006629007021063191, + "loss": 1.516, + "theoretical_loss": 3.8731490166688056, + "tokens_seen": 592904192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628846539618857, + "loss": 1.4817, + "theoretical_loss": 3.873104583042646, + "tokens_seen": 592969728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628686058174525, + "loss": 1.6328, + "theoretical_loss": 3.8730601557019657, + "tokens_seen": 593035264 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628525576730191, + "loss": 1.4929, + "theoretical_loss": 3.873015734645182, + "tokens_seen": 593100800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628365095285859, + "loss": 1.5605, + "theoretical_loss": 3.872971319870711, + "tokens_seen": 593166336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628204613841525, + "loss": 1.5236, + "theoretical_loss": 3.872926911376971, + "tokens_seen": 593231872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006628044132397193, + "loss": 1.6331, + "theoretical_loss": 3.8728825091623786, + "tokens_seen": 593297408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627883650952859, + "loss": 1.4228, + "theoretical_loss": 3.872838113225354, + "tokens_seen": 593362944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627723169508527, + "loss": 1.4968, + "theoretical_loss": 3.8727937235643157, + "tokens_seen": 593428480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627562688064193, + "loss": 1.5322, + "theoretical_loss": 3.872749340177685, + "tokens_seen": 593494016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627402206619861, + "loss": 1.589, + "theoretical_loss": 3.87270496306388, + "tokens_seen": 593559552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627241725175527, + "loss": 1.4518, + "theoretical_loss": 3.8726605922213233, + "tokens_seen": 593625088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006627081243731195, + "loss": 1.4568, + "theoretical_loss": 3.872616227648436, + "tokens_seen": 593690624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626920762286861, + "loss": 1.3985, + "theoretical_loss": 3.8725718693436404, + "tokens_seen": 593756160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626760280842529, + "loss": 1.4634, + "theoretical_loss": 3.8725275173053593, + "tokens_seen": 593821696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626599799398195, + "loss": 1.4963, + "theoretical_loss": 3.872483171532015, + "tokens_seen": 593887232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626439317953863, + "loss": 1.4958, + "theoretical_loss": 3.8724388320220333, + "tokens_seen": 593952768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626278836509529, + "loss": 1.5004, + "theoretical_loss": 3.872394498773837, + "tokens_seen": 594018304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006626118355065197, + "loss": 1.4517, + "theoretical_loss": 3.8723501717858513, + "tokens_seen": 594083840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625957873620863, + "loss": 1.4561, + "theoretical_loss": 3.872305851056502, + "tokens_seen": 594149376 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662579739217653, + "loss": 1.4439, + "theoretical_loss": 3.8722615365842157, + "tokens_seen": 594214912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625636910732197, + "loss": 1.4387, + "theoretical_loss": 3.8722172283674183, + "tokens_seen": 594280448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625476429287864, + "loss": 1.5314, + "theoretical_loss": 3.8721729264045375, + "tokens_seen": 594345984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625315947843531, + "loss": 1.605, + "theoretical_loss": 3.872128630694001, + "tokens_seen": 594411520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006625155466399198, + "loss": 1.5348, + "theoretical_loss": 3.8720843412342374, + "tokens_seen": 594477056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624994984954865, + "loss": 1.3706, + "theoretical_loss": 3.872040058023676, + "tokens_seen": 594542592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624834503510532, + "loss": 1.5329, + "theoretical_loss": 3.871995781060745, + "tokens_seen": 594608128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624674022066199, + "loss": 1.3876, + "theoretical_loss": 3.8719515103438757, + "tokens_seen": 594673664 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624513540621866, + "loss": 1.4423, + "theoretical_loss": 3.871907245871499, + "tokens_seen": 594739200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624353059177532, + "loss": 1.4371, + "theoretical_loss": 3.871862987642045, + "tokens_seen": 594804736 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066241925777332, + "loss": 1.5197, + "theoretical_loss": 3.871818735653947, + "tokens_seen": 594870272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006624032096288866, + "loss": 1.5188, + "theoretical_loss": 3.8717744899056354, + "tokens_seen": 594935808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623871614844534, + "loss": 1.5513, + "theoretical_loss": 3.8717302503955446, + "tokens_seen": 595001344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00066237111334002, + "loss": 1.512, + "theoretical_loss": 3.8716860171221086, + "tokens_seen": 595066880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623550651955868, + "loss": 1.4901, + "theoretical_loss": 3.8716417900837596, + "tokens_seen": 595132416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623390170511534, + "loss": 1.4464, + "theoretical_loss": 3.8715975692789337, + "tokens_seen": 595197952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623229689067202, + "loss": 1.5011, + "theoretical_loss": 3.871553354706066, + "tokens_seen": 595263488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006623069207622868, + "loss": 1.4773, + "theoretical_loss": 3.871509146363592, + "tokens_seen": 595329024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006622908726178536, + "loss": 1.5368, + "theoretical_loss": 3.8714649442499476, + "tokens_seen": 595394560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006622748244734202, + "loss": 1.5308, + "theoretical_loss": 3.8714207483635703, + "tokens_seen": 595460096 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662258776328987, + "loss": 1.6056, + "theoretical_loss": 3.8713765587028974, + "tokens_seen": 595525632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006622427281845536, + "loss": 1.5008, + "theoretical_loss": 3.871332375266367, + "tokens_seen": 595591168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006622266800401204, + "loss": 1.4114, + "theoretical_loss": 3.871288198052418, + "tokens_seen": 595656704 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662210631895687, + "loss": 1.4543, + "theoretical_loss": 3.871244027059489, + "tokens_seen": 595722240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621945837512538, + "loss": 1.5004, + "theoretical_loss": 3.8711998622860193, + "tokens_seen": 595787776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621785356068204, + "loss": 1.4265, + "theoretical_loss": 3.8711557037304507, + "tokens_seen": 595853312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621624874623872, + "loss": 1.4438, + "theoretical_loss": 3.8711115513912224, + "tokens_seen": 595918848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621464393179538, + "loss": 1.5851, + "theoretical_loss": 3.8710674052667766, + "tokens_seen": 595984384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621303911735206, + "loss": 1.5015, + "theoretical_loss": 3.8710232653555554, + "tokens_seen": 596049920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006621143430290872, + "loss": 1.5851, + "theoretical_loss": 3.870979131656001, + "tokens_seen": 596115456 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662098294884654, + "loss": 1.4908, + "theoretical_loss": 3.870935004166557, + "tokens_seen": 596180992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620822467402206, + "loss": 1.4731, + "theoretical_loss": 3.870890882885667, + "tokens_seen": 596246528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620661985957874, + "loss": 1.5147, + "theoretical_loss": 3.8708467678117744, + "tokens_seen": 596312064 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 227080, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5281542539596558, + "objective/train/theoretical_loss": 3.870802658943324, + "objective/train/tokens_used": 616837600, + "theoretical_loss": 3.870802658943324, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.000662050150451354, + "loss": 1.5282, + "theoretical_loss": 3.870802658943324, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620341023069208, + "loss": 1.4594, + "theoretical_loss": 3.8707585562787625, + "tokens_seen": 596443136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620180541624874, + "loss": 1.4781, + "theoretical_loss": 3.8707144598165346, + "tokens_seen": 596508672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006620020060180542, + "loss": 1.5619, + "theoretical_loss": 3.8706703695550875, + "tokens_seen": 596574208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619859578736208, + "loss": 1.5506, + "theoretical_loss": 3.8706262854928672, + "tokens_seen": 596639744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619699097291876, + "loss": 1.4662, + "theoretical_loss": 3.8705822076283223, + "tokens_seen": 596705280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619538615847542, + "loss": 1.5435, + "theoretical_loss": 3.870538135959901, + "tokens_seen": 596770816 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661937813440321, + "loss": 1.4336, + "theoretical_loss": 3.8704940704860507, + "tokens_seen": 596836352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619217652958876, + "loss": 1.4817, + "theoretical_loss": 3.8704500112052216, + "tokens_seen": 596901888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006619057171514544, + "loss": 1.5595, + "theoretical_loss": 3.870405958115864, + "tokens_seen": 596967424 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661889669007021, + "loss": 1.5231, + "theoretical_loss": 3.870361911216427, + "tokens_seen": 597032960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006618736208625878, + "loss": 1.5683, + "theoretical_loss": 3.8703178705053625, + "tokens_seen": 597098496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006618575727181545, + "loss": 1.5525, + "theoretical_loss": 3.8702738359811217, + "tokens_seen": 597164032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006618415245737212, + "loss": 1.4554, + "theoretical_loss": 3.8702298076421564, + "tokens_seen": 597229568 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661825476429288, + "loss": 1.417, + "theoretical_loss": 3.87018578548692, + "tokens_seen": 597295104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006618094282848546, + "loss": 1.52, + "theoretical_loss": 3.8701417695138645, + "tokens_seen": 597360640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617933801404213, + "loss": 1.5164, + "theoretical_loss": 3.8700977597214443, + "tokens_seen": 597426176 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661777331995988, + "loss": 1.4719, + "theoretical_loss": 3.870053756108114, + "tokens_seen": 597491712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617612838515548, + "loss": 1.5466, + "theoretical_loss": 3.8700097586723277, + "tokens_seen": 597557248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617452357071214, + "loss": 1.4743, + "theoretical_loss": 3.869965767412541, + "tokens_seen": 597622784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617291875626882, + "loss": 1.593, + "theoretical_loss": 3.8699217823272103, + "tokens_seen": 597688320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006617131394182548, + "loss": 1.5358, + "theoretical_loss": 3.8698778034147914, + "tokens_seen": 597753856 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616970912738216, + "loss": 1.4686, + "theoretical_loss": 3.869833830673742, + "tokens_seen": 597819392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616810431293882, + "loss": 1.4634, + "theoretical_loss": 3.8697898641025192, + "tokens_seen": 597884928 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661664994984955, + "loss": 1.5701, + "theoretical_loss": 3.8697459036995814, + "tokens_seen": 597950464 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616489468405216, + "loss": 1.5299, + "theoretical_loss": 3.869701949463387, + "tokens_seen": 598016000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616328986960884, + "loss": 1.3448, + "theoretical_loss": 3.869658001392396, + "tokens_seen": 598081536 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661616850551655, + "loss": 1.4215, + "theoretical_loss": 3.869614059485068, + "tokens_seen": 598147072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006616008024072218, + "loss": 1.4182, + "theoretical_loss": 3.8695701237398623, + "tokens_seen": 598212608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615847542627884, + "loss": 1.3361, + "theoretical_loss": 3.869526194155241, + "tokens_seen": 598278144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615687061183552, + "loss": 1.4508, + "theoretical_loss": 3.869482270729665, + "tokens_seen": 598343680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615526579739218, + "loss": 1.4183, + "theoretical_loss": 3.8694383534615966, + "tokens_seen": 598409216 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615366098294886, + "loss": 1.5407, + "theoretical_loss": 3.8693944423494986, + "tokens_seen": 598474752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006615205616850552, + "loss": 1.544, + "theoretical_loss": 3.869350537391834, + "tokens_seen": 598540288 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661504513540622, + "loss": 1.4599, + "theoretical_loss": 3.869306638587066, + "tokens_seen": 598605824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614884653961886, + "loss": 1.5609, + "theoretical_loss": 3.8692627459336597, + "tokens_seen": 598671360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614724172517554, + "loss": 1.5291, + "theoretical_loss": 3.8692188594300787, + "tokens_seen": 598736896 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661456369107322, + "loss": 1.543, + "theoretical_loss": 3.8691749790747894, + "tokens_seen": 598802432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614403209628888, + "loss": 1.5152, + "theoretical_loss": 3.8691311048662573, + "tokens_seen": 598867968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614242728184554, + "loss": 1.581, + "theoretical_loss": 3.869087236802949, + "tokens_seen": 598933504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006614082246740222, + "loss": 1.4561, + "theoretical_loss": 3.869043374883331, + "tokens_seen": 598999040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613921765295888, + "loss": 1.4824, + "theoretical_loss": 3.8689995191058717, + "tokens_seen": 599064576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613761283851556, + "loss": 1.4596, + "theoretical_loss": 3.868955669469039, + "tokens_seen": 599130112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613600802407222, + "loss": 1.4913, + "theoretical_loss": 3.868911825971301, + "tokens_seen": 599195648 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661344032096289, + "loss": 1.4978, + "theoretical_loss": 3.868867988611127, + "tokens_seen": 599261184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613279839518556, + "loss": 1.4524, + "theoretical_loss": 3.868824157386987, + "tokens_seen": 599326720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006613119358074224, + "loss": 1.4661, + "theoretical_loss": 3.8687803322973506, + "tokens_seen": 599392256 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661295887662989, + "loss": 1.506, + "theoretical_loss": 3.86873651334069, + "tokens_seen": 599457792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612798395185558, + "loss": 1.5035, + "theoretical_loss": 3.8686927005154756, + "tokens_seen": 599523328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612637913741224, + "loss": 1.5931, + "theoretical_loss": 3.8686488938201795, + "tokens_seen": 599588864 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612477432296892, + "loss": 1.4399, + "theoretical_loss": 3.868605093253274, + "tokens_seen": 599654400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612316950852558, + "loss": 1.4608, + "theoretical_loss": 3.8685612988132334, + "tokens_seen": 599719936 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006612156469408226, + "loss": 1.626, + "theoretical_loss": 3.8685175104985294, + "tokens_seen": 599785472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611995987963892, + "loss": 1.5903, + "theoretical_loss": 3.868473728307637, + "tokens_seen": 599851008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611835506519559, + "loss": 1.468, + "theoretical_loss": 3.868429952239031, + "tokens_seen": 599916544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611675025075226, + "loss": 1.4763, + "theoretical_loss": 3.8683861822911867, + "tokens_seen": 599982080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611514543630893, + "loss": 1.5109, + "theoretical_loss": 3.8683424184625794, + "tokens_seen": 600047616 + }, + { + "epoch": 0.18, + "learning_rate": 0.000661135406218656, + "loss": 1.4836, + "theoretical_loss": 3.8682986607516856, + "tokens_seen": 600113152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611193580742227, + "loss": 1.5816, + "theoretical_loss": 3.8682549091569824, + "tokens_seen": 600178688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006611033099297894, + "loss": 1.539, + "theoretical_loss": 3.868211163676947, + "tokens_seen": 600244224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610872617853561, + "loss": 1.4748, + "theoretical_loss": 3.868167424310057, + "tokens_seen": 600309760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610712136409228, + "loss": 1.4814, + "theoretical_loss": 3.868123691054792, + "tokens_seen": 600375296 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610551654964895, + "loss": 1.4616, + "theoretical_loss": 3.8680799639096297, + "tokens_seen": 600440832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610391173520561, + "loss": 1.4678, + "theoretical_loss": 3.8680362428730506, + "tokens_seen": 600506368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610230692076229, + "loss": 1.5051, + "theoretical_loss": 3.8679925279435343, + "tokens_seen": 600571904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006610070210631895, + "loss": 1.4959, + "theoretical_loss": 3.8679488191195617, + "tokens_seen": 600637440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609909729187563, + "loss": 1.4156, + "theoretical_loss": 3.8679051163996143, + "tokens_seen": 600702976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609749247743229, + "loss": 1.4571, + "theoretical_loss": 3.867861419782173, + "tokens_seen": 600768512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609588766298897, + "loss": 1.4012, + "theoretical_loss": 3.867817729265721, + "tokens_seen": 600834048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609428284854563, + "loss": 1.5339, + "theoretical_loss": 3.8677740448487414, + "tokens_seen": 600899584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609267803410231, + "loss": 1.4184, + "theoretical_loss": 3.867730366529716, + "tokens_seen": 600965120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006609107321965897, + "loss": 1.5489, + "theoretical_loss": 3.86768669430713, + "tokens_seen": 601030656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608946840521565, + "loss": 1.422, + "theoretical_loss": 3.867643028179468, + "tokens_seen": 601096192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608786359077231, + "loss": 1.5319, + "theoretical_loss": 3.867599368145214, + "tokens_seen": 601161728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608625877632899, + "loss": 1.3483, + "theoretical_loss": 3.8675557142028545, + "tokens_seen": 601227264 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608465396188565, + "loss": 1.588, + "theoretical_loss": 3.8675120663508755, + "tokens_seen": 601292800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608304914744233, + "loss": 1.6159, + "theoretical_loss": 3.867468424587763, + "tokens_seen": 601358336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006608144433299899, + "loss": 1.4923, + "theoretical_loss": 3.867424788912005, + "tokens_seen": 601423872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607983951855567, + "loss": 1.4813, + "theoretical_loss": 3.8673811593220884, + "tokens_seen": 601489408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607823470411233, + "loss": 1.6039, + "theoretical_loss": 3.8673375358165014, + "tokens_seen": 601554944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607662988966901, + "loss": 1.5274, + "theoretical_loss": 3.867293918393734, + "tokens_seen": 601620480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607502507522567, + "loss": 1.5696, + "theoretical_loss": 3.8672503070522746, + "tokens_seen": 601686016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607342026078235, + "loss": 1.4567, + "theoretical_loss": 3.8672067017906135, + "tokens_seen": 601751552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607181544633901, + "loss": 1.5342, + "theoretical_loss": 3.8671631026072406, + "tokens_seen": 601817088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006607021063189569, + "loss": 1.4782, + "theoretical_loss": 3.8671195095006468, + "tokens_seen": 601882624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606860581745235, + "loss": 1.4275, + "theoretical_loss": 3.8670759224693247, + "tokens_seen": 601948160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606700100300903, + "loss": 1.3553, + "theoretical_loss": 3.867032341511765, + "tokens_seen": 602013696 + }, + { + "epoch": 0.18, + "learning_rate": 0.000660653961885657, + "loss": 1.4876, + "theoretical_loss": 3.866988766626461, + "tokens_seen": 602079232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606379137412237, + "loss": 1.4571, + "theoretical_loss": 3.866945197811906, + "tokens_seen": 602144768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606218655967904, + "loss": 1.5518, + "theoretical_loss": 3.866901635066593, + "tokens_seen": 602210304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006606058174523571, + "loss": 1.4272, + "theoretical_loss": 3.8668580783890167, + "tokens_seen": 602275840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605897693079238, + "loss": 1.5456, + "theoretical_loss": 3.866814527777671, + "tokens_seen": 602341376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605737211634905, + "loss": 1.4301, + "theoretical_loss": 3.8667709832310524, + "tokens_seen": 602406912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605576730190572, + "loss": 1.4862, + "theoretical_loss": 3.866727444747656, + "tokens_seen": 602472448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605416248746239, + "loss": 1.4958, + "theoretical_loss": 3.8666839123259784, + "tokens_seen": 602537984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605255767301906, + "loss": 1.5496, + "theoretical_loss": 3.8666403859645158, + "tokens_seen": 602603520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006605095285857573, + "loss": 1.5064, + "theoretical_loss": 3.8665968656617666, + "tokens_seen": 602669056 + }, + { + "epoch": 0.18, + "learning_rate": 0.000660493480441324, + "loss": 1.4862, + "theoretical_loss": 3.8665533514162282, + "tokens_seen": 602734592 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604774322968907, + "loss": 1.4623, + "theoretical_loss": 3.8665098432263987, + "tokens_seen": 602800128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604613841524574, + "loss": 1.4348, + "theoretical_loss": 3.866466341090778, + "tokens_seen": 602865664 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 229402, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4810528755187988, + "objective/train/theoretical_loss": 3.866422845007865, + "objective/train/tokens_used": 623391200, + "theoretical_loss": 3.866422845007865, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604453360080241, + "loss": 1.4811, + "theoretical_loss": 3.866422845007865, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604292878635908, + "loss": 1.4513, + "theoretical_loss": 3.86637935497616, + "tokens_seen": 602996736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006604132397191575, + "loss": 1.5567, + "theoretical_loss": 3.8663358709941633, + "tokens_seen": 603062272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603971915747243, + "loss": 1.4105, + "theoretical_loss": 3.8662923930603768, + "tokens_seen": 603127808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603811434302909, + "loss": 1.5311, + "theoretical_loss": 3.8662489211733018, + "tokens_seen": 603193344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603650952858577, + "loss": 1.4507, + "theoretical_loss": 3.8662054553314404, + "tokens_seen": 603258880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603490471414243, + "loss": 1.4347, + "theoretical_loss": 3.866161995533295, + "tokens_seen": 603324416 + }, + { + "epoch": 0.18, + "learning_rate": 0.000660332998996991, + "loss": 1.3418, + "theoretical_loss": 3.86611854177737, + "tokens_seen": 603389952 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603169508525577, + "loss": 1.4388, + "theoretical_loss": 3.866075094062168, + "tokens_seen": 603455488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006603009027081245, + "loss": 1.4705, + "theoretical_loss": 3.8660316523861944, + "tokens_seen": 603521024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602848545636911, + "loss": 1.5836, + "theoretical_loss": 3.8659882167479527, + "tokens_seen": 603586560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602688064192579, + "loss": 1.4806, + "theoretical_loss": 3.86594478714595, + "tokens_seen": 603652096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602527582748245, + "loss": 1.535, + "theoretical_loss": 3.8659013635786907, + "tokens_seen": 603717632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602367101303913, + "loss": 1.5487, + "theoretical_loss": 3.8658579460446827, + "tokens_seen": 603783168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602206619859579, + "loss": 1.4269, + "theoretical_loss": 3.865814534542432, + "tokens_seen": 603848704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006602046138415247, + "loss": 1.4414, + "theoretical_loss": 3.8657711290704473, + "tokens_seen": 603914240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601885656970913, + "loss": 1.4035, + "theoretical_loss": 3.8657277296272348, + "tokens_seen": 603979776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601725175526581, + "loss": 1.5045, + "theoretical_loss": 3.865684336211305, + "tokens_seen": 604045312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601564694082247, + "loss": 1.5491, + "theoretical_loss": 3.8656409488211656, + "tokens_seen": 604110848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601404212637915, + "loss": 1.427, + "theoretical_loss": 3.8655975674553273, + "tokens_seen": 604176384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601243731193581, + "loss": 1.3722, + "theoretical_loss": 3.8655541921123, + "tokens_seen": 604241920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006601083249749249, + "loss": 1.3949, + "theoretical_loss": 3.8655108227905943, + "tokens_seen": 604307456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600922768304915, + "loss": 1.5123, + "theoretical_loss": 3.8654674594887215, + "tokens_seen": 604372992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600762286860583, + "loss": 1.4374, + "theoretical_loss": 3.8654241022051927, + "tokens_seen": 604438528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600601805416249, + "loss": 1.48, + "theoretical_loss": 3.865380750938522, + "tokens_seen": 604504064 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600441323971917, + "loss": 1.5199, + "theoretical_loss": 3.86533740568722, + "tokens_seen": 604569600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600280842527583, + "loss": 1.4523, + "theoretical_loss": 3.8652940664498026, + "tokens_seen": 604635136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006600120361083251, + "loss": 1.4881, + "theoretical_loss": 3.865250733224781, + "tokens_seen": 604700672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599959879638917, + "loss": 1.5799, + "theoretical_loss": 3.865207406010672, + "tokens_seen": 604766208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599799398194585, + "loss": 1.604, + "theoretical_loss": 3.865164084805989, + "tokens_seen": 604831744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599638916750251, + "loss": 1.432, + "theoretical_loss": 3.865120769609248, + "tokens_seen": 604897280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599478435305919, + "loss": 1.2784, + "theoretical_loss": 3.8650774604189655, + "tokens_seen": 604962816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599317953861585, + "loss": 1.415, + "theoretical_loss": 3.8650341572336573, + "tokens_seen": 605028352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006599157472417253, + "loss": 1.4423, + "theoretical_loss": 3.8649908600518406, + "tokens_seen": 605093888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598996990972919, + "loss": 1.4921, + "theoretical_loss": 3.8649475688720334, + "tokens_seen": 605159424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598836509528587, + "loss": 1.3126, + "theoretical_loss": 3.864904283692754, + "tokens_seen": 605224960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598676028084253, + "loss": 1.4577, + "theoretical_loss": 3.8648610045125196, + "tokens_seen": 605290496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598515546639921, + "loss": 1.4708, + "theoretical_loss": 3.8648177313298513, + "tokens_seen": 605356032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598355065195587, + "loss": 1.4856, + "theoretical_loss": 3.8647744641432675, + "tokens_seen": 605421568 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598194583751255, + "loss": 1.5001, + "theoretical_loss": 3.8647312029512886, + "tokens_seen": 605487104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006598034102306921, + "loss": 1.4058, + "theoretical_loss": 3.864687947752436, + "tokens_seen": 605552640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597873620862588, + "loss": 1.4884, + "theoretical_loss": 3.8646446985452307, + "tokens_seen": 605618176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597713139418255, + "loss": 1.5691, + "theoretical_loss": 3.864601455328194, + "tokens_seen": 605683712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597552657973922, + "loss": 1.469, + "theoretical_loss": 3.864558218099849, + "tokens_seen": 605749248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597392176529589, + "loss": 1.4928, + "theoretical_loss": 3.864514986858718, + "tokens_seen": 605814784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597231695085256, + "loss": 1.5442, + "theoretical_loss": 3.8644717616033244, + "tokens_seen": 605880320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006597071213640923, + "loss": 1.5, + "theoretical_loss": 3.8644285423321927, + "tokens_seen": 605945856 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659691073219659, + "loss": 1.4006, + "theoretical_loss": 3.864385329043847, + "tokens_seen": 606011392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006596750250752257, + "loss": 1.4536, + "theoretical_loss": 3.8643421217368115, + "tokens_seen": 606076928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006596589769307924, + "loss": 1.385, + "theoretical_loss": 3.8642989204096123, + "tokens_seen": 606142464 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659642928786359, + "loss": 1.4934, + "theoretical_loss": 3.8642557250607754, + "tokens_seen": 606208000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006596268806419258, + "loss": 1.5254, + "theoretical_loss": 3.8642125356888277, + "tokens_seen": 606273536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006596108324974924, + "loss": 1.4264, + "theoretical_loss": 3.8641693522922953, + "tokens_seen": 606339072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595947843530592, + "loss": 1.5391, + "theoretical_loss": 3.8641261748697064, + "tokens_seen": 606404608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595787362086258, + "loss": 1.5554, + "theoretical_loss": 3.8640830034195894, + "tokens_seen": 606470144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595626880641926, + "loss": 1.5367, + "theoretical_loss": 3.864039837940472, + "tokens_seen": 606535680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595466399197592, + "loss": 1.5226, + "theoretical_loss": 3.863996678430884, + "tokens_seen": 606601216 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659530591775326, + "loss": 1.4232, + "theoretical_loss": 3.8639535248893555, + "tokens_seen": 606666752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006595145436308926, + "loss": 1.4901, + "theoretical_loss": 3.863910377314415, + "tokens_seen": 606732288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594984954864594, + "loss": 1.4522, + "theoretical_loss": 3.863867235704595, + "tokens_seen": 606797824 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659482447342026, + "loss": 1.4461, + "theoretical_loss": 3.8638241000584257, + "tokens_seen": 606863360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594663991975928, + "loss": 1.4217, + "theoretical_loss": 3.863780970374439, + "tokens_seen": 606928896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594503510531595, + "loss": 1.4885, + "theoretical_loss": 3.863737846651167, + "tokens_seen": 606994432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594343029087262, + "loss": 1.5661, + "theoretical_loss": 3.8636947288871424, + "tokens_seen": 607059968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594182547642929, + "loss": 1.486, + "theoretical_loss": 3.863651617080899, + "tokens_seen": 607125504 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006594022066198596, + "loss": 1.5127, + "theoretical_loss": 3.8636085112309715, + "tokens_seen": 607191040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593861584754263, + "loss": 1.4154, + "theoretical_loss": 3.863565411335892, + "tokens_seen": 607256576 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659370110330993, + "loss": 1.5451, + "theoretical_loss": 3.8635223173941964, + "tokens_seen": 607322112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593540621865597, + "loss": 1.5381, + "theoretical_loss": 3.8634792294044202, + "tokens_seen": 607387648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593380140421264, + "loss": 1.3601, + "theoretical_loss": 3.8634361473650993, + "tokens_seen": 607453184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593219658976931, + "loss": 1.4987, + "theoretical_loss": 3.8633930712747704, + "tokens_seen": 607518720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006593059177532598, + "loss": 1.4245, + "theoretical_loss": 3.8633500011319692, + "tokens_seen": 607584256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592898696088265, + "loss": 1.4196, + "theoretical_loss": 3.8633069369352344, + "tokens_seen": 607649792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592738214643932, + "loss": 1.49, + "theoretical_loss": 3.8632638786831035, + "tokens_seen": 607715328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592577733199599, + "loss": 1.5793, + "theoretical_loss": 3.863220826374115, + "tokens_seen": 607780864 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592417251755266, + "loss": 1.4327, + "theoretical_loss": 3.863177780006808, + "tokens_seen": 607846400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006592256770310933, + "loss": 1.4829, + "theoretical_loss": 3.8631347395797215, + "tokens_seen": 607911936 + }, + { + "epoch": 0.18, + "learning_rate": 0.00065920962888666, + "loss": 1.5589, + "theoretical_loss": 3.8630917050913958, + "tokens_seen": 607977472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591935807422267, + "loss": 1.4632, + "theoretical_loss": 3.863048676540372, + "tokens_seen": 608043008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591775325977934, + "loss": 1.5037, + "theoretical_loss": 3.8630056539251902, + "tokens_seen": 608108544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591614844533601, + "loss": 1.5156, + "theoretical_loss": 3.862962637244393, + "tokens_seen": 608174080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591454363089268, + "loss": 1.4229, + "theoretical_loss": 3.862919626496521, + "tokens_seen": 608239616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591293881644935, + "loss": 1.4636, + "theoretical_loss": 3.862876621680119, + "tokens_seen": 608305152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006591133400200602, + "loss": 1.5377, + "theoretical_loss": 3.862833622793728, + "tokens_seen": 608370688 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659097291875627, + "loss": 1.4708, + "theoretical_loss": 3.8627906298358923, + "tokens_seen": 608436224 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590812437311936, + "loss": 1.4673, + "theoretical_loss": 3.862747642805157, + "tokens_seen": 608501760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590651955867603, + "loss": 1.4817, + "theoretical_loss": 3.8627046617000653, + "tokens_seen": 608567296 + }, + { + "epoch": 0.18, + "learning_rate": 0.000659049147442327, + "loss": 1.4881, + "theoretical_loss": 3.862661686519163, + "tokens_seen": 608632832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590330992978938, + "loss": 1.5743, + "theoretical_loss": 3.862618717260996, + "tokens_seen": 608698368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590170511534604, + "loss": 1.5369, + "theoretical_loss": 3.8625757539241103, + "tokens_seen": 608763904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006590010030090272, + "loss": 1.4867, + "theoretical_loss": 3.862532796507053, + "tokens_seen": 608829440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589849548645938, + "loss": 1.5357, + "theoretical_loss": 3.8624898450083704, + "tokens_seen": 608894976 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589689067201606, + "loss": 1.4585, + "theoretical_loss": 3.862446899426611, + "tokens_seen": 608960512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589528585757272, + "loss": 1.5151, + "theoretical_loss": 3.8624039597603232, + "tokens_seen": 609026048 + }, + { + "epoch": 0.18, + "learning_rate": 0.000658936810431294, + "loss": 1.4925, + "theoretical_loss": 3.862361026008055, + "tokens_seen": 609091584 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589207622868606, + "loss": 1.5731, + "theoretical_loss": 3.8623180981683563, + "tokens_seen": 609157120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006589047141424274, + "loss": 1.457, + "theoretical_loss": 3.8622751762397765, + "tokens_seen": 609222656 + }, + { + "epoch": 0.18, + "learning_rate": 0.000658888665997994, + "loss": 1.4431, + "theoretical_loss": 3.862232260220866, + "tokens_seen": 609288192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588726178535608, + "loss": 1.5381, + "theoretical_loss": 3.8621893501101763, + "tokens_seen": 609353728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588565697091274, + "loss": 1.5019, + "theoretical_loss": 3.8621464459062573, + "tokens_seen": 609419264 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 231726, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3975192308425903, + "objective/train/theoretical_loss": 3.8621035476076617, + "objective/train/tokens_used": 629944800, + "theoretical_loss": 3.8621035476076617, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588405215646942, + "loss": 1.3975, + "theoretical_loss": 3.8621035476076617, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588244734202608, + "loss": 1.4884, + "theoretical_loss": 3.862060655212942, + "tokens_seen": 609550336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006588084252758276, + "loss": 1.427, + "theoretical_loss": 3.8620177687206514, + "tokens_seen": 609615872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006587923771313942, + "loss": 1.4763, + "theoretical_loss": 3.861974888129342, + "tokens_seen": 609681408 + }, + { + "epoch": 0.18, + "learning_rate": 0.000658776328986961, + "loss": 1.6136, + "theoretical_loss": 3.8619320134375688, + "tokens_seen": 609746944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006587602808425276, + "loss": 1.4716, + "theoretical_loss": 3.861889144643885, + "tokens_seen": 609812480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006587442326980944, + "loss": 1.4962, + "theoretical_loss": 3.861846281746847, + "tokens_seen": 609878016 + }, + { + "epoch": 0.18, + "learning_rate": 0.000658728184553661, + "loss": 1.5667, + "theoretical_loss": 3.861803424745009, + "tokens_seen": 609943552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006587121364092278, + "loss": 1.4759, + "theoretical_loss": 3.8617605736369276, + "tokens_seen": 610009088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586960882647944, + "loss": 1.3991, + "theoretical_loss": 3.8617177284211595, + "tokens_seen": 610074624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586800401203612, + "loss": 1.5148, + "theoretical_loss": 3.8616748890962604, + "tokens_seen": 610140160 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586639919759278, + "loss": 1.6696, + "theoretical_loss": 3.8616320556607886, + "tokens_seen": 610205696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586479438314946, + "loss": 1.4129, + "theoretical_loss": 3.861589228113302, + "tokens_seen": 610271232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006586318956870612, + "loss": 1.5109, + "theoretical_loss": 3.8615464064523586, + "tokens_seen": 610336768 + }, + { + "epoch": 0.18, + "learning_rate": 0.000658615847542628, + "loss": 1.5367, + "theoretical_loss": 3.861503590676518, + "tokens_seen": 610402304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0006585997993981946, + "loss": 1.4632, + "theoretical_loss": 3.861460780784339, + "tokens_seen": 610467840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585837512537614, + "loss": 1.4715, + "theoretical_loss": 3.8614179767743826, + "tokens_seen": 610533376 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658567703109328, + "loss": 1.5612, + "theoretical_loss": 3.861375178645208, + "tokens_seen": 610598912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585516549648948, + "loss": 1.5919, + "theoretical_loss": 3.861332386395377, + "tokens_seen": 610664448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585356068204614, + "loss": 1.445, + "theoretical_loss": 3.8612896000234516, + "tokens_seen": 610729984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585195586760282, + "loss": 1.4747, + "theoretical_loss": 3.861246819527992, + "tokens_seen": 610795520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006585035105315948, + "loss": 1.4545, + "theoretical_loss": 3.861204044907563, + "tokens_seen": 610861056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006584874623871616, + "loss": 1.4335, + "theoretical_loss": 3.8611612761607255, + "tokens_seen": 610926592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006584714142427282, + "loss": 1.435, + "theoretical_loss": 3.861118513286044, + "tokens_seen": 610992128 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658455366098295, + "loss": 1.5239, + "theoretical_loss": 3.8610757562820828, + "tokens_seen": 611057664 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006584393179538616, + "loss": 1.5572, + "theoretical_loss": 3.861033005147406, + "tokens_seen": 611123200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006584232698094284, + "loss": 1.3909, + "theoretical_loss": 3.8609902598805785, + "tokens_seen": 611188736 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658407221664995, + "loss": 1.4564, + "theoretical_loss": 3.860947520480167, + "tokens_seen": 611254272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583911735205618, + "loss": 1.4265, + "theoretical_loss": 3.8609047869447357, + "tokens_seen": 611319808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583751253761284, + "loss": 1.5361, + "theoretical_loss": 3.8608620592728524, + "tokens_seen": 611385344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583590772316951, + "loss": 1.5215, + "theoretical_loss": 3.860819337463084, + "tokens_seen": 611450880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583430290872618, + "loss": 1.4732, + "theoretical_loss": 3.860776621513998, + "tokens_seen": 611516416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583269809428285, + "loss": 1.5267, + "theoretical_loss": 3.8607339114241617, + "tokens_seen": 611581952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006583109327983952, + "loss": 1.4262, + "theoretical_loss": 3.860691207192145, + "tokens_seen": 611647488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582948846539619, + "loss": 1.5706, + "theoretical_loss": 3.860648508816516, + "tokens_seen": 611713024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582788365095286, + "loss": 1.4578, + "theoretical_loss": 3.8606058162958443, + "tokens_seen": 611778560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582627883650953, + "loss": 1.5504, + "theoretical_loss": 3.8605631296287006, + "tokens_seen": 611844096 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658246740220662, + "loss": 1.4197, + "theoretical_loss": 3.860520448813655, + "tokens_seen": 611909632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582306920762287, + "loss": 1.5413, + "theoretical_loss": 3.8604777738492784, + "tokens_seen": 611975168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006582146439317954, + "loss": 1.5059, + "theoretical_loss": 3.860435104734143, + "tokens_seen": 612040704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581985957873621, + "loss": 1.4627, + "theoretical_loss": 3.86039244146682, + "tokens_seen": 612106240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581825476429288, + "loss": 1.4236, + "theoretical_loss": 3.860349784045883, + "tokens_seen": 612171776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581664994984955, + "loss": 1.585, + "theoretical_loss": 3.8603071324699045, + "tokens_seen": 612237312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581504513540622, + "loss": 1.3829, + "theoretical_loss": 3.8602644867374583, + "tokens_seen": 612302848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581344032096289, + "loss": 1.4311, + "theoretical_loss": 3.860221846847118, + "tokens_seen": 612368384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581183550651956, + "loss": 1.4248, + "theoretical_loss": 3.8601792127974583, + "tokens_seen": 612433920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006581023069207623, + "loss": 1.3677, + "theoretical_loss": 3.860136584587055, + "tokens_seen": 612499456 + }, + { + "epoch": 0.19, + "learning_rate": 0.000658086258776329, + "loss": 1.4529, + "theoretical_loss": 3.860093962214483, + "tokens_seen": 612564992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580702106318957, + "loss": 1.4504, + "theoretical_loss": 3.860051345678319, + "tokens_seen": 612630528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580541624874624, + "loss": 1.4847, + "theoretical_loss": 3.860008734977139, + "tokens_seen": 612696064 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580381143430291, + "loss": 1.4682, + "theoretical_loss": 3.8599661301095196, + "tokens_seen": 612761600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580220661985958, + "loss": 1.4137, + "theoretical_loss": 3.85992353107404, + "tokens_seen": 612827136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006580060180541625, + "loss": 1.4064, + "theoretical_loss": 3.8598809378692764, + "tokens_seen": 612892672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579899699097292, + "loss": 1.3893, + "theoretical_loss": 3.859838350493809, + "tokens_seen": 612958208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579739217652959, + "loss": 1.6019, + "theoretical_loss": 3.8597957689462157, + "tokens_seen": 613023744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579578736208626, + "loss": 1.516, + "theoretical_loss": 3.859753193225077, + "tokens_seen": 613089280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579418254764293, + "loss": 1.3498, + "theoretical_loss": 3.859710623328972, + "tokens_seen": 613154816 + }, + { + "epoch": 0.19, + "learning_rate": 0.000657925777331996, + "loss": 1.5548, + "theoretical_loss": 3.8596680592564816, + "tokens_seen": 613220352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006579097291875627, + "loss": 1.4885, + "theoretical_loss": 3.859625501006188, + "tokens_seen": 613285888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578936810431294, + "loss": 1.5065, + "theoretical_loss": 3.859582948576671, + "tokens_seen": 613351424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578776328986961, + "loss": 1.4128, + "theoretical_loss": 3.8595404019665143, + "tokens_seen": 613416960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578615847542628, + "loss": 1.5617, + "theoretical_loss": 3.859497861174299, + "tokens_seen": 613482496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578455366098295, + "loss": 1.5243, + "theoretical_loss": 3.859455326198609, + "tokens_seen": 613548032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578294884653962, + "loss": 1.4311, + "theoretical_loss": 3.8594127970380274, + "tokens_seen": 613613568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006578134403209629, + "loss": 1.4152, + "theoretical_loss": 3.859370273691139, + "tokens_seen": 613679104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577973921765296, + "loss": 1.4637, + "theoretical_loss": 3.8593277561565276, + "tokens_seen": 613744640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577813440320963, + "loss": 1.4941, + "theoretical_loss": 3.8592852444327788, + "tokens_seen": 613810176 + }, + { + "epoch": 0.19, + "learning_rate": 0.000657765295887663, + "loss": 1.5828, + "theoretical_loss": 3.8592427385184775, + "tokens_seen": 613875712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577492477432297, + "loss": 1.545, + "theoretical_loss": 3.85920023841221, + "tokens_seen": 613941248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577331995987964, + "loss": 1.499, + "theoretical_loss": 3.859157744112563, + "tokens_seen": 614006784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577171514543631, + "loss": 1.5473, + "theoretical_loss": 3.859115255618123, + "tokens_seen": 614072320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006577011033099298, + "loss": 1.385, + "theoretical_loss": 3.8590727729274787, + "tokens_seen": 614137856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576850551654965, + "loss": 1.4685, + "theoretical_loss": 3.859030296039217, + "tokens_seen": 614203392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576690070210633, + "loss": 1.41, + "theoretical_loss": 3.8589878249519263, + "tokens_seen": 614268928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576529588766299, + "loss": 1.5613, + "theoretical_loss": 3.8589453596641965, + "tokens_seen": 614334464 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576369107321967, + "loss": 1.4799, + "theoretical_loss": 3.858902900174617, + "tokens_seen": 614400000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006576208625877633, + "loss": 1.5341, + "theoretical_loss": 3.858860446481777, + "tokens_seen": 614465536 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065760481444333, + "loss": 1.4246, + "theoretical_loss": 3.8588179985842674, + "tokens_seen": 614531072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575887662988967, + "loss": 1.5047, + "theoretical_loss": 3.8587755564806794, + "tokens_seen": 614596608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575727181544635, + "loss": 1.5207, + "theoretical_loss": 3.8587331201696045, + "tokens_seen": 614662144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575566700100301, + "loss": 1.4442, + "theoretical_loss": 3.8586906896496336, + "tokens_seen": 614727680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575406218655969, + "loss": 1.4993, + "theoretical_loss": 3.8586482649193607, + "tokens_seen": 614793216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575245737211635, + "loss": 1.462, + "theoretical_loss": 3.8586058459773778, + "tokens_seen": 614858752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006575085255767303, + "loss": 1.4891, + "theoretical_loss": 3.8585634328222778, + "tokens_seen": 614924288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574924774322969, + "loss": 1.4033, + "theoretical_loss": 3.8585210254526565, + "tokens_seen": 614989824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574764292878637, + "loss": 1.4692, + "theoretical_loss": 3.8584786238671063, + "tokens_seen": 615055360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574603811434303, + "loss": 1.4669, + "theoretical_loss": 3.8584362280642233, + "tokens_seen": 615120896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574443329989971, + "loss": 1.5183, + "theoretical_loss": 3.858393838042603, + "tokens_seen": 615186432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574282848545637, + "loss": 1.4978, + "theoretical_loss": 3.8583514538008403, + "tokens_seen": 615251968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006574122367101305, + "loss": 1.4938, + "theoretical_loss": 3.858309075337532, + "tokens_seen": 615317504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573961885656971, + "loss": 1.5426, + "theoretical_loss": 3.8582667026512754, + "tokens_seen": 615383040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573801404212639, + "loss": 1.5026, + "theoretical_loss": 3.858224335740668, + "tokens_seen": 615448576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573640922768305, + "loss": 1.3606, + "theoretical_loss": 3.8581819746043067, + "tokens_seen": 615514112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573480441323973, + "loss": 1.5304, + "theoretical_loss": 3.8581396192407906, + "tokens_seen": 615579648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573319959879639, + "loss": 1.4867, + "theoretical_loss": 3.8580972696487184, + "tokens_seen": 615645184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006573159478435307, + "loss": 1.4308, + "theoretical_loss": 3.858054925826689, + "tokens_seen": 615710720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572998996990973, + "loss": 1.4457, + "theoretical_loss": 3.8580125877733025, + "tokens_seen": 615776256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572838515546641, + "loss": 1.3745, + "theoretical_loss": 3.8579702554871593, + "tokens_seen": 615841792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572678034102307, + "loss": 1.4266, + "theoretical_loss": 3.8579279289668604, + "tokens_seen": 615907328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572517552657975, + "loss": 1.5408, + "theoretical_loss": 3.8578856082110065, + "tokens_seen": 615972864 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 233987, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3476626873016357, + "objective/train/theoretical_loss": 3.8578432932182, + "objective/train/tokens_used": 636498400, + "theoretical_loss": 3.8578432932182, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572357071213641, + "loss": 1.3477, + "theoretical_loss": 3.8578432932182, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572196589769309, + "loss": 1.5229, + "theoretical_loss": 3.8578009839870426, + "tokens_seen": 616103936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006572036108324975, + "loss": 1.4509, + "theoretical_loss": 3.8577586805161372, + "tokens_seen": 616169472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571875626880643, + "loss": 1.3571, + "theoretical_loss": 3.857716382804087, + "tokens_seen": 616235008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571715145436309, + "loss": 1.4995, + "theoretical_loss": 3.8576740908494958, + "tokens_seen": 616300544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571554663991977, + "loss": 1.5121, + "theoretical_loss": 3.8576318046509677, + "tokens_seen": 616366080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571394182547643, + "loss": 1.4906, + "theoretical_loss": 3.857589524207108, + "tokens_seen": 616431616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571233701103311, + "loss": 1.43, + "theoretical_loss": 3.8575472495165206, + "tokens_seen": 616497152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006571073219658977, + "loss": 1.3856, + "theoretical_loss": 3.8575049805778123, + "tokens_seen": 616562688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570912738214645, + "loss": 1.445, + "theoretical_loss": 3.8574627173895886, + "tokens_seen": 616628224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570752256770311, + "loss": 1.3733, + "theoretical_loss": 3.8574204599504567, + "tokens_seen": 616693760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570591775325979, + "loss": 1.5245, + "theoretical_loss": 3.857378208259023, + "tokens_seen": 616759296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570431293881645, + "loss": 1.4039, + "theoretical_loss": 3.8573359623138956, + "tokens_seen": 616824832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570270812437313, + "loss": 1.2809, + "theoretical_loss": 3.857293722113683, + "tokens_seen": 616890368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006570110330992979, + "loss": 1.3977, + "theoretical_loss": 3.8572514876569923, + "tokens_seen": 616955904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569949849548647, + "loss": 1.5074, + "theoretical_loss": 3.857209258942434, + "tokens_seen": 617021440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569789368104313, + "loss": 1.4586, + "theoretical_loss": 3.857167035968617, + "tokens_seen": 617086976 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656962888665998, + "loss": 1.4476, + "theoretical_loss": 3.8571248187341522, + "tokens_seen": 617152512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569468405215647, + "loss": 1.3853, + "theoretical_loss": 3.8570826072376487, + "tokens_seen": 617218048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569307923771314, + "loss": 1.4916, + "theoretical_loss": 3.8570404014777178, + "tokens_seen": 617283584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006569147442326981, + "loss": 1.4576, + "theoretical_loss": 3.8569982014529716, + "tokens_seen": 617349120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568986960882648, + "loss": 1.34, + "theoretical_loss": 3.8569560071620224, + "tokens_seen": 617414656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568826479438315, + "loss": 1.4096, + "theoretical_loss": 3.8569138186034815, + "tokens_seen": 617480192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568665997993982, + "loss": 1.4814, + "theoretical_loss": 3.8568716357759625, + "tokens_seen": 617545728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568505516549649, + "loss": 1.4248, + "theoretical_loss": 3.856829458678079, + "tokens_seen": 617611264 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568345035105316, + "loss": 1.5354, + "theoretical_loss": 3.856787287308444, + "tokens_seen": 617676800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006568184553660983, + "loss": 1.4483, + "theoretical_loss": 3.8567451216656723, + "tokens_seen": 617742336 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656802407221665, + "loss": 1.37, + "theoretical_loss": 3.8567029617483795, + "tokens_seen": 617807872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567863590772317, + "loss": 1.4955, + "theoretical_loss": 3.8566608075551803, + "tokens_seen": 617873408 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567703109327984, + "loss": 1.4801, + "theoretical_loss": 3.8566186590846905, + "tokens_seen": 617938944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567542627883651, + "loss": 1.4284, + "theoretical_loss": 3.856576516335526, + "tokens_seen": 618004480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567382146439318, + "loss": 1.4738, + "theoretical_loss": 3.8565343793063045, + "tokens_seen": 618070016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567221664994985, + "loss": 1.3652, + "theoretical_loss": 3.856492247995643, + "tokens_seen": 618135552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006567061183550652, + "loss": 1.4268, + "theoretical_loss": 3.856450122402159, + "tokens_seen": 618201088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566900702106319, + "loss": 1.4744, + "theoretical_loss": 3.8564080025244705, + "tokens_seen": 618266624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566740220661986, + "loss": 1.4557, + "theoretical_loss": 3.8563658883611964, + "tokens_seen": 618332160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566579739217653, + "loss": 1.3793, + "theoretical_loss": 3.856323779910956, + "tokens_seen": 618397696 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656641925777332, + "loss": 1.4128, + "theoretical_loss": 3.856281677172369, + "tokens_seen": 618463232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566258776328987, + "loss": 1.4341, + "theoretical_loss": 3.856239580144056, + "tokens_seen": 618528768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006566098294884654, + "loss": 1.3387, + "theoretical_loss": 3.8561974888246366, + "tokens_seen": 618594304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565937813440321, + "loss": 1.5483, + "theoretical_loss": 3.8561554032127323, + "tokens_seen": 618659840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565777331995988, + "loss": 1.4321, + "theoretical_loss": 3.856113323306965, + "tokens_seen": 618725376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565616850551655, + "loss": 1.3964, + "theoretical_loss": 3.8560712491059563, + "tokens_seen": 618790912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565456369107322, + "loss": 1.2976, + "theoretical_loss": 3.8560291806083296, + "tokens_seen": 618856448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565295887662989, + "loss": 1.4369, + "theoretical_loss": 3.855987117812707, + "tokens_seen": 618921984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006565135406218656, + "loss": 1.5007, + "theoretical_loss": 3.8559450607177124, + "tokens_seen": 618987520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564974924774323, + "loss": 1.4973, + "theoretical_loss": 3.8559030093219695, + "tokens_seen": 619053056 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656481444332999, + "loss": 1.4793, + "theoretical_loss": 3.855860963624103, + "tokens_seen": 619118592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564653961885657, + "loss": 1.5169, + "theoretical_loss": 3.855818923622738, + "tokens_seen": 619184128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564493480441324, + "loss": 1.4591, + "theoretical_loss": 3.855776889316499, + "tokens_seen": 619249664 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564332998996991, + "loss": 1.5005, + "theoretical_loss": 3.855734860704014, + "tokens_seen": 619315200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564172517552658, + "loss": 1.4985, + "theoretical_loss": 3.8556928377839066, + "tokens_seen": 619380736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006564012036108325, + "loss": 1.509, + "theoretical_loss": 3.8556508205548057, + "tokens_seen": 619446272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006563851554663992, + "loss": 1.499, + "theoretical_loss": 3.8556088090153375, + "tokens_seen": 619511808 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656369107321966, + "loss": 1.4283, + "theoretical_loss": 3.8555668031641304, + "tokens_seen": 619577344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006563530591775326, + "loss": 1.4596, + "theoretical_loss": 3.8555248029998124, + "tokens_seen": 619642880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006563370110330993, + "loss": 1.3701, + "theoretical_loss": 3.855482808521012, + "tokens_seen": 619708416 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656320962888666, + "loss": 1.4237, + "theoretical_loss": 3.855440819726359, + "tokens_seen": 619773952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006563049147442327, + "loss": 1.423, + "theoretical_loss": 3.8553988366144827, + "tokens_seen": 619839488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562888665997994, + "loss": 1.5008, + "theoretical_loss": 3.8553568591840133, + "tokens_seen": 619905024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562728184553662, + "loss": 1.4077, + "theoretical_loss": 3.8553148874335816, + "tokens_seen": 619970560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562567703109328, + "loss": 1.4044, + "theoretical_loss": 3.855272921361819, + "tokens_seen": 620036096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562407221664996, + "loss": 1.525, + "theoretical_loss": 3.855230960967356, + "tokens_seen": 620101632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006562246740220662, + "loss": 1.4341, + "theoretical_loss": 3.8551890062488265, + "tokens_seen": 620167168 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656208625877633, + "loss": 1.3847, + "theoretical_loss": 3.855147057204861, + "tokens_seen": 620232704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561925777331996, + "loss": 1.3527, + "theoretical_loss": 3.8551051138340937, + "tokens_seen": 620298240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561765295887664, + "loss": 1.5466, + "theoretical_loss": 3.8550631761351575, + "tokens_seen": 620363776 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656160481444333, + "loss": 1.3631, + "theoretical_loss": 3.8550212441066876, + "tokens_seen": 620429312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561444332998998, + "loss": 1.3913, + "theoretical_loss": 3.8549793177473166, + "tokens_seen": 620494848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561283851554664, + "loss": 1.4873, + "theoretical_loss": 3.8549373970556813, + "tokens_seen": 620560384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006561123370110332, + "loss": 1.4034, + "theoretical_loss": 3.8548954820304155, + "tokens_seen": 620625920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560962888665998, + "loss": 1.4051, + "theoretical_loss": 3.854853572670156, + "tokens_seen": 620691456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560802407221666, + "loss": 1.5519, + "theoretical_loss": 3.8548116689735386, + "tokens_seen": 620756992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560641925777332, + "loss": 1.3682, + "theoretical_loss": 3.8547697709392006, + "tokens_seen": 620822528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560481444333, + "loss": 1.5423, + "theoretical_loss": 3.854727878565779, + "tokens_seen": 620888064 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560320962888666, + "loss": 1.4073, + "theoretical_loss": 3.8546859918519107, + "tokens_seen": 620953600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006560160481444334, + "loss": 1.4303, + "theoretical_loss": 3.8546441107962357, + "tokens_seen": 621019136 + }, + { + "epoch": 0.19, + "learning_rate": 0.000656, + "loss": 1.5123, + "theoretical_loss": 3.854602235397391, + "tokens_seen": 621084672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559839518555668, + "loss": 1.4909, + "theoretical_loss": 3.8545603656540175, + "tokens_seen": 621150208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559679037111334, + "loss": 1.4048, + "theoretical_loss": 3.854518501564753, + "tokens_seen": 621215744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559518555667002, + "loss": 1.4729, + "theoretical_loss": 3.8544766431282387, + "tokens_seen": 621281280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559358074222668, + "loss": 1.4247, + "theoretical_loss": 3.854434790343115, + "tokens_seen": 621346816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559197592778336, + "loss": 1.4424, + "theoretical_loss": 3.8543929432080226, + "tokens_seen": 621412352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006559037111334002, + "loss": 1.4373, + "theoretical_loss": 3.8543511017216034, + "tokens_seen": 621477888 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655887662988967, + "loss": 1.4371, + "theoretical_loss": 3.8543092658825, + "tokens_seen": 621543424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558716148445336, + "loss": 1.474, + "theoretical_loss": 3.8542674356893527, + "tokens_seen": 621608960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558555667001004, + "loss": 1.4411, + "theoretical_loss": 3.8542256111408064, + "tokens_seen": 621674496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558395185556671, + "loss": 1.499, + "theoretical_loss": 3.854183792235504, + "tokens_seen": 621740032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558234704112338, + "loss": 1.4171, + "theoretical_loss": 3.8541419789720894, + "tokens_seen": 621805568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006558074222668005, + "loss": 1.3786, + "theoretical_loss": 3.854100171349206, + "tokens_seen": 621871104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557913741223672, + "loss": 1.5505, + "theoretical_loss": 3.8540583693655, + "tokens_seen": 621936640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557753259779339, + "loss": 1.4827, + "theoretical_loss": 3.854016573019616, + "tokens_seen": 622002176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557592778335006, + "loss": 1.5031, + "theoretical_loss": 3.8539747823101997, + "tokens_seen": 622067712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557432296890673, + "loss": 1.3803, + "theoretical_loss": 3.853932997235897, + "tokens_seen": 622133248 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655727181544634, + "loss": 1.5023, + "theoretical_loss": 3.8538912177953555, + "tokens_seen": 622198784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006557111334002006, + "loss": 1.4838, + "theoretical_loss": 3.8538494439872215, + "tokens_seen": 622264320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556950852557674, + "loss": 1.4595, + "theoretical_loss": 3.8538076758101427, + "tokens_seen": 622329856 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655679037111334, + "loss": 1.4574, + "theoretical_loss": 3.8537659132627677, + "tokens_seen": 622395392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556629889669008, + "loss": 1.4997, + "theoretical_loss": 3.853724156343744, + "tokens_seen": 622460928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556469408224674, + "loss": 1.4373, + "theoretical_loss": 3.853682405051722, + "tokens_seen": 622526464 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 236377, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.464072823524475, + "objective/train/theoretical_loss": 3.8536406593853503, + "objective/train/tokens_used": 643052000, + "theoretical_loss": 3.8536406593853503, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556308926780342, + "loss": 1.4641, + "theoretical_loss": 3.8536406593853503, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006556148445336008, + "loss": 1.4193, + "theoretical_loss": 3.8535989193432787, + "tokens_seen": 622657536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555987963891676, + "loss": 1.4324, + "theoretical_loss": 3.8535571849241577, + "tokens_seen": 622723072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555827482447342, + "loss": 1.4849, + "theoretical_loss": 3.8535154561266385, + "tokens_seen": 622788608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555667001003009, + "loss": 1.4491, + "theoretical_loss": 3.8534737329493725, + "tokens_seen": 622854144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555506519558676, + "loss": 1.3691, + "theoretical_loss": 3.853432015391011, + "tokens_seen": 622919680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555346038114343, + "loss": 1.5258, + "theoretical_loss": 3.8533903034502064, + "tokens_seen": 622985216 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655518555667001, + "loss": 1.4963, + "theoretical_loss": 3.8533485971256125, + "tokens_seen": 623050752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006555025075225677, + "loss": 1.4393, + "theoretical_loss": 3.8533068964158805, + "tokens_seen": 623116288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554864593781344, + "loss": 1.5515, + "theoretical_loss": 3.8532652013196653, + "tokens_seen": 623181824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554704112337011, + "loss": 1.5238, + "theoretical_loss": 3.853223511835621, + "tokens_seen": 623247360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554543630892678, + "loss": 1.3742, + "theoretical_loss": 3.853181827962402, + "tokens_seen": 623312896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554383149448345, + "loss": 1.4825, + "theoretical_loss": 3.853140149698664, + "tokens_seen": 623378432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554222668004012, + "loss": 1.4356, + "theoretical_loss": 3.8530984770430607, + "tokens_seen": 623443968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006554062186559679, + "loss": 1.3851, + "theoretical_loss": 3.85305680999425, + "tokens_seen": 623509504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553901705115346, + "loss": 1.4169, + "theoretical_loss": 3.8530151485508872, + "tokens_seen": 623575040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553741223671013, + "loss": 1.4313, + "theoretical_loss": 3.85297349271163, + "tokens_seen": 623640576 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655358074222668, + "loss": 1.4901, + "theoretical_loss": 3.852931842475135, + "tokens_seen": 623706112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553420260782347, + "loss": 1.403, + "theoretical_loss": 3.8528901978400603, + "tokens_seen": 623771648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553259779338014, + "loss": 1.4506, + "theoretical_loss": 3.852848558805065, + "tokens_seen": 623837184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006553099297893681, + "loss": 1.5516, + "theoretical_loss": 3.8528069253688066, + "tokens_seen": 623902720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552938816449348, + "loss": 1.5217, + "theoretical_loss": 3.852765297529945, + "tokens_seen": 623968256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552778335005015, + "loss": 1.4394, + "theoretical_loss": 3.8527236752871397, + "tokens_seen": 624033792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552617853560682, + "loss": 1.5209, + "theoretical_loss": 3.8526820586390507, + "tokens_seen": 624099328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552457372116349, + "loss": 1.4172, + "theoretical_loss": 3.8526404475843385, + "tokens_seen": 624164864 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552296890672016, + "loss": 1.4668, + "theoretical_loss": 3.8525988421216653, + "tokens_seen": 624230400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006552136409227683, + "loss": 1.4188, + "theoretical_loss": 3.852557242249691, + "tokens_seen": 624295936 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655197592778335, + "loss": 1.4189, + "theoretical_loss": 3.8525156479670786, + "tokens_seen": 624361472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551815446339017, + "loss": 1.4364, + "theoretical_loss": 3.8524740592724904, + "tokens_seen": 624427008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551654964894684, + "loss": 1.5731, + "theoretical_loss": 3.8524324761645894, + "tokens_seen": 624492544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551494483450351, + "loss": 1.4315, + "theoretical_loss": 3.8523908986420383, + "tokens_seen": 624558080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551334002006018, + "loss": 1.4882, + "theoretical_loss": 3.8523493267035014, + "tokens_seen": 624623616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551173520561685, + "loss": 1.4664, + "theoretical_loss": 3.852307760347643, + "tokens_seen": 624689152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006551013039117352, + "loss": 1.4041, + "theoretical_loss": 3.852266199573128, + "tokens_seen": 624754688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550852557673019, + "loss": 1.4822, + "theoretical_loss": 3.852224644378621, + "tokens_seen": 624820224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550692076228686, + "loss": 1.5086, + "theoretical_loss": 3.852183094762789, + "tokens_seen": 624885760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550531594784353, + "loss": 1.4318, + "theoretical_loss": 3.852141550724297, + "tokens_seen": 624951296 + }, + { + "epoch": 0.19, + "learning_rate": 0.000655037111334002, + "loss": 1.484, + "theoretical_loss": 3.852100012261811, + "tokens_seen": 625016832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550210631895687, + "loss": 1.4621, + "theoretical_loss": 3.8520584793739996, + "tokens_seen": 625082368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006550050150451354, + "loss": 1.3767, + "theoretical_loss": 3.852016952059529, + "tokens_seen": 625147904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549889669007021, + "loss": 1.458, + "theoretical_loss": 3.8519754303170677, + "tokens_seen": 625213440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549729187562688, + "loss": 1.6429, + "theoretical_loss": 3.8519339141452846, + "tokens_seen": 625278976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549568706118355, + "loss": 1.438, + "theoretical_loss": 3.8518924035428475, + "tokens_seen": 625344512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549408224674022, + "loss": 1.3489, + "theoretical_loss": 3.8518508985084265, + "tokens_seen": 625410048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549247743229689, + "loss": 1.4193, + "theoretical_loss": 3.8518093990406914, + "tokens_seen": 625475584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006549087261785357, + "loss": 1.4188, + "theoretical_loss": 3.8517679051383116, + "tokens_seen": 625541120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548926780341023, + "loss": 1.3499, + "theoretical_loss": 3.851726416799959, + "tokens_seen": 625606656 + }, + { + "epoch": 0.19, + "learning_rate": 0.000654876629889669, + "loss": 1.4187, + "theoretical_loss": 3.8516849340243042, + "tokens_seen": 625672192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548605817452357, + "loss": 1.4074, + "theoretical_loss": 3.8516434568100184, + "tokens_seen": 625737728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548445336008025, + "loss": 1.4769, + "theoretical_loss": 3.8516019851557743, + "tokens_seen": 625803264 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548284854563691, + "loss": 1.5079, + "theoretical_loss": 3.8515605190602447, + "tokens_seen": 625868800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006548124373119359, + "loss": 1.3993, + "theoretical_loss": 3.8515190585221015, + "tokens_seen": 625934336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547963891675025, + "loss": 1.4678, + "theoretical_loss": 3.851477603540019, + "tokens_seen": 625999872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547803410230693, + "loss": 1.5355, + "theoretical_loss": 3.8514361541126703, + "tokens_seen": 626065408 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547642928786359, + "loss": 1.3893, + "theoretical_loss": 3.85139471023873, + "tokens_seen": 626130944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547482447342027, + "loss": 1.3727, + "theoretical_loss": 3.8513532719168744, + "tokens_seen": 626196480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547321965897693, + "loss": 1.4838, + "theoretical_loss": 3.851311839145777, + "tokens_seen": 626262016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547161484453361, + "loss": 1.538, + "theoretical_loss": 3.851270411924114, + "tokens_seen": 626327552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006547001003009027, + "loss": 1.4383, + "theoretical_loss": 3.8512289902505605, + "tokens_seen": 626393088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546840521564695, + "loss": 1.4609, + "theoretical_loss": 3.8511875741237955, + "tokens_seen": 626458624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546680040120361, + "loss": 1.4424, + "theoretical_loss": 3.851146163542495, + "tokens_seen": 626524160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546519558676029, + "loss": 1.3712, + "theoretical_loss": 3.8511047585053353, + "tokens_seen": 626589696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546359077231696, + "loss": 1.4463, + "theoretical_loss": 3.851063359010996, + "tokens_seen": 626655232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006546198595787363, + "loss": 1.5962, + "theoretical_loss": 3.851021965058155, + "tokens_seen": 626720768 + }, + { + "epoch": 0.19, + "learning_rate": 0.000654603811434303, + "loss": 1.5317, + "theoretical_loss": 3.850980576645491, + "tokens_seen": 626786304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545877632898697, + "loss": 1.4761, + "theoretical_loss": 3.8509391937716835, + "tokens_seen": 626851840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545717151454364, + "loss": 1.336, + "theoretical_loss": 3.8508978164354124, + "tokens_seen": 626917376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545556670010031, + "loss": 1.5293, + "theoretical_loss": 3.8508564446353573, + "tokens_seen": 626982912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545396188565698, + "loss": 1.4883, + "theoretical_loss": 3.8508150783702, + "tokens_seen": 627048448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545235707121365, + "loss": 1.3518, + "theoretical_loss": 3.850773717638621, + "tokens_seen": 627113984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006545075225677032, + "loss": 1.4156, + "theoretical_loss": 3.850732362439302, + "tokens_seen": 627179520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544914744232699, + "loss": 1.3947, + "theoretical_loss": 3.850691012770925, + "tokens_seen": 627245056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544754262788366, + "loss": 1.4855, + "theoretical_loss": 3.8506496686321725, + "tokens_seen": 627310592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544593781344033, + "loss": 1.4213, + "theoretical_loss": 3.8506083300217275, + "tokens_seen": 627376128 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065444332998997, + "loss": 1.5636, + "theoretical_loss": 3.850566996938274, + "tokens_seen": 627441664 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544272818455367, + "loss": 1.4927, + "theoretical_loss": 3.8505256693804943, + "tokens_seen": 627507200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006544112337011034, + "loss": 1.4701, + "theoretical_loss": 3.8504843473470745, + "tokens_seen": 627572736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543951855566701, + "loss": 1.4471, + "theoretical_loss": 3.850443030836698, + "tokens_seen": 627638272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543791374122368, + "loss": 1.3399, + "theoretical_loss": 3.850401719848051, + "tokens_seen": 627703808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543630892678035, + "loss": 1.435, + "theoretical_loss": 3.8503604143798187, + "tokens_seen": 627769344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543470411233702, + "loss": 1.5191, + "theoretical_loss": 3.850319114430687, + "tokens_seen": 627834880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543309929789369, + "loss": 1.4382, + "theoretical_loss": 3.8502778199993433, + "tokens_seen": 627900416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006543149448345035, + "loss": 1.4974, + "theoretical_loss": 3.850236531084473, + "tokens_seen": 627965952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542988966900703, + "loss": 1.4842, + "theoretical_loss": 3.8501952476847654, + "tokens_seen": 628031488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542828485456369, + "loss": 1.4614, + "theoretical_loss": 3.8501539697989076, + "tokens_seen": 628097024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542668004012037, + "loss": 1.3796, + "theoretical_loss": 3.8501126974255877, + "tokens_seen": 628162560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542507522567703, + "loss": 1.4269, + "theoretical_loss": 3.850071430563495, + "tokens_seen": 628228096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542347041123371, + "loss": 1.5455, + "theoretical_loss": 3.850030169211319, + "tokens_seen": 628293632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542186559679037, + "loss": 1.4604, + "theoretical_loss": 3.8499889133677483, + "tokens_seen": 628359168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006542026078234705, + "loss": 1.4758, + "theoretical_loss": 3.8499476630314735, + "tokens_seen": 628424704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541865596790371, + "loss": 1.4967, + "theoretical_loss": 3.849906418201186, + "tokens_seen": 628490240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541705115346038, + "loss": 1.5274, + "theoretical_loss": 3.849865178875576, + "tokens_seen": 628555776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541544633901705, + "loss": 1.454, + "theoretical_loss": 3.8498239450533354, + "tokens_seen": 628621312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541384152457372, + "loss": 1.4732, + "theoretical_loss": 3.8497827167331558, + "tokens_seen": 628686848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541223671013039, + "loss": 1.4902, + "theoretical_loss": 3.84974149391373, + "tokens_seen": 628752384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006541063189568706, + "loss": 1.5017, + "theoretical_loss": 3.849700276593751, + "tokens_seen": 628817920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540902708124373, + "loss": 1.4148, + "theoretical_loss": 3.8496590647719113, + "tokens_seen": 628883456 + }, + { + "epoch": 0.19, + "learning_rate": 0.000654074222668004, + "loss": 1.4403, + "theoretical_loss": 3.849617858446905, + "tokens_seen": 628948992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540581745235707, + "loss": 1.4523, + "theoretical_loss": 3.8495766576174266, + "tokens_seen": 629014528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540421263791374, + "loss": 1.4045, + "theoretical_loss": 3.8495354622821703, + "tokens_seen": 629080064 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 239008, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5397480726242065, + "objective/train/theoretical_loss": 3.8494942724398316, + "objective/train/tokens_used": 649605600, + "theoretical_loss": 3.8494942724398316, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540260782347041, + "loss": 1.5397, + "theoretical_loss": 3.8494942724398316, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006540100300902708, + "loss": 1.4836, + "theoretical_loss": 3.8494530880891054, + "tokens_seen": 629211136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539939819458375, + "loss": 1.5317, + "theoretical_loss": 3.8494119092286887, + "tokens_seen": 629276672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539779338014042, + "loss": 1.4168, + "theoretical_loss": 3.849370735857277, + "tokens_seen": 629342208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539618856569709, + "loss": 1.4724, + "theoretical_loss": 3.849329567973567, + "tokens_seen": 629407744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539458375125376, + "loss": 1.5542, + "theoretical_loss": 3.8492884055762566, + "tokens_seen": 629473280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006539297893681043, + "loss": 1.4512, + "theoretical_loss": 3.849247248664043, + "tokens_seen": 629538816 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653913741223671, + "loss": 1.5047, + "theoretical_loss": 3.8492060972356255, + "tokens_seen": 629604352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538976930792377, + "loss": 1.4959, + "theoretical_loss": 3.8491649512897017, + "tokens_seen": 629669888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538816449348044, + "loss": 1.4179, + "theoretical_loss": 3.849123810824971, + "tokens_seen": 629735424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538655967903711, + "loss": 1.5006, + "theoretical_loss": 3.8490826758401324, + "tokens_seen": 629800960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538495486459378, + "loss": 1.4271, + "theoretical_loss": 3.849041546333887, + "tokens_seen": 629866496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538335005015045, + "loss": 1.5192, + "theoretical_loss": 3.8490004223049343, + "tokens_seen": 629932032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538174523570712, + "loss": 1.4858, + "theoretical_loss": 3.8489593037519754, + "tokens_seen": 629997568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006538014042126379, + "loss": 1.4999, + "theoretical_loss": 3.8489181906737118, + "tokens_seen": 630063104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537853560682046, + "loss": 1.4046, + "theoretical_loss": 3.848877083068845, + "tokens_seen": 630128640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537693079237713, + "loss": 1.4861, + "theoretical_loss": 3.8488359809360766, + "tokens_seen": 630194176 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653753259779338, + "loss": 1.4462, + "theoretical_loss": 3.8487948842741107, + "tokens_seen": 630259712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537372116349047, + "loss": 1.5446, + "theoretical_loss": 3.8487537930816487, + "tokens_seen": 630325248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537211634904714, + "loss": 1.411, + "theoretical_loss": 3.8487127073573957, + "tokens_seen": 630390784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006537051153460381, + "loss": 1.4352, + "theoretical_loss": 3.8486716271000545, + "tokens_seen": 630456320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536890672016048, + "loss": 1.5108, + "theoretical_loss": 3.8486305523083297, + "tokens_seen": 630521856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536730190571715, + "loss": 1.478, + "theoretical_loss": 3.848589482980927, + "tokens_seen": 630587392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536569709127382, + "loss": 1.5343, + "theoretical_loss": 3.8485484191165504, + "tokens_seen": 630652928 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653640922768305, + "loss": 1.4571, + "theoretical_loss": 3.8485073607139064, + "tokens_seen": 630718464 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536248746238716, + "loss": 1.513, + "theoretical_loss": 3.8484663077717007, + "tokens_seen": 630784000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006536088264794383, + "loss": 1.4518, + "theoretical_loss": 3.84842526028864, + "tokens_seen": 630849536 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653592778335005, + "loss": 1.4537, + "theoretical_loss": 3.8483842182634325, + "tokens_seen": 630915072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535767301905717, + "loss": 1.5103, + "theoretical_loss": 3.848343181694784, + "tokens_seen": 630980608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535606820461384, + "loss": 1.5592, + "theoretical_loss": 3.8483021505814032, + "tokens_seen": 631046144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535446339017052, + "loss": 1.5209, + "theoretical_loss": 3.8482611249219985, + "tokens_seen": 631111680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535285857572718, + "loss": 1.4993, + "theoretical_loss": 3.8482201047152786, + "tokens_seen": 631177216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006535125376128386, + "loss": 1.548, + "theoretical_loss": 3.8481790899599524, + "tokens_seen": 631242752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534964894684052, + "loss": 1.3678, + "theoretical_loss": 3.84813808065473, + "tokens_seen": 631308288 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653480441323972, + "loss": 1.5076, + "theoretical_loss": 3.8480970767983216, + "tokens_seen": 631373824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534643931795386, + "loss": 1.525, + "theoretical_loss": 3.8480560783894373, + "tokens_seen": 631439360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534483450351054, + "loss": 1.4616, + "theoretical_loss": 3.8480150854267885, + "tokens_seen": 631504896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534322968906721, + "loss": 1.513, + "theoretical_loss": 3.847974097909087, + "tokens_seen": 631570432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534162487462388, + "loss": 1.3295, + "theoretical_loss": 3.8479331158350436, + "tokens_seen": 631635968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006534002006018055, + "loss": 1.3569, + "theoretical_loss": 3.8478921392033714, + "tokens_seen": 631701504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533841524573722, + "loss": 1.5612, + "theoretical_loss": 3.8478511680127827, + "tokens_seen": 631767040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533681043129389, + "loss": 1.4165, + "theoretical_loss": 3.847810202261991, + "tokens_seen": 631832576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533520561685056, + "loss": 1.4827, + "theoretical_loss": 3.8477692419497105, + "tokens_seen": 631898112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533360080240723, + "loss": 1.452, + "theoretical_loss": 3.847728287074654, + "tokens_seen": 631963648 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653319959879639, + "loss": 1.4611, + "theoretical_loss": 3.847687337635537, + "tokens_seen": 632029184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006533039117352057, + "loss": 1.485, + "theoretical_loss": 3.8476463936310736, + "tokens_seen": 632094720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532878635907724, + "loss": 1.4913, + "theoretical_loss": 3.8476054550599796, + "tokens_seen": 632160256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532718154463391, + "loss": 1.4961, + "theoretical_loss": 3.8475645219209715, + "tokens_seen": 632225792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532557673019058, + "loss": 1.5826, + "theoretical_loss": 3.847523594212765, + "tokens_seen": 632291328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532397191574725, + "loss": 1.5162, + "theoretical_loss": 3.8474826719340762, + "tokens_seen": 632356864 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532236710130392, + "loss": 1.4682, + "theoretical_loss": 3.847441755083623, + "tokens_seen": 632422400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006532076228686059, + "loss": 1.4152, + "theoretical_loss": 3.8474008436601226, + "tokens_seen": 632487936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531915747241726, + "loss": 1.5854, + "theoretical_loss": 3.8473599376622927, + "tokens_seen": 632553472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531755265797393, + "loss": 1.4468, + "theoretical_loss": 3.8473190370888526, + "tokens_seen": 632619008 + }, + { + "epoch": 0.19, + "learning_rate": 0.000653159478435306, + "loss": 1.4058, + "theoretical_loss": 3.8472781419385207, + "tokens_seen": 632684544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531434302908727, + "loss": 1.5046, + "theoretical_loss": 3.8472372522100162, + "tokens_seen": 632750080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531273821464394, + "loss": 1.4847, + "theoretical_loss": 3.8471963679020593, + "tokens_seen": 632815616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006531113340020061, + "loss": 1.4791, + "theoretical_loss": 3.8471554890133692, + "tokens_seen": 632881152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530952858575728, + "loss": 1.5217, + "theoretical_loss": 3.847114615542667, + "tokens_seen": 632946688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530792377131395, + "loss": 1.5603, + "theoretical_loss": 3.8470737474886745, + "tokens_seen": 633012224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530631895687062, + "loss": 1.5288, + "theoretical_loss": 3.847032884850112, + "tokens_seen": 633077760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530471414242729, + "loss": 1.4746, + "theoretical_loss": 3.846992027625702, + "tokens_seen": 633143296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530310932798396, + "loss": 1.5531, + "theoretical_loss": 3.8469511758141666, + "tokens_seen": 633208832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006530150451354063, + "loss": 1.5026, + "theoretical_loss": 3.8469103294142286, + "tokens_seen": 633274368 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652998996990973, + "loss": 1.457, + "theoretical_loss": 3.8468694884246117, + "tokens_seen": 633339904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529829488465397, + "loss": 1.4605, + "theoretical_loss": 3.8468286528440396, + "tokens_seen": 633405440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529669007021064, + "loss": 1.4575, + "theoretical_loss": 3.846787822671235, + "tokens_seen": 633470976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529508525576731, + "loss": 1.3704, + "theoretical_loss": 3.8467469979049245, + "tokens_seen": 633536512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529348044132398, + "loss": 1.3262, + "theoretical_loss": 3.8467061785438315, + "tokens_seen": 633602048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529187562688064, + "loss": 1.5512, + "theoretical_loss": 3.8466653645866815, + "tokens_seen": 633667584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006529027081243732, + "loss": 1.4508, + "theoretical_loss": 3.8466245560322005, + "tokens_seen": 633733120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528866599799398, + "loss": 1.5556, + "theoretical_loss": 3.8465837528791154, + "tokens_seen": 633798656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528706118355066, + "loss": 1.5369, + "theoretical_loss": 3.846542955126152, + "tokens_seen": 633864192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528545636910732, + "loss": 1.4073, + "theoretical_loss": 3.8465021627720377, + "tokens_seen": 633929728 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065283851554664, + "loss": 1.3882, + "theoretical_loss": 3.8464613758155, + "tokens_seen": 633995264 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528224674022066, + "loss": 1.4174, + "theoretical_loss": 3.8464205942552665, + "tokens_seen": 634060800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006528064192577734, + "loss": 1.4423, + "theoretical_loss": 3.8463798180900666, + "tokens_seen": 634126336 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065279037111334, + "loss": 1.4535, + "theoretical_loss": 3.8463390473186285, + "tokens_seen": 634191872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527743229689067, + "loss": 1.5122, + "theoretical_loss": 3.8462982819396814, + "tokens_seen": 634257408 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527582748244734, + "loss": 1.5491, + "theoretical_loss": 3.846257521951955, + "tokens_seen": 634322944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527422266800401, + "loss": 1.4143, + "theoretical_loss": 3.84621676735418, + "tokens_seen": 634388480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527261785356068, + "loss": 1.5147, + "theoretical_loss": 3.8461760181450857, + "tokens_seen": 634454016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006527101303911735, + "loss": 1.3593, + "theoretical_loss": 3.8461352743234043, + "tokens_seen": 634519552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526940822467402, + "loss": 1.4563, + "theoretical_loss": 3.8460945358878664, + "tokens_seen": 634585088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526780341023069, + "loss": 1.452, + "theoretical_loss": 3.846053802837204, + "tokens_seen": 634650624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526619859578736, + "loss": 1.48, + "theoretical_loss": 3.8460130751701502, + "tokens_seen": 634716160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526459378134403, + "loss": 1.402, + "theoretical_loss": 3.845972352885437, + "tokens_seen": 634781696 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652629889669007, + "loss": 1.5052, + "theoretical_loss": 3.845931635981797, + "tokens_seen": 634847232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006526138415245737, + "loss": 1.4657, + "theoretical_loss": 3.845890924457965, + "tokens_seen": 634912768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525977933801404, + "loss": 1.4623, + "theoretical_loss": 3.845850218312674, + "tokens_seen": 634978304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525817452357071, + "loss": 1.4168, + "theoretical_loss": 3.8458095175446587, + "tokens_seen": 635043840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525656970912738, + "loss": 1.4256, + "theoretical_loss": 3.8457688221526545, + "tokens_seen": 635109376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525496489468405, + "loss": 1.4972, + "theoretical_loss": 3.8457281321353953, + "tokens_seen": 635174912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525336008024072, + "loss": 1.4882, + "theoretical_loss": 3.8456874474916183, + "tokens_seen": 635240448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525175526579739, + "loss": 1.4392, + "theoretical_loss": 3.8456467682200586, + "tokens_seen": 635305984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006525015045135406, + "loss": 1.4399, + "theoretical_loss": 3.8456060943194537, + "tokens_seen": 635371520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524854563691073, + "loss": 1.5155, + "theoretical_loss": 3.845565425788539, + "tokens_seen": 635437056 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652469408224674, + "loss": 1.5251, + "theoretical_loss": 3.8455247626260536, + "tokens_seen": 635502592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524533600802407, + "loss": 1.413, + "theoretical_loss": 3.845484104830734, + "tokens_seen": 635568128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524373119358074, + "loss": 1.6216, + "theoretical_loss": 3.84544345240132, + "tokens_seen": 635633664 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 241399, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5089519023895264, + "objective/train/theoretical_loss": 3.8454028053365494, + "objective/train/tokens_used": 656159200, + "theoretical_loss": 3.8454028053365494, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524212637913741, + "loss": 1.509, + "theoretical_loss": 3.8454028053365494, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006524052156469408, + "loss": 1.5375, + "theoretical_loss": 3.8453621636351607, + "tokens_seen": 635764736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523891675025075, + "loss": 1.4249, + "theoretical_loss": 3.8453215272958943, + "tokens_seen": 635830272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523731193580742, + "loss": 1.5183, + "theoretical_loss": 3.84528089631749, + "tokens_seen": 635895808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523570712136409, + "loss": 1.4245, + "theoretical_loss": 3.845240270698688, + "tokens_seen": 635961344 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523410230692076, + "loss": 1.471, + "theoretical_loss": 3.845199650438229, + "tokens_seen": 636026880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006523249749247743, + "loss": 1.6113, + "theoretical_loss": 3.845159035534855, + "tokens_seen": 636092416 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652308926780341, + "loss": 1.4316, + "theoretical_loss": 3.845118425987307, + "tokens_seen": 636157952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522928786359077, + "loss": 1.4959, + "theoretical_loss": 3.845077821794327, + "tokens_seen": 636223488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522768304914744, + "loss": 1.4369, + "theoretical_loss": 3.845037222954658, + "tokens_seen": 636289024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522607823470411, + "loss": 1.4542, + "theoretical_loss": 3.8449966294670426, + "tokens_seen": 636354560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522447342026078, + "loss": 1.4349, + "theoretical_loss": 3.8449560413302244, + "tokens_seen": 636420096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522286860581746, + "loss": 1.3773, + "theoretical_loss": 3.844915458542947, + "tokens_seen": 636485632 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006522126379137412, + "loss": 1.5649, + "theoretical_loss": 3.844874881103955, + "tokens_seen": 636551168 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652196589769308, + "loss": 1.5049, + "theoretical_loss": 3.844834309011992, + "tokens_seen": 636616704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521805416248747, + "loss": 1.4924, + "theoretical_loss": 3.8447937422658045, + "tokens_seen": 636682240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521644934804414, + "loss": 1.3832, + "theoretical_loss": 3.8447531808641373, + "tokens_seen": 636747776 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652148445336008, + "loss": 1.6124, + "theoretical_loss": 3.844712624805736, + "tokens_seen": 636813312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521323971915748, + "loss": 1.4039, + "theoretical_loss": 3.844672074089347, + "tokens_seen": 636878848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521163490471415, + "loss": 1.4502, + "theoretical_loss": 3.844631528713718, + "tokens_seen": 636944384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006521003009027082, + "loss": 1.4077, + "theoretical_loss": 3.844590988677595, + "tokens_seen": 637009920 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520842527582749, + "loss": 1.4884, + "theoretical_loss": 3.844550453979726, + "tokens_seen": 637075456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520682046138416, + "loss": 1.5514, + "theoretical_loss": 3.8445099246188597, + "tokens_seen": 637140992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520521564694083, + "loss": 1.4682, + "theoretical_loss": 3.8444694005937436, + "tokens_seen": 637206528 + }, + { + "epoch": 0.19, + "learning_rate": 0.000652036108324975, + "loss": 1.538, + "theoretical_loss": 3.844428881903127, + "tokens_seen": 637272064 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520200601805417, + "loss": 1.506, + "theoretical_loss": 3.844388368545759, + "tokens_seen": 637337600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006520040120361084, + "loss": 1.5039, + "theoretical_loss": 3.84434786052039, + "tokens_seen": 637403136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519879638916751, + "loss": 1.494, + "theoretical_loss": 3.8443073578257687, + "tokens_seen": 637468672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519719157472418, + "loss": 1.5839, + "theoretical_loss": 3.8442668604606474, + "tokens_seen": 637534208 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519558676028085, + "loss": 1.4902, + "theoretical_loss": 3.844226368423776, + "tokens_seen": 637599744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519398194583752, + "loss": 1.4963, + "theoretical_loss": 3.844185881713906, + "tokens_seen": 637665280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519237713139419, + "loss": 1.5346, + "theoretical_loss": 3.844145400329789, + "tokens_seen": 637730816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006519077231695086, + "loss": 1.4716, + "theoretical_loss": 3.8441049242701784, + "tokens_seen": 637796352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518916750250753, + "loss": 1.5676, + "theoretical_loss": 3.8440644535338255, + "tokens_seen": 637861888 + }, + { + "epoch": 0.19, + "learning_rate": 0.000651875626880642, + "loss": 1.4952, + "theoretical_loss": 3.844023988119484, + "tokens_seen": 637927424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518595787362087, + "loss": 1.4717, + "theoretical_loss": 3.8439835280259076, + "tokens_seen": 637992960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518435305917754, + "loss": 1.4799, + "theoretical_loss": 3.84394307325185, + "tokens_seen": 638058496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518274824473421, + "loss": 1.5232, + "theoretical_loss": 3.8439026237960654, + "tokens_seen": 638124032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006518114343029088, + "loss": 1.4734, + "theoretical_loss": 3.8438621796573087, + "tokens_seen": 638189568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517953861584755, + "loss": 1.5293, + "theoretical_loss": 3.843821740834335, + "tokens_seen": 638255104 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517793380140422, + "loss": 1.42, + "theoretical_loss": 3.8437813073258997, + "tokens_seen": 638320640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517632898696089, + "loss": 1.5198, + "theoretical_loss": 3.8437408791307597, + "tokens_seen": 638386176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517472417251756, + "loss": 1.5068, + "theoretical_loss": 3.843700456247671, + "tokens_seen": 638451712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006517311935807423, + "loss": 1.447, + "theoretical_loss": 3.84366003867539, + "tokens_seen": 638517248 + }, + { + "epoch": 0.19, + "learning_rate": 0.000651715145436309, + "loss": 1.4394, + "theoretical_loss": 3.8436196264126736, + "tokens_seen": 638582784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516990972918757, + "loss": 1.4495, + "theoretical_loss": 3.843579219458281, + "tokens_seen": 638648320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516830491474424, + "loss": 1.4755, + "theoretical_loss": 3.843538817810969, + "tokens_seen": 638713856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516670010030091, + "loss": 1.4184, + "theoretical_loss": 3.843498421469497, + "tokens_seen": 638779392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516509528585758, + "loss": 1.4083, + "theoretical_loss": 3.843458030432623, + "tokens_seen": 638844928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516349047141425, + "loss": 1.5011, + "theoretical_loss": 3.8434176446991075, + "tokens_seen": 638910464 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516188565697092, + "loss": 1.4321, + "theoretical_loss": 3.8433772642677093, + "tokens_seen": 638976000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006516028084252759, + "loss": 1.5657, + "theoretical_loss": 3.8433368891371895, + "tokens_seen": 639041536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515867602808426, + "loss": 1.5398, + "theoretical_loss": 3.843296519306308, + "tokens_seen": 639107072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515707121364093, + "loss": 1.449, + "theoretical_loss": 3.843256154773826, + "tokens_seen": 639172608 + }, + { + "epoch": 0.19, + "learning_rate": 0.000651554663991976, + "loss": 1.4311, + "theoretical_loss": 3.843215795538505, + "tokens_seen": 639238144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515386158475427, + "loss": 1.4788, + "theoretical_loss": 3.843175441599107, + "tokens_seen": 639303680 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515225677031094, + "loss": 1.5347, + "theoretical_loss": 3.8431350929543946, + "tokens_seen": 639369216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006515065195586761, + "loss": 1.4341, + "theoretical_loss": 3.8430947496031296, + "tokens_seen": 639434752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514904714142427, + "loss": 1.3922, + "theoretical_loss": 3.8430544115440752, + "tokens_seen": 639500288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514744232698095, + "loss": 1.37, + "theoretical_loss": 3.8430140787759957, + "tokens_seen": 639565824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514583751253761, + "loss": 1.3849, + "theoretical_loss": 3.842973751297655, + "tokens_seen": 639631360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514423269809429, + "loss": 1.48, + "theoretical_loss": 3.8429334291078168, + "tokens_seen": 639696896 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514262788365095, + "loss": 1.4006, + "theoretical_loss": 3.842893112205246, + "tokens_seen": 639762432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006514102306920763, + "loss": 1.4109, + "theoretical_loss": 3.842852800588709, + "tokens_seen": 639827968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513941825476429, + "loss": 1.4504, + "theoretical_loss": 3.8428124942569695, + "tokens_seen": 639893504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513781344032097, + "loss": 1.4214, + "theoretical_loss": 3.842772193208795, + "tokens_seen": 639959040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513620862587763, + "loss": 1.4009, + "theoretical_loss": 3.8427318974429507, + "tokens_seen": 640024576 + }, + { + "epoch": 0.19, + "learning_rate": 0.000651346038114343, + "loss": 1.5065, + "theoretical_loss": 3.8426916069582044, + "tokens_seen": 640090112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513299899699097, + "loss": 1.4115, + "theoretical_loss": 3.842651321753323, + "tokens_seen": 640155648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006513139418254764, + "loss": 1.5884, + "theoretical_loss": 3.8426110418270745, + "tokens_seen": 640221184 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512978936810431, + "loss": 1.4323, + "theoretical_loss": 3.842570767178227, + "tokens_seen": 640286720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512818455366098, + "loss": 1.4177, + "theoretical_loss": 3.8425304978055483, + "tokens_seen": 640352256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512657973921765, + "loss": 1.5901, + "theoretical_loss": 3.8424902337078084, + "tokens_seen": 640417792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512497492477432, + "loss": 1.3942, + "theoretical_loss": 3.8424499748837753, + "tokens_seen": 640483328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512337011033099, + "loss": 1.4784, + "theoretical_loss": 3.8424097213322197, + "tokens_seen": 640548864 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512176529588766, + "loss": 1.3875, + "theoretical_loss": 3.8423694730519116, + "tokens_seen": 640614400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006512016048144433, + "loss": 1.4447, + "theoretical_loss": 3.8423292300416216, + "tokens_seen": 640679936 + }, + { + "epoch": 0.19, + "learning_rate": 0.00065118555667001, + "loss": 1.409, + "theoretical_loss": 3.8422889923001207, + "tokens_seen": 640745472 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511695085255767, + "loss": 1.5282, + "theoretical_loss": 3.84224875982618, + "tokens_seen": 640811008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511534603811434, + "loss": 1.3874, + "theoretical_loss": 3.8422085326185718, + "tokens_seen": 640876544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511374122367101, + "loss": 1.4922, + "theoretical_loss": 3.842168310676067, + "tokens_seen": 640942080 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511213640922768, + "loss": 1.5043, + "theoretical_loss": 3.8421280939974407, + "tokens_seen": 641007616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006511053159478435, + "loss": 1.4879, + "theoretical_loss": 3.8420878825814637, + "tokens_seen": 641073152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510892678034102, + "loss": 1.4488, + "theoretical_loss": 3.84204767642691, + "tokens_seen": 641138688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510732196589769, + "loss": 1.5199, + "theoretical_loss": 3.8420074755325544, + "tokens_seen": 641204224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510571715145436, + "loss": 1.425, + "theoretical_loss": 3.8419672798971707, + "tokens_seen": 641269760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510411233701103, + "loss": 1.4399, + "theoretical_loss": 3.8419270895195323, + "tokens_seen": 641335296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510250752256771, + "loss": 1.4412, + "theoretical_loss": 3.8418869043984163, + "tokens_seen": 641400832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006510090270812437, + "loss": 1.4629, + "theoretical_loss": 3.841846724532597, + "tokens_seen": 641466368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509929789368105, + "loss": 1.4795, + "theoretical_loss": 3.8418065499208502, + "tokens_seen": 641531904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509769307923771, + "loss": 1.5173, + "theoretical_loss": 3.8417663805619533, + "tokens_seen": 641597440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509608826479439, + "loss": 1.4075, + "theoretical_loss": 3.841726216454682, + "tokens_seen": 641662976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509448345035105, + "loss": 1.429, + "theoretical_loss": 3.8416860575978142, + "tokens_seen": 641728512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006509287863590773, + "loss": 1.5094, + "theoretical_loss": 3.841645903990127, + "tokens_seen": 641794048 + }, + { + "epoch": 0.19, + "learning_rate": 0.000650912738214644, + "loss": 1.3706, + "theoretical_loss": 3.841605755630398, + "tokens_seen": 641859584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508966900702107, + "loss": 1.434, + "theoretical_loss": 3.8415656125174067, + "tokens_seen": 641925120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508806419257773, + "loss": 1.5509, + "theoretical_loss": 3.8415254746499308, + "tokens_seen": 641990656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508645937813441, + "loss": 1.4528, + "theoretical_loss": 3.8414853420267496, + "tokens_seen": 642056192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508485456369107, + "loss": 1.5741, + "theoretical_loss": 3.841445214646644, + "tokens_seen": 642121728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508324974924775, + "loss": 1.5186, + "theoretical_loss": 3.8414050925083925, + "tokens_seen": 642187264 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 243591, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.479576587677002, + "objective/train/theoretical_loss": 3.8413649756107757, + "objective/train/tokens_used": 662712800, + "theoretical_loss": 3.8413649756107757, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508164493480441, + "loss": 1.4796, + "theoretical_loss": 3.8413649756107757, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006508004012036109, + "loss": 1.4658, + "theoretical_loss": 3.8413248639525754, + "tokens_seen": 642318336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507843530591776, + "loss": 1.3487, + "theoretical_loss": 3.8412847575325717, + "tokens_seen": 642383872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507683049147443, + "loss": 1.505, + "theoretical_loss": 3.8412446563495477, + "tokens_seen": 642449408 + }, + { + "epoch": 0.19, + "learning_rate": 0.000650752256770311, + "loss": 1.3866, + "theoretical_loss": 3.8412045604022835, + "tokens_seen": 642514944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507362086258777, + "loss": 1.4406, + "theoretical_loss": 3.8411644696895633, + "tokens_seen": 642580480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507201604814444, + "loss": 1.553, + "theoretical_loss": 3.8411243842101688, + "tokens_seen": 642646016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006507041123370111, + "loss": 1.5225, + "theoretical_loss": 3.841084303962884, + "tokens_seen": 642711552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506880641925778, + "loss": 1.5059, + "theoretical_loss": 3.841044228946492, + "tokens_seen": 642777088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506720160481445, + "loss": 1.4548, + "theoretical_loss": 3.8410041591597777, + "tokens_seen": 642842624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506559679037112, + "loss": 1.3804, + "theoretical_loss": 3.840964094601525, + "tokens_seen": 642908160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506399197592779, + "loss": 1.3935, + "theoretical_loss": 3.8409240352705187, + "tokens_seen": 642973696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506238716148446, + "loss": 1.435, + "theoretical_loss": 3.840883981165544, + "tokens_seen": 643039232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006506078234704113, + "loss": 1.5695, + "theoretical_loss": 3.8408439322853876, + "tokens_seen": 643104768 + }, + { + "epoch": 0.19, + "learning_rate": 0.000650591775325978, + "loss": 1.3768, + "theoretical_loss": 3.8408038886288347, + "tokens_seen": 643170304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006505757271815447, + "loss": 1.4821, + "theoretical_loss": 3.840763850194672, + "tokens_seen": 643235840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006505596790371114, + "loss": 1.3644, + "theoretical_loss": 3.8407238169816864, + "tokens_seen": 643301376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006505436308926781, + "loss": 1.4589, + "theoretical_loss": 3.840683788988666, + "tokens_seen": 643366912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006505275827482448, + "loss": 1.4501, + "theoretical_loss": 3.840643766214397, + "tokens_seen": 643432448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0006505115346038115, + "loss": 1.4082, + "theoretical_loss": 3.840603748657669, + "tokens_seen": 643497984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504954864593782, + "loss": 1.4583, + "theoretical_loss": 3.8405637363172693, + "tokens_seen": 643563520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504794383149449, + "loss": 1.4884, + "theoretical_loss": 3.8405237291919883, + "tokens_seen": 643629056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504633901705116, + "loss": 1.4412, + "theoretical_loss": 3.8404837272806143, + "tokens_seen": 643694592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504473420260783, + "loss": 1.3987, + "theoretical_loss": 3.840443730581937, + "tokens_seen": 643760128 + }, + { + "epoch": 0.2, + "learning_rate": 0.000650431293881645, + "loss": 1.5839, + "theoretical_loss": 3.8404037390947474, + "tokens_seen": 643825664 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006504152457372117, + "loss": 1.3561, + "theoretical_loss": 3.8403637528178356, + "tokens_seen": 643891200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503991975927784, + "loss": 1.3912, + "theoretical_loss": 3.8403237717499925, + "tokens_seen": 643956736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503831494483451, + "loss": 1.5135, + "theoretical_loss": 3.8402837958900093, + "tokens_seen": 644022272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503671013039118, + "loss": 1.5536, + "theoretical_loss": 3.840243825236678, + "tokens_seen": 644087808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503510531594785, + "loss": 1.4526, + "theoretical_loss": 3.840203859788791, + "tokens_seen": 644153344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503350050150452, + "loss": 1.4381, + "theoretical_loss": 3.840163899545141, + "tokens_seen": 644218880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503189568706119, + "loss": 1.3929, + "theoretical_loss": 3.8401239445045205, + "tokens_seen": 644284416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006503029087261786, + "loss": 1.4056, + "theoretical_loss": 3.840083994665723, + "tokens_seen": 644349952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502868605817453, + "loss": 1.3765, + "theoretical_loss": 3.840044050027543, + "tokens_seen": 644415488 + }, + { + "epoch": 0.2, + "learning_rate": 0.000650270812437312, + "loss": 1.4899, + "theoretical_loss": 3.8400041105887732, + "tokens_seen": 644481024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502547642928787, + "loss": 1.4017, + "theoretical_loss": 3.8399641763482095, + "tokens_seen": 644546560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502387161484454, + "loss": 1.4771, + "theoretical_loss": 3.8399242473046464, + "tokens_seen": 644612096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502226680040121, + "loss": 1.4126, + "theoretical_loss": 3.8398843234568796, + "tokens_seen": 644677632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006502066198595788, + "loss": 1.5008, + "theoretical_loss": 3.8398444048037046, + "tokens_seen": 644743168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501905717151455, + "loss": 1.5392, + "theoretical_loss": 3.8398044913439175, + "tokens_seen": 644808704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501745235707122, + "loss": 1.4901, + "theoretical_loss": 3.839764583076316, + "tokens_seen": 644874240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501584754262789, + "loss": 1.4638, + "theoretical_loss": 3.839724679999695, + "tokens_seen": 644939776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501424272818456, + "loss": 1.4991, + "theoretical_loss": 3.8396847821128537, + "tokens_seen": 645005312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006501263791374123, + "loss": 1.5162, + "theoretical_loss": 3.83964488941459, + "tokens_seen": 645070848 + }, + { + "epoch": 0.2, + "learning_rate": 0.000650110330992979, + "loss": 1.3765, + "theoretical_loss": 3.8396050019037005, + "tokens_seen": 645136384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500942828485456, + "loss": 1.4156, + "theoretical_loss": 3.8395651195789853, + "tokens_seen": 645201920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500782347041124, + "loss": 1.4406, + "theoretical_loss": 3.839525242439243, + "tokens_seen": 645267456 + }, + { + "epoch": 0.2, + "learning_rate": 0.000650062186559679, + "loss": 1.3824, + "theoretical_loss": 3.8394853704832723, + "tokens_seen": 645332992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500461384152458, + "loss": 1.4198, + "theoretical_loss": 3.8394455037098743, + "tokens_seen": 645398528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500300902708124, + "loss": 1.4507, + "theoretical_loss": 3.8394056421178484, + "tokens_seen": 645464064 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006500140421263792, + "loss": 1.3978, + "theoretical_loss": 3.8393657857059953, + "tokens_seen": 645529600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499979939819458, + "loss": 1.4655, + "theoretical_loss": 3.839325934473116, + "tokens_seen": 645595136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499819458375126, + "loss": 1.4654, + "theoretical_loss": 3.839286088418012, + "tokens_seen": 645660672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499658976930792, + "loss": 1.4025, + "theoretical_loss": 3.839246247539485, + "tokens_seen": 645726208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499498495486459, + "loss": 1.4471, + "theoretical_loss": 3.8392064118363374, + "tokens_seen": 645791744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499338014042126, + "loss": 1.4104, + "theoretical_loss": 3.839166581307372, + "tokens_seen": 645857280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006499177532597793, + "loss": 1.4305, + "theoretical_loss": 3.8391267559513915, + "tokens_seen": 645922816 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649901705115346, + "loss": 1.492, + "theoretical_loss": 3.839086935767199, + "tokens_seen": 645988352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498856569709127, + "loss": 1.4454, + "theoretical_loss": 3.839047120753599, + "tokens_seen": 646053888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498696088264794, + "loss": 1.3641, + "theoretical_loss": 3.8390073109093947, + "tokens_seen": 646119424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498535606820461, + "loss": 1.4589, + "theoretical_loss": 3.8389675062333923, + "tokens_seen": 646184960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498375125376128, + "loss": 1.5123, + "theoretical_loss": 3.838927706724395, + "tokens_seen": 646250496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498214643931796, + "loss": 1.5459, + "theoretical_loss": 3.8388879123812094, + "tokens_seen": 646316032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006498054162487462, + "loss": 1.5181, + "theoretical_loss": 3.8388481232026406, + "tokens_seen": 646381568 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649789368104313, + "loss": 1.4206, + "theoretical_loss": 3.838808339187495, + "tokens_seen": 646447104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006497733199598796, + "loss": 1.4735, + "theoretical_loss": 3.8387685603345805, + "tokens_seen": 646512640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006497572718154464, + "loss": 1.4834, + "theoretical_loss": 3.838728786642702, + "tokens_seen": 646578176 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649741223671013, + "loss": 1.4354, + "theoretical_loss": 3.838689018110667, + "tokens_seen": 646643712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006497251755265798, + "loss": 1.4467, + "theoretical_loss": 3.838649254737285, + "tokens_seen": 646709248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006497091273821464, + "loss": 1.4152, + "theoretical_loss": 3.838609496521363, + "tokens_seen": 646774784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496930792377132, + "loss": 1.3575, + "theoretical_loss": 3.8385697434617096, + "tokens_seen": 646840320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496770310932798, + "loss": 1.3781, + "theoretical_loss": 3.838529995557134, + "tokens_seen": 646905856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496609829488466, + "loss": 1.4292, + "theoretical_loss": 3.8384902528064457, + "tokens_seen": 646971392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496449348044132, + "loss": 1.5414, + "theoretical_loss": 3.8384505152084545, + "tokens_seen": 647036928 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064962888665998, + "loss": 1.5301, + "theoretical_loss": 3.8384107827619696, + "tokens_seen": 647102464 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006496128385155466, + "loss": 1.5103, + "theoretical_loss": 3.8383710554658026, + "tokens_seen": 647168000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495967903711134, + "loss": 1.4195, + "theoretical_loss": 3.838331333318764, + "tokens_seen": 647233536 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064958074222668, + "loss": 1.5543, + "theoretical_loss": 3.8382916163196654, + "tokens_seen": 647299072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495646940822468, + "loss": 1.5018, + "theoretical_loss": 3.8382519044673176, + "tokens_seen": 647364608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495486459378134, + "loss": 1.3748, + "theoretical_loss": 3.8382121977605346, + "tokens_seen": 647430144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495325977933802, + "loss": 1.5563, + "theoretical_loss": 3.8381724961981267, + "tokens_seen": 647495680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495165496489468, + "loss": 1.5555, + "theoretical_loss": 3.8381327997789088, + "tokens_seen": 647561216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006495005015045136, + "loss": 1.4346, + "theoretical_loss": 3.8380931085016927, + "tokens_seen": 647626752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006494844533600802, + "loss": 1.3791, + "theoretical_loss": 3.8380534223652933, + "tokens_seen": 647692288 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649468405215647, + "loss": 1.4617, + "theoretical_loss": 3.838013741368523, + "tokens_seen": 647757824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006494523570712136, + "loss": 1.4555, + "theoretical_loss": 3.837974065510198, + "tokens_seen": 647823360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006494363089267804, + "loss": 1.435, + "theoretical_loss": 3.837934394789133, + "tokens_seen": 647888896 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649420260782347, + "loss": 1.5084, + "theoretical_loss": 3.8378947292041428, + "tokens_seen": 647954432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006494042126379138, + "loss": 1.5735, + "theoretical_loss": 3.8378550687540427, + "tokens_seen": 648019968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493881644934805, + "loss": 1.376, + "theoretical_loss": 3.8378154134376494, + "tokens_seen": 648085504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493721163490472, + "loss": 1.5218, + "theoretical_loss": 3.837775763253779, + "tokens_seen": 648151040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493560682046139, + "loss": 1.4522, + "theoretical_loss": 3.8377361182012484, + "tokens_seen": 648216576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493400200601806, + "loss": 1.3984, + "theoretical_loss": 3.8376964782788754, + "tokens_seen": 648282112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006493239719157473, + "loss": 1.487, + "theoretical_loss": 3.8376568434854765, + "tokens_seen": 648347648 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649307923771314, + "loss": 1.3561, + "theoretical_loss": 3.837617213819871, + "tokens_seen": 648413184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492918756268807, + "loss": 1.4425, + "theoretical_loss": 3.837577589280876, + "tokens_seen": 648478720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492758274824474, + "loss": 1.3625, + "theoretical_loss": 3.8375379698673115, + "tokens_seen": 648544256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492597793380141, + "loss": 1.4914, + "theoretical_loss": 3.837498355577996, + "tokens_seen": 648609792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492437311935808, + "loss": 1.4376, + "theoretical_loss": 3.837458746411749, + "tokens_seen": 648675328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492276830491475, + "loss": 1.533, + "theoretical_loss": 3.8374191423673913, + "tokens_seen": 648740864 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 245985, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5545860528945923, + "objective/train/theoretical_loss": 3.8373795434437423, + "objective/train/tokens_used": 669266400, + "theoretical_loss": 3.8373795434437423, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006492116349047142, + "loss": 1.5546, + "theoretical_loss": 3.8373795434437423, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491955867602809, + "loss": 1.4275, + "theoretical_loss": 3.8373399496396234, + "tokens_seen": 648871936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491795386158476, + "loss": 1.3472, + "theoretical_loss": 3.8373003609538556, + "tokens_seen": 648937472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491634904714143, + "loss": 1.5125, + "theoretical_loss": 3.8372607773852603, + "tokens_seen": 649003008 + }, + { + "epoch": 0.2, + "learning_rate": 0.000649147442326981, + "loss": 1.5282, + "theoretical_loss": 3.8372211989326592, + "tokens_seen": 649068544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491313941825477, + "loss": 1.4661, + "theoretical_loss": 3.8371816255948756, + "tokens_seen": 649134080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006491153460381144, + "loss": 1.5063, + "theoretical_loss": 3.837142057370731, + "tokens_seen": 649199616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490992978936811, + "loss": 1.3638, + "theoretical_loss": 3.837102494259049, + "tokens_seen": 649265152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490832497492478, + "loss": 1.4794, + "theoretical_loss": 3.8370629362586532, + "tokens_seen": 649330688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490672016048145, + "loss": 1.5194, + "theoretical_loss": 3.8370233833683676, + "tokens_seen": 649396224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490511534603812, + "loss": 1.5512, + "theoretical_loss": 3.8369838355870165, + "tokens_seen": 649461760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490351053159479, + "loss": 1.4867, + "theoretical_loss": 3.836944292913424, + "tokens_seen": 649527296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490190571715146, + "loss": 1.4556, + "theoretical_loss": 3.8369047553464153, + "tokens_seen": 649592832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006490030090270813, + "loss": 1.3596, + "theoretical_loss": 3.836865222884817, + "tokens_seen": 649658368 + }, + { + "epoch": 0.2, + "learning_rate": 0.000648986960882648, + "loss": 1.5339, + "theoretical_loss": 3.8368256955274527, + "tokens_seen": 649723904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489709127382147, + "loss": 1.4392, + "theoretical_loss": 3.8367861732731505, + "tokens_seen": 649789440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489548645937814, + "loss": 1.4641, + "theoretical_loss": 3.836746656120737, + "tokens_seen": 649854976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489388164493481, + "loss": 1.4491, + "theoretical_loss": 3.8367071440690372, + "tokens_seen": 649920512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489227683049148, + "loss": 1.456, + "theoretical_loss": 3.836667637116881, + "tokens_seen": 649986048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006489067201604815, + "loss": 1.5543, + "theoretical_loss": 3.8366281352630947, + "tokens_seen": 650051584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488906720160482, + "loss": 1.4315, + "theoretical_loss": 3.8365886385065067, + "tokens_seen": 650117120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488746238716149, + "loss": 1.425, + "theoretical_loss": 3.836549146845946, + "tokens_seen": 650182656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488585757271816, + "loss": 1.4434, + "theoretical_loss": 3.8365096602802407, + "tokens_seen": 650248192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488425275827483, + "loss": 1.5287, + "theoretical_loss": 3.83647017880822, + "tokens_seen": 650313728 + }, + { + "epoch": 0.2, + "learning_rate": 0.000648826479438315, + "loss": 1.3891, + "theoretical_loss": 3.836430702428715, + "tokens_seen": 650379264 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006488104312938817, + "loss": 1.3616, + "theoretical_loss": 3.8363912311405546, + "tokens_seen": 650444800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487943831494484, + "loss": 1.4846, + "theoretical_loss": 3.8363517649425702, + "tokens_seen": 650510336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487783350050151, + "loss": 1.4366, + "theoretical_loss": 3.8363123038335916, + "tokens_seen": 650575872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487622868605818, + "loss": 1.5076, + "theoretical_loss": 3.8362728478124506, + "tokens_seen": 650641408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487462387161485, + "loss": 1.5003, + "theoretical_loss": 3.8362333968779794, + "tokens_seen": 650706944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487301905717152, + "loss": 1.4369, + "theoretical_loss": 3.836193951029008, + "tokens_seen": 650772480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006487141424272819, + "loss": 1.4745, + "theoretical_loss": 3.8361545102643717, + "tokens_seen": 650838016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486980942828485, + "loss": 1.5247, + "theoretical_loss": 3.836115074582901, + "tokens_seen": 650903552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486820461384153, + "loss": 1.4435, + "theoretical_loss": 3.8360756439834303, + "tokens_seen": 650969088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486659979939819, + "loss": 1.5009, + "theoretical_loss": 3.836036218464792, + "tokens_seen": 651034624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486499498495487, + "loss": 1.5951, + "theoretical_loss": 3.8359967980258216, + "tokens_seen": 651100160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486339017051153, + "loss": 1.5729, + "theoretical_loss": 3.8359573826653524, + "tokens_seen": 651165696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486178535606821, + "loss": 1.5142, + "theoretical_loss": 3.8359179723822194, + "tokens_seen": 651231232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006486018054162487, + "loss": 1.4476, + "theoretical_loss": 3.835878567175257, + "tokens_seen": 651296768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485857572718155, + "loss": 1.4938, + "theoretical_loss": 3.8358391670433023, + "tokens_seen": 651362304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485697091273821, + "loss": 1.4937, + "theoretical_loss": 3.8357997719851893, + "tokens_seen": 651427840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485536609829489, + "loss": 1.4299, + "theoretical_loss": 3.835760381999756, + "tokens_seen": 651493376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485376128385155, + "loss": 1.5207, + "theoretical_loss": 3.835720997085838, + "tokens_seen": 651558912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485215646940823, + "loss": 1.3665, + "theoretical_loss": 3.835681617242272, + "tokens_seen": 651624448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006485055165496489, + "loss": 1.4855, + "theoretical_loss": 3.8356422424678964, + "tokens_seen": 651689984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484894684052157, + "loss": 1.4631, + "theoretical_loss": 3.835602872761549, + "tokens_seen": 651755520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484734202607823, + "loss": 1.4003, + "theoretical_loss": 3.8355635081220667, + "tokens_seen": 651821056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484573721163491, + "loss": 1.4674, + "theoretical_loss": 3.8355241485482887, + "tokens_seen": 651886592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484413239719157, + "loss": 1.4219, + "theoretical_loss": 3.835484794039055, + "tokens_seen": 651952128 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484252758274825, + "loss": 1.4485, + "theoretical_loss": 3.835445444593203, + "tokens_seen": 652017664 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006484092276830491, + "loss": 1.5308, + "theoretical_loss": 3.8354061002095743, + "tokens_seen": 652083200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483931795386159, + "loss": 1.4267, + "theoretical_loss": 3.835366760887008, + "tokens_seen": 652148736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483771313941825, + "loss": 1.3844, + "theoretical_loss": 3.8353274266243442, + "tokens_seen": 652214272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483610832497493, + "loss": 1.5607, + "theoretical_loss": 3.835288097420425, + "tokens_seen": 652279808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483450351053159, + "loss": 1.4874, + "theoretical_loss": 3.83524877327409, + "tokens_seen": 652345344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483289869608827, + "loss": 1.3353, + "theoretical_loss": 3.8352094541841817, + "tokens_seen": 652410880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006483129388164493, + "loss": 1.5799, + "theoretical_loss": 3.8351701401495424, + "tokens_seen": 652476416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482968906720161, + "loss": 1.4345, + "theoretical_loss": 3.8351308311690144, + "tokens_seen": 652541952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482808425275827, + "loss": 1.4646, + "theoretical_loss": 3.83509152724144, + "tokens_seen": 652607488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482647943831495, + "loss": 1.4711, + "theoretical_loss": 3.8350522283656625, + "tokens_seen": 652673024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482487462387161, + "loss": 1.4266, + "theoretical_loss": 3.8350129345405257, + "tokens_seen": 652738560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482326980942829, + "loss": 1.4721, + "theoretical_loss": 3.8349736457648733, + "tokens_seen": 652804096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482166499498495, + "loss": 1.4875, + "theoretical_loss": 3.83493436203755, + "tokens_seen": 652869632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006482006018054163, + "loss": 1.5229, + "theoretical_loss": 3.834895083357399, + "tokens_seen": 652935168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481845536609829, + "loss": 1.4348, + "theoretical_loss": 3.8348558097232672, + "tokens_seen": 653000704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481685055165497, + "loss": 1.4832, + "theoretical_loss": 3.834816541133999, + "tokens_seen": 653066240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481524573721163, + "loss": 1.447, + "theoretical_loss": 3.83477727758844, + "tokens_seen": 653131776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481364092276831, + "loss": 1.4818, + "theoretical_loss": 3.8347380190854374, + "tokens_seen": 653197312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481203610832497, + "loss": 1.4478, + "theoretical_loss": 3.8346987656238367, + "tokens_seen": 653262848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006481043129388165, + "loss": 1.5024, + "theoretical_loss": 3.8346595172024855, + "tokens_seen": 653328384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480882647943831, + "loss": 1.48, + "theoretical_loss": 3.834620273820231, + "tokens_seen": 653393920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480722166499499, + "loss": 1.5584, + "theoretical_loss": 3.834581035475921, + "tokens_seen": 653459456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480561685055166, + "loss": 1.484, + "theoretical_loss": 3.8345418021684035, + "tokens_seen": 653524992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480401203610833, + "loss": 1.4577, + "theoretical_loss": 3.8345025738965264, + "tokens_seen": 653590528 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064802407221665, + "loss": 1.4302, + "theoretical_loss": 3.8344633506591395, + "tokens_seen": 653656064 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006480080240722167, + "loss": 1.41, + "theoretical_loss": 3.8344241324550907, + "tokens_seen": 653721600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479919759277834, + "loss": 1.4335, + "theoretical_loss": 3.8343849192832313, + "tokens_seen": 653787136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479759277833501, + "loss": 1.3559, + "theoretical_loss": 3.83434571114241, + "tokens_seen": 653852672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479598796389168, + "loss": 1.5692, + "theoretical_loss": 3.834306508031478, + "tokens_seen": 653918208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479438314944835, + "loss": 1.3775, + "theoretical_loss": 3.8342673099492854, + "tokens_seen": 653983744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479277833500502, + "loss": 1.5192, + "theoretical_loss": 3.8342281168946832, + "tokens_seen": 654049280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006479117352056169, + "loss": 1.4671, + "theoretical_loss": 3.8341889288665234, + "tokens_seen": 654114816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478956870611836, + "loss": 1.4257, + "theoretical_loss": 3.8341497458636575, + "tokens_seen": 654180352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478796389167503, + "loss": 1.4124, + "theoretical_loss": 3.834110567884938, + "tokens_seen": 654245888 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647863590772317, + "loss": 1.4574, + "theoretical_loss": 3.8340713949292176, + "tokens_seen": 654311424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478475426278837, + "loss": 1.5902, + "theoretical_loss": 3.834032226995349, + "tokens_seen": 654376960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478314944834504, + "loss": 1.5316, + "theoretical_loss": 3.8339930640821853, + "tokens_seen": 654442496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006478154463390171, + "loss": 1.4885, + "theoretical_loss": 3.8339539061885812, + "tokens_seen": 654508032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477993981945838, + "loss": 1.4326, + "theoretical_loss": 3.83391475331339, + "tokens_seen": 654573568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477833500501505, + "loss": 1.3492, + "theoretical_loss": 3.833875605455466, + "tokens_seen": 654639104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477673019057172, + "loss": 1.473, + "theoretical_loss": 3.8338364626136654, + "tokens_seen": 654704640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477512537612839, + "loss": 1.5465, + "theoretical_loss": 3.833797324786842, + "tokens_seen": 654770176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477352056168506, + "loss": 1.3759, + "theoretical_loss": 3.8337581919738524, + "tokens_seen": 654835712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006477191574724173, + "loss": 1.5621, + "theoretical_loss": 3.8337190641735512, + "tokens_seen": 654901248 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647703109327984, + "loss": 1.5794, + "theoretical_loss": 3.833679941384797, + "tokens_seen": 654966784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476870611835507, + "loss": 1.4584, + "theoretical_loss": 3.833640823606445, + "tokens_seen": 655032320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476710130391174, + "loss": 1.5169, + "theoretical_loss": 3.8336017108373523, + "tokens_seen": 655097856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476549648946841, + "loss": 1.4352, + "theoretical_loss": 3.833562603076377, + "tokens_seen": 655163392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476389167502508, + "loss": 1.5129, + "theoretical_loss": 3.833523500322377, + "tokens_seen": 655228928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476228686058175, + "loss": 1.4624, + "theoretical_loss": 3.83348440257421, + "tokens_seen": 655294464 + }, + { + "debugging/Compilability": 1.0, + "debugging/distinct-1-grams": 0.720533615064761, + "debugging/entropy-1-grams": 5.441721251993373, + "debugging/length": 493.11538461538464, + "debugging/num_segments": 26, + "debugging/score": 0.002968524506531717, + "debugging/score_std": 0.0023907544761418357, + "epoch": 0.2, + "objective/train/docs_used": 248293, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4819377660751343, + "objective/train/theoretical_loss": 3.833445309830735, + "objective/train/tokens_used": 675820000, + "theoretical_loss": 3.833445309830735, + "tokens_seen": 655360000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006476068204613842, + "loss": 1.4819, + "theoretical_loss": 3.833445309830735, + "tokens_seen": 655360000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006475907723169509, + "loss": 1.5181, + "theoretical_loss": 3.833406222090811, + "tokens_seen": 655425536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006475747241725176, + "loss": 1.5724, + "theoretical_loss": 3.8333671393532973, + "tokens_seen": 655491072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006475586760280843, + "loss": 1.525, + "theoretical_loss": 3.8333280616170535, + "tokens_seen": 655556608 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647542627883651, + "loss": 1.5106, + "theoretical_loss": 3.8332889888809403, + "tokens_seen": 655622144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006475265797392177, + "loss": 1.4123, + "theoretical_loss": 3.8332499211438176, + "tokens_seen": 655687680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006475105315947844, + "loss": 1.4325, + "theoretical_loss": 3.8332108584045463, + "tokens_seen": 655753216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006474944834503511, + "loss": 1.4515, + "theoretical_loss": 3.8331718006619884, + "tokens_seen": 655818752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006474784353059178, + "loss": 1.4993, + "theoretical_loss": 3.833132747915004, + "tokens_seen": 655884288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006474623871614845, + "loss": 1.3545, + "theoretical_loss": 3.8330937001624568, + "tokens_seen": 655949824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006474463390170512, + "loss": 1.4027, + "theoretical_loss": 3.833054657403208, + "tokens_seen": 656015360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006474302908726179, + "loss": 1.4349, + "theoretical_loss": 3.8330156196361207, + "tokens_seen": 656080896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006474142427281846, + "loss": 1.4546, + "theoretical_loss": 3.8329765868600583, + "tokens_seen": 656146432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006473981945837513, + "loss": 1.3289, + "theoretical_loss": 3.832937559073884, + "tokens_seen": 656211968 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647382146439318, + "loss": 1.5639, + "theoretical_loss": 3.8328985362764616, + "tokens_seen": 656277504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006473660982948847, + "loss": 1.4154, + "theoretical_loss": 3.8328595184666554, + "tokens_seen": 656343040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006473500501504514, + "loss": 1.3872, + "theoretical_loss": 3.8328205056433298, + "tokens_seen": 656408576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006473340020060181, + "loss": 1.4889, + "theoretical_loss": 3.8327814978053505, + "tokens_seen": 656474112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006473179538615848, + "loss": 1.4498, + "theoretical_loss": 3.8327424949515816, + "tokens_seen": 656539648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006473019057171514, + "loss": 1.4404, + "theoretical_loss": 3.83270349708089, + "tokens_seen": 656605184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006472858575727182, + "loss": 1.5596, + "theoretical_loss": 3.832664504192141, + "tokens_seen": 656670720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006472698094282848, + "loss": 1.4844, + "theoretical_loss": 3.8326255162842022, + "tokens_seen": 656736256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006472537612838516, + "loss": 1.5223, + "theoretical_loss": 3.832586533355939, + "tokens_seen": 656801792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006472377131394182, + "loss": 1.2955, + "theoretical_loss": 3.832547555406219, + "tokens_seen": 656867328 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647221664994985, + "loss": 1.5255, + "theoretical_loss": 3.8325085824339102, + "tokens_seen": 656932864 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006472056168505516, + "loss": 1.4322, + "theoretical_loss": 3.8324696144378803, + "tokens_seen": 656998400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006471895687061184, + "loss": 1.5131, + "theoretical_loss": 3.8324306514169977, + "tokens_seen": 657063936 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647173520561685, + "loss": 1.453, + "theoretical_loss": 3.832391693370131, + "tokens_seen": 657129472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006471574724172518, + "loss": 1.4607, + "theoretical_loss": 3.832352740296149, + "tokens_seen": 657195008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006471414242728184, + "loss": 1.4163, + "theoretical_loss": 3.8323137921939217, + "tokens_seen": 657260544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006471253761283852, + "loss": 1.4338, + "theoretical_loss": 3.832274849062318, + "tokens_seen": 657326080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006471093279839518, + "loss": 1.3972, + "theoretical_loss": 3.8322359109002093, + "tokens_seen": 657391616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006470932798395186, + "loss": 1.4953, + "theoretical_loss": 3.8321969777064653, + "tokens_seen": 657457152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006470772316950852, + "loss": 1.3967, + "theoretical_loss": 3.8321580494799568, + "tokens_seen": 657522688 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647061183550652, + "loss": 1.4272, + "theoretical_loss": 3.8321191262195553, + "tokens_seen": 657588224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006470451354062186, + "loss": 1.4333, + "theoretical_loss": 3.832080207924133, + "tokens_seen": 657653760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006470290872617854, + "loss": 1.3768, + "theoretical_loss": 3.832041294592561, + "tokens_seen": 657719296 + }, + { + "epoch": 0.2, + "learning_rate": 0.000647013039117352, + "loss": 1.4894, + "theoretical_loss": 3.832002386223712, + "tokens_seen": 657784832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006469969909729188, + "loss": 1.502, + "theoretical_loss": 3.831963482816459, + "tokens_seen": 657850368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006469809428284854, + "loss": 1.4374, + "theoretical_loss": 3.8319245843696743, + "tokens_seen": 657915904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006469648946840522, + "loss": 1.4651, + "theoretical_loss": 3.8318856908822325, + "tokens_seen": 657981440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006469488465396188, + "loss": 1.4638, + "theoretical_loss": 3.831846802353007, + "tokens_seen": 658046976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006469327983951856, + "loss": 1.4591, + "theoretical_loss": 3.8318079187808713, + "tokens_seen": 658112512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006469167502507522, + "loss": 1.5921, + "theoretical_loss": 3.831769040164701, + "tokens_seen": 658178048 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646900702106319, + "loss": 1.4298, + "theoretical_loss": 3.8317301665033705, + "tokens_seen": 658243584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006468846539618856, + "loss": 1.4376, + "theoretical_loss": 3.8316912977957553, + "tokens_seen": 658309120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006468686058174524, + "loss": 1.4354, + "theoretical_loss": 3.8316524340407314, + "tokens_seen": 658374656 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646852557673019, + "loss": 1.4528, + "theoretical_loss": 3.8316135752371743, + "tokens_seen": 658440192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006468365095285858, + "loss": 1.4728, + "theoretical_loss": 3.8315747213839604, + "tokens_seen": 658505728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006468204613841524, + "loss": 1.4027, + "theoretical_loss": 3.831535872479967, + "tokens_seen": 658571264 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006468044132397192, + "loss": 1.4114, + "theoretical_loss": 3.831497028524071, + "tokens_seen": 658636800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006467883650952858, + "loss": 1.5779, + "theoretical_loss": 3.83145818951515, + "tokens_seen": 658702336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006467723169508526, + "loss": 1.3208, + "theoretical_loss": 3.8314193554520806, + "tokens_seen": 658767872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006467562688064192, + "loss": 1.3667, + "theoretical_loss": 3.831380526333743, + "tokens_seen": 658833408 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646740220661986, + "loss": 1.4175, + "theoretical_loss": 3.8313417021590155, + "tokens_seen": 658898944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006467241725175526, + "loss": 1.385, + "theoretical_loss": 3.831302882926776, + "tokens_seen": 658964480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006467081243731194, + "loss": 1.4629, + "theoretical_loss": 3.831264068635905, + "tokens_seen": 659030016 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646692076228686, + "loss": 1.5108, + "theoretical_loss": 3.831225259285281, + "tokens_seen": 659095552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006466760280842528, + "loss": 1.5288, + "theoretical_loss": 3.8311864548737855, + "tokens_seen": 659161088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006466599799398195, + "loss": 1.3354, + "theoretical_loss": 3.831147655400298, + "tokens_seen": 659226624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006466439317953862, + "loss": 1.5348, + "theoretical_loss": 3.831108860863699, + "tokens_seen": 659292160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006466278836509529, + "loss": 1.5496, + "theoretical_loss": 3.8310700712628707, + "tokens_seen": 659357696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006466118355065196, + "loss": 1.573, + "theoretical_loss": 3.8310312865966942, + "tokens_seen": 659423232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006465957873620863, + "loss": 1.4868, + "theoretical_loss": 3.830992506864051, + "tokens_seen": 659488768 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646579739217653, + "loss": 1.3556, + "theoretical_loss": 3.830953732063824, + "tokens_seen": 659554304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006465636910732197, + "loss": 1.5203, + "theoretical_loss": 3.830914962194895, + "tokens_seen": 659619840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006465476429287864, + "loss": 1.4971, + "theoretical_loss": 3.830876197256148, + "tokens_seen": 659685376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006465315947843531, + "loss": 1.4675, + "theoretical_loss": 3.8308374372464655, + "tokens_seen": 659750912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006465155466399198, + "loss": 1.3648, + "theoretical_loss": 3.830798682164732, + "tokens_seen": 659816448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006464994984954865, + "loss": 1.4014, + "theoretical_loss": 3.8307599320098307, + "tokens_seen": 659881984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006464834503510532, + "loss": 1.5246, + "theoretical_loss": 3.8307211867806465, + "tokens_seen": 659947520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006464674022066199, + "loss": 1.3775, + "theoretical_loss": 3.8306824464760645, + "tokens_seen": 660013056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006464513540621866, + "loss": 1.6433, + "theoretical_loss": 3.83064371109497, + "tokens_seen": 660078592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006464353059177533, + "loss": 1.525, + "theoretical_loss": 3.830604980636248, + "tokens_seen": 660144128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064641925777332, + "loss": 1.4815, + "theoretical_loss": 3.8305662550987845, + "tokens_seen": 660209664 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006464032096288867, + "loss": 1.4589, + "theoretical_loss": 3.8305275344814658, + "tokens_seen": 660275200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006463871614844534, + "loss": 1.5249, + "theoretical_loss": 3.8304888187831785, + "tokens_seen": 660340736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006463711133400201, + "loss": 1.4021, + "theoretical_loss": 3.8304501080028097, + "tokens_seen": 660406272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006463550651955868, + "loss": 1.3538, + "theoretical_loss": 3.830411402139247, + "tokens_seen": 660471808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006463390170511535, + "loss": 1.5667, + "theoretical_loss": 3.830372701191377, + "tokens_seen": 660537344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006463229689067202, + "loss": 1.4984, + "theoretical_loss": 3.8303340051580896, + "tokens_seen": 660602880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006463069207622869, + "loss": 1.5137, + "theoretical_loss": 3.8302953140382714, + "tokens_seen": 660668416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006462908726178536, + "loss": 1.5396, + "theoretical_loss": 3.8302566278308126, + "tokens_seen": 660733952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006462748244734203, + "loss": 1.5304, + "theoretical_loss": 3.830217946534602, + "tokens_seen": 660799488 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646258776328987, + "loss": 1.3623, + "theoretical_loss": 3.830179270148528, + "tokens_seen": 660865024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006462427281845538, + "loss": 1.4956, + "theoretical_loss": 3.8301405986714823, + "tokens_seen": 660930560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006462266800401204, + "loss": 1.4504, + "theoretical_loss": 3.8301019321023544, + "tokens_seen": 660996096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006462106318956872, + "loss": 1.3777, + "theoretical_loss": 3.830063270440034, + "tokens_seen": 661061632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006461945837512538, + "loss": 1.5454, + "theoretical_loss": 3.8300246136834133, + "tokens_seen": 661127168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006461785356068206, + "loss": 1.4418, + "theoretical_loss": 3.829985961831383, + "tokens_seen": 661192704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006461624874623872, + "loss": 1.406, + "theoretical_loss": 3.829947314882835, + "tokens_seen": 661258240 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646146439317954, + "loss": 1.4561, + "theoretical_loss": 3.829908672836661, + "tokens_seen": 661323776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006461303911735206, + "loss": 1.523, + "theoretical_loss": 3.829870035691754, + "tokens_seen": 661389312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006461143430290874, + "loss": 1.5152, + "theoretical_loss": 3.829831403447007, + "tokens_seen": 661454848 + }, + { + "epoch": 0.2, + "learning_rate": 0.000646098294884654, + "loss": 1.4649, + "theoretical_loss": 3.8297927761013115, + "tokens_seen": 661520384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006460822467402208, + "loss": 1.4576, + "theoretical_loss": 3.829754153653563, + "tokens_seen": 661585920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006460661985957874, + "loss": 1.4542, + "theoretical_loss": 3.829715536102654, + "tokens_seen": 661651456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006460501504513542, + "loss": 1.4534, + "theoretical_loss": 3.8296769234474795, + "tokens_seen": 661716992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006460341023069208, + "loss": 1.4472, + "theoretical_loss": 3.829638315686933, + "tokens_seen": 661782528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006460180541624875, + "loss": 1.459, + "theoretical_loss": 3.8295997128199106, + "tokens_seen": 661848064 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 251048, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4656431674957275, + "objective/train/theoretical_loss": 3.8295611148453075, + "objective/train/tokens_used": 682373600, + "theoretical_loss": 3.8295611148453075, + "tokens_seen": 661913600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006460020060180542, + "loss": 1.4656, + "theoretical_loss": 3.8295611148453075, + "tokens_seen": 661913600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006459859578736209, + "loss": 1.4711, + "theoretical_loss": 3.8295225217620183, + "tokens_seen": 661979136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006459699097291876, + "loss": 1.5151, + "theoretical_loss": 3.82948393356894, + "tokens_seen": 662044672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006459538615847543, + "loss": 1.4518, + "theoretical_loss": 3.829445350264969, + "tokens_seen": 662110208 + }, + { + "epoch": 0.2, + "learning_rate": 0.000645937813440321, + "loss": 1.4811, + "theoretical_loss": 3.8294067718490012, + "tokens_seen": 662175744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006459217652958877, + "loss": 1.4783, + "theoretical_loss": 3.829368198319934, + "tokens_seen": 662241280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006459057171514543, + "loss": 1.6445, + "theoretical_loss": 3.829329629676665, + "tokens_seen": 662306816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006458896690070211, + "loss": 1.4921, + "theoretical_loss": 3.829291065918092, + "tokens_seen": 662372352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006458736208625877, + "loss": 1.4392, + "theoretical_loss": 3.8292525070431136, + "tokens_seen": 662437888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006458575727181545, + "loss": 1.3938, + "theoretical_loss": 3.829213953050627, + "tokens_seen": 662503424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006458415245737211, + "loss": 1.4164, + "theoretical_loss": 3.8291754039395323, + "tokens_seen": 662568960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006458254764292879, + "loss": 1.4304, + "theoretical_loss": 3.8291368597087283, + "tokens_seen": 662634496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006458094282848545, + "loss": 1.4666, + "theoretical_loss": 3.8290983203571147, + "tokens_seen": 662700032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006457933801404213, + "loss": 1.44, + "theoretical_loss": 3.829059785883591, + "tokens_seen": 662765568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006457773319959879, + "loss": 1.3826, + "theoretical_loss": 3.8290212562870583, + "tokens_seen": 662831104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006457612838515547, + "loss": 1.3801, + "theoretical_loss": 3.8289827315664158, + "tokens_seen": 662896640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006457452357071213, + "loss": 1.4399, + "theoretical_loss": 3.8289442117205663, + "tokens_seen": 662962176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006457291875626881, + "loss": 1.379, + "theoretical_loss": 3.82890569674841, + "tokens_seen": 663027712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006457131394182547, + "loss": 1.3771, + "theoretical_loss": 3.828867186648849, + "tokens_seen": 663093248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456970912738215, + "loss": 1.4082, + "theoretical_loss": 3.8288286814207853, + "tokens_seen": 663158784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456810431293881, + "loss": 1.4927, + "theoretical_loss": 3.828790181063121, + "tokens_seen": 663224320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456649949849549, + "loss": 1.4435, + "theoretical_loss": 3.828751685574759, + "tokens_seen": 663289856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456489468405215, + "loss": 1.4762, + "theoretical_loss": 3.828713194954604, + "tokens_seen": 663355392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456328986960883, + "loss": 1.4037, + "theoretical_loss": 3.8286747092015574, + "tokens_seen": 663420928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456168505516549, + "loss": 1.302, + "theoretical_loss": 3.828636228314523, + "tokens_seen": 663486464 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006456008024072217, + "loss": 1.5484, + "theoretical_loss": 3.8285977522924073, + "tokens_seen": 663552000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006455847542627883, + "loss": 1.4774, + "theoretical_loss": 3.8285592811341127, + "tokens_seen": 663617536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006455687061183551, + "loss": 1.3935, + "theoretical_loss": 3.8285208148385443, + "tokens_seen": 663683072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006455526579739217, + "loss": 1.5153, + "theoretical_loss": 3.8284823534046084, + "tokens_seen": 663748608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006455366098294885, + "loss": 1.4043, + "theoretical_loss": 3.8284438968312102, + "tokens_seen": 663814144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006455205616850551, + "loss": 1.5063, + "theoretical_loss": 3.828405445117255, + "tokens_seen": 663879680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006455045135406219, + "loss": 1.4025, + "theoretical_loss": 3.82836699826165, + "tokens_seen": 663945216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006454884653961885, + "loss": 1.4684, + "theoretical_loss": 3.828328556263302, + "tokens_seen": 664010752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006454724172517553, + "loss": 1.4941, + "theoretical_loss": 3.8282901191211174, + "tokens_seen": 664076288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006454563691073219, + "loss": 1.3229, + "theoretical_loss": 3.8282516868340037, + "tokens_seen": 664141824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006454403209628887, + "loss": 1.5087, + "theoretical_loss": 3.8282132594008687, + "tokens_seen": 664207360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006454242728184553, + "loss": 1.4969, + "theoretical_loss": 3.828174836820621, + "tokens_seen": 664272896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006454082246740221, + "loss": 1.5302, + "theoretical_loss": 3.8281364190921683, + "tokens_seen": 664338432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006453921765295887, + "loss": 1.531, + "theoretical_loss": 3.82809800621442, + "tokens_seen": 664403968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006453761283851555, + "loss": 1.561, + "theoretical_loss": 3.8280595981862846, + "tokens_seen": 664469504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006453600802407221, + "loss": 1.4447, + "theoretical_loss": 3.8280211950066723, + "tokens_seen": 664535040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006453440320962889, + "loss": 1.4819, + "theoretical_loss": 3.827982796674493, + "tokens_seen": 664600576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006453279839518556, + "loss": 1.4775, + "theoretical_loss": 3.8279444031886563, + "tokens_seen": 664666112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006453119358074223, + "loss": 1.4321, + "theoretical_loss": 3.827906014548073, + "tokens_seen": 664731648 + }, + { + "epoch": 0.2, + "learning_rate": 0.000645295887662989, + "loss": 1.4916, + "theoretical_loss": 3.827867630751654, + "tokens_seen": 664797184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006452798395185557, + "loss": 1.3779, + "theoretical_loss": 3.827829251798311, + "tokens_seen": 664862720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006452637913741224, + "loss": 1.5305, + "theoretical_loss": 3.8277908776869554, + "tokens_seen": 664928256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006452477432296891, + "loss": 1.437, + "theoretical_loss": 3.8277525084164994, + "tokens_seen": 664993792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006452316950852558, + "loss": 1.4119, + "theoretical_loss": 3.8277141439858546, + "tokens_seen": 665059328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006452156469408225, + "loss": 1.5956, + "theoretical_loss": 3.827675784393934, + "tokens_seen": 665124864 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006451995987963892, + "loss": 1.4413, + "theoretical_loss": 3.827637429639651, + "tokens_seen": 665190400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006451835506519559, + "loss": 1.4879, + "theoretical_loss": 3.827599079721918, + "tokens_seen": 665255936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006451675025075226, + "loss": 1.3641, + "theoretical_loss": 3.8275607346396505, + "tokens_seen": 665321472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006451514543630893, + "loss": 1.4581, + "theoretical_loss": 3.827522394391761, + "tokens_seen": 665387008 + }, + { + "epoch": 0.2, + "learning_rate": 0.000645135406218656, + "loss": 1.4461, + "theoretical_loss": 3.8274840589771655, + "tokens_seen": 665452544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006451193580742227, + "loss": 1.4798, + "theoretical_loss": 3.8274457283947765, + "tokens_seen": 665518080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006451033099297894, + "loss": 1.4307, + "theoretical_loss": 3.8274074026435114, + "tokens_seen": 665583616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006450872617853561, + "loss": 1.5289, + "theoretical_loss": 3.8273690817222845, + "tokens_seen": 665649152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006450712136409228, + "loss": 1.4446, + "theoretical_loss": 3.8273307656300117, + "tokens_seen": 665714688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006450551654964895, + "loss": 1.3357, + "theoretical_loss": 3.8272924543656095, + "tokens_seen": 665780224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006450391173520563, + "loss": 1.583, + "theoretical_loss": 3.8272541479279942, + "tokens_seen": 665845760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006450230692076229, + "loss": 1.4468, + "theoretical_loss": 3.827215846316083, + "tokens_seen": 665911296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006450070210631897, + "loss": 1.5415, + "theoretical_loss": 3.827177549528793, + "tokens_seen": 665976832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006449909729187563, + "loss": 1.4231, + "theoretical_loss": 3.8271392575650416, + "tokens_seen": 666042368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006449749247743231, + "loss": 1.4286, + "theoretical_loss": 3.827100970423747, + "tokens_seen": 666107904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006449588766298897, + "loss": 1.4574, + "theoretical_loss": 3.827062688103827, + "tokens_seen": 666173440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006449428284854565, + "loss": 1.4079, + "theoretical_loss": 3.827024410604201, + "tokens_seen": 666238976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006449267803410231, + "loss": 1.3217, + "theoretical_loss": 3.8269861379237877, + "tokens_seen": 666304512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006449107321965899, + "loss": 1.5236, + "theoretical_loss": 3.826947870061506, + "tokens_seen": 666370048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006448946840521565, + "loss": 1.4383, + "theoretical_loss": 3.826909607016276, + "tokens_seen": 666435584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006448786359077233, + "loss": 1.5482, + "theoretical_loss": 3.8268713487870176, + "tokens_seen": 666501120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006448625877632899, + "loss": 1.3688, + "theoretical_loss": 3.8268330953726513, + "tokens_seen": 666566656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006448465396188567, + "loss": 1.4253, + "theoretical_loss": 3.826794846772098, + "tokens_seen": 666632192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006448304914744233, + "loss": 1.4314, + "theoretical_loss": 3.826756602984278, + "tokens_seen": 666697728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006448144433299901, + "loss": 1.4466, + "theoretical_loss": 3.826718364008114, + "tokens_seen": 666763264 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447983951855567, + "loss": 1.4245, + "theoretical_loss": 3.826680129842526, + "tokens_seen": 666828800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447823470411235, + "loss": 1.4653, + "theoretical_loss": 3.826641900486438, + "tokens_seen": 666894336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447662988966901, + "loss": 1.4317, + "theoretical_loss": 3.8266036759387716, + "tokens_seen": 666959872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447502507522569, + "loss": 1.4173, + "theoretical_loss": 3.8265654561984492, + "tokens_seen": 667025408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447342026078235, + "loss": 1.4304, + "theoretical_loss": 3.8265272412643947, + "tokens_seen": 667090944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447181544633903, + "loss": 1.4287, + "theoretical_loss": 3.8264890311355315, + "tokens_seen": 667156480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006447021063189569, + "loss": 1.5302, + "theoretical_loss": 3.8264508258107828, + "tokens_seen": 667222016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006446860581745237, + "loss": 1.4458, + "theoretical_loss": 3.8264126252890738, + "tokens_seen": 667287552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006446700100300903, + "loss": 1.4648, + "theoretical_loss": 3.826374429569328, + "tokens_seen": 667353088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006446539618856571, + "loss": 1.4563, + "theoretical_loss": 3.8263362386504713, + "tokens_seen": 667418624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006446379137412237, + "loss": 1.5038, + "theoretical_loss": 3.826298052531428, + "tokens_seen": 667484160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006446218655967904, + "loss": 1.4522, + "theoretical_loss": 3.8262598712111244, + "tokens_seen": 667549696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006446058174523571, + "loss": 1.4917, + "theoretical_loss": 3.8262216946884866, + "tokens_seen": 667615232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006445897693079238, + "loss": 1.3278, + "theoretical_loss": 3.8261835229624395, + "tokens_seen": 667680768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006445737211634905, + "loss": 1.391, + "theoretical_loss": 3.8261453560319114, + "tokens_seen": 667746304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006445576730190572, + "loss": 1.4028, + "theoretical_loss": 3.8261071938958278, + "tokens_seen": 667811840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006445416248746239, + "loss": 1.4081, + "theoretical_loss": 3.826069036553118, + "tokens_seen": 667877376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006445255767301906, + "loss": 1.4501, + "theoretical_loss": 3.8260308840027073, + "tokens_seen": 667942912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006445095285857573, + "loss": 1.4055, + "theoretical_loss": 3.825992736243525, + "tokens_seen": 668008448 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644493480441324, + "loss": 1.5276, + "theoretical_loss": 3.8259545932745, + "tokens_seen": 668073984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006444774322968906, + "loss": 1.4461, + "theoretical_loss": 3.8259164550945597, + "tokens_seen": 668139520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006444613841524574, + "loss": 1.5221, + "theoretical_loss": 3.8258783217026338, + "tokens_seen": 668205056 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644445336008024, + "loss": 1.4351, + "theoretical_loss": 3.825840193097652, + "tokens_seen": 668270592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006444292878635908, + "loss": 1.3219, + "theoretical_loss": 3.8258020692785433, + "tokens_seen": 668336128 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006444132397191574, + "loss": 1.4793, + "theoretical_loss": 3.825763950244238, + "tokens_seen": 668401664 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 253423, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3911454677581787, + "objective/train/theoretical_loss": 3.8257258359936666, + "objective/train/tokens_used": 688927200, + "theoretical_loss": 3.8257258359936666, + "tokens_seen": 668467200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006443971915747242, + "loss": 1.3911, + "theoretical_loss": 3.8257258359936666, + "tokens_seen": 668467200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006443811434302908, + "loss": 1.3948, + "theoretical_loss": 3.82568772652576, + "tokens_seen": 668532736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006443650952858576, + "loss": 1.5572, + "theoretical_loss": 3.8256496218394496, + "tokens_seen": 668598272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006443490471414242, + "loss": 1.4988, + "theoretical_loss": 3.825611521933666, + "tokens_seen": 668663808 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644332998996991, + "loss": 1.5753, + "theoretical_loss": 3.8255734268073414, + "tokens_seen": 668729344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006443169508525576, + "loss": 1.525, + "theoretical_loss": 3.8255353364594082, + "tokens_seen": 668794880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006443009027081244, + "loss": 1.3792, + "theoretical_loss": 3.825497250888799, + "tokens_seen": 668860416 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644284854563691, + "loss": 1.4945, + "theoretical_loss": 3.8254591700944456, + "tokens_seen": 668925952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006442688064192578, + "loss": 1.3719, + "theoretical_loss": 3.825421094075282, + "tokens_seen": 668991488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006442527582748244, + "loss": 1.5215, + "theoretical_loss": 3.8253830228302417, + "tokens_seen": 669057024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006442367101303912, + "loss": 1.4796, + "theoretical_loss": 3.825344956358258, + "tokens_seen": 669122560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006442206619859578, + "loss": 1.3992, + "theoretical_loss": 3.825306894658266, + "tokens_seen": 669188096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006442046138415246, + "loss": 1.3552, + "theoretical_loss": 3.8252688377292, + "tokens_seen": 669253632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006441885656970912, + "loss": 1.3949, + "theoretical_loss": 3.8252307855699943, + "tokens_seen": 669319168 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644172517552658, + "loss": 1.4571, + "theoretical_loss": 3.8251927381795845, + "tokens_seen": 669384704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006441564694082246, + "loss": 1.501, + "theoretical_loss": 3.825154695556906, + "tokens_seen": 669450240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006441404212637914, + "loss": 1.4459, + "theoretical_loss": 3.825116657700894, + "tokens_seen": 669515776 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644124373119358, + "loss": 1.4188, + "theoretical_loss": 3.8250786246104864, + "tokens_seen": 669581312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006441083249749248, + "loss": 1.3845, + "theoretical_loss": 3.825040596284619, + "tokens_seen": 669646848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006440922768304914, + "loss": 1.3524, + "theoretical_loss": 3.8250025727222283, + "tokens_seen": 669712384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006440762286860582, + "loss": 1.4997, + "theoretical_loss": 3.8249645539222525, + "tokens_seen": 669777920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006440601805416248, + "loss": 1.4777, + "theoretical_loss": 3.824926539883628, + "tokens_seen": 669843456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006440441323971916, + "loss": 1.4311, + "theoretical_loss": 3.8248885306052935, + "tokens_seen": 669908992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006440280842527582, + "loss": 1.4545, + "theoretical_loss": 3.8248505260861876, + "tokens_seen": 669974528 + }, + { + "epoch": 0.2, + "learning_rate": 0.000644012036108325, + "loss": 1.5047, + "theoretical_loss": 3.824812526325248, + "tokens_seen": 670040064 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006439959879638916, + "loss": 1.4194, + "theoretical_loss": 3.824774531321414, + "tokens_seen": 670105600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006439799398194584, + "loss": 1.4616, + "theoretical_loss": 3.8247365410736256, + "tokens_seen": 670171136 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643963891675025, + "loss": 1.449, + "theoretical_loss": 3.824698555580822, + "tokens_seen": 670236672 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006439478435305918, + "loss": 1.4941, + "theoretical_loss": 3.8246605748419427, + "tokens_seen": 670302208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006439317953861585, + "loss": 1.4648, + "theoretical_loss": 3.824622598855929, + "tokens_seen": 670367744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006439157472417252, + "loss": 1.5464, + "theoretical_loss": 3.824584627621721, + "tokens_seen": 670433280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006438996990972919, + "loss": 1.4615, + "theoretical_loss": 3.8245466611382595, + "tokens_seen": 670498816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006438836509528586, + "loss": 1.4937, + "theoretical_loss": 3.824508699404486, + "tokens_seen": 670564352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006438676028084253, + "loss": 1.3652, + "theoretical_loss": 3.8244707424193427, + "tokens_seen": 670629888 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643851554663992, + "loss": 1.3394, + "theoretical_loss": 3.8244327901817705, + "tokens_seen": 670695424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006438355065195588, + "loss": 1.4806, + "theoretical_loss": 3.8243948426907135, + "tokens_seen": 670760960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006438194583751254, + "loss": 1.5021, + "theoretical_loss": 3.824356899945113, + "tokens_seen": 670826496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006438034102306922, + "loss": 1.4077, + "theoretical_loss": 3.824318961943912, + "tokens_seen": 670892032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006437873620862588, + "loss": 1.3698, + "theoretical_loss": 3.8242810286860545, + "tokens_seen": 670957568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006437713139418256, + "loss": 1.4493, + "theoretical_loss": 3.8242431001704844, + "tokens_seen": 671023104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006437552657973922, + "loss": 1.4623, + "theoretical_loss": 3.8242051763961458, + "tokens_seen": 671088640 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643739217652959, + "loss": 1.4465, + "theoretical_loss": 3.824167257361982, + "tokens_seen": 671154176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006437231695085256, + "loss": 1.4497, + "theoretical_loss": 3.8241293430669385, + "tokens_seen": 671219712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006437071213640924, + "loss": 1.4534, + "theoretical_loss": 3.824091433509961, + "tokens_seen": 671285248 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643691073219659, + "loss": 1.4898, + "theoretical_loss": 3.824053528689994, + "tokens_seen": 671350784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006436750250752258, + "loss": 1.4757, + "theoretical_loss": 3.8240156286059834, + "tokens_seen": 671416320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006436589769307924, + "loss": 1.3737, + "theoretical_loss": 3.823977733256875, + "tokens_seen": 671481856 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006436429287863592, + "loss": 1.4475, + "theoretical_loss": 3.8239398426416162, + "tokens_seen": 671547392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006436268806419258, + "loss": 1.4758, + "theoretical_loss": 3.823901956759153, + "tokens_seen": 671612928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006436108324974926, + "loss": 1.4277, + "theoretical_loss": 3.8238640756084328, + "tokens_seen": 671678464 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006435947843530592, + "loss": 1.5101, + "theoretical_loss": 3.823826199188403, + "tokens_seen": 671744000 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643578736208626, + "loss": 1.3972, + "theoretical_loss": 3.8237883274980113, + "tokens_seen": 671809536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006435626880641926, + "loss": 1.5516, + "theoretical_loss": 3.823750460536206, + "tokens_seen": 671875072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006435466399197594, + "loss": 1.4647, + "theoretical_loss": 3.8237125983019347, + "tokens_seen": 671940608 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643530591775326, + "loss": 1.4529, + "theoretical_loss": 3.8236747407941474, + "tokens_seen": 672006144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006435145436308928, + "loss": 1.4181, + "theoretical_loss": 3.8236368880117926, + "tokens_seen": 672071680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006434984954864594, + "loss": 1.3701, + "theoretical_loss": 3.82359903995382, + "tokens_seen": 672137216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006434824473420262, + "loss": 1.5339, + "theoretical_loss": 3.823561196619179, + "tokens_seen": 672202752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006434663991975928, + "loss": 1.4403, + "theoretical_loss": 3.82352335800682, + "tokens_seen": 672268288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006434503510531596, + "loss": 1.3814, + "theoretical_loss": 3.8234855241156938, + "tokens_seen": 672333824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006434343029087262, + "loss": 1.4819, + "theoretical_loss": 3.8234476949447505, + "tokens_seen": 672399360 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643418254764293, + "loss": 1.4458, + "theoretical_loss": 3.8234098704929416, + "tokens_seen": 672464896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006434022066198596, + "loss": 1.3617, + "theoretical_loss": 3.823372050759218, + "tokens_seen": 672530432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006433861584754264, + "loss": 1.3267, + "theoretical_loss": 3.823334235742533, + "tokens_seen": 672595968 + }, + { + "epoch": 0.2, + "learning_rate": 0.000643370110330993, + "loss": 1.6313, + "theoretical_loss": 3.8232964254418373, + "tokens_seen": 672661504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006433540621865598, + "loss": 1.4105, + "theoretical_loss": 3.823258619856084, + "tokens_seen": 672727040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006433380140421264, + "loss": 1.4961, + "theoretical_loss": 3.8232208189842254, + "tokens_seen": 672792576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006433219658976932, + "loss": 1.4937, + "theoretical_loss": 3.8231830228252157, + "tokens_seen": 672858112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006433059177532598, + "loss": 1.4487, + "theoretical_loss": 3.8231452313780068, + "tokens_seen": 672923648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006432898696088266, + "loss": 1.481, + "theoretical_loss": 3.823107444641554, + "tokens_seen": 672989184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006432738214643932, + "loss": 1.4759, + "theoretical_loss": 3.8230696626148113, + "tokens_seen": 673054720 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064325777331996, + "loss": 1.5078, + "theoretical_loss": 3.823031885296732, + "tokens_seen": 673120256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006432417251755266, + "loss": 1.4645, + "theoretical_loss": 3.822994112686272, + "tokens_seen": 673185792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006432256770310933, + "loss": 1.5027, + "theoretical_loss": 3.8229563447823858, + "tokens_seen": 673251328 + }, + { + "epoch": 0.2, + "learning_rate": 0.00064320962888666, + "loss": 1.5369, + "theoretical_loss": 3.8229185815840294, + "tokens_seen": 673316864 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006431935807422267, + "loss": 1.4078, + "theoretical_loss": 3.8228808230901583, + "tokens_seen": 673382400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006431775325977934, + "loss": 1.501, + "theoretical_loss": 3.822843069299729, + "tokens_seen": 673447936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006431614844533601, + "loss": 1.3339, + "theoretical_loss": 3.822805320211698, + "tokens_seen": 673513472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006431454363089268, + "loss": 1.4359, + "theoretical_loss": 3.8227675758250212, + "tokens_seen": 673579008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006431293881644935, + "loss": 1.4524, + "theoretical_loss": 3.822729836138657, + "tokens_seen": 673644544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006431133400200602, + "loss": 1.5686, + "theoretical_loss": 3.8226921011515618, + "tokens_seen": 673710080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430972918756269, + "loss": 1.487, + "theoretical_loss": 3.822654370862694, + "tokens_seen": 673775616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430812437311935, + "loss": 1.4516, + "theoretical_loss": 3.8226166452710117, + "tokens_seen": 673841152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430651955867603, + "loss": 1.5144, + "theoretical_loss": 3.822578924375473, + "tokens_seen": 673906688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430491474423269, + "loss": 1.4431, + "theoretical_loss": 3.8225412081750374, + "tokens_seen": 673972224 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430330992978937, + "loss": 1.36, + "theoretical_loss": 3.8225034966686637, + "tokens_seen": 674037760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430170511534603, + "loss": 1.4488, + "theoretical_loss": 3.822465789855311, + "tokens_seen": 674103296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006430010030090271, + "loss": 1.3974, + "theoretical_loss": 3.82242808773394, + "tokens_seen": 674168832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006429849548645937, + "loss": 1.3749, + "theoretical_loss": 3.82239039030351, + "tokens_seen": 674234368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006429689067201605, + "loss": 1.4578, + "theoretical_loss": 3.822352697562982, + "tokens_seen": 674299904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006429528585757271, + "loss": 1.3355, + "theoretical_loss": 3.8223150095113163, + "tokens_seen": 674365440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006429368104312939, + "loss": 1.4898, + "theoretical_loss": 3.822277326147475, + "tokens_seen": 674430976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006429207622868605, + "loss": 1.4613, + "theoretical_loss": 3.822239647470418, + "tokens_seen": 674496512 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006429047141424273, + "loss": 1.4978, + "theoretical_loss": 3.822201973479108, + "tokens_seen": 674562048 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006428886659979939, + "loss": 1.447, + "theoretical_loss": 3.8221643041725075, + "tokens_seen": 674627584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006428726178535607, + "loss": 1.4329, + "theoretical_loss": 3.8221266395495785, + "tokens_seen": 674693120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006428565697091273, + "loss": 1.4829, + "theoretical_loss": 3.822088979609284, + "tokens_seen": 674758656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006428405215646941, + "loss": 1.4044, + "theoretical_loss": 3.8220513243505865, + "tokens_seen": 674824192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006428244734202607, + "loss": 1.3436, + "theoretical_loss": 3.8220136737724504, + "tokens_seen": 674889728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006428084252758275, + "loss": 1.44, + "theoretical_loss": 3.8219760278738386, + "tokens_seen": 674955264 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 255766, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.49822998046875, + "objective/train/theoretical_loss": 3.821938386653716, + "objective/train/tokens_used": 695480800, + "theoretical_loss": 3.821938386653716, + "tokens_seen": 675020800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006427923771313941, + "loss": 1.4982, + "theoretical_loss": 3.821938386653716, + "tokens_seen": 675020800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006427763289869609, + "loss": 1.4225, + "theoretical_loss": 3.8219007501110465, + "tokens_seen": 675086336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006427602808425275, + "loss": 1.3116, + "theoretical_loss": 3.8218631182447944, + "tokens_seen": 675151872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006427442326980943, + "loss": 1.4599, + "theoretical_loss": 3.821825491053926, + "tokens_seen": 675217408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006427281845536609, + "loss": 1.4841, + "theoretical_loss": 3.821787868537406, + "tokens_seen": 675282944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006427121364092277, + "loss": 1.4674, + "theoretical_loss": 3.8217502506942003, + "tokens_seen": 675348480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006426960882647943, + "loss": 1.5434, + "theoretical_loss": 3.821712637523275, + "tokens_seen": 675414016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006426800401203611, + "loss": 1.4472, + "theoretical_loss": 3.8216750290235963, + "tokens_seen": 675479552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006426639919759277, + "loss": 1.5112, + "theoretical_loss": 3.8216374251941314, + "tokens_seen": 675545088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006426479438314945, + "loss": 1.46, + "theoretical_loss": 3.8215998260338466, + "tokens_seen": 675610624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006426318956870613, + "loss": 1.4324, + "theoretical_loss": 3.8215622315417104, + "tokens_seen": 675676160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006426158475426279, + "loss": 1.4402, + "theoretical_loss": 3.8215246417166893, + "tokens_seen": 675741696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425997993981947, + "loss": 1.4747, + "theoretical_loss": 3.821487056557752, + "tokens_seen": 675807232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425837512537613, + "loss": 1.4789, + "theoretical_loss": 3.8214494760638673, + "tokens_seen": 675872768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425677031093281, + "loss": 1.5034, + "theoretical_loss": 3.8214119002340032, + "tokens_seen": 675938304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425516549648947, + "loss": 1.4781, + "theoretical_loss": 3.821374329067129, + "tokens_seen": 676003840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425356068204615, + "loss": 1.4622, + "theoretical_loss": 3.8213367625622143, + "tokens_seen": 676069376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425195586760281, + "loss": 1.4422, + "theoretical_loss": 3.8212992007182285, + "tokens_seen": 676134912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006425035105315949, + "loss": 1.3767, + "theoretical_loss": 3.8212616435341413, + "tokens_seen": 676200448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006424874623871615, + "loss": 1.3889, + "theoretical_loss": 3.821224091008924, + "tokens_seen": 676265984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006424714142427283, + "loss": 1.3664, + "theoretical_loss": 3.821186543141547, + "tokens_seen": 676331520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006424553660982949, + "loss": 1.4213, + "theoretical_loss": 3.8211489999309807, + "tokens_seen": 676397056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0006424393179538617, + "loss": 1.482, + "theoretical_loss": 3.821111461376196, + "tokens_seen": 676462592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006424232698094283, + "loss": 1.407, + "theoretical_loss": 3.8210739274761663, + "tokens_seen": 676528128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006424072216649951, + "loss": 1.4529, + "theoretical_loss": 3.8210363982298623, + "tokens_seen": 676593664 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006423911735205617, + "loss": 1.4119, + "theoretical_loss": 3.820998873636257, + "tokens_seen": 676659200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006423751253761285, + "loss": 1.4802, + "theoretical_loss": 3.8209613536943223, + "tokens_seen": 676724736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006423590772316951, + "loss": 1.445, + "theoretical_loss": 3.8209238384030315, + "tokens_seen": 676790272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006423430290872619, + "loss": 1.484, + "theoretical_loss": 3.820886327761358, + "tokens_seen": 676855808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006423269809428285, + "loss": 1.5019, + "theoretical_loss": 3.820848821768275, + "tokens_seen": 676921344 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006423109327983953, + "loss": 1.4392, + "theoretical_loss": 3.8208113204227576, + "tokens_seen": 676986880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006422948846539619, + "loss": 1.4155, + "theoretical_loss": 3.8207738237237785, + "tokens_seen": 677052416 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006422788365095287, + "loss": 1.3028, + "theoretical_loss": 3.8207363316703136, + "tokens_seen": 677117952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006422627883650953, + "loss": 1.4238, + "theoretical_loss": 3.820698844261337, + "tokens_seen": 677183488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006422467402206621, + "loss": 1.5176, + "theoretical_loss": 3.8206613614958242, + "tokens_seen": 677249024 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006422306920762287, + "loss": 1.6216, + "theoretical_loss": 3.820623883372751, + "tokens_seen": 677314560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006422146439317955, + "loss": 1.4701, + "theoretical_loss": 3.820586409891093, + "tokens_seen": 677380096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421985957873621, + "loss": 1.5553, + "theoretical_loss": 3.820548941049826, + "tokens_seen": 677445632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421825476429289, + "loss": 1.4916, + "theoretical_loss": 3.820511476847928, + "tokens_seen": 677511168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421664994984955, + "loss": 1.3735, + "theoretical_loss": 3.8204740172843747, + "tokens_seen": 677576704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421504513540623, + "loss": 1.4842, + "theoretical_loss": 3.820436562358143, + "tokens_seen": 677642240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421344032096289, + "loss": 1.4138, + "theoretical_loss": 3.8203991120682117, + "tokens_seen": 677707776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421183550651957, + "loss": 1.48, + "theoretical_loss": 3.8203616664135573, + "tokens_seen": 677773312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006421023069207623, + "loss": 1.3928, + "theoretical_loss": 3.8203242253931586, + "tokens_seen": 677838848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006420862587763291, + "loss": 1.4028, + "theoretical_loss": 3.820286789005994, + "tokens_seen": 677904384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006420702106318957, + "loss": 1.425, + "theoretical_loss": 3.820249357251043, + "tokens_seen": 677969920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006420541624874625, + "loss": 1.3696, + "theoretical_loss": 3.8202119301272837, + "tokens_seen": 678035456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006420381143430291, + "loss": 1.6002, + "theoretical_loss": 3.8201745076336957, + "tokens_seen": 678100992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006420220661985959, + "loss": 1.4544, + "theoretical_loss": 3.8201370897692595, + "tokens_seen": 678166528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006420060180541625, + "loss": 1.4562, + "theoretical_loss": 3.820099676532955, + "tokens_seen": 678232064 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006419899699097293, + "loss": 1.4991, + "theoretical_loss": 3.8200622679237615, + "tokens_seen": 678297600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006419739217652959, + "loss": 1.4277, + "theoretical_loss": 3.8200248639406613, + "tokens_seen": 678363136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006419578736208627, + "loss": 1.4796, + "theoretical_loss": 3.819987464582635, + "tokens_seen": 678428672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006419418254764293, + "loss": 1.355, + "theoretical_loss": 3.8199500698486633, + "tokens_seen": 678494208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006419257773319961, + "loss": 1.4865, + "theoretical_loss": 3.8199126797377287, + "tokens_seen": 678559744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006419097291875627, + "loss": 1.3883, + "theoretical_loss": 3.8198752942488134, + "tokens_seen": 678625280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006418936810431295, + "loss": 1.4149, + "theoretical_loss": 3.8198379133808995, + "tokens_seen": 678690816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006418776328986961, + "loss": 1.447, + "theoretical_loss": 3.819800537132969, + "tokens_seen": 678756352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006418615847542629, + "loss": 1.3905, + "theoretical_loss": 3.8197631655040056, + "tokens_seen": 678821888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006418455366098295, + "loss": 1.4758, + "theoretical_loss": 3.8197257984929927, + "tokens_seen": 678887424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006418294884653962, + "loss": 1.573, + "theoretical_loss": 3.8196884360989136, + "tokens_seen": 678952960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006418134403209629, + "loss": 1.4217, + "theoretical_loss": 3.819651078320753, + "tokens_seen": 679018496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006417973921765296, + "loss": 1.3761, + "theoretical_loss": 3.819613725157494, + "tokens_seen": 679084032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006417813440320963, + "loss": 1.449, + "theoretical_loss": 3.819576376608122, + "tokens_seen": 679149568 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641765295887663, + "loss": 1.3973, + "theoretical_loss": 3.8195390326716225, + "tokens_seen": 679215104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006417492477432297, + "loss": 1.449, + "theoretical_loss": 3.8195016933469796, + "tokens_seen": 679280640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006417331995987964, + "loss": 1.351, + "theoretical_loss": 3.819464358633179, + "tokens_seen": 679346176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006417171514543631, + "loss": 1.5313, + "theoretical_loss": 3.819427028529208, + "tokens_seen": 679411712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006417011033099298, + "loss": 1.4955, + "theoretical_loss": 3.8193897030340516, + "tokens_seen": 679477248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006416850551654964, + "loss": 1.5166, + "theoretical_loss": 3.819352382146696, + "tokens_seen": 679542784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006416690070210632, + "loss": 1.4533, + "theoretical_loss": 3.819315065866129, + "tokens_seen": 679608320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006416529588766298, + "loss": 1.5197, + "theoretical_loss": 3.8192777541913374, + "tokens_seen": 679673856 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006416369107321966, + "loss": 1.4249, + "theoretical_loss": 3.819240447121309, + "tokens_seen": 679739392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006416208625877632, + "loss": 1.4677, + "theoretical_loss": 3.8192031446550314, + "tokens_seen": 679804928 + }, + { + "epoch": 0.21, + "learning_rate": 0.00064160481444333, + "loss": 1.5862, + "theoretical_loss": 3.819165846791493, + "tokens_seen": 679870464 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006415887662988966, + "loss": 1.4963, + "theoretical_loss": 3.8191285535296813, + "tokens_seen": 679936000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006415727181544634, + "loss": 1.4743, + "theoretical_loss": 3.819091264868586, + "tokens_seen": 680001536 + }, + { + "epoch": 0.21, + "learning_rate": 0.00064155667001003, + "loss": 1.5455, + "theoretical_loss": 3.8190539808071966, + "tokens_seen": 680067072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006415406218655968, + "loss": 1.4972, + "theoretical_loss": 3.819016701344501, + "tokens_seen": 680132608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006415245737211634, + "loss": 1.3704, + "theoretical_loss": 3.8189794264794905, + "tokens_seen": 680198144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006415085255767302, + "loss": 1.4696, + "theoretical_loss": 3.8189421562111545, + "tokens_seen": 680263680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006414924774322968, + "loss": 1.4398, + "theoretical_loss": 3.8189048905384837, + "tokens_seen": 680329216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006414764292878636, + "loss": 1.4797, + "theoretical_loss": 3.8188676294604686, + "tokens_seen": 680394752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006414603811434302, + "loss": 1.4531, + "theoretical_loss": 3.8188303729761, + "tokens_seen": 680460288 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641444332998997, + "loss": 1.5019, + "theoretical_loss": 3.818793121084369, + "tokens_seen": 680525824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006414282848545638, + "loss": 1.5117, + "theoretical_loss": 3.818755873784268, + "tokens_seen": 680591360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006414122367101304, + "loss": 1.4771, + "theoretical_loss": 3.8187186310747885, + "tokens_seen": 680656896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006413961885656972, + "loss": 1.4212, + "theoretical_loss": 3.8186813929549235, + "tokens_seen": 680722432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006413801404212638, + "loss": 1.4327, + "theoretical_loss": 3.818644159423665, + "tokens_seen": 680787968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006413640922768306, + "loss": 1.5288, + "theoretical_loss": 3.818606930480006, + "tokens_seen": 680853504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006413480441323972, + "loss": 1.41, + "theoretical_loss": 3.8185697061229398, + "tokens_seen": 680919040 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641331995987964, + "loss": 1.4027, + "theoretical_loss": 3.8185324863514594, + "tokens_seen": 680984576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006413159478435306, + "loss": 1.5207, + "theoretical_loss": 3.8184952711645597, + "tokens_seen": 681050112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006412998996990974, + "loss": 1.534, + "theoretical_loss": 3.8184580605612344, + "tokens_seen": 681115648 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641283851554664, + "loss": 1.4747, + "theoretical_loss": 3.818420854540478, + "tokens_seen": 681181184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006412678034102308, + "loss": 1.3608, + "theoretical_loss": 3.818383653101286, + "tokens_seen": 681246720 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006412517552657974, + "loss": 1.4838, + "theoretical_loss": 3.8183464562426526, + "tokens_seen": 681312256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006412357071213642, + "loss": 1.5294, + "theoretical_loss": 3.8183092639635734, + "tokens_seen": 681377792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006412196589769308, + "loss": 1.3433, + "theoretical_loss": 3.8182720762630447, + "tokens_seen": 681443328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006412036108324976, + "loss": 1.4816, + "theoretical_loss": 3.8182348931400627, + "tokens_seen": 681508864 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 258203, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.421541690826416, + "objective/train/theoretical_loss": 3.818197714593623, + "objective/train/tokens_used": 702034400, + "theoretical_loss": 3.818197714593623, + "tokens_seen": 681574400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006411875626880642, + "loss": 1.4215, + "theoretical_loss": 3.818197714593623, + "tokens_seen": 681574400 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641171514543631, + "loss": 1.5006, + "theoretical_loss": 3.818160540622723, + "tokens_seen": 681639936 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006411554663991976, + "loss": 1.4045, + "theoretical_loss": 3.8181233712263594, + "tokens_seen": 681705472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006411394182547644, + "loss": 1.4083, + "theoretical_loss": 3.81808620640353, + "tokens_seen": 681771008 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641123370110331, + "loss": 1.4453, + "theoretical_loss": 3.8180490461532317, + "tokens_seen": 681836544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006411073219658978, + "loss": 1.4927, + "theoretical_loss": 3.818011890474464, + "tokens_seen": 681902080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006410912738214644, + "loss": 1.4768, + "theoretical_loss": 3.817974739366224, + "tokens_seen": 681967616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006410752256770312, + "loss": 1.4475, + "theoretical_loss": 3.8179375928275103, + "tokens_seen": 682033152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006410591775325978, + "loss": 1.4331, + "theoretical_loss": 3.817900450857322, + "tokens_seen": 682098688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006410431293881646, + "loss": 1.4782, + "theoretical_loss": 3.817863313454659, + "tokens_seen": 682164224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006410270812437312, + "loss": 1.3593, + "theoretical_loss": 3.8178261806185207, + "tokens_seen": 682229760 + }, + { + "epoch": 0.21, + "learning_rate": 0.000641011033099298, + "loss": 1.4741, + "theoretical_loss": 3.817789052347906, + "tokens_seen": 682295296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006409949849548646, + "loss": 1.374, + "theoretical_loss": 3.8177519286418162, + "tokens_seen": 682360832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006409789368104314, + "loss": 1.5386, + "theoretical_loss": 3.817714809499252, + "tokens_seen": 682426368 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640962888665998, + "loss": 1.4878, + "theoretical_loss": 3.8176776949192135, + "tokens_seen": 682491904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006409468405215648, + "loss": 1.419, + "theoretical_loss": 3.8176405849007016, + "tokens_seen": 682557440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006409307923771314, + "loss": 1.3988, + "theoretical_loss": 3.8176034794427185, + "tokens_seen": 682622976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006409147442326982, + "loss": 1.4276, + "theoretical_loss": 3.8175663785442655, + "tokens_seen": 682688512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006408986960882648, + "loss": 1.4501, + "theoretical_loss": 3.8175292822043456, + "tokens_seen": 682754048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006408826479438316, + "loss": 1.4382, + "theoretical_loss": 3.81749219042196, + "tokens_seen": 682819584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006408665997993982, + "loss": 1.6076, + "theoretical_loss": 3.8174551031961124, + "tokens_seen": 682885120 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640850551654965, + "loss": 1.5218, + "theoretical_loss": 3.817418020525805, + "tokens_seen": 682950656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006408345035105316, + "loss": 1.3998, + "theoretical_loss": 3.817380942410042, + "tokens_seen": 683016192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006408184553660984, + "loss": 1.419, + "theoretical_loss": 3.8173438688478267, + "tokens_seen": 683081728 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640802407221665, + "loss": 1.4713, + "theoretical_loss": 3.817306799838163, + "tokens_seen": 683147264 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006407863590772318, + "loss": 1.4609, + "theoretical_loss": 3.817269735380055, + "tokens_seen": 683212800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006407703109327984, + "loss": 1.3927, + "theoretical_loss": 3.817232675472508, + "tokens_seen": 683278336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006407542627883652, + "loss": 1.3993, + "theoretical_loss": 3.817195620114526, + "tokens_seen": 683343872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006407382146439318, + "loss": 1.4514, + "theoretical_loss": 3.817158569305115, + "tokens_seen": 683409408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006407221664994986, + "loss": 1.4626, + "theoretical_loss": 3.817121523043281, + "tokens_seen": 683474944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006407061183550652, + "loss": 1.4377, + "theoretical_loss": 3.8170844813280285, + "tokens_seen": 683540480 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640690070210632, + "loss": 1.4334, + "theoretical_loss": 3.817047444158364, + "tokens_seen": 683606016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006406740220661986, + "loss": 1.4374, + "theoretical_loss": 3.8170104115332943, + "tokens_seen": 683671552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006406579739217654, + "loss": 1.5512, + "theoretical_loss": 3.816973383451827, + "tokens_seen": 683737088 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640641925777332, + "loss": 1.317, + "theoretical_loss": 3.816936359912968, + "tokens_seen": 683802624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006406258776328988, + "loss": 1.4007, + "theoretical_loss": 3.8168993409157252, + "tokens_seen": 683868160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006406098294884654, + "loss": 1.3103, + "theoretical_loss": 3.8168623264591064, + "tokens_seen": 683933696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006405937813440322, + "loss": 1.446, + "theoretical_loss": 3.8168253165421198, + "tokens_seen": 683999232 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006405777331995988, + "loss": 1.445, + "theoretical_loss": 3.816788311163773, + "tokens_seen": 684064768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006405616850551656, + "loss": 1.4927, + "theoretical_loss": 3.8167513103230757, + "tokens_seen": 684130304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006405456369107322, + "loss": 1.4894, + "theoretical_loss": 3.816714314019036, + "tokens_seen": 684195840 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640529588766299, + "loss": 1.4655, + "theoretical_loss": 3.816677322250664, + "tokens_seen": 684261376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006405135406218656, + "loss": 1.4337, + "theoretical_loss": 3.8166403350169684, + "tokens_seen": 684326912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006404974924774324, + "loss": 1.5661, + "theoretical_loss": 3.81660335231696, + "tokens_seen": 684392448 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640481444332999, + "loss": 1.4464, + "theoretical_loss": 3.816566374149648, + "tokens_seen": 684457984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006404653961885658, + "loss": 1.5329, + "theoretical_loss": 3.816529400514044, + "tokens_seen": 684523520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006404493480441324, + "loss": 1.472, + "theoretical_loss": 3.8164924314091584, + "tokens_seen": 684589056 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006404332998996991, + "loss": 1.4696, + "theoretical_loss": 3.816455466834003, + "tokens_seen": 684654592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006404172517552658, + "loss": 1.404, + "theoretical_loss": 3.8164185067875875, + "tokens_seen": 684720128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006404012036108325, + "loss": 1.3387, + "theoretical_loss": 3.8163815512689254, + "tokens_seen": 684785664 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006403851554663992, + "loss": 1.4977, + "theoretical_loss": 3.816344600277029, + "tokens_seen": 684851200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006403691073219659, + "loss": 1.4655, + "theoretical_loss": 3.8163076538109086, + "tokens_seen": 684916736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006403530591775326, + "loss": 1.3902, + "theoretical_loss": 3.816270711869579, + "tokens_seen": 684982272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006403370110330993, + "loss": 1.4608, + "theoretical_loss": 3.8162337744520523, + "tokens_seen": 685047808 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640320962888666, + "loss": 1.4216, + "theoretical_loss": 3.816196841557342, + "tokens_seen": 685113344 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006403049147442327, + "loss": 1.4353, + "theoretical_loss": 3.816159913184462, + "tokens_seen": 685178880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006402888665997993, + "loss": 1.4822, + "theoretical_loss": 3.8161229893324258, + "tokens_seen": 685244416 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006402728184553661, + "loss": 1.5553, + "theoretical_loss": 3.8160860700002477, + "tokens_seen": 685309952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006402567703109327, + "loss": 1.4583, + "theoretical_loss": 3.816049155186943, + "tokens_seen": 685375488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006402407221664995, + "loss": 1.491, + "theoretical_loss": 3.8160122448915255, + "tokens_seen": 685441024 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006402246740220662, + "loss": 1.3492, + "theoretical_loss": 3.815975339113011, + "tokens_seen": 685506560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006402086258776329, + "loss": 1.4449, + "theoretical_loss": 3.8159384378504146, + "tokens_seen": 685572096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006401925777331996, + "loss": 1.4511, + "theoretical_loss": 3.8159015411027526, + "tokens_seen": 685637632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006401765295887663, + "loss": 1.4732, + "theoretical_loss": 3.815864648869041, + "tokens_seen": 685703168 + }, + { + "epoch": 0.21, + "learning_rate": 0.000640160481444333, + "loss": 1.3874, + "theoretical_loss": 3.815827761148296, + "tokens_seen": 685768704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006401444332998997, + "loss": 1.405, + "theoretical_loss": 3.8157908779395346, + "tokens_seen": 685834240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006401283851554664, + "loss": 1.4224, + "theoretical_loss": 3.8157539992417737, + "tokens_seen": 685899776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006401123370110331, + "loss": 1.4894, + "theoretical_loss": 3.8157171250540305, + "tokens_seen": 685965312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006400962888665998, + "loss": 1.4269, + "theoretical_loss": 3.815680255375323, + "tokens_seen": 686030848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006400802407221665, + "loss": 1.4044, + "theoretical_loss": 3.815643390204669, + "tokens_seen": 686096384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006400641925777332, + "loss": 1.5028, + "theoretical_loss": 3.8156065295410864, + "tokens_seen": 686161920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006400481444332999, + "loss": 1.3964, + "theoretical_loss": 3.8155696733835947, + "tokens_seen": 686227456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006400320962888667, + "loss": 1.4116, + "theoretical_loss": 3.8155328217312117, + "tokens_seen": 686292992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006400160481444333, + "loss": 1.4049, + "theoretical_loss": 3.815495974582957, + "tokens_seen": 686358528 + }, + { + "epoch": 0.21, + "learning_rate": 0.00064, + "loss": 1.4762, + "theoretical_loss": 3.8154591319378506, + "tokens_seen": 686424064 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006399839518555667, + "loss": 1.4171, + "theoretical_loss": 3.815422293794912, + "tokens_seen": 686489600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006399679037111335, + "loss": 1.4429, + "theoretical_loss": 3.8153854601531605, + "tokens_seen": 686555136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006399518555667001, + "loss": 1.459, + "theoretical_loss": 3.815348631011618, + "tokens_seen": 686620672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006399358074222669, + "loss": 1.4823, + "theoretical_loss": 3.8153118063693037, + "tokens_seen": 686686208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006399197592778335, + "loss": 1.3997, + "theoretical_loss": 3.8152749862252398, + "tokens_seen": 686751744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006399037111334003, + "loss": 1.4431, + "theoretical_loss": 3.8152381705784473, + "tokens_seen": 686817280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006398876629889669, + "loss": 1.487, + "theoretical_loss": 3.815201359427948, + "tokens_seen": 686882816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006398716148445337, + "loss": 1.447, + "theoretical_loss": 3.815164552772763, + "tokens_seen": 686948352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006398555667001003, + "loss": 1.4343, + "theoretical_loss": 3.8151277506119152, + "tokens_seen": 687013888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006398395185556671, + "loss": 1.4085, + "theoretical_loss": 3.8150909529444275, + "tokens_seen": 687079424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006398234704112337, + "loss": 1.5031, + "theoretical_loss": 3.815054159769322, + "tokens_seen": 687144960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006398074222668005, + "loss": 1.4931, + "theoretical_loss": 3.8150173710856228, + "tokens_seen": 687210496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006397913741223671, + "loss": 1.2858, + "theoretical_loss": 3.8149805868923528, + "tokens_seen": 687276032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006397753259779339, + "loss": 1.5146, + "theoretical_loss": 3.814943807188536, + "tokens_seen": 687341568 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006397592778335005, + "loss": 1.4681, + "theoretical_loss": 3.8149070319731955, + "tokens_seen": 687407104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006397432296890673, + "loss": 1.3412, + "theoretical_loss": 3.8148702612453573, + "tokens_seen": 687472640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006397271815446339, + "loss": 1.4148, + "theoretical_loss": 3.814833495004045, + "tokens_seen": 687538176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006397111334002007, + "loss": 1.5088, + "theoretical_loss": 3.8147967332482846, + "tokens_seen": 687603712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006396950852557673, + "loss": 1.5177, + "theoretical_loss": 3.8147599759771, + "tokens_seen": 687669248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006396790371113341, + "loss": 1.4575, + "theoretical_loss": 3.814723223189518, + "tokens_seen": 687734784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006396629889669007, + "loss": 1.4852, + "theoretical_loss": 3.8146864748845637, + "tokens_seen": 687800320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006396469408224675, + "loss": 1.5093, + "theoretical_loss": 3.814649731061264, + "tokens_seen": 687865856 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006396308926780341, + "loss": 1.4693, + "theoretical_loss": 3.8146129917186453, + "tokens_seen": 687931392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006396148445336009, + "loss": 1.4313, + "theoretical_loss": 3.8145762568557338, + "tokens_seen": 687996928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395987963891675, + "loss": 1.5436, + "theoretical_loss": 3.814539526471558, + "tokens_seen": 688062464 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 260284, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4174013137817383, + "objective/train/theoretical_loss": 3.8145028005651436, + "objective/train/tokens_used": 708588000, + "theoretical_loss": 3.8145028005651436, + "tokens_seen": 688128000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395827482447343, + "loss": 1.4174, + "theoretical_loss": 3.8145028005651436, + "tokens_seen": 688128000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395667001003009, + "loss": 1.4887, + "theoretical_loss": 3.8144660791355194, + "tokens_seen": 688193536 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395506519558677, + "loss": 1.4351, + "theoretical_loss": 3.8144293621817136, + "tokens_seen": 688259072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395346038114343, + "loss": 1.4604, + "theoretical_loss": 3.8143926497027536, + "tokens_seen": 688324608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395185556670011, + "loss": 1.5529, + "theoretical_loss": 3.814355941697669, + "tokens_seen": 688390144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006395025075225677, + "loss": 1.3497, + "theoretical_loss": 3.814319238165489, + "tokens_seen": 688455680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006394864593781345, + "loss": 1.5023, + "theoretical_loss": 3.8142825391052417, + "tokens_seen": 688521216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006394704112337011, + "loss": 1.4963, + "theoretical_loss": 3.8142458445159573, + "tokens_seen": 688586752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006394543630892679, + "loss": 1.4773, + "theoretical_loss": 3.8142091543966656, + "tokens_seen": 688652288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006394383149448345, + "loss": 1.4716, + "theoretical_loss": 3.814172468746397, + "tokens_seen": 688717824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006394222668004013, + "loss": 1.531, + "theoretical_loss": 3.8141357875641813, + "tokens_seen": 688783360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006394062186559679, + "loss": 1.4234, + "theoretical_loss": 3.81409911084905, + "tokens_seen": 688848896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006393901705115347, + "loss": 1.4357, + "theoretical_loss": 3.8140624386000344, + "tokens_seen": 688914432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006393741223671013, + "loss": 1.4271, + "theoretical_loss": 3.8140257708161647, + "tokens_seen": 688979968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006393580742226681, + "loss": 1.4019, + "theoretical_loss": 3.8139891074964734, + "tokens_seen": 689045504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006393420260782347, + "loss": 1.4006, + "theoretical_loss": 3.8139524486399927, + "tokens_seen": 689111040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006393259779338015, + "loss": 1.4786, + "theoretical_loss": 3.813915794245754, + "tokens_seen": 689176576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006393099297893681, + "loss": 1.4514, + "theoretical_loss": 3.813879144312791, + "tokens_seen": 689242112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006392938816449349, + "loss": 1.507, + "theoretical_loss": 3.8138424988401365, + "tokens_seen": 689307648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006392778335005015, + "loss": 1.4973, + "theoretical_loss": 3.8138058578268224, + "tokens_seen": 689373184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006392617853560683, + "loss": 1.4639, + "theoretical_loss": 3.8137692212718832, + "tokens_seen": 689438720 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006392457372116349, + "loss": 1.3891, + "theoretical_loss": 3.8137325891743528, + "tokens_seen": 689504256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006392296890672017, + "loss": 1.4572, + "theoretical_loss": 3.8136959615332646, + "tokens_seen": 689569792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006392136409227683, + "loss": 1.4703, + "theoretical_loss": 3.813659338347654, + "tokens_seen": 689635328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391975927783351, + "loss": 1.4791, + "theoretical_loss": 3.8136227196165544, + "tokens_seen": 689700864 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391815446339017, + "loss": 1.5133, + "theoretical_loss": 3.813586105339002, + "tokens_seen": 689766400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391654964894685, + "loss": 1.4081, + "theoretical_loss": 3.813549495514031, + "tokens_seen": 689831936 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391494483450351, + "loss": 1.4089, + "theoretical_loss": 3.8135128901406783, + "tokens_seen": 689897472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391334002006019, + "loss": 1.5003, + "theoretical_loss": 3.813476289217979, + "tokens_seen": 689963008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391173520561685, + "loss": 1.5088, + "theoretical_loss": 3.813439692744969, + "tokens_seen": 690028544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006391013039117353, + "loss": 1.4252, + "theoretical_loss": 3.813403100720686, + "tokens_seen": 690094080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006390852557673019, + "loss": 1.384, + "theoretical_loss": 3.813366513144165, + "tokens_seen": 690159616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006390692076228687, + "loss": 1.3944, + "theoretical_loss": 3.8133299300144445, + "tokens_seen": 690225152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006390531594784353, + "loss": 1.3052, + "theoretical_loss": 3.813293351330562, + "tokens_seen": 690290688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006390371113340021, + "loss": 1.5375, + "theoretical_loss": 3.8132567770915546, + "tokens_seen": 690356224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006390210631895687, + "loss": 1.5073, + "theoretical_loss": 3.8132202072964603, + "tokens_seen": 690421760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006390210631895687, + "loss": 1.4873, + "theoretical_loss": 3.8131836419443172, + "tokens_seen": 690487296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006390050150451354, + "loss": 1.4545, + "theoretical_loss": 3.8131470810341646, + "tokens_seen": 690552832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006389889669007021, + "loss": 1.4685, + "theoretical_loss": 3.8131105245650403, + "tokens_seen": 690618368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006389729187562688, + "loss": 1.4233, + "theoretical_loss": 3.8130739725359852, + "tokens_seen": 690683904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006389568706118355, + "loss": 1.5129, + "theoretical_loss": 3.8130374249460375, + "tokens_seen": 690749440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006389408224674022, + "loss": 1.4762, + "theoretical_loss": 3.8130008817942374, + "tokens_seen": 690814976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006389247743229689, + "loss": 1.3981, + "theoretical_loss": 3.8129643430796243, + "tokens_seen": 690880512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006389087261785356, + "loss": 1.453, + "theoretical_loss": 3.8129278088012395, + "tokens_seen": 690946048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006388926780341023, + "loss": 1.491, + "theoretical_loss": 3.8128912789581237, + "tokens_seen": 691011584 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638876629889669, + "loss": 1.522, + "theoretical_loss": 3.8128547535493174, + "tokens_seen": 691077120 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006388605817452357, + "loss": 1.3295, + "theoretical_loss": 3.8128182325738624, + "tokens_seen": 691142656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006388445336008024, + "loss": 1.4969, + "theoretical_loss": 3.8127817160307993, + "tokens_seen": 691208192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006388284854563691, + "loss": 1.4792, + "theoretical_loss": 3.812745203919171, + "tokens_seen": 691273728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006388124373119358, + "loss": 1.3818, + "theoretical_loss": 3.8127086962380194, + "tokens_seen": 691339264 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006387963891675025, + "loss": 1.4467, + "theoretical_loss": 3.812672192986387, + "tokens_seen": 691404800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006387803410230692, + "loss": 1.4559, + "theoretical_loss": 3.8126356941633164, + "tokens_seen": 691470336 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638764292878636, + "loss": 1.4865, + "theoretical_loss": 3.8125991997678503, + "tokens_seen": 691535872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006387482447342026, + "loss": 1.5313, + "theoretical_loss": 3.8125627097990327, + "tokens_seen": 691601408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006387321965897693, + "loss": 1.4187, + "theoretical_loss": 3.8125262242559073, + "tokens_seen": 691666944 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638716148445336, + "loss": 1.3765, + "theoretical_loss": 3.812489743137518, + "tokens_seen": 691732480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006387001003009027, + "loss": 1.3662, + "theoretical_loss": 3.8124532664429087, + "tokens_seen": 691798016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006386840521564694, + "loss": 1.5504, + "theoretical_loss": 3.8124167941711242, + "tokens_seen": 691863552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006386680040120362, + "loss": 1.4837, + "theoretical_loss": 3.8123803263212097, + "tokens_seen": 691929088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006386519558676028, + "loss": 1.4086, + "theoretical_loss": 3.8123438628922095, + "tokens_seen": 691994624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006386359077231696, + "loss": 1.4718, + "theoretical_loss": 3.8123074038831697, + "tokens_seen": 692060160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006386198595787362, + "loss": 1.3195, + "theoretical_loss": 3.8122709492931364, + "tokens_seen": 692125696 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638603811434303, + "loss": 1.4105, + "theoretical_loss": 3.8122344991211543, + "tokens_seen": 692191232 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006385877632898696, + "loss": 1.438, + "theoretical_loss": 3.8121980533662714, + "tokens_seen": 692256768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006385717151454364, + "loss": 1.5916, + "theoretical_loss": 3.8121616120275332, + "tokens_seen": 692322304 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638555667001003, + "loss": 1.4713, + "theoretical_loss": 3.812125175103987, + "tokens_seen": 692387840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006385396188565698, + "loss": 1.4163, + "theoretical_loss": 3.81208874259468, + "tokens_seen": 692453376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006385235707121364, + "loss": 1.4942, + "theoretical_loss": 3.8120523144986596, + "tokens_seen": 692518912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006385075225677032, + "loss": 1.5215, + "theoretical_loss": 3.8120158908149735, + "tokens_seen": 692584448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006384914744232698, + "loss": 1.4311, + "theoretical_loss": 3.8119794715426707, + "tokens_seen": 692649984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006384754262788366, + "loss": 1.4015, + "theoretical_loss": 3.8119430566807986, + "tokens_seen": 692715520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006384593781344032, + "loss": 1.4903, + "theoretical_loss": 3.8119066462284064, + "tokens_seen": 692781056 + }, + { + "epoch": 0.21, + "learning_rate": 0.00063844332998997, + "loss": 1.3537, + "theoretical_loss": 3.8118702401845423, + "tokens_seen": 692846592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006384272818455366, + "loss": 1.4603, + "theoretical_loss": 3.8118338385482575, + "tokens_seen": 692912128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006384112337011034, + "loss": 1.4575, + "theoretical_loss": 3.8117974413185993, + "tokens_seen": 692977664 + }, + { + "epoch": 0.21, + "learning_rate": 0.00063839518555667, + "loss": 1.5261, + "theoretical_loss": 3.8117610484946187, + "tokens_seen": 693043200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006383791374122368, + "loss": 1.4881, + "theoretical_loss": 3.8117246600753667, + "tokens_seen": 693108736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006383630892678034, + "loss": 1.49, + "theoretical_loss": 3.8116882760598925, + "tokens_seen": 693174272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006383470411233702, + "loss": 1.5443, + "theoretical_loss": 3.811651896447247, + "tokens_seen": 693239808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006383309929789368, + "loss": 1.5125, + "theoretical_loss": 3.8116155212364817, + "tokens_seen": 693305344 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006383149448345036, + "loss": 1.4691, + "theoretical_loss": 3.811579150426648, + "tokens_seen": 693370880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006382988966900702, + "loss": 1.4517, + "theoretical_loss": 3.811542784016797, + "tokens_seen": 693436416 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638282848545637, + "loss": 1.3483, + "theoretical_loss": 3.811506422005981, + "tokens_seen": 693501952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006382668004012036, + "loss": 1.4534, + "theoretical_loss": 3.8114700643932524, + "tokens_seen": 693567488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006382507522567704, + "loss": 1.446, + "theoretical_loss": 3.8114337111776635, + "tokens_seen": 693633024 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638234704112337, + "loss": 1.501, + "theoretical_loss": 3.8113973623582673, + "tokens_seen": 693698560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006382186559679038, + "loss": 1.5282, + "theoretical_loss": 3.8113610179341166, + "tokens_seen": 693764096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006382026078234704, + "loss": 1.4272, + "theoretical_loss": 3.811324677904265, + "tokens_seen": 693829632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006381865596790372, + "loss": 1.5657, + "theoretical_loss": 3.811288342267767, + "tokens_seen": 693895168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006381705115346038, + "loss": 1.4459, + "theoretical_loss": 3.811252011023675, + "tokens_seen": 693960704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006381544633901706, + "loss": 1.422, + "theoretical_loss": 3.811215684171044, + "tokens_seen": 694026240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006381384152457372, + "loss": 1.4608, + "theoretical_loss": 3.8111793617089296, + "tokens_seen": 694091776 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638122367101304, + "loss": 1.436, + "theoretical_loss": 3.8111430436363856, + "tokens_seen": 694157312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006381063189568706, + "loss": 1.4678, + "theoretical_loss": 3.8111067299524666, + "tokens_seen": 694222848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006380902708124374, + "loss": 1.5609, + "theoretical_loss": 3.81107042065623, + "tokens_seen": 694288384 + }, + { + "epoch": 0.21, + "learning_rate": 0.000638074222668004, + "loss": 1.6249, + "theoretical_loss": 3.811034115746729, + "tokens_seen": 694353920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006380581745235708, + "loss": 1.345, + "theoretical_loss": 3.810997815223022, + "tokens_seen": 694419456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006380421263791374, + "loss": 1.4964, + "theoretical_loss": 3.810961519084164, + "tokens_seen": 694484992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006380260782347042, + "loss": 1.3931, + "theoretical_loss": 3.8109252273292125, + "tokens_seen": 694550528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006380100300902708, + "loss": 1.384, + "theoretical_loss": 3.8108889399572234, + "tokens_seen": 694616064 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 263013, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4728338718414307, + "objective/train/theoretical_loss": 3.810852656967255, + "objective/train/tokens_used": 715141600, + "theoretical_loss": 3.810852656967255, + "tokens_seen": 694681600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006379939819458376, + "loss": 1.4728, + "theoretical_loss": 3.810852656967255, + "tokens_seen": 694681600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006379779338014042, + "loss": 1.3561, + "theoretical_loss": 3.8108163783583637, + "tokens_seen": 694747136 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637961885656971, + "loss": 1.5104, + "theoretical_loss": 3.8107801041296083, + "tokens_seen": 694812672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006379458375125376, + "loss": 1.5279, + "theoretical_loss": 3.8107438342800464, + "tokens_seen": 694878208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006379297893681044, + "loss": 1.4215, + "theoretical_loss": 3.810707568808736, + "tokens_seen": 694943744 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637913741223671, + "loss": 1.3405, + "theoretical_loss": 3.8106713077147365, + "tokens_seen": 695009280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006378976930792378, + "loss": 1.4085, + "theoretical_loss": 3.810635050997107, + "tokens_seen": 695074816 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006378816449348044, + "loss": 1.3904, + "theoretical_loss": 3.810598798654906, + "tokens_seen": 695140352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006378655967903712, + "loss": 1.462, + "theoretical_loss": 3.8105625506871936, + "tokens_seen": 695205888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006378495486459378, + "loss": 1.3635, + "theoretical_loss": 3.810526307093029, + "tokens_seen": 695271424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006378335005015046, + "loss": 1.4507, + "theoretical_loss": 3.810490067871473, + "tokens_seen": 695336960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006378174523570713, + "loss": 1.5045, + "theoretical_loss": 3.810453833021586, + "tokens_seen": 695402496 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637801404212638, + "loss": 1.4347, + "theoretical_loss": 3.810417602542428, + "tokens_seen": 695468032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006377853560682047, + "loss": 1.4481, + "theoretical_loss": 3.8103813764330607, + "tokens_seen": 695533568 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006377693079237714, + "loss": 1.5116, + "theoretical_loss": 3.8103451546925453, + "tokens_seen": 695599104 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637753259779338, + "loss": 1.5182, + "theoretical_loss": 3.8103089373199435, + "tokens_seen": 695664640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006377372116349048, + "loss": 1.409, + "theoretical_loss": 3.8102727243143164, + "tokens_seen": 695730176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006377211634904714, + "loss": 1.4753, + "theoretical_loss": 3.8102365156747267, + "tokens_seen": 695795712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006377051153460382, + "loss": 1.4418, + "theoretical_loss": 3.810200311400237, + "tokens_seen": 695861248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006376890672016048, + "loss": 1.3818, + "theoretical_loss": 3.8101641114899105, + "tokens_seen": 695926784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006376730190571716, + "loss": 1.4955, + "theoretical_loss": 3.810127915942809, + "tokens_seen": 695992320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006376569709127382, + "loss": 1.4058, + "theoretical_loss": 3.8100917247579966, + "tokens_seen": 696057856 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637640922768305, + "loss": 1.3733, + "theoretical_loss": 3.8100555379345367, + "tokens_seen": 696123392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006376248746238716, + "loss": 1.4756, + "theoretical_loss": 3.810019355471493, + "tokens_seen": 696188928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006376088264794383, + "loss": 1.4036, + "theoretical_loss": 3.80998317736793, + "tokens_seen": 696254464 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637592778335005, + "loss": 1.481, + "theoretical_loss": 3.809947003622913, + "tokens_seen": 696320000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006375767301905717, + "loss": 1.4431, + "theoretical_loss": 3.8099108342355046, + "tokens_seen": 696385536 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006375606820461384, + "loss": 1.4946, + "theoretical_loss": 3.809874669204772, + "tokens_seen": 696451072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006375446339017051, + "loss": 1.428, + "theoretical_loss": 3.8098385085297792, + "tokens_seen": 696516608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006375285857572718, + "loss": 1.5256, + "theoretical_loss": 3.809802352209592, + "tokens_seen": 696582144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006375125376128385, + "loss": 1.4599, + "theoretical_loss": 3.809766200243277, + "tokens_seen": 696647680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006374964894684052, + "loss": 1.4741, + "theoretical_loss": 3.8097300526299, + "tokens_seen": 696713216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006374804413239719, + "loss": 1.4358, + "theoretical_loss": 3.8096939093685274, + "tokens_seen": 696778752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006374643931795386, + "loss": 1.4519, + "theoretical_loss": 3.8096577704582266, + "tokens_seen": 696844288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006374483450351053, + "loss": 1.4547, + "theoretical_loss": 3.8096216358980635, + "tokens_seen": 696909824 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637432296890672, + "loss": 1.4836, + "theoretical_loss": 3.809585505687106, + "tokens_seen": 696975360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006374162487462387, + "loss": 1.3587, + "theoretical_loss": 3.809549379824422, + "tokens_seen": 697040896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006374002006018054, + "loss": 1.4484, + "theoretical_loss": 3.80951325830908, + "tokens_seen": 697106432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006373841524573721, + "loss": 1.3909, + "theoretical_loss": 3.809477141140147, + "tokens_seen": 697171968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006373681043129388, + "loss": 1.3511, + "theoretical_loss": 3.8094410283166917, + "tokens_seen": 697237504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006373520561685055, + "loss": 1.4979, + "theoretical_loss": 3.809404919837784, + "tokens_seen": 697303040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006373360080240722, + "loss": 1.572, + "theoretical_loss": 3.8093688157024914, + "tokens_seen": 697368576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006373199598796389, + "loss": 1.5325, + "theoretical_loss": 3.8093327159098846, + "tokens_seen": 697434112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006373039117352057, + "loss": 1.4989, + "theoretical_loss": 3.8092966204590324, + "tokens_seen": 697499648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006372878635907723, + "loss": 1.4933, + "theoretical_loss": 3.8092605293490056, + "tokens_seen": 697565184 + }, + { + "epoch": 0.21, + "learning_rate": 0.000637271815446339, + "loss": 1.5409, + "theoretical_loss": 3.809224442578873, + "tokens_seen": 697630720 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006372557673019057, + "loss": 1.3217, + "theoretical_loss": 3.809188360147707, + "tokens_seen": 697696256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006372397191574725, + "loss": 1.3888, + "theoretical_loss": 3.8091522820545762, + "tokens_seen": 697761792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006372236710130391, + "loss": 1.3911, + "theoretical_loss": 3.809116208298554, + "tokens_seen": 697827328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006372076228686059, + "loss": 1.3186, + "theoretical_loss": 3.80908013887871, + "tokens_seen": 697892864 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006371915747241725, + "loss": 1.482, + "theoretical_loss": 3.8090440737941176, + "tokens_seen": 697958400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006371755265797393, + "loss": 1.357, + "theoretical_loss": 3.8090080130438464, + "tokens_seen": 698023936 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006371594784353059, + "loss": 1.3123, + "theoretical_loss": 3.80897195662697, + "tokens_seen": 698089472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006371434302908727, + "loss": 1.4227, + "theoretical_loss": 3.8089359045425617, + "tokens_seen": 698155008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006371273821464393, + "loss": 1.4981, + "theoretical_loss": 3.808899856789693, + "tokens_seen": 698220544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006371113340020061, + "loss": 1.4198, + "theoretical_loss": 3.808863813367437, + "tokens_seen": 698286080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006370952858575727, + "loss": 1.4526, + "theoretical_loss": 3.808827774274868, + "tokens_seen": 698351616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006370792377131395, + "loss": 1.5908, + "theoretical_loss": 3.808791739511059, + "tokens_seen": 698417152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006370631895687061, + "loss": 1.4059, + "theoretical_loss": 3.808755709075084, + "tokens_seen": 698482688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006370471414242729, + "loss": 1.4946, + "theoretical_loss": 3.8087196829660175, + "tokens_seen": 698548224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006370310932798395, + "loss": 1.5356, + "theoretical_loss": 3.8086836611829336, + "tokens_seen": 698613760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006370150451354063, + "loss": 1.4602, + "theoretical_loss": 3.8086476437249077, + "tokens_seen": 698679296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369989969909729, + "loss": 1.4548, + "theoretical_loss": 3.808611630591014, + "tokens_seen": 698744832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369829488465397, + "loss": 1.5249, + "theoretical_loss": 3.808575621780329, + "tokens_seen": 698810368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369669007021063, + "loss": 1.406, + "theoretical_loss": 3.8085396172919275, + "tokens_seen": 698875904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369508525576731, + "loss": 1.4615, + "theoretical_loss": 3.8085036171248854, + "tokens_seen": 698941440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369348044132397, + "loss": 1.425, + "theoretical_loss": 3.8084676212782793, + "tokens_seen": 699006976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369187562688065, + "loss": 1.4811, + "theoretical_loss": 3.808431629751186, + "tokens_seen": 699072512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006369027081243731, + "loss": 1.4794, + "theoretical_loss": 3.8083956425426813, + "tokens_seen": 699138048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006368866599799399, + "loss": 1.5493, + "theoretical_loss": 3.808359659651843, + "tokens_seen": 699203584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006368706118355065, + "loss": 1.4024, + "theoretical_loss": 3.8083236810777485, + "tokens_seen": 699269120 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006368545636910733, + "loss": 1.4595, + "theoretical_loss": 3.808287706819475, + "tokens_seen": 699334656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006368385155466399, + "loss": 1.44, + "theoretical_loss": 3.808251736876101, + "tokens_seen": 699400192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006368224674022067, + "loss": 1.5531, + "theoretical_loss": 3.8082157712467044, + "tokens_seen": 699465728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006368064192577733, + "loss": 1.3713, + "theoretical_loss": 3.8081798099303636, + "tokens_seen": 699531264 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006367903711133401, + "loss": 1.449, + "theoretical_loss": 3.808143852926157, + "tokens_seen": 699596800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006367743229689067, + "loss": 1.3707, + "theoretical_loss": 3.8081079002331646, + "tokens_seen": 699662336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006367582748244735, + "loss": 1.474, + "theoretical_loss": 3.808071951850465, + "tokens_seen": 699727872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006367422266800401, + "loss": 1.4811, + "theoretical_loss": 3.8080360077771385, + "tokens_seen": 699793408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006367261785356069, + "loss": 1.3756, + "theoretical_loss": 3.808000068012264, + "tokens_seen": 699858944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006367101303911735, + "loss": 1.3242, + "theoretical_loss": 3.8079641325549223, + "tokens_seen": 699924480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006366940822467403, + "loss": 1.3966, + "theoretical_loss": 3.807928201404194, + "tokens_seen": 699990016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006366780341023069, + "loss": 1.4719, + "theoretical_loss": 3.8078922745591597, + "tokens_seen": 700055552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006366619859578737, + "loss": 1.4324, + "theoretical_loss": 3.8078563520189004, + "tokens_seen": 700121088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006366459378134403, + "loss": 1.4687, + "theoretical_loss": 3.8078204337824975, + "tokens_seen": 700186624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006366298896690071, + "loss": 1.3743, + "theoretical_loss": 3.807784519849032, + "tokens_seen": 700252160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006366138415245738, + "loss": 1.3854, + "theoretical_loss": 3.807748610217587, + "tokens_seen": 700317696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006365977933801405, + "loss": 1.5134, + "theoretical_loss": 3.8077127048872437, + "tokens_seen": 700383232 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006365817452357072, + "loss": 1.3864, + "theoretical_loss": 3.8076768038570847, + "tokens_seen": 700448768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006365656970912739, + "loss": 1.5112, + "theoretical_loss": 3.807640907126193, + "tokens_seen": 700514304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006365496489468406, + "loss": 1.4755, + "theoretical_loss": 3.8076050146936513, + "tokens_seen": 700579840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006365336008024073, + "loss": 1.3768, + "theoretical_loss": 3.807569126558543, + "tokens_seen": 700645376 + }, + { + "epoch": 0.21, + "learning_rate": 0.000636517552657974, + "loss": 1.5132, + "theoretical_loss": 3.807533242719952, + "tokens_seen": 700710912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006365015045135407, + "loss": 1.4584, + "theoretical_loss": 3.807497363176962, + "tokens_seen": 700776448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006364854563691074, + "loss": 1.3921, + "theoretical_loss": 3.8074614879286566, + "tokens_seen": 700841984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006364694082246741, + "loss": 1.3594, + "theoretical_loss": 3.8074256169741205, + "tokens_seen": 700907520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006364533600802408, + "loss": 1.5407, + "theoretical_loss": 3.8073897503124385, + "tokens_seen": 700973056 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006364373119358075, + "loss": 1.4546, + "theoretical_loss": 3.807353887942696, + "tokens_seen": 701038592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006364212637913742, + "loss": 1.4251, + "theoretical_loss": 3.8073180298639775, + "tokens_seen": 701104128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006364052156469409, + "loss": 1.4813, + "theoretical_loss": 3.8072821760753683, + "tokens_seen": 701169664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 265490, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5426807403564453, + "objective/train/theoretical_loss": 3.8072463265759557, + "objective/train/tokens_used": 721695200, + "theoretical_loss": 3.8072463265759557, + "tokens_seen": 701235200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006363891675025076, + "loss": 1.5427, + "theoretical_loss": 3.8072463265759557, + "tokens_seen": 701235200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006363731193580743, + "loss": 1.413, + "theoretical_loss": 3.8072104813648244, + "tokens_seen": 701300736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006363570712136409, + "loss": 1.4842, + "theoretical_loss": 3.8071746404410614, + "tokens_seen": 701366272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006363410230692077, + "loss": 1.4012, + "theoretical_loss": 3.807138803803753, + "tokens_seen": 701431808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006363249749247743, + "loss": 1.3996, + "theoretical_loss": 3.807102971451986, + "tokens_seen": 701497344 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006363089267803411, + "loss": 1.4591, + "theoretical_loss": 3.807067143384849, + "tokens_seen": 701562880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006362928786359077, + "loss": 1.4282, + "theoretical_loss": 3.8070313196014274, + "tokens_seen": 701628416 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006362768304914745, + "loss": 1.434, + "theoretical_loss": 3.8069955001008102, + "tokens_seen": 701693952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006362607823470411, + "loss": 1.473, + "theoretical_loss": 3.8069596848820852, + "tokens_seen": 701759488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006362447342026079, + "loss": 1.377, + "theoretical_loss": 3.806923873944341, + "tokens_seen": 701825024 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006362286860581745, + "loss": 1.413, + "theoretical_loss": 3.8068880672866667, + "tokens_seen": 701890560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006362126379137412, + "loss": 1.6195, + "theoretical_loss": 3.8068522649081498, + "tokens_seen": 701956096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006361965897693079, + "loss": 1.4501, + "theoretical_loss": 3.80681646680788, + "tokens_seen": 702021632 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006361805416248746, + "loss": 1.422, + "theoretical_loss": 3.8067806729849467, + "tokens_seen": 702087168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006361644934804413, + "loss": 1.4327, + "theoretical_loss": 3.8067448834384408, + "tokens_seen": 702152704 + }, + { + "epoch": 0.21, + "learning_rate": 0.000636148445336008, + "loss": 1.5188, + "theoretical_loss": 3.8067090981674507, + "tokens_seen": 702218240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006361323971915747, + "loss": 1.4471, + "theoretical_loss": 3.806673317171067, + "tokens_seen": 702283776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006361163490471414, + "loss": 1.4731, + "theoretical_loss": 3.8066375404483814, + "tokens_seen": 702349312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006361003009027081, + "loss": 1.4922, + "theoretical_loss": 3.806601767998483, + "tokens_seen": 702414848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006360842527582748, + "loss": 1.4149, + "theoretical_loss": 3.806565999820465, + "tokens_seen": 702480384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006360682046138415, + "loss": 1.5456, + "theoretical_loss": 3.806530235913417, + "tokens_seen": 702545920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006360521564694082, + "loss": 1.4229, + "theoretical_loss": 3.8064944762764314, + "tokens_seen": 702611456 + }, + { + "epoch": 0.21, + "learning_rate": 0.000636036108324975, + "loss": 1.5108, + "theoretical_loss": 3.8064587209086005, + "tokens_seen": 702676992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006360200601805416, + "loss": 1.4486, + "theoretical_loss": 3.8064229698090157, + "tokens_seen": 702742528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006360040120361083, + "loss": 1.5336, + "theoretical_loss": 3.80638722297677, + "tokens_seen": 702808064 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635987963891675, + "loss": 1.4812, + "theoretical_loss": 3.8063514804109557, + "tokens_seen": 702873600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006359719157472417, + "loss": 1.4266, + "theoretical_loss": 3.806315742110667, + "tokens_seen": 702939136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006359558676028084, + "loss": 1.4178, + "theoretical_loss": 3.806280008074996, + "tokens_seen": 703004672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006359398194583752, + "loss": 1.5052, + "theoretical_loss": 3.806244278303037, + "tokens_seen": 703070208 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006359237713139418, + "loss": 1.3948, + "theoretical_loss": 3.806208552793884, + "tokens_seen": 703135744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006359077231695086, + "loss": 1.419, + "theoretical_loss": 3.8061728315466308, + "tokens_seen": 703201280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006358916750250752, + "loss": 1.53, + "theoretical_loss": 3.8061371145603715, + "tokens_seen": 703266816 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635875626880642, + "loss": 1.4521, + "theoretical_loss": 3.806101401834202, + "tokens_seen": 703332352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006358595787362086, + "loss": 1.4161, + "theoretical_loss": 3.8060656933672163, + "tokens_seen": 703397888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006358435305917754, + "loss": 1.3942, + "theoretical_loss": 3.8060299891585094, + "tokens_seen": 703463424 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635827482447342, + "loss": 1.4801, + "theoretical_loss": 3.8059942892071783, + "tokens_seen": 703528960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006358114343029088, + "loss": 1.4581, + "theoretical_loss": 3.8059585935123175, + "tokens_seen": 703594496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006357953861584754, + "loss": 1.4621, + "theoretical_loss": 3.8059229020730236, + "tokens_seen": 703660032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006357793380140422, + "loss": 1.3571, + "theoretical_loss": 3.805887214888393, + "tokens_seen": 703725568 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006357632898696088, + "loss": 1.4339, + "theoretical_loss": 3.8058515319575226, + "tokens_seen": 703791104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006357472417251756, + "loss": 1.4399, + "theoretical_loss": 3.8058158532795088, + "tokens_seen": 703856640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006357311935807422, + "loss": 1.416, + "theoretical_loss": 3.805780178853449, + "tokens_seen": 703922176 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635715145436309, + "loss": 1.4226, + "theoretical_loss": 3.8057445086784405, + "tokens_seen": 703987712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006356990972918756, + "loss": 1.4262, + "theoretical_loss": 3.8057088427535817, + "tokens_seen": 704053248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006356830491474424, + "loss": 1.4375, + "theoretical_loss": 3.80567318107797, + "tokens_seen": 704118784 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635667001003009, + "loss": 1.4768, + "theoretical_loss": 3.8056375236507045, + "tokens_seen": 704184320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006356509528585758, + "loss": 1.4472, + "theoretical_loss": 3.805601870470883, + "tokens_seen": 704249856 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006356349047141424, + "loss": 1.4487, + "theoretical_loss": 3.805566221537604, + "tokens_seen": 704315392 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006356188565697092, + "loss": 1.4167, + "theoretical_loss": 3.805530576849968, + "tokens_seen": 704380928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006356028084252758, + "loss": 1.3656, + "theoretical_loss": 3.8054949364070736, + "tokens_seen": 704446464 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006355867602808426, + "loss": 1.3924, + "theoretical_loss": 3.8054593002080206, + "tokens_seen": 704512000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006355707121364092, + "loss": 1.383, + "theoretical_loss": 3.805423668251909, + "tokens_seen": 704577536 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635554663991976, + "loss": 1.3968, + "theoretical_loss": 3.8053880405378386, + "tokens_seen": 704643072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006355386158475426, + "loss": 1.4719, + "theoretical_loss": 3.8053524170649107, + "tokens_seen": 704708608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006355225677031094, + "loss": 1.3405, + "theoretical_loss": 3.805316797832225, + "tokens_seen": 704774144 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635506519558676, + "loss": 1.524, + "theoretical_loss": 3.805281182838884, + "tokens_seen": 704839680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006354904714142428, + "loss": 1.5249, + "theoretical_loss": 3.8052455720839875, + "tokens_seen": 704905216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006354744232698094, + "loss": 1.5797, + "theoretical_loss": 3.8052099655666387, + "tokens_seen": 704970752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006354583751253762, + "loss": 1.4054, + "theoretical_loss": 3.8051743632859383, + "tokens_seen": 705036288 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006354423269809428, + "loss": 1.3585, + "theoretical_loss": 3.8051387652409887, + "tokens_seen": 705101824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006354262788365096, + "loss": 1.4993, + "theoretical_loss": 3.8051031714308925, + "tokens_seen": 705167360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006354102306920763, + "loss": 1.5183, + "theoretical_loss": 3.8050675818547526, + "tokens_seen": 705232896 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635394182547643, + "loss": 1.402, + "theoretical_loss": 3.8050319965116715, + "tokens_seen": 705298432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006353781344032097, + "loss": 1.3846, + "theoretical_loss": 3.804996415400753, + "tokens_seen": 705363968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006353620862587764, + "loss": 1.4966, + "theoretical_loss": 3.8049608385211, + "tokens_seen": 705429504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006353460381143431, + "loss": 1.4174, + "theoretical_loss": 3.804925265871817, + "tokens_seen": 705495040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006353299899699098, + "loss": 1.5106, + "theoretical_loss": 3.804889697452007, + "tokens_seen": 705560576 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006353139418254765, + "loss": 1.5223, + "theoretical_loss": 3.8048541332607755, + "tokens_seen": 705626112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006352978936810432, + "loss": 1.4228, + "theoretical_loss": 3.8048185732972266, + "tokens_seen": 705691648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006352818455366099, + "loss": 1.4822, + "theoretical_loss": 3.8047830175604656, + "tokens_seen": 705757184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006352657973921766, + "loss": 1.498, + "theoretical_loss": 3.804747466049597, + "tokens_seen": 705822720 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006352497492477433, + "loss": 1.4656, + "theoretical_loss": 3.804711918763726, + "tokens_seen": 705888256 + }, + { + "epoch": 0.21, + "learning_rate": 0.00063523370110331, + "loss": 1.4805, + "theoretical_loss": 3.8046763757019595, + "tokens_seen": 705953792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006352176529588767, + "loss": 1.4861, + "theoretical_loss": 3.8046408368634026, + "tokens_seen": 706019328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006352016048144434, + "loss": 1.4615, + "theoretical_loss": 3.804605302247162, + "tokens_seen": 706084864 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006351855566700101, + "loss": 1.4218, + "theoretical_loss": 3.8045697718523437, + "tokens_seen": 706150400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006351695085255768, + "loss": 1.5278, + "theoretical_loss": 3.804534245678055, + "tokens_seen": 706215936 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006351534603811435, + "loss": 1.4023, + "theoretical_loss": 3.8044987237234027, + "tokens_seen": 706281472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006351374122367102, + "loss": 1.4111, + "theoretical_loss": 3.804463205987494, + "tokens_seen": 706347008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006351213640922769, + "loss": 1.4588, + "theoretical_loss": 3.804427692469437, + "tokens_seen": 706412544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006351053159478436, + "loss": 1.5057, + "theoretical_loss": 3.8043921831683396, + "tokens_seen": 706478080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006350892678034103, + "loss": 1.3696, + "theoretical_loss": 3.804356678083309, + "tokens_seen": 706543616 + }, + { + "epoch": 0.21, + "learning_rate": 0.000635073219658977, + "loss": 1.5083, + "theoretical_loss": 3.8043211772134544, + "tokens_seen": 706609152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006350571715145437, + "loss": 1.4317, + "theoretical_loss": 3.804285680557885, + "tokens_seen": 706674688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006350411233701104, + "loss": 1.4335, + "theoretical_loss": 3.8042501881157085, + "tokens_seen": 706740224 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006350250752256771, + "loss": 1.3647, + "theoretical_loss": 3.804214699886035, + "tokens_seen": 706805760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006350090270812438, + "loss": 1.3602, + "theoretical_loss": 3.8041792158679737, + "tokens_seen": 706871296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006349929789368105, + "loss": 1.3525, + "theoretical_loss": 3.804143736060635, + "tokens_seen": 706936832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006349769307923772, + "loss": 1.4603, + "theoretical_loss": 3.8041082604631278, + "tokens_seen": 707002368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006349608826479438, + "loss": 1.4847, + "theoretical_loss": 3.804072789074563, + "tokens_seen": 707067904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006349448345035106, + "loss": 1.4811, + "theoretical_loss": 3.8040373218940515, + "tokens_seen": 707133440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006349287863590772, + "loss": 1.4983, + "theoretical_loss": 3.804001858920704, + "tokens_seen": 707198976 + }, + { + "epoch": 0.21, + "learning_rate": 0.000634912738214644, + "loss": 1.4131, + "theoretical_loss": 3.8039664001536315, + "tokens_seen": 707264512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006348966900702106, + "loss": 1.4471, + "theoretical_loss": 3.8039309455919454, + "tokens_seen": 707330048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006348806419257774, + "loss": 1.3745, + "theoretical_loss": 3.803895495234757, + "tokens_seen": 707395584 + }, + { + "epoch": 0.21, + "learning_rate": 0.000634864593781344, + "loss": 1.5989, + "theoretical_loss": 3.8038600490811794, + "tokens_seen": 707461120 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006348485456369108, + "loss": 1.5303, + "theoretical_loss": 3.8038246071303234, + "tokens_seen": 707526656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006348324974924774, + "loss": 1.4357, + "theoretical_loss": 3.803789169381303, + "tokens_seen": 707592192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006348164493480441, + "loss": 1.3748, + "theoretical_loss": 3.8037537358332294, + "tokens_seen": 707657728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006348004012036108, + "loss": 1.4979, + "theoretical_loss": 3.8037183064852162, + "tokens_seen": 707723264 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 267822, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4624550342559814, + "objective/train/theoretical_loss": 3.8036828813363774, + "objective/train/tokens_used": 728248800, + "theoretical_loss": 3.8036828813363774, + "tokens_seen": 707788800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006347843530591775, + "loss": 1.4625, + "theoretical_loss": 3.8036828813363774, + "tokens_seen": 707788800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006347683049147442, + "loss": 1.478, + "theoretical_loss": 3.8036474603858252, + "tokens_seen": 707854336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006347522567703109, + "loss": 1.4514, + "theoretical_loss": 3.803612043632675, + "tokens_seen": 707919872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006347362086258776, + "loss": 1.4327, + "theoretical_loss": 3.80357663107604, + "tokens_seen": 707985408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006347201604814443, + "loss": 1.3922, + "theoretical_loss": 3.803541222715034, + "tokens_seen": 708050944 + }, + { + "epoch": 0.21, + "learning_rate": 0.000634704112337011, + "loss": 1.5106, + "theoretical_loss": 3.803505818548773, + "tokens_seen": 708116480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006346880641925777, + "loss": 1.4595, + "theoretical_loss": 3.803470418576371, + "tokens_seen": 708182016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006346720160481444, + "loss": 1.3954, + "theoretical_loss": 3.8034350227969433, + "tokens_seen": 708247552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006346559679037111, + "loss": 1.5501, + "theoretical_loss": 3.8033996312096052, + "tokens_seen": 708313088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006346399197592778, + "loss": 1.5247, + "theoretical_loss": 3.803364243813473, + "tokens_seen": 708378624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006346238716148445, + "loss": 1.4985, + "theoretical_loss": 3.803328860607662, + "tokens_seen": 708444160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006346078234704112, + "loss": 1.5289, + "theoretical_loss": 3.8032934815912895, + "tokens_seen": 708509696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006345917753259779, + "loss": 1.4466, + "theoretical_loss": 3.8032581067634705, + "tokens_seen": 708575232 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006345757271815446, + "loss": 1.4743, + "theoretical_loss": 3.803222736123323, + "tokens_seen": 708640768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006345596790371113, + "loss": 1.3385, + "theoretical_loss": 3.8031873696699634, + "tokens_seen": 708706304 + }, + { + "epoch": 0.21, + "learning_rate": 0.000634543630892678, + "loss": 1.3498, + "theoretical_loss": 3.8031520074025096, + "tokens_seen": 708771840 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006345275827482447, + "loss": 1.3754, + "theoretical_loss": 3.8031166493200788, + "tokens_seen": 708837376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006345115346038115, + "loss": 1.4299, + "theoretical_loss": 3.803081295421788, + "tokens_seen": 708902912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006344954864593781, + "loss": 1.4544, + "theoretical_loss": 3.8030459457067574, + "tokens_seen": 708968448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006344794383149449, + "loss": 1.3495, + "theoretical_loss": 3.803010600174103, + "tokens_seen": 709033984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006344633901705115, + "loss": 1.4578, + "theoretical_loss": 3.8029752588229453, + "tokens_seen": 709099520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006344473420260783, + "loss": 1.4614, + "theoretical_loss": 3.8029399216524027, + "tokens_seen": 709165056 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006344312938816449, + "loss": 1.5498, + "theoretical_loss": 3.802904588661594, + "tokens_seen": 709230592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006344152457372117, + "loss": 1.554, + "theoretical_loss": 3.802869259849639, + "tokens_seen": 709296128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006343991975927783, + "loss": 1.4749, + "theoretical_loss": 3.8028339352156575, + "tokens_seen": 709361664 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006343831494483451, + "loss": 1.4278, + "theoretical_loss": 3.802798614758769, + "tokens_seen": 709427200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0006343671013039117, + "loss": 1.5911, + "theoretical_loss": 3.8027632984780944, + "tokens_seen": 709492736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006343510531594785, + "loss": 1.4853, + "theoretical_loss": 3.8027279863727537, + "tokens_seen": 709558272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006343350050150451, + "loss": 1.4358, + "theoretical_loss": 3.8026926784418675, + "tokens_seen": 709623808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006343189568706119, + "loss": 1.4554, + "theoretical_loss": 3.802657374684558, + "tokens_seen": 709689344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006343029087261785, + "loss": 1.342, + "theoretical_loss": 3.8026220750999453, + "tokens_seen": 709754880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006342868605817453, + "loss": 1.3879, + "theoretical_loss": 3.802586779687152, + "tokens_seen": 709820416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006342708124373119, + "loss": 1.4332, + "theoretical_loss": 3.8025514884452987, + "tokens_seen": 709885952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006342547642928787, + "loss": 1.4558, + "theoretical_loss": 3.8025162013735088, + "tokens_seen": 709951488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006342387161484453, + "loss": 1.4395, + "theoretical_loss": 3.802480918470904, + "tokens_seen": 710017024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006342226680040121, + "loss": 1.45, + "theoretical_loss": 3.8024456397366073, + "tokens_seen": 710082560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006342066198595788, + "loss": 1.4876, + "theoretical_loss": 3.802410365169741, + "tokens_seen": 710148096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006341905717151455, + "loss": 1.4237, + "theoretical_loss": 3.802375094769429, + "tokens_seen": 710213632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006341745235707122, + "loss": 1.3341, + "theoretical_loss": 3.8023398285347945, + "tokens_seen": 710279168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006341584754262789, + "loss": 1.4731, + "theoretical_loss": 3.8023045664649606, + "tokens_seen": 710344704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006341424272818456, + "loss": 1.3805, + "theoretical_loss": 3.802269308559052, + "tokens_seen": 710410240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006341263791374123, + "loss": 1.498, + "theoretical_loss": 3.8022340548161933, + "tokens_seen": 710475776 + }, + { + "epoch": 0.22, + "learning_rate": 0.000634110330992979, + "loss": 1.5303, + "theoretical_loss": 3.8021988052355082, + "tokens_seen": 710541312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006340942828485457, + "loss": 1.4483, + "theoretical_loss": 3.8021635598161216, + "tokens_seen": 710606848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006340782347041124, + "loss": 1.5473, + "theoretical_loss": 3.802128318557159, + "tokens_seen": 710672384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006340782347041124, + "loss": 1.4566, + "theoretical_loss": 3.802093081457745, + "tokens_seen": 710737920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006340621865596791, + "loss": 1.5172, + "theoretical_loss": 3.8020578485170056, + "tokens_seen": 710803456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006340461384152458, + "loss": 1.4352, + "theoretical_loss": 3.8020226197340667, + "tokens_seen": 710868992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006340300902708125, + "loss": 1.4934, + "theoretical_loss": 3.801987395108055, + "tokens_seen": 710934528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006340140421263792, + "loss": 1.5166, + "theoretical_loss": 3.8019521746380955, + "tokens_seen": 711000064 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006339979939819459, + "loss": 1.5226, + "theoretical_loss": 3.801916958323315, + "tokens_seen": 711065600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006339819458375126, + "loss": 1.4886, + "theoretical_loss": 3.8018817461628416, + "tokens_seen": 711131136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006339658976930793, + "loss": 1.5122, + "theoretical_loss": 3.801846538155802, + "tokens_seen": 711196672 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633949849548646, + "loss": 1.4615, + "theoretical_loss": 3.801811334301323, + "tokens_seen": 711262208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006339338014042127, + "loss": 1.5207, + "theoretical_loss": 3.8017761345985326, + "tokens_seen": 711327744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006339177532597794, + "loss": 1.5536, + "theoretical_loss": 3.8017409390465584, + "tokens_seen": 711393280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006339017051153461, + "loss": 1.422, + "theoretical_loss": 3.8017057476445295, + "tokens_seen": 711458816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006338856569709128, + "loss": 1.4986, + "theoretical_loss": 3.8016705603915737, + "tokens_seen": 711524352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006338696088264795, + "loss": 1.4191, + "theoretical_loss": 3.8016353772868197, + "tokens_seen": 711589888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006338535606820462, + "loss": 1.4324, + "theoretical_loss": 3.801600198329397, + "tokens_seen": 711655424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006338375125376129, + "loss": 1.4581, + "theoretical_loss": 3.8015650235184344, + "tokens_seen": 711720960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006338214643931796, + "loss": 1.4864, + "theoretical_loss": 3.8015298528530614, + "tokens_seen": 711786496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006338054162487463, + "loss": 1.5053, + "theoretical_loss": 3.801494686332408, + "tokens_seen": 711852032 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633789368104313, + "loss": 1.4672, + "theoretical_loss": 3.8014595239556046, + "tokens_seen": 711917568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006337733199598797, + "loss": 1.5904, + "theoretical_loss": 3.8014243657217808, + "tokens_seen": 711983104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006337572718154464, + "loss": 1.4954, + "theoretical_loss": 3.8013892116300676, + "tokens_seen": 712048640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006337412236710131, + "loss": 1.4744, + "theoretical_loss": 3.8013540616795956, + "tokens_seen": 712114176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006337251755265798, + "loss": 1.428, + "theoretical_loss": 3.801318915869496, + "tokens_seen": 712179712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006337091273821465, + "loss": 1.417, + "theoretical_loss": 3.8012837741989003, + "tokens_seen": 712245248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006336930792377132, + "loss": 1.4956, + "theoretical_loss": 3.80124863666694, + "tokens_seen": 712310784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006336770310932799, + "loss": 1.3525, + "theoretical_loss": 3.8012135032727468, + "tokens_seen": 712376320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006336609829488466, + "loss": 1.5225, + "theoretical_loss": 3.8011783740154534, + "tokens_seen": 712441856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006336449348044133, + "loss": 1.5795, + "theoretical_loss": 3.801143248894192, + "tokens_seen": 712507392 + }, + { + "epoch": 0.22, + "learning_rate": 0.00063362888665998, + "loss": 1.4768, + "theoretical_loss": 3.801108127908095, + "tokens_seen": 712572928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006336128385155467, + "loss": 1.5102, + "theoretical_loss": 3.801073011056295, + "tokens_seen": 712638464 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335967903711134, + "loss": 1.3734, + "theoretical_loss": 3.801037898337926, + "tokens_seen": 712704000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335807422266801, + "loss": 1.4811, + "theoretical_loss": 3.801002789752121, + "tokens_seen": 712769536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335646940822468, + "loss": 1.4119, + "theoretical_loss": 3.800967685298014, + "tokens_seen": 712835072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335486459378135, + "loss": 1.477, + "theoretical_loss": 3.8009325849747384, + "tokens_seen": 712900608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335325977933801, + "loss": 1.4521, + "theoretical_loss": 3.8008974887814295, + "tokens_seen": 712966144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335165496489469, + "loss": 1.4321, + "theoretical_loss": 3.8008623967172204, + "tokens_seen": 713031680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006335005015045135, + "loss": 1.4095, + "theoretical_loss": 3.800827308781247, + "tokens_seen": 713097216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006334844533600803, + "loss": 1.5166, + "theoretical_loss": 3.8007922249726445, + "tokens_seen": 713162752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006334684052156469, + "loss": 1.3974, + "theoretical_loss": 3.8007571452905466, + "tokens_seen": 713228288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006334523570712137, + "loss": 1.4486, + "theoretical_loss": 3.800722069734091, + "tokens_seen": 713293824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006334363089267803, + "loss": 1.4616, + "theoretical_loss": 3.800686998302411, + "tokens_seen": 713359360 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633420260782347, + "loss": 1.4593, + "theoretical_loss": 3.8006519309946447, + "tokens_seen": 713424896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006334042126379137, + "loss": 1.4266, + "theoretical_loss": 3.8006168678099277, + "tokens_seen": 713490432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006333881644934804, + "loss": 1.4203, + "theoretical_loss": 3.800581808747397, + "tokens_seen": 713555968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006333721163490471, + "loss": 1.5814, + "theoretical_loss": 3.800546753806189, + "tokens_seen": 713621504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006333560682046138, + "loss": 1.3719, + "theoretical_loss": 3.8005117029854403, + "tokens_seen": 713687040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006333400200601805, + "loss": 1.4672, + "theoretical_loss": 3.8004766562842893, + "tokens_seen": 713752576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006333239719157472, + "loss": 1.5235, + "theoretical_loss": 3.800441613701873, + "tokens_seen": 713818112 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633307923771314, + "loss": 1.4168, + "theoretical_loss": 3.8004065752373295, + "tokens_seen": 713883648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006332918756268806, + "loss": 1.4268, + "theoretical_loss": 3.800371540889797, + "tokens_seen": 713949184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006332758274824473, + "loss": 1.4721, + "theoretical_loss": 3.800336510658414, + "tokens_seen": 714014720 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633259779338014, + "loss": 1.4459, + "theoretical_loss": 3.8003014845423184, + "tokens_seen": 714080256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006332437311935807, + "loss": 1.4202, + "theoretical_loss": 3.80026646254065, + "tokens_seen": 714145792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006332276830491474, + "loss": 1.3794, + "theoretical_loss": 3.800231444652548, + "tokens_seen": 714211328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006332116349047141, + "loss": 1.4708, + "theoretical_loss": 3.800196430877151, + "tokens_seen": 714276864 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 270178, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5306047201156616, + "objective/train/theoretical_loss": 3.8001614212135997, + "objective/train/tokens_used": 734802400, + "theoretical_loss": 3.8001614212135997, + "tokens_seen": 714342400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006331955867602808, + "loss": 1.5306, + "theoretical_loss": 3.8001614212135997, + "tokens_seen": 714342400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006331795386158476, + "loss": 1.4953, + "theoretical_loss": 3.8001264156610333, + "tokens_seen": 714407936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006331634904714142, + "loss": 1.4254, + "theoretical_loss": 3.800091414218592, + "tokens_seen": 714473472 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633147442326981, + "loss": 1.3534, + "theoretical_loss": 3.800056416885417, + "tokens_seen": 714539008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006331313941825476, + "loss": 1.4833, + "theoretical_loss": 3.8000214236606484, + "tokens_seen": 714604544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006331153460381144, + "loss": 1.3678, + "theoretical_loss": 3.7999864345434275, + "tokens_seen": 714670080 + }, + { + "epoch": 0.22, + "learning_rate": 0.000633099297893681, + "loss": 1.4747, + "theoretical_loss": 3.7999514495328954, + "tokens_seen": 714735616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006330832497492478, + "loss": 1.4691, + "theoretical_loss": 3.7999164686281937, + "tokens_seen": 714801152 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006330672016048144, + "loss": 1.3934, + "theoretical_loss": 3.7998814918284642, + "tokens_seen": 714866688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006330511534603812, + "loss": 1.5197, + "theoretical_loss": 3.7998465191328483, + "tokens_seen": 714932224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006330351053159478, + "loss": 1.4904, + "theoretical_loss": 3.7998115505404892, + "tokens_seen": 714997760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006330190571715146, + "loss": 1.4561, + "theoretical_loss": 3.7997765860505295, + "tokens_seen": 715063296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006330030090270813, + "loss": 1.5689, + "theoretical_loss": 3.7997416256621106, + "tokens_seen": 715128832 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632986960882648, + "loss": 1.415, + "theoretical_loss": 3.7997066693743773, + "tokens_seen": 715194368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006329709127382147, + "loss": 1.3747, + "theoretical_loss": 3.799671717186472, + "tokens_seen": 715259904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006329548645937814, + "loss": 1.4835, + "theoretical_loss": 3.799636769097538, + "tokens_seen": 715325440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006329388164493481, + "loss": 1.4312, + "theoretical_loss": 3.7996018251067203, + "tokens_seen": 715390976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006329227683049148, + "loss": 1.4936, + "theoretical_loss": 3.799566885213162, + "tokens_seen": 715456512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006329067201604815, + "loss": 1.4079, + "theoretical_loss": 3.799531949416007, + "tokens_seen": 715522048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006328906720160482, + "loss": 1.3911, + "theoretical_loss": 3.799497017714401, + "tokens_seen": 715587584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006328746238716149, + "loss": 1.4271, + "theoretical_loss": 3.7994620901074887, + "tokens_seen": 715653120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006328585757271816, + "loss": 1.5503, + "theoretical_loss": 3.7994271665944144, + "tokens_seen": 715718656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006328425275827483, + "loss": 1.4699, + "theoretical_loss": 3.799392247174324, + "tokens_seen": 715784192 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632826479438315, + "loss": 1.5553, + "theoretical_loss": 3.799357331846364, + "tokens_seen": 715849728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006328104312938817, + "loss": 1.5208, + "theoretical_loss": 3.7993224206096787, + "tokens_seen": 715915264 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006327943831494484, + "loss": 1.4603, + "theoretical_loss": 3.7992875134634154, + "tokens_seen": 715980800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006327783350050151, + "loss": 1.4422, + "theoretical_loss": 3.79925261040672, + "tokens_seen": 716046336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006327622868605818, + "loss": 1.492, + "theoretical_loss": 3.7992177114387387, + "tokens_seen": 716111872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006327462387161485, + "loss": 1.4466, + "theoretical_loss": 3.799182816558619, + "tokens_seen": 716177408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006327301905717152, + "loss": 1.5052, + "theoretical_loss": 3.7991479257655083, + "tokens_seen": 716242944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006327141424272819, + "loss": 1.4583, + "theoretical_loss": 3.7991130390585535, + "tokens_seen": 716308480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006326980942828486, + "loss": 1.5007, + "theoretical_loss": 3.7990781564369023, + "tokens_seen": 716374016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006326820461384153, + "loss": 1.4656, + "theoretical_loss": 3.7990432778997034, + "tokens_seen": 716439552 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632665997993982, + "loss": 1.5198, + "theoretical_loss": 3.799008403446104, + "tokens_seen": 716505088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006326499498495487, + "loss": 1.3618, + "theoretical_loss": 3.798973533075253, + "tokens_seen": 716570624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006326339017051154, + "loss": 1.3291, + "theoretical_loss": 3.798938666786299, + "tokens_seen": 716636160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006326178535606821, + "loss": 1.4304, + "theoretical_loss": 3.798903804578391, + "tokens_seen": 716701696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006326018054162488, + "loss": 1.3708, + "theoretical_loss": 3.7988689464506784, + "tokens_seen": 716767232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006325857572718155, + "loss": 1.4802, + "theoretical_loss": 3.798834092402311, + "tokens_seen": 716832768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006325697091273822, + "loss": 1.5245, + "theoretical_loss": 3.7987992424324366, + "tokens_seen": 716898304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006325536609829489, + "loss": 1.3806, + "theoretical_loss": 3.7987643965402076, + "tokens_seen": 716963840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006325376128385156, + "loss": 1.409, + "theoretical_loss": 3.7987295547247735, + "tokens_seen": 717029376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006325215646940823, + "loss": 1.5383, + "theoretical_loss": 3.7986947169852834, + "tokens_seen": 717094912 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632505516549649, + "loss": 1.5265, + "theoretical_loss": 3.79865988332089, + "tokens_seen": 717160448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006324894684052157, + "loss": 1.3946, + "theoretical_loss": 3.798625053730743, + "tokens_seen": 717225984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006324734202607824, + "loss": 1.4415, + "theoretical_loss": 3.7985902282139943, + "tokens_seen": 717291520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006324573721163491, + "loss": 1.3839, + "theoretical_loss": 3.7985554067697946, + "tokens_seen": 717357056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006324413239719158, + "loss": 1.5592, + "theoretical_loss": 3.798520589397297, + "tokens_seen": 717422592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006324252758274825, + "loss": 1.3409, + "theoretical_loss": 3.798485776095652, + "tokens_seen": 717488128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006324092276830492, + "loss": 1.5184, + "theoretical_loss": 3.7984509668640136, + "tokens_seen": 717553664 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006323931795386159, + "loss": 1.552, + "theoretical_loss": 3.7984161617015326, + "tokens_seen": 717619200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006323771313941826, + "loss": 1.4175, + "theoretical_loss": 3.7983813606073626, + "tokens_seen": 717684736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006323610832497493, + "loss": 1.5719, + "theoretical_loss": 3.7983465635806564, + "tokens_seen": 717750272 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632345035105316, + "loss": 1.3737, + "theoretical_loss": 3.7983117706205674, + "tokens_seen": 717815808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006323289869608827, + "loss": 1.5743, + "theoretical_loss": 3.798276981726249, + "tokens_seen": 717881344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006323129388164494, + "loss": 1.4498, + "theoretical_loss": 3.798242196896856, + "tokens_seen": 717946880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322968906720161, + "loss": 1.4375, + "theoretical_loss": 3.798207416131541, + "tokens_seen": 718012416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322808425275828, + "loss": 1.4418, + "theoretical_loss": 3.798172639429459, + "tokens_seen": 718077952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322647943831495, + "loss": 1.4574, + "theoretical_loss": 3.798137866789764, + "tokens_seen": 718143488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322487462387162, + "loss": 1.4074, + "theoretical_loss": 3.798103098211611, + "tokens_seen": 718209024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322326980942829, + "loss": 1.5307, + "theoretical_loss": 3.798068333694156, + "tokens_seen": 718274560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322166499498496, + "loss": 1.4016, + "theoretical_loss": 3.798033573236553, + "tokens_seen": 718340096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006322006018054163, + "loss": 1.4302, + "theoretical_loss": 3.797998816837959, + "tokens_seen": 718405632 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632184553660983, + "loss": 1.4214, + "theoretical_loss": 3.7979640644975277, + "tokens_seen": 718471168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006321685055165497, + "loss": 1.5332, + "theoretical_loss": 3.797929316214417, + "tokens_seen": 718536704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006321524573721164, + "loss": 1.3572, + "theoretical_loss": 3.797894571987783, + "tokens_seen": 718602240 + }, + { + "epoch": 0.22, + "learning_rate": 0.000632136409227683, + "loss": 1.3685, + "theoretical_loss": 3.7978598318167816, + "tokens_seen": 718667776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006321203610832498, + "loss": 1.491, + "theoretical_loss": 3.7978250957005697, + "tokens_seen": 718733312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006321043129388164, + "loss": 1.4102, + "theoretical_loss": 3.7977903636383044, + "tokens_seen": 718798848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006320882647943832, + "loss": 1.5293, + "theoretical_loss": 3.7977556356291435, + "tokens_seen": 718864384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006320722166499498, + "loss": 1.3949, + "theoretical_loss": 3.7977209116722443, + "tokens_seen": 718929920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006320561685055166, + "loss": 1.407, + "theoretical_loss": 3.797686191766765, + "tokens_seen": 718995456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006320401203610832, + "loss": 1.4705, + "theoretical_loss": 3.797651475911863, + "tokens_seen": 719060992 + }, + { + "epoch": 0.22, + "learning_rate": 0.00063202407221665, + "loss": 1.4232, + "theoretical_loss": 3.7976167641066962, + "tokens_seen": 719126528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006320080240722166, + "loss": 1.4364, + "theoretical_loss": 3.7975820563504246, + "tokens_seen": 719192064 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006319919759277833, + "loss": 1.3355, + "theoretical_loss": 3.797547352642206, + "tokens_seen": 719257600 + }, + { + "epoch": 0.22, + "learning_rate": 0.00063197592778335, + "loss": 1.4609, + "theoretical_loss": 3.7975126529812, + "tokens_seen": 719323136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006319598796389167, + "loss": 1.4248, + "theoretical_loss": 3.797477957366566, + "tokens_seen": 719388672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006319438314944834, + "loss": 1.4822, + "theoretical_loss": 3.797443265797463, + "tokens_seen": 719454208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006319277833500501, + "loss": 1.4633, + "theoretical_loss": 3.7974085782730516, + "tokens_seen": 719519744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006319117352056168, + "loss": 1.5676, + "theoretical_loss": 3.797373894792491, + "tokens_seen": 719585280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006318956870611835, + "loss": 1.4182, + "theoretical_loss": 3.797339215354942, + "tokens_seen": 719650816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006318796389167502, + "loss": 1.4146, + "theoretical_loss": 3.7973045399595655, + "tokens_seen": 719716352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006318635907723169, + "loss": 1.469, + "theoretical_loss": 3.797269868605522, + "tokens_seen": 719781888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006318475426278836, + "loss": 1.4892, + "theoretical_loss": 3.797235201291973, + "tokens_seen": 719847424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006318314944834504, + "loss": 1.3983, + "theoretical_loss": 3.797200538018079, + "tokens_seen": 719912960 + }, + { + "epoch": 0.22, + "learning_rate": 0.000631815446339017, + "loss": 1.3776, + "theoretical_loss": 3.7971658787830025, + "tokens_seen": 719978496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006317993981945838, + "loss": 1.413, + "theoretical_loss": 3.7971312235859047, + "tokens_seen": 720044032 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006317833500501505, + "loss": 1.3984, + "theoretical_loss": 3.797096572425948, + "tokens_seen": 720109568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006317673019057172, + "loss": 1.3545, + "theoretical_loss": 3.7970619253022946, + "tokens_seen": 720175104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006317512537612839, + "loss": 1.46, + "theoretical_loss": 3.797027282214107, + "tokens_seen": 720240640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006317352056168506, + "loss": 1.4242, + "theoretical_loss": 3.7969926431605483, + "tokens_seen": 720306176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006317191574724173, + "loss": 1.436, + "theoretical_loss": 3.796958008140782, + "tokens_seen": 720371712 + }, + { + "epoch": 0.22, + "learning_rate": 0.000631703109327984, + "loss": 1.3751, + "theoretical_loss": 3.7969233771539708, + "tokens_seen": 720437248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006316870611835507, + "loss": 1.4672, + "theoretical_loss": 3.796888750199278, + "tokens_seen": 720502784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006316710130391174, + "loss": 1.513, + "theoretical_loss": 3.796854127275868, + "tokens_seen": 720568320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006316549648946841, + "loss": 1.5122, + "theoretical_loss": 3.796819508382905, + "tokens_seen": 720633856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006316389167502508, + "loss": 1.3706, + "theoretical_loss": 3.796784893519553, + "tokens_seen": 720699392 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006316228686058175, + "loss": 1.3873, + "theoretical_loss": 3.796750282684977, + "tokens_seen": 720764928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006316068204613842, + "loss": 1.4056, + "theoretical_loss": 3.7967156758783416, + "tokens_seen": 720830464 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 272585, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3715122938156128, + "objective/train/theoretical_loss": 3.7966810730988114, + "objective/train/tokens_used": 741356000, + "theoretical_loss": 3.7966810730988114, + "tokens_seen": 720896000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006315907723169509, + "loss": 1.3715, + "theoretical_loss": 3.7966810730988114, + "tokens_seen": 720896000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006315747241725176, + "loss": 1.3752, + "theoretical_loss": 3.7966464743455526, + "tokens_seen": 720961536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006315586760280843, + "loss": 1.4037, + "theoretical_loss": 3.79661187961773, + "tokens_seen": 721027072 + }, + { + "epoch": 0.22, + "learning_rate": 0.000631542627883651, + "loss": 1.424, + "theoretical_loss": 3.79657728891451, + "tokens_seen": 721092608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006315265797392177, + "loss": 1.5034, + "theoretical_loss": 3.796542702235059, + "tokens_seen": 721158144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006315105315947844, + "loss": 1.4959, + "theoretical_loss": 3.7965081195785424, + "tokens_seen": 721223680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006314944834503511, + "loss": 1.4613, + "theoretical_loss": 3.7964735409441275, + "tokens_seen": 721289216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006314784353059178, + "loss": 1.3696, + "theoretical_loss": 3.7964389663309808, + "tokens_seen": 721354752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006314623871614845, + "loss": 1.5149, + "theoretical_loss": 3.796404395738269, + "tokens_seen": 721420288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006314463390170512, + "loss": 1.4964, + "theoretical_loss": 3.79636982916516, + "tokens_seen": 721485824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006314302908726179, + "loss": 1.5068, + "theoretical_loss": 3.7963352666108214, + "tokens_seen": 721551360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006314142427281846, + "loss": 1.4855, + "theoretical_loss": 3.796300708074421, + "tokens_seen": 721616896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006313981945837513, + "loss": 1.4715, + "theoretical_loss": 3.796266153555127, + "tokens_seen": 721682432 + }, + { + "epoch": 0.22, + "learning_rate": 0.000631382146439318, + "loss": 1.4005, + "theoretical_loss": 3.7962316030521066, + "tokens_seen": 721747968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006313660982948847, + "loss": 1.3893, + "theoretical_loss": 3.79619705656453, + "tokens_seen": 721813504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006313500501504514, + "loss": 1.4309, + "theoretical_loss": 3.7961625140915647, + "tokens_seen": 721879040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006313340020060181, + "loss": 1.4155, + "theoretical_loss": 3.7961279756323805, + "tokens_seen": 721944576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006313179538615848, + "loss": 1.3355, + "theoretical_loss": 3.7960934411861462, + "tokens_seen": 722010112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006313019057171515, + "loss": 1.4941, + "theoretical_loss": 3.7960589107520324, + "tokens_seen": 722075648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006312858575727182, + "loss": 1.5064, + "theoretical_loss": 3.7960243843292076, + "tokens_seen": 722141184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006312698094282849, + "loss": 1.4063, + "theoretical_loss": 3.795989861916843, + "tokens_seen": 722206720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006312537612838516, + "loss": 1.3798, + "theoretical_loss": 3.7959553435141076, + "tokens_seen": 722272256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006312377131394183, + "loss": 1.4387, + "theoretical_loss": 3.795920829120173, + "tokens_seen": 722337792 + }, + { + "epoch": 0.22, + "learning_rate": 0.000631221664994985, + "loss": 1.4556, + "theoretical_loss": 3.79588631873421, + "tokens_seen": 722403328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006312056168505517, + "loss": 1.455, + "theoretical_loss": 3.7958518123553886, + "tokens_seen": 722468864 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006311895687061184, + "loss": 1.5685, + "theoretical_loss": 3.7958173099828816, + "tokens_seen": 722534400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006311735205616851, + "loss": 1.429, + "theoretical_loss": 3.795782811615859, + "tokens_seen": 722599936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006311574724172518, + "loss": 1.3827, + "theoretical_loss": 3.7957483172534934, + "tokens_seen": 722665472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006311414242728185, + "loss": 1.381, + "theoretical_loss": 3.795713826894957, + "tokens_seen": 722731008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006311253761283852, + "loss": 1.5152, + "theoretical_loss": 3.7956793405394214, + "tokens_seen": 722796544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006311093279839519, + "loss": 1.3877, + "theoretical_loss": 3.79564485818606, + "tokens_seen": 722862080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006310932798395186, + "loss": 1.4714, + "theoretical_loss": 3.7956103798340446, + "tokens_seen": 722927616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006310772316950853, + "loss": 1.547, + "theoretical_loss": 3.795575905482549, + "tokens_seen": 722993152 + }, + { + "epoch": 0.22, + "learning_rate": 0.000631061183550652, + "loss": 1.3982, + "theoretical_loss": 3.7955414351307457, + "tokens_seen": 723058688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006310451354062187, + "loss": 1.4217, + "theoretical_loss": 3.7955069687778087, + "tokens_seen": 723124224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006310290872617854, + "loss": 1.5079, + "theoretical_loss": 3.795472506422912, + "tokens_seen": 723189760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006310130391173521, + "loss": 1.4948, + "theoretical_loss": 3.795438048065229, + "tokens_seen": 723255296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006309969909729188, + "loss": 1.472, + "theoretical_loss": 3.7954035937039348, + "tokens_seen": 723320832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006309809428284855, + "loss": 1.4482, + "theoretical_loss": 3.7953691433382026, + "tokens_seen": 723386368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006309648946840522, + "loss": 1.4982, + "theoretical_loss": 3.795334696967208, + "tokens_seen": 723451904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006309488465396189, + "loss": 1.4358, + "theoretical_loss": 3.795300254590126, + "tokens_seen": 723517440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006309327983951856, + "loss": 1.3647, + "theoretical_loss": 3.7952658162061317, + "tokens_seen": 723582976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006309167502507523, + "loss": 1.3928, + "theoretical_loss": 3.7952313818144, + "tokens_seen": 723648512 + }, + { + "epoch": 0.22, + "learning_rate": 0.000630900702106319, + "loss": 1.4658, + "theoretical_loss": 3.7951969514141073, + "tokens_seen": 723714048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006308846539618857, + "loss": 1.4329, + "theoretical_loss": 3.795162525004429, + "tokens_seen": 723779584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006308686058174524, + "loss": 1.4697, + "theoretical_loss": 3.795128102584542, + "tokens_seen": 723845120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006308525576730191, + "loss": 1.4753, + "theoretical_loss": 3.7950936841536222, + "tokens_seen": 723910656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006308365095285858, + "loss": 1.3821, + "theoretical_loss": 3.7950592697108467, + "tokens_seen": 723976192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006308204613841525, + "loss": 1.4378, + "theoretical_loss": 3.795024859255392, + "tokens_seen": 724041728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006308044132397192, + "loss": 1.4715, + "theoretical_loss": 3.7949904527864353, + "tokens_seen": 724107264 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006307883650952859, + "loss": 1.4084, + "theoretical_loss": 3.7949560503031545, + "tokens_seen": 724172800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006307723169508526, + "loss": 1.4111, + "theoretical_loss": 3.794921651804727, + "tokens_seen": 724238336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006307562688064193, + "loss": 1.4577, + "theoretical_loss": 3.7948872572903305, + "tokens_seen": 724303872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006307402206619859, + "loss": 1.4141, + "theoretical_loss": 3.7948528667591432, + "tokens_seen": 724369408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006307241725175527, + "loss": 1.3726, + "theoretical_loss": 3.7948184802103437, + "tokens_seen": 724434944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006307081243731193, + "loss": 1.4355, + "theoretical_loss": 3.7947840976431104, + "tokens_seen": 724500480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006306920762286861, + "loss": 1.5371, + "theoretical_loss": 3.7947497190566226, + "tokens_seen": 724566016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006306760280842527, + "loss": 1.4615, + "theoretical_loss": 3.7947153444500588, + "tokens_seen": 724631552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006306599799398195, + "loss": 1.4447, + "theoretical_loss": 3.794680973822599, + "tokens_seen": 724697088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006306439317953861, + "loss": 1.3955, + "theoretical_loss": 3.7946466071734224, + "tokens_seen": 724762624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006306278836509529, + "loss": 1.4801, + "theoretical_loss": 3.794612244501709, + "tokens_seen": 724828160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006306118355065195, + "loss": 1.4749, + "theoretical_loss": 3.794577885806639, + "tokens_seen": 724893696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006305957873620863, + "loss": 1.5056, + "theoretical_loss": 3.7945435310873927, + "tokens_seen": 724959232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006305797392176529, + "loss": 1.5203, + "theoretical_loss": 3.7945091803431503, + "tokens_seen": 725024768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006305636910732197, + "loss": 1.3562, + "theoretical_loss": 3.7944748335730933, + "tokens_seen": 725090304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006305476429287863, + "loss": 1.5318, + "theoretical_loss": 3.794440490776402, + "tokens_seen": 725155840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006305315947843531, + "loss": 1.4413, + "theoretical_loss": 3.7944061519522583, + "tokens_seen": 725221376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006305155466399197, + "loss": 1.4646, + "theoretical_loss": 3.7943718170998437, + "tokens_seen": 725286912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006304994984954865, + "loss": 1.4539, + "theoretical_loss": 3.79433748621834, + "tokens_seen": 725352448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006304834503510531, + "loss": 1.5967, + "theoretical_loss": 3.7943031593069287, + "tokens_seen": 725417984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006304674022066199, + "loss": 1.4228, + "theoretical_loss": 3.794268836364793, + "tokens_seen": 725483520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006304513540621866, + "loss": 1.5755, + "theoretical_loss": 3.7942345173911147, + "tokens_seen": 725549056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006304353059177533, + "loss": 1.451, + "theoretical_loss": 3.7942002023850767, + "tokens_seen": 725614592 + }, + { + "epoch": 0.22, + "learning_rate": 0.00063041925777332, + "loss": 1.4127, + "theoretical_loss": 3.7941658913458625, + "tokens_seen": 725680128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006304032096288867, + "loss": 1.4606, + "theoretical_loss": 3.794131584272655, + "tokens_seen": 725745664 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006303871614844534, + "loss": 1.5038, + "theoretical_loss": 3.794097281164637, + "tokens_seen": 725811200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006303711133400201, + "loss": 1.368, + "theoretical_loss": 3.7940629820209937, + "tokens_seen": 725876736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006303550651955868, + "loss": 1.3847, + "theoretical_loss": 3.794028686840908, + "tokens_seen": 725942272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006303390170511535, + "loss": 1.3695, + "theoretical_loss": 3.7939943956235647, + "tokens_seen": 726007808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006303229689067202, + "loss": 1.4505, + "theoretical_loss": 3.7939601083681476, + "tokens_seen": 726073344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006303069207622869, + "loss": 1.4339, + "theoretical_loss": 3.793925825073842, + "tokens_seen": 726138880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006302908726178536, + "loss": 1.4142, + "theoretical_loss": 3.7938915457398323, + "tokens_seen": 726204416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006302748244734203, + "loss": 1.4737, + "theoretical_loss": 3.793857270365304, + "tokens_seen": 726269952 + }, + { + "epoch": 0.22, + "learning_rate": 0.000630258776328987, + "loss": 1.3534, + "theoretical_loss": 3.7938229989494427, + "tokens_seen": 726335488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006302427281845537, + "loss": 1.4629, + "theoretical_loss": 3.793788731491434, + "tokens_seen": 726401024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006302266800401204, + "loss": 1.3992, + "theoretical_loss": 3.7937544679904636, + "tokens_seen": 726466560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006302106318956871, + "loss": 1.457, + "theoretical_loss": 3.793720208445718, + "tokens_seen": 726532096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006301945837512538, + "loss": 1.4345, + "theoretical_loss": 3.793685952856383, + "tokens_seen": 726597632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006301785356068205, + "loss": 1.4774, + "theoretical_loss": 3.7936517012216457, + "tokens_seen": 726663168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006301624874623872, + "loss": 1.4124, + "theoretical_loss": 3.7936174535406932, + "tokens_seen": 726728704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006301464393179539, + "loss": 1.4703, + "theoretical_loss": 3.793583209812712, + "tokens_seen": 726794240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006301303911735206, + "loss": 1.3926, + "theoretical_loss": 3.7935489700368894, + "tokens_seen": 726859776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006301143430290873, + "loss": 1.4017, + "theoretical_loss": 3.7935147342124136, + "tokens_seen": 726925312 + }, + { + "epoch": 0.22, + "learning_rate": 0.000630098294884654, + "loss": 1.4982, + "theoretical_loss": 3.7934805023384723, + "tokens_seen": 726990848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006300822467402207, + "loss": 1.5153, + "theoretical_loss": 3.7934462744142534, + "tokens_seen": 727056384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006300661985957874, + "loss": 1.4617, + "theoretical_loss": 3.7934120504389455, + "tokens_seen": 727121920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006300501504513541, + "loss": 1.4228, + "theoretical_loss": 3.7933778304117363, + "tokens_seen": 727187456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006300341023069208, + "loss": 1.4933, + "theoretical_loss": 3.7933436143318153, + "tokens_seen": 727252992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006300180541624875, + "loss": 1.443, + "theoretical_loss": 3.7933094021983713, + "tokens_seen": 727318528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006300020060180542, + "loss": 1.4966, + "theoretical_loss": 3.793275194010594, + "tokens_seen": 727384064 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 275342, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4494743347167969, + "objective/train/theoretical_loss": 3.7932409897676727, + "objective/train/tokens_used": 747909600, + "theoretical_loss": 3.7932409897676727, + "tokens_seen": 727449600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006299859578736209, + "loss": 1.4495, + "theoretical_loss": 3.7932409897676727, + "tokens_seen": 727449600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006299699097291876, + "loss": 1.492, + "theoretical_loss": 3.793206789468797, + "tokens_seen": 727515136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006299538615847543, + "loss": 1.5006, + "theoretical_loss": 3.793172593113157, + "tokens_seen": 727580672 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629937813440321, + "loss": 1.4992, + "theoretical_loss": 3.7931384006999425, + "tokens_seen": 727646208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006299217652958877, + "loss": 1.3625, + "theoretical_loss": 3.7931042122283447, + "tokens_seen": 727711744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006299057171514544, + "loss": 1.4162, + "theoretical_loss": 3.7930700276975537, + "tokens_seen": 727777280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006298896690070211, + "loss": 1.5035, + "theoretical_loss": 3.793035847106761, + "tokens_seen": 727842816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006298736208625878, + "loss": 1.4326, + "theoretical_loss": 3.7930016704551575, + "tokens_seen": 727908352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006298575727181545, + "loss": 1.4584, + "theoretical_loss": 3.7929674977419343, + "tokens_seen": 727973888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006298415245737212, + "loss": 1.609, + "theoretical_loss": 3.792933328966283, + "tokens_seen": 728039424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006298254764292879, + "loss": 1.4986, + "theoretical_loss": 3.792899164127397, + "tokens_seen": 728104960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006298094282848546, + "loss": 1.523, + "theoretical_loss": 3.792865003224467, + "tokens_seen": 728170496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006297933801404213, + "loss": 1.3996, + "theoretical_loss": 3.792830846256685, + "tokens_seen": 728236032 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629777331995988, + "loss": 1.3913, + "theoretical_loss": 3.792796693223245, + "tokens_seen": 728301568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006297612838515547, + "loss": 1.401, + "theoretical_loss": 3.792762544123339, + "tokens_seen": 728367104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006297452357071214, + "loss": 1.5095, + "theoretical_loss": 3.79272839895616, + "tokens_seen": 728432640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006297291875626881, + "loss": 1.5882, + "theoretical_loss": 3.792694257720902, + "tokens_seen": 728498176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006297131394182548, + "loss": 1.5392, + "theoretical_loss": 3.792660120416758, + "tokens_seen": 728563712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006296970912738215, + "loss": 1.4208, + "theoretical_loss": 3.792625987042922, + "tokens_seen": 728629248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006296810431293882, + "loss": 1.4444, + "theoretical_loss": 3.7925918575985884, + "tokens_seen": 728694784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006296649949849549, + "loss": 1.343, + "theoretical_loss": 3.7925577320829507, + "tokens_seen": 728760320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006296489468405216, + "loss": 1.436, + "theoretical_loss": 3.792523610495204, + "tokens_seen": 728825856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006296328986960883, + "loss": 1.4136, + "theoretical_loss": 3.7924894928345427, + "tokens_seen": 728891392 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629616850551655, + "loss": 1.4828, + "theoretical_loss": 3.7924553791001623, + "tokens_seen": 728956928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006296008024072217, + "loss": 1.3601, + "theoretical_loss": 3.792421269291258, + "tokens_seen": 729022464 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006295847542627884, + "loss": 1.3892, + "theoretical_loss": 3.7923871634070245, + "tokens_seen": 729088000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006295687061183551, + "loss": 1.395, + "theoretical_loss": 3.792353061446658, + "tokens_seen": 729153536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006295526579739218, + "loss": 1.5615, + "theoretical_loss": 3.792318963409355, + "tokens_seen": 729219072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006295366098294885, + "loss": 1.6161, + "theoretical_loss": 3.7922848692943103, + "tokens_seen": 729284608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006295205616850552, + "loss": 1.4737, + "theoretical_loss": 3.792250779100722, + "tokens_seen": 729350144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006295045135406219, + "loss": 1.5072, + "theoretical_loss": 3.792216692827785, + "tokens_seen": 729415680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006294884653961886, + "loss": 1.4276, + "theoretical_loss": 3.792182610474698, + "tokens_seen": 729481216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006294724172517553, + "loss": 1.4225, + "theoretical_loss": 3.792148532040657, + "tokens_seen": 729546752 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629456369107322, + "loss": 1.3485, + "theoretical_loss": 3.7921144575248595, + "tokens_seen": 729612288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006294403209628887, + "loss": 1.4318, + "theoretical_loss": 3.7920803869265027, + "tokens_seen": 729677824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006294242728184554, + "loss": 1.4262, + "theoretical_loss": 3.792046320244786, + "tokens_seen": 729743360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006294082246740221, + "loss": 1.5412, + "theoretical_loss": 3.792012257478906, + "tokens_seen": 729808896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006293921765295888, + "loss": 1.4047, + "theoretical_loss": 3.791978198628061, + "tokens_seen": 729874432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006293761283851555, + "loss": 1.5526, + "theoretical_loss": 3.7919441436914507, + "tokens_seen": 729939968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006293600802407222, + "loss": 1.4606, + "theoretical_loss": 3.7919100926682727, + "tokens_seen": 730005504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006293440320962888, + "loss": 1.4872, + "theoretical_loss": 3.7918760455577267, + "tokens_seen": 730071040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006293279839518556, + "loss": 1.4847, + "theoretical_loss": 3.791842002359012, + "tokens_seen": 730136576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006293119358074222, + "loss": 1.4309, + "theoretical_loss": 3.7918079630713275, + "tokens_seen": 730202112 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629295887662989, + "loss": 1.5166, + "theoretical_loss": 3.7917739276938733, + "tokens_seen": 730267648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006292798395185556, + "loss": 1.5745, + "theoretical_loss": 3.7917398962258493, + "tokens_seen": 730333184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006292637913741224, + "loss": 1.3598, + "theoretical_loss": 3.791705868666456, + "tokens_seen": 730398720 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629247743229689, + "loss": 1.3668, + "theoretical_loss": 3.791671845014893, + "tokens_seen": 730464256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006292316950852558, + "loss": 1.3982, + "theoretical_loss": 3.7916378252703624, + "tokens_seen": 730529792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006292156469408224, + "loss": 1.3571, + "theoretical_loss": 3.791603809432064, + "tokens_seen": 730595328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006291995987963892, + "loss": 1.3952, + "theoretical_loss": 3.7915697974991986, + "tokens_seen": 730660864 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006291835506519558, + "loss": 1.4274, + "theoretical_loss": 3.7915357894709687, + "tokens_seen": 730726400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006291675025075226, + "loss": 1.4525, + "theoretical_loss": 3.791501785346575, + "tokens_seen": 730791936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006291514543630892, + "loss": 1.4447, + "theoretical_loss": 3.79146778512522, + "tokens_seen": 730857472 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629135406218656, + "loss": 1.4634, + "theoretical_loss": 3.7914337888061054, + "tokens_seen": 730923008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006291193580742226, + "loss": 1.3733, + "theoretical_loss": 3.7913997963884336, + "tokens_seen": 730988544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006291033099297894, + "loss": 1.5551, + "theoretical_loss": 3.791365807871407, + "tokens_seen": 731054080 + }, + { + "epoch": 0.22, + "learning_rate": 0.000629087261785356, + "loss": 1.4611, + "theoretical_loss": 3.791331823254229, + "tokens_seen": 731119616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006290712136409228, + "loss": 1.3431, + "theoretical_loss": 3.791297842536102, + "tokens_seen": 731185152 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006290551654964895, + "loss": 1.4224, + "theoretical_loss": 3.791263865716229, + "tokens_seen": 731250688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006290391173520562, + "loss": 1.4312, + "theoretical_loss": 3.7912298927938144, + "tokens_seen": 731316224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006290230692076229, + "loss": 1.3999, + "theoretical_loss": 3.7911959237680612, + "tokens_seen": 731381760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006290070210631896, + "loss": 1.436, + "theoretical_loss": 3.791161958638173, + "tokens_seen": 731447296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006289909729187563, + "loss": 1.4566, + "theoretical_loss": 3.7911279974033554, + "tokens_seen": 731512832 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628974924774323, + "loss": 1.4634, + "theoretical_loss": 3.7910940400628115, + "tokens_seen": 731578368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006289588766298897, + "loss": 1.5085, + "theoretical_loss": 3.7910600866157464, + "tokens_seen": 731643904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006289428284854564, + "loss": 1.3468, + "theoretical_loss": 3.7910261370613654, + "tokens_seen": 731709440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006289267803410231, + "loss": 1.4295, + "theoretical_loss": 3.790992191398873, + "tokens_seen": 731774976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006289107321965898, + "loss": 1.4144, + "theoretical_loss": 3.790958249627475, + "tokens_seen": 731840512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006288946840521565, + "loss": 1.4215, + "theoretical_loss": 3.7909243117463767, + "tokens_seen": 731906048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006288786359077232, + "loss": 1.4129, + "theoretical_loss": 3.7908903777547835, + "tokens_seen": 731971584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006288625877632899, + "loss": 1.4902, + "theoretical_loss": 3.7908564476519024, + "tokens_seen": 732037120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006288465396188566, + "loss": 1.491, + "theoretical_loss": 3.7908225214369393, + "tokens_seen": 732102656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006288304914744233, + "loss": 1.5219, + "theoretical_loss": 3.7907885991091, + "tokens_seen": 732168192 + }, + { + "epoch": 0.22, + "learning_rate": 0.00062881444332999, + "loss": 1.4002, + "theoretical_loss": 3.790754680667593, + "tokens_seen": 732233728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287983951855567, + "loss": 1.3788, + "theoretical_loss": 3.7907207661116233, + "tokens_seen": 732299264 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287823470411234, + "loss": 1.3572, + "theoretical_loss": 3.7906868554403994, + "tokens_seen": 732364800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287662988966901, + "loss": 1.4918, + "theoretical_loss": 3.7906529486531286, + "tokens_seen": 732430336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287502507522568, + "loss": 1.4663, + "theoretical_loss": 3.7906190457490183, + "tokens_seen": 732495872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287342026078235, + "loss": 1.4833, + "theoretical_loss": 3.790585146727276, + "tokens_seen": 732561408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287181544633902, + "loss": 1.4647, + "theoretical_loss": 3.790551251587111, + "tokens_seen": 732626944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006287021063189569, + "loss": 1.4109, + "theoretical_loss": 3.7905173603277307, + "tokens_seen": 732692480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006286860581745236, + "loss": 1.5004, + "theoretical_loss": 3.790483472948344, + "tokens_seen": 732758016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006286700100300903, + "loss": 1.3769, + "theoretical_loss": 3.79044958944816, + "tokens_seen": 732823552 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628653961885657, + "loss": 1.374, + "theoretical_loss": 3.7904157098263873, + "tokens_seen": 732889088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006286379137412237, + "loss": 1.3799, + "theoretical_loss": 3.7903818340822353, + "tokens_seen": 732954624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006286218655967904, + "loss": 1.5315, + "theoretical_loss": 3.790347962214914, + "tokens_seen": 733020160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006286058174523571, + "loss": 1.5101, + "theoretical_loss": 3.790314094223633, + "tokens_seen": 733085696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006285897693079238, + "loss": 1.4272, + "theoretical_loss": 3.7902802301076024, + "tokens_seen": 733151232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006285737211634905, + "loss": 1.4918, + "theoretical_loss": 3.790246369866032, + "tokens_seen": 733216768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006285576730190572, + "loss": 1.4247, + "theoretical_loss": 3.790212513498132, + "tokens_seen": 733282304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006285416248746239, + "loss": 1.4944, + "theoretical_loss": 3.7901786610031145, + "tokens_seen": 733347840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006285255767301906, + "loss": 1.5114, + "theoretical_loss": 3.7901448123801886, + "tokens_seen": 733413376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006285095285857573, + "loss": 1.4701, + "theoretical_loss": 3.790110967628567, + "tokens_seen": 733478912 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628493480441324, + "loss": 1.3806, + "theoretical_loss": 3.7900771267474607, + "tokens_seen": 733544448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006284774322968907, + "loss": 1.3518, + "theoretical_loss": 3.7900432897360803, + "tokens_seen": 733609984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006284613841524574, + "loss": 1.3985, + "theoretical_loss": 3.790009456593639, + "tokens_seen": 733675520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006284453360080241, + "loss": 1.4925, + "theoretical_loss": 3.7899756273193486, + "tokens_seen": 733741056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006284292878635908, + "loss": 1.4379, + "theoretical_loss": 3.7899418019124207, + "tokens_seen": 733806592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006284132397191575, + "loss": 1.4446, + "theoretical_loss": 3.7899079803720688, + "tokens_seen": 733872128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006283971915747242, + "loss": 1.4944, + "theoretical_loss": 3.7898741626975054, + "tokens_seen": 733937664 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 277594, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3950543403625488, + "objective/train/theoretical_loss": 3.7898403488879424, + "objective/train/tokens_used": 754463200, + "theoretical_loss": 3.7898403488879424, + "tokens_seen": 734003200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006283811434302909, + "loss": 1.3951, + "theoretical_loss": 3.7898403488879424, + "tokens_seen": 734003200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006283650952858576, + "loss": 1.3408, + "theoretical_loss": 3.7898065389425946, + "tokens_seen": 734068736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006283490471414243, + "loss": 1.5184, + "theoretical_loss": 3.789772732860675, + "tokens_seen": 734134272 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628332998996991, + "loss": 1.3913, + "theoretical_loss": 3.7897389306413967, + "tokens_seen": 734199808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006283169508525577, + "loss": 1.3981, + "theoretical_loss": 3.7897051322839745, + "tokens_seen": 734265344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006283009027081244, + "loss": 1.4943, + "theoretical_loss": 3.7896713377876217, + "tokens_seen": 734330880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006282848545636911, + "loss": 1.3503, + "theoretical_loss": 3.7896375471515533, + "tokens_seen": 734396416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006282688064192578, + "loss": 1.5872, + "theoretical_loss": 3.789603760374984, + "tokens_seen": 734461952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006282527582748245, + "loss": 1.3946, + "theoretical_loss": 3.789569977457128, + "tokens_seen": 734527488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006282367101303912, + "loss": 1.4741, + "theoretical_loss": 3.789536198397201, + "tokens_seen": 734593024 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628220661985958, + "loss": 1.5592, + "theoretical_loss": 3.789502423194418, + "tokens_seen": 734658560 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006282046138415246, + "loss": 1.4533, + "theoretical_loss": 3.7894686518479945, + "tokens_seen": 734724096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006281885656970914, + "loss": 1.4046, + "theoretical_loss": 3.789434884357147, + "tokens_seen": 734789632 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628172517552658, + "loss": 1.3307, + "theoretical_loss": 3.7894011207210907, + "tokens_seen": 734855168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006281564694082248, + "loss": 1.5058, + "theoretical_loss": 3.7893673609390417, + "tokens_seen": 734920704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006281404212637914, + "loss": 1.3784, + "theoretical_loss": 3.789333605010217, + "tokens_seen": 734986240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006281243731193582, + "loss": 1.5779, + "theoretical_loss": 3.7892998529338326, + "tokens_seen": 735051776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006281083249749248, + "loss": 1.4499, + "theoretical_loss": 3.7892661047091067, + "tokens_seen": 735117312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006280922768304916, + "loss": 1.4755, + "theoretical_loss": 3.789232360335255, + "tokens_seen": 735182848 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006280762286860582, + "loss": 1.5546, + "theoretical_loss": 3.789198619811496, + "tokens_seen": 735248384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006280601805416249, + "loss": 1.3681, + "theoretical_loss": 3.7891648831370466, + "tokens_seen": 735313920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006280441323971916, + "loss": 1.5576, + "theoretical_loss": 3.7891311503111247, + "tokens_seen": 735379456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006280280842527583, + "loss": 1.4988, + "theoretical_loss": 3.789097421332949, + "tokens_seen": 735444992 + }, + { + "epoch": 0.22, + "learning_rate": 0.000628012036108325, + "loss": 1.4416, + "theoretical_loss": 3.7890636962017368, + "tokens_seen": 735510528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006279959879638917, + "loss": 1.43, + "theoretical_loss": 3.7890299749167076, + "tokens_seen": 735576064 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006279799398194584, + "loss": 1.3844, + "theoretical_loss": 3.7889962574770792, + "tokens_seen": 735641600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006279638916750251, + "loss": 1.4261, + "theoretical_loss": 3.7889625438820715, + "tokens_seen": 735707136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006279478435305917, + "loss": 1.4297, + "theoretical_loss": 3.788928834130903, + "tokens_seen": 735772672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006279317953861585, + "loss": 1.5187, + "theoretical_loss": 3.7888951282227934, + "tokens_seen": 735838208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006279157472417251, + "loss": 1.4685, + "theoretical_loss": 3.788861426156962, + "tokens_seen": 735903744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278996990972919, + "loss": 1.4279, + "theoretical_loss": 3.7888277279326292, + "tokens_seen": 735969280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278836509528585, + "loss": 1.4114, + "theoretical_loss": 3.7887940335490153, + "tokens_seen": 736034816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278676028084253, + "loss": 1.3598, + "theoretical_loss": 3.7887603430053396, + "tokens_seen": 736100352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278515546639919, + "loss": 1.6882, + "theoretical_loss": 3.788726656300824, + "tokens_seen": 736165888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278355065195587, + "loss": 1.4531, + "theoretical_loss": 3.7886929734346886, + "tokens_seen": 736231424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278194583751253, + "loss": 1.4688, + "theoretical_loss": 3.788659294406154, + "tokens_seen": 736296960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006278034102306921, + "loss": 1.4123, + "theoretical_loss": 3.788625619214442, + "tokens_seen": 736362496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006277873620862587, + "loss": 1.4739, + "theoretical_loss": 3.7885919478587744, + "tokens_seen": 736428032 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006277713139418255, + "loss": 1.3712, + "theoretical_loss": 3.788558280338372, + "tokens_seen": 736493568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006277552657973921, + "loss": 1.4728, + "theoretical_loss": 3.7885246166524578, + "tokens_seen": 736559104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006277392176529589, + "loss": 1.5351, + "theoretical_loss": 3.7884909568002536, + "tokens_seen": 736624640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006277231695085255, + "loss": 1.4573, + "theoretical_loss": 3.7884573007809808, + "tokens_seen": 736690176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006277071213640923, + "loss": 1.3739, + "theoretical_loss": 3.788423648593863, + "tokens_seen": 736755712 + }, + { + "epoch": 0.22, + "learning_rate": 0.000627691073219659, + "loss": 1.4566, + "theoretical_loss": 3.788390000238123, + "tokens_seen": 736821248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006276750250752257, + "loss": 1.5113, + "theoretical_loss": 3.788356355712984, + "tokens_seen": 736886784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006276589769307924, + "loss": 1.4041, + "theoretical_loss": 3.788322715017668, + "tokens_seen": 736952320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006276429287863591, + "loss": 1.4999, + "theoretical_loss": 3.7882890781514007, + "tokens_seen": 737017856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006276268806419258, + "loss": 1.5089, + "theoretical_loss": 3.7882554451134043, + "tokens_seen": 737083392 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006276108324974925, + "loss": 1.5377, + "theoretical_loss": 3.7882218159029026, + "tokens_seen": 737148928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006275947843530592, + "loss": 1.4385, + "theoretical_loss": 3.7881881905191213, + "tokens_seen": 737214464 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006275787362086259, + "loss": 1.4934, + "theoretical_loss": 3.788154568961283, + "tokens_seen": 737280000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006275626880641926, + "loss": 1.413, + "theoretical_loss": 3.788120951228614, + "tokens_seen": 737345536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006275466399197593, + "loss": 1.3029, + "theoretical_loss": 3.788087337320338, + "tokens_seen": 737411072 + }, + { + "epoch": 0.22, + "learning_rate": 0.000627530591775326, + "loss": 1.398, + "theoretical_loss": 3.7880537272356802, + "tokens_seen": 737476608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006275145436308927, + "loss": 1.3945, + "theoretical_loss": 3.788020120973867, + "tokens_seen": 737542144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274984954864594, + "loss": 1.3777, + "theoretical_loss": 3.7879865185341224, + "tokens_seen": 737607680 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274824473420261, + "loss": 1.5799, + "theoretical_loss": 3.787952919915673, + "tokens_seen": 737673216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274663991975928, + "loss": 1.3696, + "theoretical_loss": 3.7879193251177457, + "tokens_seen": 737738752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274503510531595, + "loss": 1.4853, + "theoretical_loss": 3.7878857341395653, + "tokens_seen": 737804288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274343029087262, + "loss": 1.3801, + "theoretical_loss": 3.7878521469803585, + "tokens_seen": 737869824 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274182547642929, + "loss": 1.5638, + "theoretical_loss": 3.7878185636393527, + "tokens_seen": 737935360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006274022066198596, + "loss": 1.4549, + "theoretical_loss": 3.787784984115774, + "tokens_seen": 738000896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006273861584754263, + "loss": 1.4346, + "theoretical_loss": 3.78775140840885, + "tokens_seen": 738066432 + }, + { + "epoch": 0.22, + "learning_rate": 0.000627370110330993, + "loss": 1.4445, + "theoretical_loss": 3.7877178365178086, + "tokens_seen": 738131968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006273540621865597, + "loss": 1.3383, + "theoretical_loss": 3.787684268441876, + "tokens_seen": 738197504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006273380140421264, + "loss": 1.503, + "theoretical_loss": 3.787650704180281, + "tokens_seen": 738263040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006273219658976931, + "loss": 1.3623, + "theoretical_loss": 3.787617143732251, + "tokens_seen": 738328576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006273059177532598, + "loss": 1.3803, + "theoretical_loss": 3.787583587097015, + "tokens_seen": 738394112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006272898696088265, + "loss": 1.5226, + "theoretical_loss": 3.7875500342738015, + "tokens_seen": 738459648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006272738214643932, + "loss": 1.4148, + "theoretical_loss": 3.787516485261839, + "tokens_seen": 738525184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006272577733199599, + "loss": 1.4439, + "theoretical_loss": 3.7874829400603556, + "tokens_seen": 738590720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006272417251755266, + "loss": 1.4708, + "theoretical_loss": 3.7874493986685813, + "tokens_seen": 738656256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006272256770310933, + "loss": 1.3589, + "theoretical_loss": 3.787415861085745, + "tokens_seen": 738721792 + }, + { + "epoch": 0.22, + "learning_rate": 0.00062720962888666, + "loss": 1.3324, + "theoretical_loss": 3.787382327311077, + "tokens_seen": 738787328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006271935807422267, + "loss": 1.4964, + "theoretical_loss": 3.7873487973438067, + "tokens_seen": 738852864 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006271775325977934, + "loss": 1.541, + "theoretical_loss": 3.7873152711831644, + "tokens_seen": 738918400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006271614844533601, + "loss": 1.3928, + "theoretical_loss": 3.7872817488283803, + "tokens_seen": 738983936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006271454363089268, + "loss": 1.435, + "theoretical_loss": 3.7872482302786845, + "tokens_seen": 739049472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006271293881644935, + "loss": 1.4267, + "theoretical_loss": 3.787214715533308, + "tokens_seen": 739115008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006271133400200602, + "loss": 1.457, + "theoretical_loss": 3.787181204591482, + "tokens_seen": 739180544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006270972918756269, + "loss": 1.4747, + "theoretical_loss": 3.787147697452437, + "tokens_seen": 739246080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006270812437311936, + "loss": 1.5003, + "theoretical_loss": 3.7871141941154054, + "tokens_seen": 739311616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006270651955867603, + "loss": 1.4814, + "theoretical_loss": 3.7870806945796174, + "tokens_seen": 739377152 + }, + { + "epoch": 0.22, + "learning_rate": 0.000627049147442327, + "loss": 1.3379, + "theoretical_loss": 3.7870471988443066, + "tokens_seen": 739442688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006270330992978937, + "loss": 1.5872, + "theoretical_loss": 3.7870137069087035, + "tokens_seen": 739508224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006270170511534605, + "loss": 1.4056, + "theoretical_loss": 3.786980218772041, + "tokens_seen": 739573760 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006270010030090271, + "loss": 1.5331, + "theoretical_loss": 3.7869467344335526, + "tokens_seen": 739639296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006269849548645939, + "loss": 1.4877, + "theoretical_loss": 3.7869132538924695, + "tokens_seen": 739704832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006269689067201605, + "loss": 1.4877, + "theoretical_loss": 3.786879777148025, + "tokens_seen": 739770368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006269528585757273, + "loss": 1.3691, + "theoretical_loss": 3.786846304199453, + "tokens_seen": 739835904 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006269368104312939, + "loss": 1.5295, + "theoretical_loss": 3.7868128350459864, + "tokens_seen": 739901440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006269207622868607, + "loss": 1.3843, + "theoretical_loss": 3.7867793696868586, + "tokens_seen": 739966976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006269047141424273, + "loss": 1.3247, + "theoretical_loss": 3.7867459081213037, + "tokens_seen": 740032512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006268886659979941, + "loss": 1.3068, + "theoretical_loss": 3.7867124503485563, + "tokens_seen": 740098048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006268726178535607, + "loss": 1.3989, + "theoretical_loss": 3.7866789963678498, + "tokens_seen": 740163584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006268565697091275, + "loss": 1.5187, + "theoretical_loss": 3.786645546178419, + "tokens_seen": 740229120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006268405215646941, + "loss": 1.4269, + "theoretical_loss": 3.7866120997794983, + "tokens_seen": 740294656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006268244734202609, + "loss": 1.4092, + "theoretical_loss": 3.786578657170324, + "tokens_seen": 740360192 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006268084252758275, + "loss": 1.4114, + "theoretical_loss": 3.7865452183501302, + "tokens_seen": 740425728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006267923771313943, + "loss": 1.5296, + "theoretical_loss": 3.7865117833181525, + "tokens_seen": 740491264 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 279627, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4995160102844238, + "objective/train/theoretical_loss": 3.786478352073626, + "objective/train/tokens_used": 761016800, + "theoretical_loss": 3.786478352073626, + "tokens_seen": 740556800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006267763289869609, + "loss": 1.4995, + "theoretical_loss": 3.786478352073626, + "tokens_seen": 740556800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006267602808425277, + "loss": 1.4852, + "theoretical_loss": 3.786444924615788, + "tokens_seen": 740622336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006267442326980943, + "loss": 1.3596, + "theoretical_loss": 3.7864115009438724, + "tokens_seen": 740687872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006267281845536611, + "loss": 1.49, + "theoretical_loss": 3.7863780810571175, + "tokens_seen": 740753408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006267121364092277, + "loss": 1.4216, + "theoretical_loss": 3.7863446649547594, + "tokens_seen": 740818944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006266960882647945, + "loss": 1.4069, + "theoretical_loss": 3.786311252636034, + "tokens_seen": 740884480 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006266800401203611, + "loss": 1.3675, + "theoretical_loss": 3.7862778441001783, + "tokens_seen": 740950016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006266639919759278, + "loss": 1.5097, + "theoretical_loss": 3.7862444393464307, + "tokens_seen": 741015552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006266479438314945, + "loss": 1.4985, + "theoretical_loss": 3.7862110383740273, + "tokens_seen": 741081088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006266318956870612, + "loss": 1.605, + "theoretical_loss": 3.7861776411822063, + "tokens_seen": 741146624 + }, + { + "epoch": 0.22, + "learning_rate": 0.000626615847542628, + "loss": 1.4839, + "theoretical_loss": 3.7861442477702063, + "tokens_seen": 741212160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006265997993981946, + "loss": 1.3988, + "theoretical_loss": 3.7861108581372633, + "tokens_seen": 741277696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006265837512537613, + "loss": 1.4261, + "theoretical_loss": 3.786077472282618, + "tokens_seen": 741343232 + }, + { + "epoch": 0.22, + "learning_rate": 0.000626567703109328, + "loss": 1.4712, + "theoretical_loss": 3.786044090205507, + "tokens_seen": 741408768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006265516549648948, + "loss": 1.5411, + "theoretical_loss": 3.7860107119051696, + "tokens_seen": 741474304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006265356068204614, + "loss": 1.44, + "theoretical_loss": 3.785977337380846, + "tokens_seen": 741539840 + }, + { + "epoch": 0.22, + "learning_rate": 0.000626519558676028, + "loss": 1.5256, + "theoretical_loss": 3.7859439666317733, + "tokens_seen": 741605376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006265035105315948, + "loss": 1.4502, + "theoretical_loss": 3.7859105996571922, + "tokens_seen": 741670912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006264874623871614, + "loss": 1.3718, + "theoretical_loss": 3.7858772364563418, + "tokens_seen": 741736448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006264714142427282, + "loss": 1.5422, + "theoretical_loss": 3.785843877028462, + "tokens_seen": 741801984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006264553660982948, + "loss": 1.4725, + "theoretical_loss": 3.7858105213727935, + "tokens_seen": 741867520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006264393179538616, + "loss": 1.4529, + "theoretical_loss": 3.7857771694885756, + "tokens_seen": 741933056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006264232698094282, + "loss": 1.3679, + "theoretical_loss": 3.785743821375049, + "tokens_seen": 741998592 + }, + { + "epoch": 0.22, + "learning_rate": 0.000626407221664995, + "loss": 1.5596, + "theoretical_loss": 3.785710477031455, + "tokens_seen": 742064128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006263911735205616, + "loss": 1.4528, + "theoretical_loss": 3.785677136457034, + "tokens_seen": 742129664 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006263751253761284, + "loss": 1.4111, + "theoretical_loss": 3.7856437996510275, + "tokens_seen": 742195200 + }, + { + "epoch": 0.22, + "learning_rate": 0.000626359077231695, + "loss": 1.5165, + "theoretical_loss": 3.7856104666126766, + "tokens_seen": 742260736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006263430290872618, + "loss": 1.6193, + "theoretical_loss": 3.785577137341223, + "tokens_seen": 742326272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006263269809428285, + "loss": 1.3567, + "theoretical_loss": 3.7855438118359084, + "tokens_seen": 742391808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0006263109327983952, + "loss": 1.4197, + "theoretical_loss": 3.785510490095975, + "tokens_seen": 742457344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006262948846539619, + "loss": 1.3827, + "theoretical_loss": 3.7854771721206646, + "tokens_seen": 742522880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006262788365095286, + "loss": 1.567, + "theoretical_loss": 3.78544385790922, + "tokens_seen": 742588416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006262627883650953, + "loss": 1.3674, + "theoretical_loss": 3.7854105474608843, + "tokens_seen": 742653952 + }, + { + "epoch": 0.23, + "learning_rate": 0.000626246740220662, + "loss": 1.6259, + "theoretical_loss": 3.785377240774899, + "tokens_seen": 742719488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006262306920762287, + "loss": 1.54, + "theoretical_loss": 3.7853439378505094, + "tokens_seen": 742785024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006262146439317954, + "loss": 1.4026, + "theoretical_loss": 3.7853106386869575, + "tokens_seen": 742850560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261985957873621, + "loss": 1.4987, + "theoretical_loss": 3.7852773432834868, + "tokens_seen": 742916096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261825476429288, + "loss": 1.4198, + "theoretical_loss": 3.7852440516393413, + "tokens_seen": 742981632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261664994984955, + "loss": 1.4083, + "theoretical_loss": 3.785210763753765, + "tokens_seen": 743047168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261504513540622, + "loss": 1.5544, + "theoretical_loss": 3.785177479626002, + "tokens_seen": 743112704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261344032096289, + "loss": 1.4902, + "theoretical_loss": 3.785144199255297, + "tokens_seen": 743178240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261183550651956, + "loss": 1.4138, + "theoretical_loss": 3.7851109226408943, + "tokens_seen": 743243776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006261023069207623, + "loss": 1.4719, + "theoretical_loss": 3.785077649782039, + "tokens_seen": 743309312 + }, + { + "epoch": 0.23, + "learning_rate": 0.000626086258776329, + "loss": 1.34, + "theoretical_loss": 3.7850443806779763, + "tokens_seen": 743374848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006260702106318957, + "loss": 1.3881, + "theoretical_loss": 3.7850111153279515, + "tokens_seen": 743440384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006260541624874624, + "loss": 1.5375, + "theoretical_loss": 3.78497785373121, + "tokens_seen": 743505920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006260381143430291, + "loss": 1.4658, + "theoretical_loss": 3.784944595886998, + "tokens_seen": 743571456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006260220661985958, + "loss": 1.4661, + "theoretical_loss": 3.7849113417945603, + "tokens_seen": 743636992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006260060180541625, + "loss": 1.3983, + "theoretical_loss": 3.7848780914531437, + "tokens_seen": 743702528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006259899699097292, + "loss": 1.4034, + "theoretical_loss": 3.784844844861995, + "tokens_seen": 743768064 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006259739217652959, + "loss": 1.4561, + "theoretical_loss": 3.784811602020361, + "tokens_seen": 743833600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006259578736208626, + "loss": 1.3627, + "theoretical_loss": 3.7847783629274874, + "tokens_seen": 743899136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006259418254764293, + "loss": 1.388, + "theoretical_loss": 3.7847451275826227, + "tokens_seen": 743964672 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625925777331996, + "loss": 1.3532, + "theoretical_loss": 3.784711895985013, + "tokens_seen": 744030208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006259097291875627, + "loss": 1.3611, + "theoretical_loss": 3.784678668133906, + "tokens_seen": 744095744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006258936810431294, + "loss": 1.5999, + "theoretical_loss": 3.7846454440285493, + "tokens_seen": 744161280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006258776328986961, + "loss": 1.3012, + "theoretical_loss": 3.7846122236681916, + "tokens_seen": 744226816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006258615847542628, + "loss": 1.4731, + "theoretical_loss": 3.7845790070520806, + "tokens_seen": 744292352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006258455366098295, + "loss": 1.521, + "theoretical_loss": 3.784545794179464, + "tokens_seen": 744357888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006258294884653962, + "loss": 1.4213, + "theoretical_loss": 3.7845125850495918, + "tokens_seen": 744423424 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625813440320963, + "loss": 1.3919, + "theoretical_loss": 3.7844793796617116, + "tokens_seen": 744488960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006257973921765296, + "loss": 1.3685, + "theoretical_loss": 3.784446178015073, + "tokens_seen": 744554496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006257813440320964, + "loss": 1.5204, + "theoretical_loss": 3.7844129801089244, + "tokens_seen": 744620032 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625765295887663, + "loss": 1.4252, + "theoretical_loss": 3.784379785942517, + "tokens_seen": 744685568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006257492477432298, + "loss": 1.4184, + "theoretical_loss": 3.784346595515098, + "tokens_seen": 744751104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006257331995987964, + "loss": 1.4169, + "theoretical_loss": 3.784313408825919, + "tokens_seen": 744816640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006257171514543632, + "loss": 1.4739, + "theoretical_loss": 3.78428022587423, + "tokens_seen": 744882176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006257011033099298, + "loss": 1.6099, + "theoretical_loss": 3.7842470466592806, + "tokens_seen": 744947712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006256850551654966, + "loss": 1.4298, + "theoretical_loss": 3.784213871180322, + "tokens_seen": 745013248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006256690070210632, + "loss": 1.5705, + "theoretical_loss": 3.7841806994366047, + "tokens_seen": 745078784 + }, + { + "epoch": 0.23, + "learning_rate": 0.00062565295887663, + "loss": 1.5225, + "theoretical_loss": 3.7841475314273794, + "tokens_seen": 745144320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006256369107321966, + "loss": 1.4157, + "theoretical_loss": 3.7841143671518975, + "tokens_seen": 745209856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006256208625877634, + "loss": 1.5072, + "theoretical_loss": 3.7840812066094105, + "tokens_seen": 745275392 + }, + { + "epoch": 0.23, + "learning_rate": 0.00062560481444333, + "loss": 1.4253, + "theoretical_loss": 3.78404804979917, + "tokens_seen": 745340928 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006255887662988968, + "loss": 1.4656, + "theoretical_loss": 3.7840148967204277, + "tokens_seen": 745406464 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006255727181544634, + "loss": 1.4842, + "theoretical_loss": 3.7839817473724353, + "tokens_seen": 745472000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006255566700100302, + "loss": 1.44, + "theoretical_loss": 3.783948601754445, + "tokens_seen": 745537536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006255406218655968, + "loss": 1.4869, + "theoretical_loss": 3.7839154598657103, + "tokens_seen": 745603072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006255245737211636, + "loss": 1.4838, + "theoretical_loss": 3.7838823217054833, + "tokens_seen": 745668608 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006255085255767302, + "loss": 1.4181, + "theoretical_loss": 3.7838491872730167, + "tokens_seen": 745734144 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625492477432297, + "loss": 1.4877, + "theoretical_loss": 3.783816056567564, + "tokens_seen": 745799680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006254764292878636, + "loss": 1.4319, + "theoretical_loss": 3.783782929588378, + "tokens_seen": 745865216 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006254603811434304, + "loss": 1.3929, + "theoretical_loss": 3.783749806334712, + "tokens_seen": 745930752 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625444332998997, + "loss": 1.3521, + "theoretical_loss": 3.783716686805821, + "tokens_seen": 745996288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006254282848545638, + "loss": 1.4806, + "theoretical_loss": 3.7836835710009584, + "tokens_seen": 746061824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006254122367101304, + "loss": 1.5587, + "theoretical_loss": 3.783650458919378, + "tokens_seen": 746127360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006253961885656972, + "loss": 1.414, + "theoretical_loss": 3.783617350560334, + "tokens_seen": 746192896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006253801404212638, + "loss": 1.5039, + "theoretical_loss": 3.7835842459230817, + "tokens_seen": 746258432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006253640922768306, + "loss": 1.3992, + "theoretical_loss": 3.783551145006876, + "tokens_seen": 746323968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006253480441323972, + "loss": 1.3266, + "theoretical_loss": 3.783518047810971, + "tokens_seen": 746389504 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625331995987964, + "loss": 1.4604, + "theoretical_loss": 3.7834849543346234, + "tokens_seen": 746455040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006253159478435306, + "loss": 1.4406, + "theoretical_loss": 3.7834518645770876, + "tokens_seen": 746520576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006252998996990974, + "loss": 1.3444, + "theoretical_loss": 3.7834187785376194, + "tokens_seen": 746586112 + }, + { + "epoch": 0.23, + "learning_rate": 0.000625283851554664, + "loss": 1.54, + "theoretical_loss": 3.7833856962154755, + "tokens_seen": 746651648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006252678034102307, + "loss": 1.4447, + "theoretical_loss": 3.7833526176099106, + "tokens_seen": 746717184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006252517552657974, + "loss": 1.5642, + "theoretical_loss": 3.7833195427201822, + "tokens_seen": 746782720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006252357071213641, + "loss": 1.4665, + "theoretical_loss": 3.783286471545547, + "tokens_seen": 746848256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006252196589769308, + "loss": 1.372, + "theoretical_loss": 3.7832534040852606, + "tokens_seen": 746913792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006252036108324975, + "loss": 1.5047, + "theoretical_loss": 3.7832203403385805, + "tokens_seen": 746979328 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006251875626880643, + "loss": 1.4664, + "theoretical_loss": 3.783187280304764, + "tokens_seen": 747044864 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 281950, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4087506532669067, + "objective/train/theoretical_loss": 3.783154223983069, + "objective/train/tokens_used": 767570400, + "theoretical_loss": 3.783154223983069, + "tokens_seen": 747110400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006251715145436309, + "loss": 1.4088, + "theoretical_loss": 3.783154223983069, + "tokens_seen": 747110400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006251554663991977, + "loss": 1.505, + "theoretical_loss": 3.783121171372753, + "tokens_seen": 747175936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006251394182547643, + "loss": 1.5048, + "theoretical_loss": 3.783088122473073, + "tokens_seen": 747241472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006251233701103309, + "loss": 1.4213, + "theoretical_loss": 3.7830550772832874, + "tokens_seen": 747307008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006251073219658977, + "loss": 1.4417, + "theoretical_loss": 3.7830220358026545, + "tokens_seen": 747372544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006250912738214643, + "loss": 1.4589, + "theoretical_loss": 3.782988998030433, + "tokens_seen": 747438080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006250752256770311, + "loss": 1.4837, + "theoretical_loss": 3.7829559639658816, + "tokens_seen": 747503616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006250591775325977, + "loss": 1.4949, + "theoretical_loss": 3.7829229336082584, + "tokens_seen": 747569152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006250431293881645, + "loss": 1.3581, + "theoretical_loss": 3.7828899069568243, + "tokens_seen": 747634688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006250270812437311, + "loss": 1.4885, + "theoretical_loss": 3.7828568840108363, + "tokens_seen": 747700224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006250110330992979, + "loss": 1.3968, + "theoretical_loss": 3.782823864769556, + "tokens_seen": 747765760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006249949849548645, + "loss": 1.4875, + "theoretical_loss": 3.7827908492322413, + "tokens_seen": 747831296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006249789368104313, + "loss": 1.3599, + "theoretical_loss": 3.7827578373981536, + "tokens_seen": 747896832 + }, + { + "epoch": 0.23, + "learning_rate": 0.000624962888665998, + "loss": 1.4671, + "theoretical_loss": 3.782724829266553, + "tokens_seen": 747962368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006249468405215647, + "loss": 1.5363, + "theoretical_loss": 3.782691824836699, + "tokens_seen": 748027904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006249307923771314, + "loss": 1.4664, + "theoretical_loss": 3.7826588241078527, + "tokens_seen": 748093440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006249147442326981, + "loss": 1.4129, + "theoretical_loss": 3.7826258270792747, + "tokens_seen": 748158976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006248986960882648, + "loss": 1.5375, + "theoretical_loss": 3.7825928337502264, + "tokens_seen": 748224512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006248826479438315, + "loss": 1.4557, + "theoretical_loss": 3.7825598441199695, + "tokens_seen": 748290048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006248665997993982, + "loss": 1.4072, + "theoretical_loss": 3.7825268581877642, + "tokens_seen": 748355584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006248505516549649, + "loss": 1.384, + "theoretical_loss": 3.7824938759528726, + "tokens_seen": 748421120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006248345035105316, + "loss": 1.6861, + "theoretical_loss": 3.7824608974145573, + "tokens_seen": 748486656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006248184553660983, + "loss": 1.4946, + "theoretical_loss": 3.782427922572079, + "tokens_seen": 748552192 + }, + { + "epoch": 0.23, + "learning_rate": 0.000624802407221665, + "loss": 1.4708, + "theoretical_loss": 3.7823949514247017, + "tokens_seen": 748617728 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006247863590772317, + "loss": 1.5068, + "theoretical_loss": 3.782361983971686, + "tokens_seen": 748683264 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006247703109327984, + "loss": 1.5854, + "theoretical_loss": 3.7823290202122966, + "tokens_seen": 748748800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006247542627883651, + "loss": 1.574, + "theoretical_loss": 3.7822960601457947, + "tokens_seen": 748814336 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006247382146439318, + "loss": 1.5317, + "theoretical_loss": 3.782263103771445, + "tokens_seen": 748879872 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006247221664994985, + "loss": 1.4062, + "theoretical_loss": 3.78223015108851, + "tokens_seen": 748945408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006247061183550652, + "loss": 1.4724, + "theoretical_loss": 3.7821972020962527, + "tokens_seen": 749010944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006246900702106319, + "loss": 1.4458, + "theoretical_loss": 3.782164256793938, + "tokens_seen": 749076480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006246740220661986, + "loss": 1.4279, + "theoretical_loss": 3.782131315180829, + "tokens_seen": 749142016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006246579739217653, + "loss": 1.4585, + "theoretical_loss": 3.782098377256191, + "tokens_seen": 749207552 + }, + { + "epoch": 0.23, + "learning_rate": 0.000624641925777332, + "loss": 1.4721, + "theoretical_loss": 3.7820654430192873, + "tokens_seen": 749273088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006246258776328987, + "loss": 1.4831, + "theoretical_loss": 3.782032512469383, + "tokens_seen": 749338624 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006246098294884655, + "loss": 1.4595, + "theoretical_loss": 3.781999585605743, + "tokens_seen": 749404160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006245937813440321, + "loss": 1.5204, + "theoretical_loss": 3.781966662427632, + "tokens_seen": 749469696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006245777331995989, + "loss": 1.445, + "theoretical_loss": 3.7819337429343154, + "tokens_seen": 749535232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006245616850551655, + "loss": 1.5239, + "theoretical_loss": 3.7819008271250594, + "tokens_seen": 749600768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006245456369107323, + "loss": 1.6305, + "theoretical_loss": 3.7818679149991286, + "tokens_seen": 749666304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006245295887662989, + "loss": 1.5468, + "theoretical_loss": 3.7818350065557893, + "tokens_seen": 749731840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006245135406218657, + "loss": 1.3974, + "theoretical_loss": 3.7818021017943075, + "tokens_seen": 749797376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244974924774323, + "loss": 1.6412, + "theoretical_loss": 3.78176920071395, + "tokens_seen": 749862912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244814443329991, + "loss": 1.3505, + "theoretical_loss": 3.7817363033139824, + "tokens_seen": 749928448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244653961885657, + "loss": 1.4821, + "theoretical_loss": 3.781703409593672, + "tokens_seen": 749993984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244493480441325, + "loss": 1.4513, + "theoretical_loss": 3.7816705195522866, + "tokens_seen": 750059520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244332998996991, + "loss": 1.4285, + "theoretical_loss": 3.7816376331890913, + "tokens_seen": 750125056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244172517552659, + "loss": 1.4804, + "theoretical_loss": 3.7816047505033556, + "tokens_seen": 750190592 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006244012036108325, + "loss": 1.4449, + "theoretical_loss": 3.7815718714943456, + "tokens_seen": 750256128 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006243851554663993, + "loss": 1.464, + "theoretical_loss": 3.7815389961613297, + "tokens_seen": 750321664 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006243691073219659, + "loss": 1.4282, + "theoretical_loss": 3.7815061245035757, + "tokens_seen": 750387200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006243530591775327, + "loss": 1.4544, + "theoretical_loss": 3.7814732565203517, + "tokens_seen": 750452736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006243370110330993, + "loss": 1.482, + "theoretical_loss": 3.7814403922109263, + "tokens_seen": 750518272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006243209628886661, + "loss": 1.4327, + "theoretical_loss": 3.781407531574568, + "tokens_seen": 750583808 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006243049147442327, + "loss": 1.451, + "theoretical_loss": 3.7813746746105465, + "tokens_seen": 750649344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006242888665997995, + "loss": 1.5022, + "theoretical_loss": 3.7813418213181293, + "tokens_seen": 750714880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006242728184553661, + "loss": 1.4825, + "theoretical_loss": 3.7813089716965864, + "tokens_seen": 750780416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006242567703109329, + "loss": 1.6059, + "theoretical_loss": 3.7812761257451872, + "tokens_seen": 750845952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006242407221664995, + "loss": 1.4072, + "theoretical_loss": 3.7812432834632017, + "tokens_seen": 750911488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006242246740220663, + "loss": 1.4685, + "theoretical_loss": 3.7812104448498998, + "tokens_seen": 750977024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006242086258776329, + "loss": 1.357, + "theoretical_loss": 3.7811776099045504, + "tokens_seen": 751042560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006241925777331997, + "loss": 1.6648, + "theoretical_loss": 3.781144778626425, + "tokens_seen": 751108096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006241765295887663, + "loss": 1.5134, + "theoretical_loss": 3.781111951014794, + "tokens_seen": 751173632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006241604814443331, + "loss": 1.6305, + "theoretical_loss": 3.781079127068928, + "tokens_seen": 751239168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006241444332998997, + "loss": 1.4094, + "theoretical_loss": 3.781046306788097, + "tokens_seen": 751304704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006241283851554665, + "loss": 1.4801, + "theoretical_loss": 3.7810134901715733, + "tokens_seen": 751370240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006241123370110331, + "loss": 1.4796, + "theoretical_loss": 3.7809806772186283, + "tokens_seen": 751435776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240962888665999, + "loss": 1.3226, + "theoretical_loss": 3.780947867928532, + "tokens_seen": 751501312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240802407221665, + "loss": 1.387, + "theoretical_loss": 3.7809150623005583, + "tokens_seen": 751566848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240641925777333, + "loss": 1.4166, + "theoretical_loss": 3.7808822603339776, + "tokens_seen": 751632384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240481444332999, + "loss": 1.4739, + "theoretical_loss": 3.780849462028063, + "tokens_seen": 751697920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240320962888667, + "loss": 1.3834, + "theoretical_loss": 3.780816667382086, + "tokens_seen": 751763456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240160481444333, + "loss": 1.3816, + "theoretical_loss": 3.7807838763953194, + "tokens_seen": 751828992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006240000000000001, + "loss": 1.4946, + "theoretical_loss": 3.780751089067037, + "tokens_seen": 751894528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006239839518555667, + "loss": 1.4588, + "theoretical_loss": 3.78071830539651, + "tokens_seen": 751960064 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006239679037111335, + "loss": 1.4928, + "theoretical_loss": 3.7806855253830136, + "tokens_seen": 752025600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006239518555667001, + "loss": 1.531, + "theoretical_loss": 3.78065274902582, + "tokens_seen": 752091136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006239358074222669, + "loss": 1.4817, + "theoretical_loss": 3.7806199763242025, + "tokens_seen": 752156672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006239197592778335, + "loss": 1.4757, + "theoretical_loss": 3.780587207277436, + "tokens_seen": 752222208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006239037111334003, + "loss": 1.4319, + "theoretical_loss": 3.7805544418847936, + "tokens_seen": 752287744 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623887662988967, + "loss": 1.4132, + "theoretical_loss": 3.7805216801455503, + "tokens_seen": 752353280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006238716148445336, + "loss": 1.4962, + "theoretical_loss": 3.7804889220589804, + "tokens_seen": 752418816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006238555667001003, + "loss": 1.4498, + "theoretical_loss": 3.780456167624358, + "tokens_seen": 752484352 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623839518555667, + "loss": 1.5219, + "theoretical_loss": 3.780423416840958, + "tokens_seen": 752549888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006238234704112337, + "loss": 1.4605, + "theoretical_loss": 3.7803906697080567, + "tokens_seen": 752615424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006238074222668004, + "loss": 1.4863, + "theoretical_loss": 3.780357926224928, + "tokens_seen": 752680960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006237913741223672, + "loss": 1.3956, + "theoretical_loss": 3.780325186390848, + "tokens_seen": 752746496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006237753259779338, + "loss": 1.4156, + "theoretical_loss": 3.780292450205092, + "tokens_seen": 752812032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006237592778335006, + "loss": 1.4429, + "theoretical_loss": 3.7802597176669366, + "tokens_seen": 752877568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006237432296890672, + "loss": 1.4436, + "theoretical_loss": 3.7802269887756577, + "tokens_seen": 752943104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006237271815446338, + "loss": 1.3938, + "theoretical_loss": 3.7801942635305306, + "tokens_seen": 753008640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006237111334002006, + "loss": 1.4437, + "theoretical_loss": 3.780161541930833, + "tokens_seen": 753074176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006236950852557672, + "loss": 1.6316, + "theoretical_loss": 3.7801288239758417, + "tokens_seen": 753139712 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623679037111334, + "loss": 1.4556, + "theoretical_loss": 3.7800961096648327, + "tokens_seen": 753205248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006236629889669006, + "loss": 1.3974, + "theoretical_loss": 3.7800633989970835, + "tokens_seen": 753270784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006236469408224674, + "loss": 1.5001, + "theoretical_loss": 3.780030691971872, + "tokens_seen": 753336320 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623630892678034, + "loss": 1.3932, + "theoretical_loss": 3.7799979885884754, + "tokens_seen": 753401856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006236148445336008, + "loss": 1.4453, + "theoretical_loss": 3.7799652888461712, + "tokens_seen": 753467392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006235987963891675, + "loss": 1.4934, + "theoretical_loss": 3.7799325927442373, + "tokens_seen": 753532928 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006235827482447342, + "loss": 1.4449, + "theoretical_loss": 3.7798999002819524, + "tokens_seen": 753598464 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 284355, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4268677234649658, + "objective/train/theoretical_loss": 3.779867211458595, + "objective/train/tokens_used": 774124000, + "theoretical_loss": 3.779867211458595, + "tokens_seen": 753664000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006235667001003009, + "loss": 1.4269, + "theoretical_loss": 3.779867211458595, + "tokens_seen": 753664000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006235506519558676, + "loss": 1.3835, + "theoretical_loss": 3.7798345262734427, + "tokens_seen": 753729536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006235346038114343, + "loss": 1.4834, + "theoretical_loss": 3.779801844725775, + "tokens_seen": 753795072 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623518555667001, + "loss": 1.4402, + "theoretical_loss": 3.779769166814871, + "tokens_seen": 753860608 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006235025075225677, + "loss": 1.3216, + "theoretical_loss": 3.7797364925400094, + "tokens_seen": 753926144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006234864593781344, + "loss": 1.3217, + "theoretical_loss": 3.77970382190047, + "tokens_seen": 753991680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006234704112337011, + "loss": 1.4561, + "theoretical_loss": 3.7796711548955324, + "tokens_seen": 754057216 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006234543630892678, + "loss": 1.3749, + "theoretical_loss": 3.7796384915244765, + "tokens_seen": 754122752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006234383149448345, + "loss": 1.4978, + "theoretical_loss": 3.7796058317865815, + "tokens_seen": 754188288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006234222668004012, + "loss": 1.511, + "theoretical_loss": 3.7795731756811284, + "tokens_seen": 754253824 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623406218655968, + "loss": 1.3987, + "theoretical_loss": 3.779540523207398, + "tokens_seen": 754319360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006233901705115346, + "loss": 1.4646, + "theoretical_loss": 3.77950787436467, + "tokens_seen": 754384896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006233741223671014, + "loss": 1.4212, + "theoretical_loss": 3.779475229152226, + "tokens_seen": 754450432 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623358074222668, + "loss": 1.461, + "theoretical_loss": 3.7794425875693465, + "tokens_seen": 754515968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006233420260782348, + "loss": 1.5224, + "theoretical_loss": 3.779409949615313, + "tokens_seen": 754581504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006233259779338014, + "loss": 1.5034, + "theoretical_loss": 3.7793773152894072, + "tokens_seen": 754647040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006233099297893682, + "loss": 1.5091, + "theoretical_loss": 3.77934468459091, + "tokens_seen": 754712576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006232938816449348, + "loss": 1.4613, + "theoretical_loss": 3.779312057519104, + "tokens_seen": 754778112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006232778335005016, + "loss": 1.4566, + "theoretical_loss": 3.7792794340732714, + "tokens_seen": 754843648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006232617853560682, + "loss": 1.3632, + "theoretical_loss": 3.7792468142526934, + "tokens_seen": 754909184 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623245737211635, + "loss": 1.5399, + "theoretical_loss": 3.7792141980566534, + "tokens_seen": 754974720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006232296890672016, + "loss": 1.4165, + "theoretical_loss": 3.7791815854844337, + "tokens_seen": 755040256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006232136409227684, + "loss": 1.5175, + "theoretical_loss": 3.7791489765353177, + "tokens_seen": 755105792 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623197592778335, + "loss": 1.4577, + "theoretical_loss": 3.7791163712085885, + "tokens_seen": 755171328 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006231815446339018, + "loss": 1.4501, + "theoretical_loss": 3.7790837695035284, + "tokens_seen": 755236864 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006231654964894684, + "loss": 1.5075, + "theoretical_loss": 3.7790511714194217, + "tokens_seen": 755302400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006231494483450352, + "loss": 1.4214, + "theoretical_loss": 3.779018576955552, + "tokens_seen": 755367936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006231334002006018, + "loss": 1.6183, + "theoretical_loss": 3.778985986111203, + "tokens_seen": 755433472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006231173520561686, + "loss": 1.3977, + "theoretical_loss": 3.778953398885659, + "tokens_seen": 755499008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006231013039117352, + "loss": 1.4507, + "theoretical_loss": 3.7789208152782043, + "tokens_seen": 755564544 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623085255767302, + "loss": 1.4285, + "theoretical_loss": 3.7788882352881235, + "tokens_seen": 755630080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006230692076228686, + "loss": 1.6022, + "theoretical_loss": 3.7788556589147015, + "tokens_seen": 755695616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006230531594784354, + "loss": 1.3981, + "theoretical_loss": 3.7788230861572227, + "tokens_seen": 755761152 + }, + { + "epoch": 0.23, + "learning_rate": 0.000623037111334002, + "loss": 1.4943, + "theoretical_loss": 3.778790517014972, + "tokens_seen": 755826688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006230210631895688, + "loss": 1.4436, + "theoretical_loss": 3.7787579514872354, + "tokens_seen": 755892224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006230050150451354, + "loss": 1.3937, + "theoretical_loss": 3.7787253895732986, + "tokens_seen": 755957760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006229889669007022, + "loss": 1.482, + "theoretical_loss": 3.778692831272447, + "tokens_seen": 756023296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006229729187562688, + "loss": 1.3875, + "theoretical_loss": 3.778660276583966, + "tokens_seen": 756088832 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006229568706118356, + "loss": 1.4086, + "theoretical_loss": 3.7786277255071425, + "tokens_seen": 756154368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006229408224674022, + "loss": 1.4017, + "theoretical_loss": 3.7785951780412628, + "tokens_seen": 756219904 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622924774322969, + "loss": 1.4859, + "theoretical_loss": 3.7785626341856133, + "tokens_seen": 756285440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006229087261785356, + "loss": 1.4639, + "theoretical_loss": 3.77853009393948, + "tokens_seen": 756350976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006228926780341024, + "loss": 1.6196, + "theoretical_loss": 3.7784975573021518, + "tokens_seen": 756416512 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622876629889669, + "loss": 1.4094, + "theoretical_loss": 3.7784650242729136, + "tokens_seen": 756482048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006228605817452358, + "loss": 1.4244, + "theoretical_loss": 3.7784324948510544, + "tokens_seen": 756547584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006228445336008024, + "loss": 1.3992, + "theoretical_loss": 3.7783999690358607, + "tokens_seen": 756613120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006228284854563692, + "loss": 1.5185, + "theoretical_loss": 3.778367446826621, + "tokens_seen": 756678656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006228124373119358, + "loss": 1.3727, + "theoretical_loss": 3.778334928222623, + "tokens_seen": 756744192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006227963891675026, + "loss": 1.4286, + "theoretical_loss": 3.7783024132231544, + "tokens_seen": 756809728 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006227803410230692, + "loss": 1.5158, + "theoretical_loss": 3.7782699018275046, + "tokens_seen": 756875264 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622764292878636, + "loss": 1.4931, + "theoretical_loss": 3.778237394034961, + "tokens_seen": 756940800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006227482447342026, + "loss": 1.4743, + "theoretical_loss": 3.7782048898448135, + "tokens_seen": 757006336 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006227321965897694, + "loss": 1.3627, + "theoretical_loss": 3.7781723892563503, + "tokens_seen": 757071872 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622716148445336, + "loss": 1.3718, + "theoretical_loss": 3.778139892268861, + "tokens_seen": 757137408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006227001003009028, + "loss": 1.5357, + "theoretical_loss": 3.7781073988816347, + "tokens_seen": 757202944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006226840521564694, + "loss": 1.4439, + "theoretical_loss": 3.7780749090939616, + "tokens_seen": 757268480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006226680040120362, + "loss": 1.5659, + "theoretical_loss": 3.7780424229051306, + "tokens_seen": 757334016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006226519558676028, + "loss": 1.3886, + "theoretical_loss": 3.778009940314432, + "tokens_seen": 757399552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006226359077231696, + "loss": 1.5053, + "theoretical_loss": 3.7779774613211563, + "tokens_seen": 757465088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006226198595787362, + "loss": 1.4244, + "theoretical_loss": 3.7779449859245933, + "tokens_seen": 757530624 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622603811434303, + "loss": 1.5621, + "theoretical_loss": 3.777912514124034, + "tokens_seen": 757596160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006225877632898696, + "loss": 1.3692, + "theoretical_loss": 3.7778800459187694, + "tokens_seen": 757661696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006225717151454364, + "loss": 1.4504, + "theoretical_loss": 3.77784758130809, + "tokens_seen": 757727232 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622555667001003, + "loss": 1.5196, + "theoretical_loss": 3.7778151202912875, + "tokens_seen": 757792768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006225396188565698, + "loss": 1.3794, + "theoretical_loss": 3.777782662867653, + "tokens_seen": 757858304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006225235707121364, + "loss": 1.4037, + "theoretical_loss": 3.777750209036478, + "tokens_seen": 757923840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006225075225677032, + "loss": 1.4543, + "theoretical_loss": 3.7777177587970545, + "tokens_seen": 757989376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006224914744232698, + "loss": 1.5288, + "theoretical_loss": 3.777685312148674, + "tokens_seen": 758054912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006224754262788365, + "loss": 1.596, + "theoretical_loss": 3.77765286909063, + "tokens_seen": 758120448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006224593781344032, + "loss": 1.4775, + "theoretical_loss": 3.777620429622213, + "tokens_seen": 758185984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006224433299899699, + "loss": 1.45, + "theoretical_loss": 3.7775879937427175, + "tokens_seen": 758251520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006224272818455367, + "loss": 1.5273, + "theoretical_loss": 3.777555561451435, + "tokens_seen": 758317056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006224112337011033, + "loss": 1.4597, + "theoretical_loss": 3.777523132747659, + "tokens_seen": 758382592 + }, + { + "epoch": 0.23, + "learning_rate": 0.00062239518555667, + "loss": 1.4332, + "theoretical_loss": 3.7774907076306823, + "tokens_seen": 758448128 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006223791374122367, + "loss": 1.4019, + "theoretical_loss": 3.777458286099799, + "tokens_seen": 758513664 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006223630892678035, + "loss": 1.5919, + "theoretical_loss": 3.7774258681543027, + "tokens_seen": 758579200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006223470411233701, + "loss": 1.4866, + "theoretical_loss": 3.7773934537934863, + "tokens_seen": 758644736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006223309929789367, + "loss": 1.4413, + "theoretical_loss": 3.7773610430166444, + "tokens_seen": 758710272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006223149448345035, + "loss": 1.4634, + "theoretical_loss": 3.7773286358230713, + "tokens_seen": 758775808 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006222988966900701, + "loss": 1.4287, + "theoretical_loss": 3.777296232212061, + "tokens_seen": 758841344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006222828485456369, + "loss": 1.5258, + "theoretical_loss": 3.777263832182909, + "tokens_seen": 758906880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006222668004012035, + "loss": 1.4601, + "theoretical_loss": 3.7772314357349086, + "tokens_seen": 758972416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006222507522567703, + "loss": 1.4812, + "theoretical_loss": 3.7771990428673563, + "tokens_seen": 759037952 + }, + { + "epoch": 0.23, + "learning_rate": 0.000622234704112337, + "loss": 1.5509, + "theoretical_loss": 3.7771666535795463, + "tokens_seen": 759103488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006222186559679037, + "loss": 1.4949, + "theoretical_loss": 3.777134267870774, + "tokens_seen": 759169024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006222026078234705, + "loss": 1.5512, + "theoretical_loss": 3.7771018857403362, + "tokens_seen": 759234560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006221865596790371, + "loss": 1.4195, + "theoretical_loss": 3.7770695071875275, + "tokens_seen": 759300096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006221705115346039, + "loss": 1.4715, + "theoretical_loss": 3.7770371322116443, + "tokens_seen": 759365632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006221544633901705, + "loss": 1.4277, + "theoretical_loss": 3.7770047608119826, + "tokens_seen": 759431168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006221384152457373, + "loss": 1.406, + "theoretical_loss": 3.7769723929878394, + "tokens_seen": 759496704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006221223671013039, + "loss": 1.3999, + "theoretical_loss": 3.77694002873851, + "tokens_seen": 759562240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006221063189568707, + "loss": 1.4038, + "theoretical_loss": 3.7769076680632927, + "tokens_seen": 759627776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006220902708124373, + "loss": 1.4118, + "theoretical_loss": 3.7768753109614837, + "tokens_seen": 759693312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006220742226680041, + "loss": 1.3896, + "theoretical_loss": 3.7768429574323803, + "tokens_seen": 759758848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006220581745235707, + "loss": 1.3394, + "theoretical_loss": 3.77681060747528, + "tokens_seen": 759824384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006220421263791375, + "loss": 1.4813, + "theoretical_loss": 3.7767782610894804, + "tokens_seen": 759889920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006220260782347041, + "loss": 1.4247, + "theoretical_loss": 3.776745918274279, + "tokens_seen": 759955456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006220100300902709, + "loss": 1.4092, + "theoretical_loss": 3.776713579028974, + "tokens_seen": 760020992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006219939819458375, + "loss": 1.3455, + "theoretical_loss": 3.776681243352864, + "tokens_seen": 760086528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006219779338014043, + "loss": 1.3928, + "theoretical_loss": 3.7766489112452466, + "tokens_seen": 760152064 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 286634, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2943229675292969, + "objective/train/theoretical_loss": 3.776616582705421, + "objective/train/tokens_used": 780677600, + "theoretical_loss": 3.776616582705421, + "tokens_seen": 760217600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006219618856569709, + "loss": 1.2943, + "theoretical_loss": 3.776616582705421, + "tokens_seen": 760217600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006219458375125377, + "loss": 1.3217, + "theoretical_loss": 3.7765842577326856, + "tokens_seen": 760283136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006219297893681043, + "loss": 1.4656, + "theoretical_loss": 3.7765519363263396, + "tokens_seen": 760348672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006219137412236711, + "loss": 1.458, + "theoretical_loss": 3.7765196184856826, + "tokens_seen": 760414208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218976930792377, + "loss": 1.4789, + "theoretical_loss": 3.776487304210012, + "tokens_seen": 760479744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218816449348045, + "loss": 1.4485, + "theoretical_loss": 3.7764549934986302, + "tokens_seen": 760545280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218655967903711, + "loss": 1.4512, + "theoretical_loss": 3.776422686350836, + "tokens_seen": 760610816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218495486459379, + "loss": 1.4212, + "theoretical_loss": 3.7763903827659275, + "tokens_seen": 760676352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218335005015045, + "loss": 1.3433, + "theoretical_loss": 3.776358082743208, + "tokens_seen": 760741888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218174523570713, + "loss": 1.4251, + "theoretical_loss": 3.776325786281975, + "tokens_seen": 760807424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006218014042126379, + "loss": 1.5181, + "theoretical_loss": 3.776293493381531, + "tokens_seen": 760872960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006217853560682047, + "loss": 1.4257, + "theoretical_loss": 3.776261204041176, + "tokens_seen": 760938496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006217693079237713, + "loss": 1.4139, + "theoretical_loss": 3.7762289182602107, + "tokens_seen": 761004032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006217532597793381, + "loss": 1.334, + "theoretical_loss": 3.7761966360379366, + "tokens_seen": 761069568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006217372116349047, + "loss": 1.5288, + "theoretical_loss": 3.776164357373655, + "tokens_seen": 761135104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006217211634904715, + "loss": 1.4715, + "theoretical_loss": 3.7761320822666677, + "tokens_seen": 761200640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006217051153460381, + "loss": 1.474, + "theoretical_loss": 3.7760998107162767, + "tokens_seen": 761266176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006216890672016049, + "loss": 1.3549, + "theoretical_loss": 3.7760675427217825, + "tokens_seen": 761331712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006216730190571715, + "loss": 1.4293, + "theoretical_loss": 3.776035278282489, + "tokens_seen": 761397248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006216569709127383, + "loss": 1.4763, + "theoretical_loss": 3.7760030173976977, + "tokens_seen": 761462784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006216409227683049, + "loss": 1.4042, + "theoretical_loss": 3.775970760066711, + "tokens_seen": 761528320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006216248746238717, + "loss": 1.4446, + "theoretical_loss": 3.7759385062888318, + "tokens_seen": 761593856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006216088264794383, + "loss": 1.4656, + "theoretical_loss": 3.7759062560633634, + "tokens_seen": 761659392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006215927783350051, + "loss": 1.4173, + "theoretical_loss": 3.775874009389608, + "tokens_seen": 761724928 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006215767301905717, + "loss": 1.5205, + "theoretical_loss": 3.77584176626687, + "tokens_seen": 761790464 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006215606820461385, + "loss": 1.4561, + "theoretical_loss": 3.7758095266944522, + "tokens_seen": 761856000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006215446339017051, + "loss": 1.5295, + "theoretical_loss": 3.775777290671659, + "tokens_seen": 761921536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006215285857572719, + "loss": 1.4454, + "theoretical_loss": 3.775745058197794, + "tokens_seen": 761987072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006215125376128385, + "loss": 1.4155, + "theoretical_loss": 3.7757128292721607, + "tokens_seen": 762052608 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214964894684053, + "loss": 1.5223, + "theoretical_loss": 3.775680603894064, + "tokens_seen": 762118144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214804413239719, + "loss": 1.488, + "theoretical_loss": 3.7756483820628084, + "tokens_seen": 762183680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214643931795387, + "loss": 1.4707, + "theoretical_loss": 3.7756161637776984, + "tokens_seen": 762249216 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214483450351053, + "loss": 1.483, + "theoretical_loss": 3.775583949038039, + "tokens_seen": 762314752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214322968906721, + "loss": 1.4926, + "theoretical_loss": 3.775551737843135, + "tokens_seen": 762380288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214162487462387, + "loss": 1.5113, + "theoretical_loss": 3.775519530192293, + "tokens_seen": 762445824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006214002006018055, + "loss": 1.4313, + "theoretical_loss": 3.775487326084816, + "tokens_seen": 762511360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006213841524573721, + "loss": 1.4934, + "theoretical_loss": 3.775455125520012, + "tokens_seen": 762576896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006213681043129389, + "loss": 1.5133, + "theoretical_loss": 3.7754229284971856, + "tokens_seen": 762642432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006213520561685055, + "loss": 1.4052, + "theoretical_loss": 3.775390735015644, + "tokens_seen": 762707968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006213360080240723, + "loss": 1.4116, + "theoretical_loss": 3.7753585450746927, + "tokens_seen": 762773504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006213199598796389, + "loss": 1.5549, + "theoretical_loss": 3.775326358673637, + "tokens_seen": 762839040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006213039117352057, + "loss": 1.5333, + "theoretical_loss": 3.775294175811786, + "tokens_seen": 762904576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006212878635907723, + "loss": 1.4597, + "theoretical_loss": 3.775261996488445, + "tokens_seen": 762970112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006212718154463391, + "loss": 1.3778, + "theoretical_loss": 3.775229820702921, + "tokens_seen": 763035648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006212557673019057, + "loss": 1.5114, + "theoretical_loss": 3.7751976484545224, + "tokens_seen": 763101184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006212397191574725, + "loss": 1.4605, + "theoretical_loss": 3.7751654797425553, + "tokens_seen": 763166720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006212236710130391, + "loss": 1.4807, + "theoretical_loss": 3.775133314566328, + "tokens_seen": 763232256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006212076228686059, + "loss": 1.3731, + "theoretical_loss": 3.7751011529251475, + "tokens_seen": 763297792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006211915747241725, + "loss": 1.4149, + "theoretical_loss": 3.7750689948183234, + "tokens_seen": 763363328 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006211755265797393, + "loss": 1.4045, + "theoretical_loss": 3.7750368402451624, + "tokens_seen": 763428864 + }, + { + "epoch": 0.23, + "learning_rate": 0.000621159478435306, + "loss": 1.404, + "theoretical_loss": 3.775004689204974, + "tokens_seen": 763494400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006211434302908727, + "loss": 1.4118, + "theoretical_loss": 3.7749725416970668, + "tokens_seen": 763559936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006211273821464393, + "loss": 1.4784, + "theoretical_loss": 3.7749403977207483, + "tokens_seen": 763625472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006211113340020061, + "loss": 1.3292, + "theoretical_loss": 3.774908257275329, + "tokens_seen": 763691008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006210952858575727, + "loss": 1.5181, + "theoretical_loss": 3.7748761203601164, + "tokens_seen": 763756544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006210792377131395, + "loss": 1.4844, + "theoretical_loss": 3.774843986974422, + "tokens_seen": 763822080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006210631895687062, + "loss": 1.4004, + "theoretical_loss": 3.774811857117554, + "tokens_seen": 763887616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006210471414242728, + "loss": 1.4275, + "theoretical_loss": 3.7747797307888225, + "tokens_seen": 763953152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006210310932798396, + "loss": 1.4496, + "theoretical_loss": 3.7747476079875373, + "tokens_seen": 764018688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006210150451354062, + "loss": 1.4264, + "theoretical_loss": 3.7747154887130088, + "tokens_seen": 764084224 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620998996990973, + "loss": 1.4503, + "theoretical_loss": 3.7746833729645473, + "tokens_seen": 764149760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006209829488465396, + "loss": 1.6087, + "theoretical_loss": 3.774651260741463, + "tokens_seen": 764215296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006209669007021064, + "loss": 1.4744, + "theoretical_loss": 3.7746191520430674, + "tokens_seen": 764280832 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620950852557673, + "loss": 1.4466, + "theoretical_loss": 3.774587046868671, + "tokens_seen": 764346368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006209348044132398, + "loss": 1.5031, + "theoretical_loss": 3.7745549452175844, + "tokens_seen": 764411904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006209187562688064, + "loss": 1.3149, + "theoretical_loss": 3.77452284708912, + "tokens_seen": 764477440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006209027081243732, + "loss": 1.5055, + "theoretical_loss": 3.7744907524825884, + "tokens_seen": 764542976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006208866599799398, + "loss": 1.4139, + "theoretical_loss": 3.774458661397302, + "tokens_seen": 764608512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006208706118355066, + "loss": 1.4644, + "theoretical_loss": 3.774426573832572, + "tokens_seen": 764674048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006208545636910732, + "loss": 1.4251, + "theoretical_loss": 3.7743944897877113, + "tokens_seen": 764739584 + }, + { + "epoch": 0.23, + "learning_rate": 0.00062083851554664, + "loss": 1.4561, + "theoretical_loss": 3.7743624092620314, + "tokens_seen": 764805120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006208224674022066, + "loss": 1.4075, + "theoretical_loss": 3.774330332254846, + "tokens_seen": 764870656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006208064192577734, + "loss": 1.5163, + "theoretical_loss": 3.774298258765466, + "tokens_seen": 764936192 + }, + { + "epoch": 0.23, + "learning_rate": 0.00062079037111334, + "loss": 1.3608, + "theoretical_loss": 3.774266188793206, + "tokens_seen": 765001728 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006207743229689068, + "loss": 1.3687, + "theoretical_loss": 3.7742341223373774, + "tokens_seen": 765067264 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006207582748244734, + "loss": 1.4045, + "theoretical_loss": 3.774202059397295, + "tokens_seen": 765132800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006207422266800402, + "loss": 1.4398, + "theoretical_loss": 3.774169999972271, + "tokens_seen": 765198336 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006207261785356068, + "loss": 1.4558, + "theoretical_loss": 3.77413794406162, + "tokens_seen": 765263872 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006207101303911736, + "loss": 1.5928, + "theoretical_loss": 3.7741058916646555, + "tokens_seen": 765329408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006206940822467402, + "loss": 1.5524, + "theoretical_loss": 3.7740738427806915, + "tokens_seen": 765394944 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620678034102307, + "loss": 1.4149, + "theoretical_loss": 3.7740417974090423, + "tokens_seen": 765460480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006206619859578736, + "loss": 1.5424, + "theoretical_loss": 3.774009755549022, + "tokens_seen": 765526016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006206459378134404, + "loss": 1.3652, + "theoretical_loss": 3.773977717199946, + "tokens_seen": 765591552 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620629889669007, + "loss": 1.4047, + "theoretical_loss": 3.773945682361128, + "tokens_seen": 765657088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006206138415245738, + "loss": 1.3888, + "theoretical_loss": 3.7739136510318834, + "tokens_seen": 765722624 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006205977933801404, + "loss": 1.5019, + "theoretical_loss": 3.7738816232115275, + "tokens_seen": 765788160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006205817452357072, + "loss": 1.4552, + "theoretical_loss": 3.773849598899376, + "tokens_seen": 765853696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006205656970912738, + "loss": 1.4108, + "theoretical_loss": 3.773817578094744, + "tokens_seen": 765919232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006205496489468406, + "loss": 1.4887, + "theoretical_loss": 3.7737855607969477, + "tokens_seen": 765984768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006205336008024072, + "loss": 1.4633, + "theoretical_loss": 3.7737535470053025, + "tokens_seen": 766050304 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620517552657974, + "loss": 1.37, + "theoretical_loss": 3.7737215367191252, + "tokens_seen": 766115840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006205015045135406, + "loss": 1.4241, + "theoretical_loss": 3.773689529937731, + "tokens_seen": 766181376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006204854563691074, + "loss": 1.5932, + "theoretical_loss": 3.773657526660438, + "tokens_seen": 766246912 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620469408224674, + "loss": 1.4708, + "theoretical_loss": 3.773625526886562, + "tokens_seen": 766312448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006204533600802408, + "loss": 1.4532, + "theoretical_loss": 3.7735935306154196, + "tokens_seen": 766377984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006204373119358074, + "loss": 1.4225, + "theoretical_loss": 3.7735615378463283, + "tokens_seen": 766443520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006204212637913742, + "loss": 1.4142, + "theoretical_loss": 3.7735295485786056, + "tokens_seen": 766509056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006204052156469408, + "loss": 1.5164, + "theoretical_loss": 3.773497562811569, + "tokens_seen": 766574592 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006203891675025076, + "loss": 1.4519, + "theoretical_loss": 3.773465580544536, + "tokens_seen": 766640128 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006203731193580742, + "loss": 1.432, + "theoretical_loss": 3.7734336017768237, + "tokens_seen": 766705664 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 289375, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4793999195098877, + "objective/train/theoretical_loss": 3.773401626507752, + "objective/train/tokens_used": 787231200, + "theoretical_loss": 3.773401626507752, + "tokens_seen": 766771200 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620357071213641, + "loss": 1.4794, + "theoretical_loss": 3.773401626507752, + "tokens_seen": 766771200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006203410230692076, + "loss": 1.3851, + "theoretical_loss": 3.773369654736637, + "tokens_seen": 766836736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006203249749247744, + "loss": 1.5244, + "theoretical_loss": 3.773337686462799, + "tokens_seen": 766902272 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620308926780341, + "loss": 1.4359, + "theoretical_loss": 3.7733057216855554, + "tokens_seen": 766967808 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006202928786359078, + "loss": 1.2812, + "theoretical_loss": 3.7732737604042255, + "tokens_seen": 767033344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006202768304914744, + "loss": 1.4247, + "theoretical_loss": 3.773241802618129, + "tokens_seen": 767098880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006202607823470412, + "loss": 1.4855, + "theoretical_loss": 3.7732098483265837, + "tokens_seen": 767164416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006202447342026078, + "loss": 1.4122, + "theoretical_loss": 3.7731778975289094, + "tokens_seen": 767229952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006202286860581746, + "loss": 1.4023, + "theoretical_loss": 3.7731459502244262, + "tokens_seen": 767295488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006202126379137412, + "loss": 1.4255, + "theoretical_loss": 3.773114006412454, + "tokens_seen": 767361024 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620196589769308, + "loss": 1.5182, + "theoretical_loss": 3.773082066092312, + "tokens_seen": 767426560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006201805416248746, + "loss": 1.4882, + "theoretical_loss": 3.773050129263321, + "tokens_seen": 767492096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006201644934804414, + "loss": 1.3748, + "theoretical_loss": 3.7730181959248013, + "tokens_seen": 767557632 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620148445336008, + "loss": 1.4161, + "theoretical_loss": 3.7729862660760727, + "tokens_seen": 767623168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006201323971915748, + "loss": 1.4566, + "theoretical_loss": 3.7729543397164567, + "tokens_seen": 767688704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006201163490471414, + "loss": 1.4378, + "theoretical_loss": 3.772922416845274, + "tokens_seen": 767754240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006201003009027082, + "loss": 1.4588, + "theoretical_loss": 3.7728904974618462, + "tokens_seen": 767819776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006200842527582748, + "loss": 1.3922, + "theoretical_loss": 3.7728585815654934, + "tokens_seen": 767885312 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006200682046138416, + "loss": 1.4311, + "theoretical_loss": 3.7728266691555383, + "tokens_seen": 767950848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006200521564694082, + "loss": 1.3693, + "theoretical_loss": 3.7727947602313017, + "tokens_seen": 768016384 + }, + { + "epoch": 0.23, + "learning_rate": 0.000620036108324975, + "loss": 1.5276, + "theoretical_loss": 3.772762854792106, + "tokens_seen": 768081920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006200200601805416, + "loss": 1.4007, + "theoretical_loss": 3.772730952837273, + "tokens_seen": 768147456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006200040120361084, + "loss": 1.471, + "theoretical_loss": 3.7726990543661256, + "tokens_seen": 768212992 + }, + { + "epoch": 0.23, + "learning_rate": 0.000619987963891675, + "loss": 1.3487, + "theoretical_loss": 3.7726671593779852, + "tokens_seen": 768278528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006199719157472418, + "loss": 1.4433, + "theoretical_loss": 3.7726352678721753, + "tokens_seen": 768344064 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006199558676028084, + "loss": 1.4619, + "theoretical_loss": 3.772603379848018, + "tokens_seen": 768409600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006199398194583752, + "loss": 1.4973, + "theoretical_loss": 3.772571495304837, + "tokens_seen": 768475136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006199237713139418, + "loss": 1.4225, + "theoretical_loss": 3.7725396142419547, + "tokens_seen": 768540672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006199077231695086, + "loss": 1.3667, + "theoretical_loss": 3.772507736658695, + "tokens_seen": 768606208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006198916750250752, + "loss": 1.427, + "theoretical_loss": 3.7724758625543817, + "tokens_seen": 768671744 + }, + { + "epoch": 0.23, + "learning_rate": 0.000619875626880642, + "loss": 1.5333, + "theoretical_loss": 3.7724439919283386, + "tokens_seen": 768737280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006198595787362086, + "loss": 1.5983, + "theoretical_loss": 3.7724121247798887, + "tokens_seen": 768802816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006198435305917754, + "loss": 1.4801, + "theoretical_loss": 3.7723802611083572, + "tokens_seen": 768868352 + }, + { + "epoch": 0.23, + "learning_rate": 0.000619827482447342, + "loss": 1.4066, + "theoretical_loss": 3.7723484009130677, + "tokens_seen": 768933888 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006198114343029088, + "loss": 1.3755, + "theoretical_loss": 3.772316544193345, + "tokens_seen": 768999424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006197953861584754, + "loss": 1.564, + "theoretical_loss": 3.772284690948514, + "tokens_seen": 769064960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006197793380140422, + "loss": 1.416, + "theoretical_loss": 3.7722528411778997, + "tokens_seen": 769130496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006197632898696088, + "loss": 1.4491, + "theoretical_loss": 3.7722209948808265, + "tokens_seen": 769196032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006197472417251756, + "loss": 1.4206, + "theoretical_loss": 3.77218915205662, + "tokens_seen": 769261568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006197311935807422, + "loss": 1.3684, + "theoretical_loss": 3.772157312704606, + "tokens_seen": 769327104 + }, + { + "epoch": 0.23, + "learning_rate": 0.000619715145436309, + "loss": 1.4622, + "theoretical_loss": 3.7721254768241104, + "tokens_seen": 769392640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006196990972918757, + "loss": 1.2457, + "theoretical_loss": 3.772093644414458, + "tokens_seen": 769458176 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006196830491474424, + "loss": 1.5573, + "theoretical_loss": 3.7720618154749754, + "tokens_seen": 769523712 + }, + { + "epoch": 0.23, + "learning_rate": 0.000619667001003009, + "loss": 1.5187, + "theoretical_loss": 3.772029990004989, + "tokens_seen": 769589248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006196509528585757, + "loss": 1.4374, + "theoretical_loss": 3.771998168003825, + "tokens_seen": 769654784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006196349047141425, + "loss": 1.4548, + "theoretical_loss": 3.77196634947081, + "tokens_seen": 769720320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006196188565697091, + "loss": 1.4242, + "theoretical_loss": 3.7719345344052706, + "tokens_seen": 769785856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006196028084252759, + "loss": 1.5952, + "theoretical_loss": 3.771902722806534, + "tokens_seen": 769851392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006195867602808425, + "loss": 1.4166, + "theoretical_loss": 3.7718709146739275, + "tokens_seen": 769916928 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006195707121364093, + "loss": 1.4267, + "theoretical_loss": 3.771839110006778, + "tokens_seen": 769982464 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006195546639919759, + "loss": 1.2916, + "theoretical_loss": 3.7718073088044135, + "tokens_seen": 770048000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006195386158475427, + "loss": 1.4347, + "theoretical_loss": 3.7717755110661617, + "tokens_seen": 770113536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006195225677031093, + "loss": 1.3548, + "theoretical_loss": 3.7717437167913497, + "tokens_seen": 770179072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006195065195586761, + "loss": 1.4381, + "theoretical_loss": 3.771711925979307, + "tokens_seen": 770244608 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006194904714142427, + "loss": 1.4394, + "theoretical_loss": 3.7716801386293604, + "tokens_seen": 770310144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006194744232698095, + "loss": 1.3796, + "theoretical_loss": 3.7716483547408393, + "tokens_seen": 770375680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006194583751253761, + "loss": 1.4561, + "theoretical_loss": 3.7716165743130725, + "tokens_seen": 770441216 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006194423269809429, + "loss": 1.3811, + "theoretical_loss": 3.7715847973453878, + "tokens_seen": 770506752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006194262788365095, + "loss": 1.4528, + "theoretical_loss": 3.7715530238371153, + "tokens_seen": 770572288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006194102306920763, + "loss": 1.4028, + "theoretical_loss": 3.7715212537875837, + "tokens_seen": 770637824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006193941825476429, + "loss": 1.4051, + "theoretical_loss": 3.7714894871961233, + "tokens_seen": 770703360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006193781344032097, + "loss": 1.5261, + "theoretical_loss": 3.7714577240620617, + "tokens_seen": 770768896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006193620862587763, + "loss": 1.4925, + "theoretical_loss": 3.771425964384731, + "tokens_seen": 770834432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006193460381143431, + "loss": 1.545, + "theoretical_loss": 3.7713942081634593, + "tokens_seen": 770899968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006193299899699097, + "loss": 1.576, + "theoretical_loss": 3.7713624553975778, + "tokens_seen": 770965504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006193139418254765, + "loss": 1.3983, + "theoretical_loss": 3.7713307060864167, + "tokens_seen": 771031040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192978936810431, + "loss": 1.5701, + "theoretical_loss": 3.771298960229306, + "tokens_seen": 771096576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192818455366099, + "loss": 1.4538, + "theoretical_loss": 3.7712672178255775, + "tokens_seen": 771162112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192657973921765, + "loss": 1.4237, + "theoretical_loss": 3.7712354788745612, + "tokens_seen": 771227648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192497492477433, + "loss": 1.4469, + "theoretical_loss": 3.771203743375588, + "tokens_seen": 771293184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192337011033099, + "loss": 1.5454, + "theoretical_loss": 3.77117201132799, + "tokens_seen": 771358720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192176529588767, + "loss": 1.4242, + "theoretical_loss": 3.771140282731098, + "tokens_seen": 771424256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006192016048144433, + "loss": 1.5196, + "theoretical_loss": 3.7711085575842436, + "tokens_seen": 771489792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006191855566700101, + "loss": 1.4421, + "theoretical_loss": 3.7710768358867597, + "tokens_seen": 771555328 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006191695085255767, + "loss": 1.4048, + "theoretical_loss": 3.771045117637977, + "tokens_seen": 771620864 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006191534603811435, + "loss": 1.4855, + "theoretical_loss": 3.7710134028372284, + "tokens_seen": 771686400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006191374122367101, + "loss": 1.3666, + "theoretical_loss": 3.770981691483846, + "tokens_seen": 771751936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006191213640922769, + "loss": 1.3975, + "theoretical_loss": 3.7709499835771627, + "tokens_seen": 771817472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006191053159478435, + "loss": 1.3464, + "theoretical_loss": 3.7709182791165112, + "tokens_seen": 771883008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006190892678034103, + "loss": 1.4218, + "theoretical_loss": 3.770886578101224, + "tokens_seen": 771948544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006190732196589769, + "loss": 1.409, + "theoretical_loss": 3.770854880530635, + "tokens_seen": 772014080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006190571715145437, + "loss": 1.4079, + "theoretical_loss": 3.770823186404077, + "tokens_seen": 772079616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006190411233701103, + "loss": 1.4733, + "theoretical_loss": 3.770791495720883, + "tokens_seen": 772145152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006190250752256771, + "loss": 1.4311, + "theoretical_loss": 3.770759808480388, + "tokens_seen": 772210688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006190090270812437, + "loss": 1.4728, + "theoretical_loss": 3.770728124681925, + "tokens_seen": 772276224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006189929789368105, + "loss": 1.4909, + "theoretical_loss": 3.7706964443248276, + "tokens_seen": 772341760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006189769307923771, + "loss": 1.4656, + "theoretical_loss": 3.7706647674084315, + "tokens_seen": 772407296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006189608826479439, + "loss": 1.4173, + "theoretical_loss": 3.77063309393207, + "tokens_seen": 772472832 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006189448345035105, + "loss": 1.3889, + "theoretical_loss": 3.7706014238950782, + "tokens_seen": 772538368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006189287863590773, + "loss": 1.5056, + "theoretical_loss": 3.7705697572967902, + "tokens_seen": 772603904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006189127382146439, + "loss": 1.271, + "theoretical_loss": 3.7705380941365423, + "tokens_seen": 772669440 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188966900702107, + "loss": 1.4789, + "theoretical_loss": 3.770506434413668, + "tokens_seen": 772734976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188806419257773, + "loss": 1.3036, + "theoretical_loss": 3.770474778127504, + "tokens_seen": 772800512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188645937813441, + "loss": 1.476, + "theoretical_loss": 3.7704431252773856, + "tokens_seen": 772866048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188485456369107, + "loss": 1.3749, + "theoretical_loss": 3.770411475862648, + "tokens_seen": 772931584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188324974924775, + "loss": 1.4597, + "theoretical_loss": 3.7703798298826277, + "tokens_seen": 772997120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188164493480441, + "loss": 1.492, + "theoretical_loss": 3.77034818733666, + "tokens_seen": 773062656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006188004012036109, + "loss": 1.4353, + "theoretical_loss": 3.7703165482240824, + "tokens_seen": 773128192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006187843530591775, + "loss": 1.3775, + "theoretical_loss": 3.77028491254423, + "tokens_seen": 773193728 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006187683049147443, + "loss": 1.3751, + "theoretical_loss": 3.7702532802964406, + "tokens_seen": 773259264 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 291637, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3960922956466675, + "objective/train/theoretical_loss": 3.7702216514800506, + "objective/train/tokens_used": 793784800, + "theoretical_loss": 3.7702216514800506, + "tokens_seen": 773324800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006187522567703109, + "loss": 1.3961, + "theoretical_loss": 3.7702216514800506, + "tokens_seen": 773324800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006187362086258777, + "loss": 1.414, + "theoretical_loss": 3.770190026094397, + "tokens_seen": 773390336 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006187201604814443, + "loss": 1.4027, + "theoretical_loss": 3.7701584041388165, + "tokens_seen": 773455872 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006187041123370111, + "loss": 1.4691, + "theoretical_loss": 3.7701267856126472, + "tokens_seen": 773521408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006186880641925777, + "loss": 1.4206, + "theoretical_loss": 3.7700951705152264, + "tokens_seen": 773586944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006186720160481445, + "loss": 1.5293, + "theoretical_loss": 3.7700635588458926, + "tokens_seen": 773652480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006186559679037111, + "loss": 1.3406, + "theoretical_loss": 3.7700319506039826, + "tokens_seen": 773718016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006186399197592779, + "loss": 1.4487, + "theoretical_loss": 3.7700003457888345, + "tokens_seen": 773783552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006186238716148445, + "loss": 1.5389, + "theoretical_loss": 3.7699687443997876, + "tokens_seen": 773849088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006186078234704113, + "loss": 1.452, + "theoretical_loss": 3.7699371464361797, + "tokens_seen": 773914624 + }, + { + "epoch": 0.23, + "learning_rate": 0.000618591775325978, + "loss": 1.3763, + "theoretical_loss": 3.7699055518973497, + "tokens_seen": 773980160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006185757271815447, + "loss": 1.4641, + "theoretical_loss": 3.769873960782636, + "tokens_seen": 774045696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006185596790371114, + "loss": 1.4362, + "theoretical_loss": 3.769842373091379, + "tokens_seen": 774111232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006185436308926781, + "loss": 1.4082, + "theoretical_loss": 3.769810788822916, + "tokens_seen": 774176768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006185275827482449, + "loss": 1.4205, + "theoretical_loss": 3.769779207976588, + "tokens_seen": 774242304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006185115346038115, + "loss": 1.3495, + "theoretical_loss": 3.7697476305517337, + "tokens_seen": 774307840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006184954864593783, + "loss": 1.454, + "theoretical_loss": 3.7697160565476935, + "tokens_seen": 774373376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006184794383149449, + "loss": 1.3371, + "theoretical_loss": 3.769684485963807, + "tokens_seen": 774438912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006184633901705117, + "loss": 1.5336, + "theoretical_loss": 3.7696529187994137, + "tokens_seen": 774504448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006184473420260783, + "loss": 1.3699, + "theoretical_loss": 3.769621355053855, + "tokens_seen": 774569984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006184312938816451, + "loss": 1.4351, + "theoretical_loss": 3.769589794726471, + "tokens_seen": 774635520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006184152457372117, + "loss": 1.4286, + "theoretical_loss": 3.7695582378166024, + "tokens_seen": 774701056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183991975927783, + "loss": 1.5472, + "theoretical_loss": 3.7695266843235897, + "tokens_seen": 774766592 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183831494483451, + "loss": 1.3918, + "theoretical_loss": 3.7694951342467746, + "tokens_seen": 774832128 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183671013039117, + "loss": 1.5575, + "theoretical_loss": 3.7694635875854985, + "tokens_seen": 774897664 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183510531594785, + "loss": 1.3781, + "theoretical_loss": 3.7694320443391023, + "tokens_seen": 774963200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183350050150452, + "loss": 1.4311, + "theoretical_loss": 3.769400504506927, + "tokens_seen": 775028736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183189568706119, + "loss": 1.4607, + "theoretical_loss": 3.7693689680883153, + "tokens_seen": 775094272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006183029087261786, + "loss": 1.4027, + "theoretical_loss": 3.769337435082609, + "tokens_seen": 775159808 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006182868605817453, + "loss": 1.4598, + "theoretical_loss": 3.769305905489151, + "tokens_seen": 775225344 + }, + { + "epoch": 0.23, + "learning_rate": 0.000618270812437312, + "loss": 1.2932, + "theoretical_loss": 3.769274379307282, + "tokens_seen": 775290880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006182547642928786, + "loss": 1.5859, + "theoretical_loss": 3.7692428565363456, + "tokens_seen": 775356416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0006182387161484454, + "loss": 1.5217, + "theoretical_loss": 3.769211337175684, + "tokens_seen": 775421952 + }, + { + "epoch": 0.23, + "learning_rate": 0.000618222668004012, + "loss": 1.4681, + "theoretical_loss": 3.7691798212246406, + "tokens_seen": 775487488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006182066198595788, + "loss": 1.44, + "theoretical_loss": 3.769148308682558, + "tokens_seen": 775553024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006181905717151454, + "loss": 1.5679, + "theoretical_loss": 3.76911679954878, + "tokens_seen": 775618560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006181745235707122, + "loss": 1.457, + "theoretical_loss": 3.7690852938226493, + "tokens_seen": 775684096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006181584754262788, + "loss": 1.4172, + "theoretical_loss": 3.7690537915035103, + "tokens_seen": 775749632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006181424272818456, + "loss": 1.3962, + "theoretical_loss": 3.769022292590706, + "tokens_seen": 775815168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006181263791374122, + "loss": 1.3591, + "theoretical_loss": 3.768990797083581, + "tokens_seen": 775880704 + }, + { + "epoch": 0.24, + "learning_rate": 0.000618110330992979, + "loss": 1.4354, + "theoretical_loss": 3.7689593049814794, + "tokens_seen": 775946240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006180942828485456, + "loss": 1.4772, + "theoretical_loss": 3.768927816283745, + "tokens_seen": 776011776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006180782347041124, + "loss": 1.5267, + "theoretical_loss": 3.768896330989723, + "tokens_seen": 776077312 + }, + { + "epoch": 0.24, + "learning_rate": 0.000618062186559679, + "loss": 1.3232, + "theoretical_loss": 3.7688648490987577, + "tokens_seen": 776142848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006180461384152458, + "loss": 1.3899, + "theoretical_loss": 3.768833370610194, + "tokens_seen": 776208384 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006180300902708124, + "loss": 1.4796, + "theoretical_loss": 3.768801895523377, + "tokens_seen": 776273920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006180140421263792, + "loss": 1.4148, + "theoretical_loss": 3.768770423837652, + "tokens_seen": 776339456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006179979939819458, + "loss": 1.4054, + "theoretical_loss": 3.7687389555523643, + "tokens_seen": 776404992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006179819458375126, + "loss": 1.5384, + "theoretical_loss": 3.7687074906668596, + "tokens_seen": 776470528 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006179658976930792, + "loss": 1.3736, + "theoretical_loss": 3.768676029180484, + "tokens_seen": 776536064 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617949849548646, + "loss": 1.4441, + "theoretical_loss": 3.768644571092583, + "tokens_seen": 776601600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006179338014042126, + "loss": 1.5099, + "theoretical_loss": 3.768613116402503, + "tokens_seen": 776667136 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006179177532597794, + "loss": 1.4317, + "theoretical_loss": 3.76858166510959, + "tokens_seen": 776732672 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617901705115346, + "loss": 1.4186, + "theoretical_loss": 3.768550217213191, + "tokens_seen": 776798208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006178856569709128, + "loss": 1.3436, + "theoretical_loss": 3.7685187727126523, + "tokens_seen": 776863744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006178696088264794, + "loss": 1.4722, + "theoretical_loss": 3.768487331607321, + "tokens_seen": 776929280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006178535606820462, + "loss": 1.3885, + "theoretical_loss": 3.7684558938965442, + "tokens_seen": 776994816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006178375125376128, + "loss": 1.3805, + "theoretical_loss": 3.768424459579669, + "tokens_seen": 777060352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006178214643931796, + "loss": 1.4343, + "theoretical_loss": 3.7683930286560425, + "tokens_seen": 777125888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006178054162487462, + "loss": 1.3872, + "theoretical_loss": 3.7683616011250134, + "tokens_seen": 777191424 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617789368104313, + "loss": 1.4163, + "theoretical_loss": 3.768330176985928, + "tokens_seen": 777256960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006177733199598796, + "loss": 1.4076, + "theoretical_loss": 3.7682987562381354, + "tokens_seen": 777322496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006177572718154464, + "loss": 1.4056, + "theoretical_loss": 3.768267338880983, + "tokens_seen": 777388032 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617741223671013, + "loss": 1.5382, + "theoretical_loss": 3.7682359249138195, + "tokens_seen": 777453568 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006177251755265798, + "loss": 1.3816, + "theoretical_loss": 3.7682045143359937, + "tokens_seen": 777519104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006177091273821464, + "loss": 1.5206, + "theoretical_loss": 3.7681731071468536, + "tokens_seen": 777584640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006176930792377132, + "loss": 1.4411, + "theoretical_loss": 3.7681417033457487, + "tokens_seen": 777650176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006176770310932798, + "loss": 1.4962, + "theoretical_loss": 3.768110302932027, + "tokens_seen": 777715712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006176609829488466, + "loss": 1.3595, + "theoretical_loss": 3.768078905905039, + "tokens_seen": 777781248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006176449348044132, + "loss": 1.4622, + "theoretical_loss": 3.7680475122641335, + "tokens_seen": 777846784 + }, + { + "epoch": 0.24, + "learning_rate": 0.00061762888665998, + "loss": 1.5397, + "theoretical_loss": 3.7680161220086603, + "tokens_seen": 777912320 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006176128385155466, + "loss": 1.3045, + "theoretical_loss": 3.7679847351379685, + "tokens_seen": 777977856 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006175967903711134, + "loss": 1.3299, + "theoretical_loss": 3.7679533516514088, + "tokens_seen": 778043392 + }, + { + "epoch": 0.24, + "learning_rate": 0.00061758074222668, + "loss": 1.4785, + "theoretical_loss": 3.767921971548331, + "tokens_seen": 778108928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006175646940822468, + "loss": 1.3438, + "theoretical_loss": 3.767890594828085, + "tokens_seen": 778174464 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006175486459378134, + "loss": 1.4244, + "theoretical_loss": 3.7678592214900224, + "tokens_seen": 778240000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006175325977933802, + "loss": 1.4446, + "theoretical_loss": 3.7678278515334926, + "tokens_seen": 778305536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006175165496489468, + "loss": 1.426, + "theoretical_loss": 3.7677964849578474, + "tokens_seen": 778371072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006175005015045136, + "loss": 1.4236, + "theoretical_loss": 3.7677651217624373, + "tokens_seen": 778436608 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006174844533600802, + "loss": 1.4591, + "theoretical_loss": 3.767733761946613, + "tokens_seen": 778502144 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617468405215647, + "loss": 1.4938, + "theoretical_loss": 3.767702405509727, + "tokens_seen": 778567680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006174523570712136, + "loss": 1.3904, + "theoretical_loss": 3.7676710524511305, + "tokens_seen": 778633216 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006174363089267804, + "loss": 1.338, + "theoretical_loss": 3.767639702770175, + "tokens_seen": 778698752 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617420260782347, + "loss": 1.3991, + "theoretical_loss": 3.767608356466212, + "tokens_seen": 778764288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006174042126379138, + "loss": 1.3979, + "theoretical_loss": 3.7675770135385944, + "tokens_seen": 778829824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006173881644934805, + "loss": 1.3204, + "theoretical_loss": 3.7675456739866746, + "tokens_seen": 778895360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006173721163490472, + "loss": 1.4996, + "theoretical_loss": 3.7675143378098035, + "tokens_seen": 778960896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006173560682046139, + "loss": 1.4999, + "theoretical_loss": 3.7674830050073354, + "tokens_seen": 779026432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006173400200601806, + "loss": 1.4464, + "theoretical_loss": 3.7674516755786227, + "tokens_seen": 779091968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006173239719157473, + "loss": 1.4405, + "theoretical_loss": 3.767420349523018, + "tokens_seen": 779157504 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617307923771314, + "loss": 1.3791, + "theoretical_loss": 3.7673890268398744, + "tokens_seen": 779223040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006172918756268807, + "loss": 1.458, + "theoretical_loss": 3.7673577075285456, + "tokens_seen": 779288576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006172758274824474, + "loss": 1.2921, + "theoretical_loss": 3.767326391588385, + "tokens_seen": 779354112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006172597793380141, + "loss": 1.4929, + "theoretical_loss": 3.767295079018746, + "tokens_seen": 779419648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006172437311935808, + "loss": 1.3387, + "theoretical_loss": 3.7672637698189835, + "tokens_seen": 779485184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006172276830491475, + "loss": 1.4506, + "theoretical_loss": 3.7672324639884502, + "tokens_seen": 779550720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006172116349047142, + "loss": 1.3751, + "theoretical_loss": 3.7672011615265015, + "tokens_seen": 779616256 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617195586760281, + "loss": 1.4637, + "theoretical_loss": 3.7671698624324907, + "tokens_seen": 779681792 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006171795386158476, + "loss": 1.41, + "theoretical_loss": 3.767138566705773, + "tokens_seen": 779747328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006171634904714144, + "loss": 1.4773, + "theoretical_loss": 3.7671072743457032, + "tokens_seen": 779812864 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 293946, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.450742244720459, + "objective/train/theoretical_loss": 3.767075985351636, + "objective/train/tokens_used": 800338400, + "theoretical_loss": 3.767075985351636, + "tokens_seen": 779878400 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617147442326981, + "loss": 1.4507, + "theoretical_loss": 3.767075985351636, + "tokens_seen": 779878400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006171313941825478, + "loss": 1.3972, + "theoretical_loss": 3.767044699722927, + "tokens_seen": 779943936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006171153460381144, + "loss": 1.325, + "theoretical_loss": 3.767013417458931, + "tokens_seen": 780009472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006170992978936812, + "loss": 1.4158, + "theoretical_loss": 3.766982138559004, + "tokens_seen": 780075008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006170832497492478, + "loss": 1.3991, + "theoretical_loss": 3.7669508630225006, + "tokens_seen": 780140544 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006170672016048146, + "loss": 1.5069, + "theoretical_loss": 3.7669195908487776, + "tokens_seen": 780206080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006170511534603812, + "loss": 1.4616, + "theoretical_loss": 3.766888322037191, + "tokens_seen": 780271616 + }, + { + "epoch": 0.24, + "learning_rate": 0.000617035105315948, + "loss": 1.4501, + "theoretical_loss": 3.7668570565870967, + "tokens_seen": 780337152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006170190571715146, + "loss": 1.4013, + "theoretical_loss": 3.7668257944978505, + "tokens_seen": 780402688 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006170030090270812, + "loss": 1.3625, + "theoretical_loss": 3.7667945357688097, + "tokens_seen": 780468224 + }, + { + "epoch": 0.24, + "learning_rate": 0.000616986960882648, + "loss": 1.4915, + "theoretical_loss": 3.7667632803993314, + "tokens_seen": 780533760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006169709127382146, + "loss": 1.43, + "theoretical_loss": 3.7667320283887715, + "tokens_seen": 780599296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006169548645937814, + "loss": 1.5025, + "theoretical_loss": 3.7667007797364875, + "tokens_seen": 780664832 + }, + { + "epoch": 0.24, + "learning_rate": 0.000616938816449348, + "loss": 1.4212, + "theoretical_loss": 3.766669534441837, + "tokens_seen": 780730368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006169227683049148, + "loss": 1.4073, + "theoretical_loss": 3.766638292504177, + "tokens_seen": 780795904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006169067201604815, + "loss": 1.4318, + "theoretical_loss": 3.766607053922865, + "tokens_seen": 780861440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006168906720160482, + "loss": 1.5392, + "theoretical_loss": 3.7665758186972584, + "tokens_seen": 780926976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006168746238716149, + "loss": 1.4278, + "theoretical_loss": 3.7665445868267162, + "tokens_seen": 780992512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006168585757271815, + "loss": 1.465, + "theoretical_loss": 3.766513358310596, + "tokens_seen": 781058048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006168425275827483, + "loss": 1.364, + "theoretical_loss": 3.7664821331482568, + "tokens_seen": 781123584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006168264794383149, + "loss": 1.4052, + "theoretical_loss": 3.7664509113390556, + "tokens_seen": 781189120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006168104312938817, + "loss": 1.4351, + "theoretical_loss": 3.7664196928823523, + "tokens_seen": 781254656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006167943831494483, + "loss": 1.4431, + "theoretical_loss": 3.7663884777775047, + "tokens_seen": 781320192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006167783350050151, + "loss": 1.4511, + "theoretical_loss": 3.7663572660238733, + "tokens_seen": 781385728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006167622868605817, + "loss": 1.4823, + "theoretical_loss": 3.766326057620816, + "tokens_seen": 781451264 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006167462387161485, + "loss": 1.455, + "theoretical_loss": 3.7662948525676927, + "tokens_seen": 781516800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006167301905717151, + "loss": 1.4336, + "theoretical_loss": 3.7662636508638627, + "tokens_seen": 781582336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006167141424272819, + "loss": 1.3475, + "theoretical_loss": 3.7662324525086865, + "tokens_seen": 781647872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166980942828485, + "loss": 1.515, + "theoretical_loss": 3.766201257501523, + "tokens_seen": 781713408 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166820461384153, + "loss": 1.4992, + "theoretical_loss": 3.7661700658417328, + "tokens_seen": 781778944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166659979939819, + "loss": 1.4595, + "theoretical_loss": 3.7661388775286753, + "tokens_seen": 781844480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166499498495487, + "loss": 1.2975, + "theoretical_loss": 3.7661076925617123, + "tokens_seen": 781910016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166339017051153, + "loss": 1.4901, + "theoretical_loss": 3.7660765109402035, + "tokens_seen": 781975552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166178535606821, + "loss": 1.4131, + "theoretical_loss": 3.76604533266351, + "tokens_seen": 782041088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006166018054162487, + "loss": 1.4479, + "theoretical_loss": 3.7660141577309925, + "tokens_seen": 782106624 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006165857572718155, + "loss": 1.4426, + "theoretical_loss": 3.7659829861420127, + "tokens_seen": 782172160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006165697091273821, + "loss": 1.3808, + "theoretical_loss": 3.765951817895931, + "tokens_seen": 782237696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006165536609829489, + "loss": 1.5585, + "theoretical_loss": 3.765920652992109, + "tokens_seen": 782303232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006165376128385155, + "loss": 1.4458, + "theoretical_loss": 3.7658894914299097, + "tokens_seen": 782368768 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006165215646940823, + "loss": 1.3465, + "theoretical_loss": 3.7658583332086932, + "tokens_seen": 782434304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006165055165496489, + "loss": 1.3807, + "theoretical_loss": 3.7658271783278225, + "tokens_seen": 782499840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006164894684052157, + "loss": 1.3933, + "theoretical_loss": 3.765796026786659, + "tokens_seen": 782565376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006164734202607823, + "loss": 1.376, + "theoretical_loss": 3.7657648785845663, + "tokens_seen": 782630912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006164573721163491, + "loss": 1.3969, + "theoretical_loss": 3.765733733720906, + "tokens_seen": 782696448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006164413239719157, + "loss": 1.3467, + "theoretical_loss": 3.7657025921950407, + "tokens_seen": 782761984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006164252758274825, + "loss": 1.3559, + "theoretical_loss": 3.765671454006334, + "tokens_seen": 782827520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006164092276830491, + "loss": 1.5712, + "theoretical_loss": 3.7656403191541483, + "tokens_seen": 782893056 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006163931795386159, + "loss": 1.5638, + "theoretical_loss": 3.765609187637847, + "tokens_seen": 782958592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006163771313941825, + "loss": 1.5393, + "theoretical_loss": 3.765578059456794, + "tokens_seen": 783024128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006163610832497493, + "loss": 1.3558, + "theoretical_loss": 3.765546934610352, + "tokens_seen": 783089664 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006163450351053159, + "loss": 1.4599, + "theoretical_loss": 3.7655158130978856, + "tokens_seen": 783155200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006163289869608827, + "loss": 1.4878, + "theoretical_loss": 3.765484694918758, + "tokens_seen": 783220736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006163129388164493, + "loss": 1.5042, + "theoretical_loss": 3.7654535800723337, + "tokens_seen": 783286272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162968906720161, + "loss": 1.4178, + "theoretical_loss": 3.7654224685579765, + "tokens_seen": 783351808 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162808425275827, + "loss": 1.5494, + "theoretical_loss": 3.765391360375052, + "tokens_seen": 783417344 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162647943831495, + "loss": 1.3482, + "theoretical_loss": 3.765360255522924, + "tokens_seen": 783482880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162487462387161, + "loss": 1.3643, + "theoretical_loss": 3.7653291540009572, + "tokens_seen": 783548416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162326980942829, + "loss": 1.4249, + "theoretical_loss": 3.7652980558085165, + "tokens_seen": 783613952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162166499498495, + "loss": 1.4299, + "theoretical_loss": 3.7652669609449676, + "tokens_seen": 783679488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006162006018054163, + "loss": 1.3594, + "theoretical_loss": 3.7652358694096755, + "tokens_seen": 783745024 + }, + { + "epoch": 0.24, + "learning_rate": 0.000616184553660983, + "loss": 1.4581, + "theoretical_loss": 3.765204781202006, + "tokens_seen": 783810560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006161685055165497, + "loss": 1.3778, + "theoretical_loss": 3.765173696321324, + "tokens_seen": 783876096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006161524573721164, + "loss": 1.3243, + "theoretical_loss": 3.7651426147669964, + "tokens_seen": 783941632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006161364092276831, + "loss": 1.3363, + "theoretical_loss": 3.765111536538388, + "tokens_seen": 784007168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006161203610832498, + "loss": 1.4554, + "theoretical_loss": 3.7650804616348665, + "tokens_seen": 784072704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006161043129388165, + "loss": 1.4199, + "theoretical_loss": 3.765049390055797, + "tokens_seen": 784138240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006160882647943832, + "loss": 1.4648, + "theoretical_loss": 3.765018321800546, + "tokens_seen": 784203776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006160722166499499, + "loss": 1.4897, + "theoretical_loss": 3.7649872568684812, + "tokens_seen": 784269312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006160561685055166, + "loss": 1.4893, + "theoretical_loss": 3.764956195258969, + "tokens_seen": 784334848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006160401203610833, + "loss": 1.366, + "theoretical_loss": 3.7649251369713763, + "tokens_seen": 784400384 + }, + { + "epoch": 0.24, + "learning_rate": 0.00061602407221665, + "loss": 1.4324, + "theoretical_loss": 3.7648940820050707, + "tokens_seen": 784465920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006160080240722167, + "loss": 1.5716, + "theoretical_loss": 3.764863030359419, + "tokens_seen": 784531456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006159919759277834, + "loss": 1.5084, + "theoretical_loss": 3.7648319820337894, + "tokens_seen": 784596992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006159759277833501, + "loss": 1.5627, + "theoretical_loss": 3.7648009370275495, + "tokens_seen": 784662528 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006159598796389168, + "loss": 1.3773, + "theoretical_loss": 3.7647698953400672, + "tokens_seen": 784728064 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006159438314944835, + "loss": 1.4182, + "theoretical_loss": 3.76473885697071, + "tokens_seen": 784793600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006159277833500502, + "loss": 1.4161, + "theoretical_loss": 3.7647078219188472, + "tokens_seen": 784859136 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006159117352056169, + "loss": 1.4003, + "theoretical_loss": 3.7646767901838465, + "tokens_seen": 784924672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006158956870611836, + "loss": 1.4396, + "theoretical_loss": 3.764645761765077, + "tokens_seen": 784990208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006158796389167503, + "loss": 1.3611, + "theoretical_loss": 3.7646147366619074, + "tokens_seen": 785055744 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615863590772317, + "loss": 1.3992, + "theoretical_loss": 3.764583714873706, + "tokens_seen": 785121280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006158475426278837, + "loss": 1.4083, + "theoretical_loss": 3.764552696399843, + "tokens_seen": 785186816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006158314944834504, + "loss": 1.4061, + "theoretical_loss": 3.764521681239687, + "tokens_seen": 785252352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006158154463390171, + "loss": 1.4494, + "theoretical_loss": 3.7644906693926075, + "tokens_seen": 785317888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006157993981945839, + "loss": 1.4257, + "theoretical_loss": 3.7644596608579746, + "tokens_seen": 785383424 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006157833500501505, + "loss": 1.4777, + "theoretical_loss": 3.764428655635158, + "tokens_seen": 785448960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006157673019057173, + "loss": 1.4569, + "theoretical_loss": 3.764397653723527, + "tokens_seen": 785514496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006157512537612839, + "loss": 1.4597, + "theoretical_loss": 3.7643666551224526, + "tokens_seen": 785580032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006157352056168507, + "loss": 1.4706, + "theoretical_loss": 3.764335659831305, + "tokens_seen": 785645568 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006157191574724173, + "loss": 1.4741, + "theoretical_loss": 3.7643046678494545, + "tokens_seen": 785711104 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615703109327984, + "loss": 1.4427, + "theoretical_loss": 3.764273679176272, + "tokens_seen": 785776640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006156870611835507, + "loss": 1.3952, + "theoretical_loss": 3.764242693811128, + "tokens_seen": 785842176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006156710130391175, + "loss": 1.5304, + "theoretical_loss": 3.764211711753394, + "tokens_seen": 785907712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006156549648946841, + "loss": 1.5857, + "theoretical_loss": 3.7641807330024406, + "tokens_seen": 785973248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006156389167502509, + "loss": 1.4904, + "theoretical_loss": 3.7641497575576404, + "tokens_seen": 786038784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006156228686058175, + "loss": 1.4673, + "theoretical_loss": 3.764118785418363, + "tokens_seen": 786104320 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006156068204613843, + "loss": 1.3818, + "theoretical_loss": 3.7640878165839817, + "tokens_seen": 786169856 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006155907723169509, + "loss": 1.4902, + "theoretical_loss": 3.7640568510538683, + "tokens_seen": 786235392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006155747241725176, + "loss": 1.4712, + "theoretical_loss": 3.764025888827394, + "tokens_seen": 786300928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006155586760280843, + "loss": 1.3981, + "theoretical_loss": 3.763994929903932, + "tokens_seen": 786366464 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 296193, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4492788314819336, + "objective/train/theoretical_loss": 3.763963974282854, + "objective/train/tokens_used": 806892000, + "theoretical_loss": 3.763963974282854, + "tokens_seen": 786432000 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615542627883651, + "loss": 1.4493, + "theoretical_loss": 3.763963974282854, + "tokens_seen": 786432000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006155265797392177, + "loss": 1.4475, + "theoretical_loss": 3.763933021963533, + "tokens_seen": 786497536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006155105315947844, + "loss": 1.36, + "theoretical_loss": 3.763902072945341, + "tokens_seen": 786563072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006154944834503511, + "loss": 1.3794, + "theoretical_loss": 3.7638711272276524, + "tokens_seen": 786628608 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006154784353059178, + "loss": 1.467, + "theoretical_loss": 3.7638401848098386, + "tokens_seen": 786694144 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006154623871614844, + "loss": 1.4188, + "theoretical_loss": 3.763809245691274, + "tokens_seen": 786759680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006154463390170512, + "loss": 1.4553, + "theoretical_loss": 3.7637783098713316, + "tokens_seen": 786825216 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006154302908726178, + "loss": 1.5525, + "theoretical_loss": 3.7637473773493855, + "tokens_seen": 786890752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006154142427281846, + "loss": 1.4655, + "theoretical_loss": 3.7637164481248084, + "tokens_seen": 786956288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006153981945837512, + "loss": 1.431, + "theoretical_loss": 3.763685522196975, + "tokens_seen": 787021824 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615382146439318, + "loss": 1.4225, + "theoretical_loss": 3.7636545995652604, + "tokens_seen": 787087360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006153660982948846, + "loss": 1.4799, + "theoretical_loss": 3.7636236802290366, + "tokens_seen": 787152896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006153500501504514, + "loss": 1.345, + "theoretical_loss": 3.7635927641876803, + "tokens_seen": 787218432 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615334002006018, + "loss": 1.4388, + "theoretical_loss": 3.7635618514405644, + "tokens_seen": 787283968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006153179538615848, + "loss": 1.4388, + "theoretical_loss": 3.7635309419870646, + "tokens_seen": 787349504 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006153019057171514, + "loss": 1.3613, + "theoretical_loss": 3.763500035826556, + "tokens_seen": 787415040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006152858575727182, + "loss": 1.4993, + "theoretical_loss": 3.7634691329584133, + "tokens_seen": 787480576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006152698094282848, + "loss": 1.4229, + "theoretical_loss": 3.763438233382012, + "tokens_seen": 787546112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006152537612838516, + "loss": 1.4666, + "theoretical_loss": 3.7634073370967274, + "tokens_seen": 787611648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006152377131394182, + "loss": 1.4503, + "theoretical_loss": 3.7633764441019357, + "tokens_seen": 787677184 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615221664994985, + "loss": 1.3743, + "theoretical_loss": 3.763345554397012, + "tokens_seen": 787742720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006152056168505516, + "loss": 1.3601, + "theoretical_loss": 3.763314667981333, + "tokens_seen": 787808256 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006151895687061184, + "loss": 1.4383, + "theoretical_loss": 3.7632837848542744, + "tokens_seen": 787873792 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615173520561685, + "loss": 1.3829, + "theoretical_loss": 3.7632529050152126, + "tokens_seen": 787939328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006151574724172518, + "loss": 1.44, + "theoretical_loss": 3.7632220284635247, + "tokens_seen": 788004864 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006151414242728184, + "loss": 1.5238, + "theoretical_loss": 3.763191155198586, + "tokens_seen": 788070400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006151253761283852, + "loss": 1.3422, + "theoretical_loss": 3.7631602852197745, + "tokens_seen": 788135936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006151093279839518, + "loss": 1.4445, + "theoretical_loss": 3.763129418526467, + "tokens_seen": 788201472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006150932798395186, + "loss": 1.3922, + "theoretical_loss": 3.7630985551180407, + "tokens_seen": 788267008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006150772316950852, + "loss": 1.4179, + "theoretical_loss": 3.7630676949938726, + "tokens_seen": 788332544 + }, + { + "epoch": 0.24, + "learning_rate": 0.000615061183550652, + "loss": 1.4367, + "theoretical_loss": 3.763036838153341, + "tokens_seen": 788398080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006150451354062186, + "loss": 1.4551, + "theoretical_loss": 3.763005984595823, + "tokens_seen": 788463616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006150290872617854, + "loss": 1.5017, + "theoretical_loss": 3.762975134320696, + "tokens_seen": 788529152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006150130391173521, + "loss": 1.4802, + "theoretical_loss": 3.762944287327339, + "tokens_seen": 788594688 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006149969909729188, + "loss": 1.4034, + "theoretical_loss": 3.7629134436151297, + "tokens_seen": 788660224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006149809428284855, + "loss": 1.4226, + "theoretical_loss": 3.7628826031834466, + "tokens_seen": 788725760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006149648946840522, + "loss": 1.3694, + "theoretical_loss": 3.7628517660316687, + "tokens_seen": 788791296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006149488465396189, + "loss": 1.5048, + "theoretical_loss": 3.7628209321591735, + "tokens_seen": 788856832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006149327983951856, + "loss": 1.3724, + "theoretical_loss": 3.762790101565341, + "tokens_seen": 788922368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006149167502507523, + "loss": 1.4845, + "theoretical_loss": 3.76275927424955, + "tokens_seen": 788987904 + }, + { + "epoch": 0.24, + "learning_rate": 0.000614900702106319, + "loss": 1.4418, + "theoretical_loss": 3.7627284502111795, + "tokens_seen": 789053440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006148846539618857, + "loss": 1.5219, + "theoretical_loss": 3.762697629449609, + "tokens_seen": 789118976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006148686058174524, + "loss": 1.3517, + "theoretical_loss": 3.762666811964218, + "tokens_seen": 789184512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006148525576730191, + "loss": 1.3978, + "theoretical_loss": 3.762635997754386, + "tokens_seen": 789250048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006148365095285858, + "loss": 1.4614, + "theoretical_loss": 3.762605186819494, + "tokens_seen": 789315584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006148204613841525, + "loss": 1.3917, + "theoretical_loss": 3.76257437915892, + "tokens_seen": 789381120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006148044132397192, + "loss": 1.4085, + "theoretical_loss": 3.762543574772047, + "tokens_seen": 789446656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006147883650952859, + "loss": 1.3531, + "theoretical_loss": 3.7625127736582527, + "tokens_seen": 789512192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006147723169508526, + "loss": 1.4477, + "theoretical_loss": 3.7624819758169195, + "tokens_seen": 789577728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006147562688064193, + "loss": 1.3799, + "theoretical_loss": 3.7624511812474273, + "tokens_seen": 789643264 + }, + { + "epoch": 0.24, + "learning_rate": 0.000614740220661986, + "loss": 1.4188, + "theoretical_loss": 3.762420389949157, + "tokens_seen": 789708800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006147241725175527, + "loss": 1.4466, + "theoretical_loss": 3.76238960192149, + "tokens_seen": 789774336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006147081243731194, + "loss": 1.4526, + "theoretical_loss": 3.762358817163808, + "tokens_seen": 789839872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006146920762286861, + "loss": 1.3476, + "theoretical_loss": 3.762328035675491, + "tokens_seen": 789905408 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006146760280842528, + "loss": 1.4258, + "theoretical_loss": 3.7622972574559226, + "tokens_seen": 789970944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006146599799398195, + "loss": 1.4731, + "theoretical_loss": 3.762266482504483, + "tokens_seen": 790036480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006146439317953862, + "loss": 1.4768, + "theoretical_loss": 3.762235710820554, + "tokens_seen": 790102016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006146278836509529, + "loss": 1.4853, + "theoretical_loss": 3.7622049424035184, + "tokens_seen": 790167552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006146118355065196, + "loss": 1.3782, + "theoretical_loss": 3.762174177252758, + "tokens_seen": 790233088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006145957873620863, + "loss": 1.4274, + "theoretical_loss": 3.7621434153676563, + "tokens_seen": 790298624 + }, + { + "epoch": 0.24, + "learning_rate": 0.000614579739217653, + "loss": 1.3877, + "theoretical_loss": 3.7621126567475947, + "tokens_seen": 790364160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006145636910732197, + "loss": 1.4037, + "theoretical_loss": 3.762081901391956, + "tokens_seen": 790429696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006145476429287864, + "loss": 1.4312, + "theoretical_loss": 3.7620511493001243, + "tokens_seen": 790495232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006145315947843531, + "loss": 1.3244, + "theoretical_loss": 3.7620204004714815, + "tokens_seen": 790560768 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006145155466399198, + "loss": 1.3378, + "theoretical_loss": 3.761989654905411, + "tokens_seen": 790626304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006144994984954865, + "loss": 1.418, + "theoretical_loss": 3.7619589126012967, + "tokens_seen": 790691840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006144834503510532, + "loss": 1.4876, + "theoretical_loss": 3.761928173558522, + "tokens_seen": 790757376 + }, + { + "epoch": 0.24, + "learning_rate": 0.00061446740220662, + "loss": 1.4118, + "theoretical_loss": 3.7618974377764705, + "tokens_seen": 790822912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006144513540621866, + "loss": 1.4512, + "theoretical_loss": 3.7618667052545263, + "tokens_seen": 790888448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006144353059177534, + "loss": 1.3091, + "theoretical_loss": 3.7618359759920734, + "tokens_seen": 790953984 + }, + { + "epoch": 0.24, + "learning_rate": 0.00061441925777332, + "loss": 1.3588, + "theoretical_loss": 3.7618052499884964, + "tokens_seen": 791019520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006144032096288868, + "loss": 1.4681, + "theoretical_loss": 3.7617745272431797, + "tokens_seen": 791085056 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006143871614844534, + "loss": 1.427, + "theoretical_loss": 3.761743807755507, + "tokens_seen": 791150592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006143711133400202, + "loss": 1.3612, + "theoretical_loss": 3.7617130915248644, + "tokens_seen": 791216128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006143550651955868, + "loss": 1.4023, + "theoretical_loss": 3.7616823785506357, + "tokens_seen": 791281664 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006143390170511536, + "loss": 1.4632, + "theoretical_loss": 3.761651668832207, + "tokens_seen": 791347200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006143229689067202, + "loss": 1.4643, + "theoretical_loss": 3.761620962368963, + "tokens_seen": 791412736 + }, + { + "epoch": 0.24, + "learning_rate": 0.000614306920762287, + "loss": 1.4261, + "theoretical_loss": 3.7615902591602888, + "tokens_seen": 791478272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006142908726178536, + "loss": 1.4255, + "theoretical_loss": 3.761559559205571, + "tokens_seen": 791543808 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006142748244734204, + "loss": 1.4106, + "theoretical_loss": 3.7615288625041945, + "tokens_seen": 791609344 + }, + { + "epoch": 0.24, + "learning_rate": 0.000614258776328987, + "loss": 1.3901, + "theoretical_loss": 3.7614981690555456, + "tokens_seen": 791674880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006142427281845538, + "loss": 1.3109, + "theoretical_loss": 3.7614674788590103, + "tokens_seen": 791740416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006142266800401204, + "loss": 1.5292, + "theoretical_loss": 3.761436791913975, + "tokens_seen": 791805952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006142106318956872, + "loss": 1.5059, + "theoretical_loss": 3.761406108219826, + "tokens_seen": 791871488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006141945837512538, + "loss": 1.3547, + "theoretical_loss": 3.76137542777595, + "tokens_seen": 791937024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006141785356068205, + "loss": 1.3849, + "theoretical_loss": 3.761344750581734, + "tokens_seen": 792002560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006141624874623872, + "loss": 1.4683, + "theoretical_loss": 3.7613140766365643, + "tokens_seen": 792068096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006141464393179539, + "loss": 1.4557, + "theoretical_loss": 3.7612834059398286, + "tokens_seen": 792133632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006141303911735206, + "loss": 1.4257, + "theoretical_loss": 3.7612527384909136, + "tokens_seen": 792199168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006141143430290873, + "loss": 1.4322, + "theoretical_loss": 3.7612220742892077, + "tokens_seen": 792264704 + }, + { + "epoch": 0.24, + "learning_rate": 0.000614098294884654, + "loss": 1.4328, + "theoretical_loss": 3.761191413334098, + "tokens_seen": 792330240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006140822467402207, + "loss": 1.3899, + "theoretical_loss": 3.7611607556249718, + "tokens_seen": 792395776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006140661985957874, + "loss": 1.3818, + "theoretical_loss": 3.761130101161217, + "tokens_seen": 792461312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006140501504513541, + "loss": 1.4745, + "theoretical_loss": 3.761099449942223, + "tokens_seen": 792526848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006140341023069207, + "loss": 1.3391, + "theoretical_loss": 3.7610688019673764, + "tokens_seen": 792592384 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006140180541624875, + "loss": 1.3971, + "theoretical_loss": 3.761038157236067, + "tokens_seen": 792657920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006140020060180541, + "loss": 1.3484, + "theoretical_loss": 3.7610075157476826, + "tokens_seen": 792723456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006139859578736209, + "loss": 1.4146, + "theoretical_loss": 3.760976877501612, + "tokens_seen": 792788992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006139699097291875, + "loss": 1.4697, + "theoretical_loss": 3.7609462424972446, + "tokens_seen": 792854528 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006139538615847543, + "loss": 1.3815, + "theoretical_loss": 3.760915610733969, + "tokens_seen": 792920064 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 298936, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.464165449142456, + "objective/train/theoretical_loss": 3.7608849822111754, + "objective/train/tokens_used": 813445600, + "theoretical_loss": 3.7608849822111754, + "tokens_seen": 792985600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006139378134403209, + "loss": 1.4642, + "theoretical_loss": 3.7608849822111754, + "tokens_seen": 792985600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006139217652958877, + "loss": 1.5003, + "theoretical_loss": 3.7608543569282515, + "tokens_seen": 793051136 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006139057171514543, + "loss": 1.5275, + "theoretical_loss": 3.7608237348845885, + "tokens_seen": 793116672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006138896690070211, + "loss": 1.4937, + "theoretical_loss": 3.760793116079576, + "tokens_seen": 793182208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006138736208625877, + "loss": 1.5334, + "theoretical_loss": 3.7607625005126026, + "tokens_seen": 793247744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006138575727181545, + "loss": 1.4422, + "theoretical_loss": 3.7607318881830594, + "tokens_seen": 793313280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006138415245737211, + "loss": 1.5173, + "theoretical_loss": 3.7607012790903367, + "tokens_seen": 793378816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006138254764292879, + "loss": 1.5348, + "theoretical_loss": 3.7606706732338253, + "tokens_seen": 793444352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006138094282848546, + "loss": 1.5778, + "theoretical_loss": 3.7606400706129146, + "tokens_seen": 793509888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006137933801404213, + "loss": 1.4604, + "theoretical_loss": 3.760609471226996, + "tokens_seen": 793575424 + }, + { + "epoch": 0.24, + "learning_rate": 0.000613777331995988, + "loss": 1.4356, + "theoretical_loss": 3.7605788750754607, + "tokens_seen": 793640960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006137612838515547, + "loss": 1.5132, + "theoretical_loss": 3.7605482821577, + "tokens_seen": 793706496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006137452357071214, + "loss": 1.6228, + "theoretical_loss": 3.760517692473104, + "tokens_seen": 793772032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006137291875626881, + "loss": 1.5889, + "theoretical_loss": 3.7604871060210643, + "tokens_seen": 793837568 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006137131394182548, + "loss": 1.4168, + "theoretical_loss": 3.7604565228009736, + "tokens_seen": 793903104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006136970912738215, + "loss": 1.4644, + "theoretical_loss": 3.760425942812223, + "tokens_seen": 793968640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006136810431293882, + "loss": 1.3748, + "theoretical_loss": 3.760395366054204, + "tokens_seen": 794034176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006136649949849549, + "loss": 1.424, + "theoretical_loss": 3.7603647925263095, + "tokens_seen": 794099712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006136489468405216, + "loss": 1.4418, + "theoretical_loss": 3.7603342222279315, + "tokens_seen": 794165248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006136328986960883, + "loss": 1.3789, + "theoretical_loss": 3.760303655158461, + "tokens_seen": 794230784 + }, + { + "epoch": 0.24, + "learning_rate": 0.000613616850551655, + "loss": 1.396, + "theoretical_loss": 3.760273091317293, + "tokens_seen": 794296320 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006136008024072217, + "loss": 1.4989, + "theoretical_loss": 3.7602425307038185, + "tokens_seen": 794361856 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006135847542627884, + "loss": 1.486, + "theoretical_loss": 3.760211973317431, + "tokens_seen": 794427392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006135687061183551, + "loss": 1.351, + "theoretical_loss": 3.760181419157523, + "tokens_seen": 794492928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006135526579739218, + "loss": 1.4642, + "theoretical_loss": 3.7601508682234885, + "tokens_seen": 794558464 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006135366098294885, + "loss": 1.5075, + "theoretical_loss": 3.7601203205147202, + "tokens_seen": 794624000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006135205616850552, + "loss": 1.3534, + "theoretical_loss": 3.760089776030613, + "tokens_seen": 794689536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006135045135406219, + "loss": 1.4512, + "theoretical_loss": 3.7600592347705586, + "tokens_seen": 794755072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006134884653961886, + "loss": 1.4186, + "theoretical_loss": 3.7600286967339525, + "tokens_seen": 794820608 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006134724172517553, + "loss": 1.4414, + "theoretical_loss": 3.759998161920187, + "tokens_seen": 794886144 + }, + { + "epoch": 0.24, + "learning_rate": 0.000613456369107322, + "loss": 1.3882, + "theoretical_loss": 3.7599676303286587, + "tokens_seen": 794951680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006134403209628887, + "loss": 1.5386, + "theoretical_loss": 3.75993710195876, + "tokens_seen": 795017216 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006134242728184554, + "loss": 1.4796, + "theoretical_loss": 3.759906576809886, + "tokens_seen": 795082752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006134082246740221, + "loss": 1.4204, + "theoretical_loss": 3.759876054881432, + "tokens_seen": 795148288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006133921765295888, + "loss": 1.479, + "theoretical_loss": 3.759845536172792, + "tokens_seen": 795213824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006133761283851555, + "loss": 1.4669, + "theoretical_loss": 3.7598150206833614, + "tokens_seen": 795279360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006133600802407222, + "loss": 1.4437, + "theoretical_loss": 3.759784508412536, + "tokens_seen": 795344896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006133440320962889, + "loss": 1.3883, + "theoretical_loss": 3.7597539993597096, + "tokens_seen": 795410432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006133279839518556, + "loss": 1.3684, + "theoretical_loss": 3.7597234935242794, + "tokens_seen": 795475968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006133119358074223, + "loss": 1.4055, + "theoretical_loss": 3.7596929909056396, + "tokens_seen": 795541504 + }, + { + "epoch": 0.24, + "learning_rate": 0.000613295887662989, + "loss": 1.3471, + "theoretical_loss": 3.759662491503187, + "tokens_seen": 795607040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006132798395185557, + "loss": 1.4267, + "theoretical_loss": 3.759631995316318, + "tokens_seen": 795672576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006132637913741224, + "loss": 1.4015, + "theoretical_loss": 3.7596015023444274, + "tokens_seen": 795738112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006132477432296891, + "loss": 1.4337, + "theoretical_loss": 3.759571012586913, + "tokens_seen": 795803648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006132316950852558, + "loss": 1.4085, + "theoretical_loss": 3.75954052604317, + "tokens_seen": 795869184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006132156469408225, + "loss": 1.5206, + "theoretical_loss": 3.759510042712596, + "tokens_seen": 795934720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006131995987963892, + "loss": 1.4628, + "theoretical_loss": 3.7594795625945867, + "tokens_seen": 796000256 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006131835506519559, + "loss": 1.4644, + "theoretical_loss": 3.75944908568854, + "tokens_seen": 796065792 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006131675025075226, + "loss": 1.4426, + "theoretical_loss": 3.759418611993853, + "tokens_seen": 796131328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006131514543630893, + "loss": 1.3447, + "theoretical_loss": 3.7593881415099233, + "tokens_seen": 796196864 + }, + { + "epoch": 0.24, + "learning_rate": 0.000613135406218656, + "loss": 1.4333, + "theoretical_loss": 3.7593576742361474, + "tokens_seen": 796262400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006131193580742227, + "loss": 1.552, + "theoretical_loss": 3.759327210171924, + "tokens_seen": 796327936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006131033099297894, + "loss": 1.3931, + "theoretical_loss": 3.7592967493166496, + "tokens_seen": 796393472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006130872617853561, + "loss": 1.4333, + "theoretical_loss": 3.7592662916697237, + "tokens_seen": 796459008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006130712136409228, + "loss": 1.4643, + "theoretical_loss": 3.759235837230543, + "tokens_seen": 796524544 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006130551654964895, + "loss": 1.4644, + "theoretical_loss": 3.7592053859985066, + "tokens_seen": 796590080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006130391173520563, + "loss": 1.3593, + "theoretical_loss": 3.759174937973013, + "tokens_seen": 796655616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006130230692076229, + "loss": 1.3333, + "theoretical_loss": 3.7591444931534603, + "tokens_seen": 796721152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006130070210631897, + "loss": 1.4518, + "theoretical_loss": 3.759114051539248, + "tokens_seen": 796786688 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006129909729187563, + "loss": 1.4433, + "theoretical_loss": 3.759083613129774, + "tokens_seen": 796852224 + }, + { + "epoch": 0.24, + "learning_rate": 0.000612974924774323, + "loss": 1.4577, + "theoretical_loss": 3.7590531779244385, + "tokens_seen": 796917760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006129588766298897, + "loss": 1.3704, + "theoretical_loss": 3.75902274592264, + "tokens_seen": 796983296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006129428284854565, + "loss": 1.3831, + "theoretical_loss": 3.758992317123778, + "tokens_seen": 797048832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006129267803410231, + "loss": 1.4317, + "theoretical_loss": 3.7589618915272522, + "tokens_seen": 797114368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006129107321965899, + "loss": 1.5445, + "theoretical_loss": 3.7589314691324622, + "tokens_seen": 797179904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006128946840521565, + "loss": 1.4091, + "theoretical_loss": 3.7589010499388085, + "tokens_seen": 797245440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006128786359077233, + "loss": 1.5017, + "theoretical_loss": 3.7588706339456905, + "tokens_seen": 797310976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006128625877632899, + "loss": 1.4897, + "theoretical_loss": 3.7588402211525085, + "tokens_seen": 797376512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006128465396188567, + "loss": 1.4441, + "theoretical_loss": 3.7588098115586632, + "tokens_seen": 797442048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006128304914744233, + "loss": 1.425, + "theoretical_loss": 3.758779405163555, + "tokens_seen": 797507584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006128144433299901, + "loss": 1.5728, + "theoretical_loss": 3.7587490019665846, + "tokens_seen": 797573120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127983951855567, + "loss": 1.3692, + "theoretical_loss": 3.7587186019671526, + "tokens_seen": 797638656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127823470411234, + "loss": 1.5427, + "theoretical_loss": 3.7586882051646606, + "tokens_seen": 797704192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127662988966901, + "loss": 1.3932, + "theoretical_loss": 3.7586578115585096, + "tokens_seen": 797769728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127502507522568, + "loss": 1.4266, + "theoretical_loss": 3.758627421148101, + "tokens_seen": 797835264 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127342026078235, + "loss": 1.4246, + "theoretical_loss": 3.758597033932836, + "tokens_seen": 797900800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127181544633902, + "loss": 1.4268, + "theoretical_loss": 3.758566649912116, + "tokens_seen": 797966336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006127021063189569, + "loss": 1.4033, + "theoretical_loss": 3.7585362690853445, + "tokens_seen": 798031872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006126860581745236, + "loss": 1.39, + "theoretical_loss": 3.758505891451921, + "tokens_seen": 798097408 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006126700100300903, + "loss": 1.3615, + "theoretical_loss": 3.7584755170112496, + "tokens_seen": 798162944 + }, + { + "epoch": 0.24, + "learning_rate": 0.000612653961885657, + "loss": 1.3507, + "theoretical_loss": 3.758445145762732, + "tokens_seen": 798228480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006126379137412236, + "loss": 1.4568, + "theoretical_loss": 3.758414777705771, + "tokens_seen": 798294016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006126218655967904, + "loss": 1.4975, + "theoretical_loss": 3.7583844128397685, + "tokens_seen": 798359552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006126058174523571, + "loss": 1.3911, + "theoretical_loss": 3.758354051164128, + "tokens_seen": 798425088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006125897693079238, + "loss": 1.4882, + "theoretical_loss": 3.7583236926782515, + "tokens_seen": 798490624 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006125737211634905, + "loss": 1.4584, + "theoretical_loss": 3.7582933373815433, + "tokens_seen": 798556160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006125576730190572, + "loss": 1.4676, + "theoretical_loss": 3.758262985273406, + "tokens_seen": 798621696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006125416248746239, + "loss": 1.3085, + "theoretical_loss": 3.7582326363532443, + "tokens_seen": 798687232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006125255767301906, + "loss": 1.4878, + "theoretical_loss": 3.75820229062046, + "tokens_seen": 798752768 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006125095285857573, + "loss": 1.4115, + "theoretical_loss": 3.758171948074457, + "tokens_seen": 798818304 + }, + { + "epoch": 0.24, + "learning_rate": 0.000612493480441324, + "loss": 1.4715, + "theoretical_loss": 3.7581416087146406, + "tokens_seen": 798883840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006124774322968907, + "loss": 1.4209, + "theoretical_loss": 3.7581112725404147, + "tokens_seen": 798949376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006124613841524574, + "loss": 1.4207, + "theoretical_loss": 3.7580809395511823, + "tokens_seen": 799014912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006124453360080241, + "loss": 1.4347, + "theoretical_loss": 3.7580506097463484, + "tokens_seen": 799080448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006124292878635908, + "loss": 1.4591, + "theoretical_loss": 3.758020283125318, + "tokens_seen": 799145984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006124132397191575, + "loss": 1.499, + "theoretical_loss": 3.757989959687496, + "tokens_seen": 799211520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006123971915747242, + "loss": 1.3894, + "theoretical_loss": 3.7579596394322863, + "tokens_seen": 799277056 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006123811434302909, + "loss": 1.3904, + "theoretical_loss": 3.757929322359095, + "tokens_seen": 799342592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006123650952858576, + "loss": 1.3727, + "theoretical_loss": 3.7578990084673256, + "tokens_seen": 799408128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006123490471414243, + "loss": 1.4144, + "theoretical_loss": 3.7578686977563858, + "tokens_seen": 799473664 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 301321, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.245927095413208, + "objective/train/theoretical_loss": 3.75783839022568, + "objective/train/tokens_used": 819999200, + "theoretical_loss": 3.75783839022568, + "tokens_seen": 799539200 + }, + { + "epoch": 0.24, + "learning_rate": 0.000612332998996991, + "loss": 1.2459, + "theoretical_loss": 3.75783839022568, + "tokens_seen": 799539200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006123169508525577, + "loss": 1.383, + "theoretical_loss": 3.757808085874613, + "tokens_seen": 799604736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006123009027081244, + "loss": 1.4364, + "theoretical_loss": 3.7577777847025926, + "tokens_seen": 799670272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006122848545636911, + "loss": 1.3681, + "theoretical_loss": 3.757747486709023, + "tokens_seen": 799735808 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006122688064192578, + "loss": 1.3975, + "theoretical_loss": 3.7577171918933114, + "tokens_seen": 799801344 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006122527582748245, + "loss": 1.461, + "theoretical_loss": 3.757686900254864, + "tokens_seen": 799866880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006122367101303912, + "loss": 1.4184, + "theoretical_loss": 3.757656611793087, + "tokens_seen": 799932416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006122206619859579, + "loss": 1.3823, + "theoretical_loss": 3.7576263265073875, + "tokens_seen": 799997952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006122046138415246, + "loss": 1.3179, + "theoretical_loss": 3.7575960443971717, + "tokens_seen": 800063488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006121885656970913, + "loss": 1.3823, + "theoretical_loss": 3.7575657654618473, + "tokens_seen": 800129024 + }, + { + "epoch": 0.24, + "learning_rate": 0.000612172517552658, + "loss": 1.4372, + "theoretical_loss": 3.75753548970082, + "tokens_seen": 800194560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006121564694082247, + "loss": 1.4028, + "theoretical_loss": 3.757505217113499, + "tokens_seen": 800260096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006121404212637914, + "loss": 1.4854, + "theoretical_loss": 3.757474947699291, + "tokens_seen": 800325632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006121243731193581, + "loss": 1.4319, + "theoretical_loss": 3.757444681457603, + "tokens_seen": 800391168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006121083249749248, + "loss": 1.423, + "theoretical_loss": 3.7574144183878433, + "tokens_seen": 800456704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006120922768304915, + "loss": 1.5111, + "theoretical_loss": 3.75738415848942, + "tokens_seen": 800522240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006120762286860582, + "loss": 1.4163, + "theoretical_loss": 3.7573539017617406, + "tokens_seen": 800587776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006120601805416249, + "loss": 1.3663, + "theoretical_loss": 3.7573236482042134, + "tokens_seen": 800653312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006120441323971916, + "loss": 1.4197, + "theoretical_loss": 3.7572933978162477, + "tokens_seen": 800718848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006120280842527583, + "loss": 1.442, + "theoretical_loss": 3.757263150597251, + "tokens_seen": 800784384 + }, + { + "epoch": 0.24, + "learning_rate": 0.000612012036108325, + "loss": 1.468, + "theoretical_loss": 3.7572329065466326, + "tokens_seen": 800849920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006119959879638917, + "loss": 1.376, + "theoretical_loss": 3.7572026656638013, + "tokens_seen": 800915456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006119799398194584, + "loss": 1.3456, + "theoretical_loss": 3.757172427948166, + "tokens_seen": 800980992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006119638916750251, + "loss": 1.3999, + "theoretical_loss": 3.7571421933991362, + "tokens_seen": 801046528 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006119478435305918, + "loss": 1.3241, + "theoretical_loss": 3.7571119620161206, + "tokens_seen": 801112064 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006119317953861585, + "loss": 1.4323, + "theoretical_loss": 3.7570817337985294, + "tokens_seen": 801177600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006119157472417252, + "loss": 1.4101, + "theoretical_loss": 3.757051508745772, + "tokens_seen": 801243136 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006118996990972919, + "loss": 1.4042, + "theoretical_loss": 3.757021286857258, + "tokens_seen": 801308672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006118836509528586, + "loss": 1.4198, + "theoretical_loss": 3.7569910681323977, + "tokens_seen": 801374208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006118676028084253, + "loss": 1.3217, + "theoretical_loss": 3.756960852570601, + "tokens_seen": 801439744 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611851554663992, + "loss": 1.3558, + "theoretical_loss": 3.7569306401712783, + "tokens_seen": 801505280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006118355065195587, + "loss": 1.4909, + "theoretical_loss": 3.7569004309338405, + "tokens_seen": 801570816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006118194583751254, + "loss": 1.4775, + "theoretical_loss": 3.756870224857698, + "tokens_seen": 801636352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006118034102306921, + "loss": 1.3895, + "theoretical_loss": 3.756840021942261, + "tokens_seen": 801701888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006117873620862588, + "loss": 1.3543, + "theoretical_loss": 3.756809822186941, + "tokens_seen": 801767424 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006117713139418255, + "loss": 1.4194, + "theoretical_loss": 3.7567796255911485, + "tokens_seen": 801832960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006117552657973922, + "loss": 1.4865, + "theoretical_loss": 3.7567494321542956, + "tokens_seen": 801898496 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611739217652959, + "loss": 1.3825, + "theoretical_loss": 3.756719241875793, + "tokens_seen": 801964032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006117231695085256, + "loss": 1.4758, + "theoretical_loss": 3.7566890547550527, + "tokens_seen": 802029568 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006117071213640923, + "loss": 1.4113, + "theoretical_loss": 3.7566588707914867, + "tokens_seen": 802095104 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611691073219659, + "loss": 1.4737, + "theoretical_loss": 3.756628689984506, + "tokens_seen": 802160640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006116750250752258, + "loss": 1.353, + "theoretical_loss": 3.756598512333523, + "tokens_seen": 802226176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006116589769307924, + "loss": 1.3367, + "theoretical_loss": 3.75656833783795, + "tokens_seen": 802291712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006116429287863592, + "loss": 1.3626, + "theoretical_loss": 3.7565381664972, + "tokens_seen": 802357248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006116268806419258, + "loss": 1.3667, + "theoretical_loss": 3.7565079983106844, + "tokens_seen": 802422784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006116108324974926, + "loss": 1.4427, + "theoretical_loss": 3.7564778332778164, + "tokens_seen": 802488320 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006115947843530592, + "loss": 1.5569, + "theoretical_loss": 3.756447671398009, + "tokens_seen": 802553856 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611578736208626, + "loss": 1.4525, + "theoretical_loss": 3.7564175126706743, + "tokens_seen": 802619392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006115626880641926, + "loss": 1.4525, + "theoretical_loss": 3.7563873570952264, + "tokens_seen": 802684928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006115466399197594, + "loss": 1.4502, + "theoretical_loss": 3.7563572046710787, + "tokens_seen": 802750464 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611530591775326, + "loss": 1.4434, + "theoretical_loss": 3.7563270553976436, + "tokens_seen": 802816000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006115145436308928, + "loss": 1.393, + "theoretical_loss": 3.7562969092743357, + "tokens_seen": 802881536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006114984954864594, + "loss": 1.444, + "theoretical_loss": 3.7562667663005684, + "tokens_seen": 802947072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006114824473420262, + "loss": 1.4487, + "theoretical_loss": 3.7562366264757556, + "tokens_seen": 803012608 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006114663991975928, + "loss": 1.3675, + "theoretical_loss": 3.756206489799311, + "tokens_seen": 803078144 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006114503510531596, + "loss": 1.4389, + "theoretical_loss": 3.7561763562706503, + "tokens_seen": 803143680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006114343029087262, + "loss": 1.4278, + "theoretical_loss": 3.7561462258891862, + "tokens_seen": 803209216 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611418254764293, + "loss": 1.4131, + "theoretical_loss": 3.756116098654334, + "tokens_seen": 803274752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006114022066198596, + "loss": 1.4228, + "theoretical_loss": 3.7560859745655084, + "tokens_seen": 803340288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006113861584754263, + "loss": 1.3832, + "theoretical_loss": 3.756055853622124, + "tokens_seen": 803405824 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611370110330993, + "loss": 1.4133, + "theoretical_loss": 3.756025735823596, + "tokens_seen": 803471360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006113540621865597, + "loss": 1.3373, + "theoretical_loss": 3.7559956211693404, + "tokens_seen": 803536896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006113380140421264, + "loss": 1.4691, + "theoretical_loss": 3.7559655096587705, + "tokens_seen": 803602432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006113219658976931, + "loss": 1.4356, + "theoretical_loss": 3.7559354012913038, + "tokens_seen": 803667968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006113059177532598, + "loss": 1.4537, + "theoretical_loss": 3.755905296066355, + "tokens_seen": 803733504 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006112898696088265, + "loss": 1.4352, + "theoretical_loss": 3.75587519398334, + "tokens_seen": 803799040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006112738214643932, + "loss": 1.397, + "theoretical_loss": 3.755845095041675, + "tokens_seen": 803864576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006112577733199599, + "loss": 1.4925, + "theoretical_loss": 3.755814999240776, + "tokens_seen": 803930112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006112417251755266, + "loss": 1.4027, + "theoretical_loss": 3.7557849065800593, + "tokens_seen": 803995648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006112256770310933, + "loss": 1.4151, + "theoretical_loss": 3.755754817058941, + "tokens_seen": 804061184 + }, + { + "epoch": 0.24, + "learning_rate": 0.00061120962888666, + "loss": 1.362, + "theoretical_loss": 3.755724730676838, + "tokens_seen": 804126720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006111935807422267, + "loss": 1.429, + "theoretical_loss": 3.7556946474331667, + "tokens_seen": 804192256 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006111775325977934, + "loss": 1.4984, + "theoretical_loss": 3.7556645673273445, + "tokens_seen": 804257792 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006111614844533601, + "loss": 1.4729, + "theoretical_loss": 3.755634490358789, + "tokens_seen": 804323328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006111454363089268, + "loss": 1.4827, + "theoretical_loss": 3.7556044165269156, + "tokens_seen": 804388864 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006111293881644935, + "loss": 1.5564, + "theoretical_loss": 3.7555743458311435, + "tokens_seen": 804454400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006111133400200602, + "loss": 1.4667, + "theoretical_loss": 3.7555442782708885, + "tokens_seen": 804519936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006110972918756269, + "loss": 1.4431, + "theoretical_loss": 3.75551421384557, + "tokens_seen": 804585472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006110812437311936, + "loss": 1.4178, + "theoretical_loss": 3.755484152554605, + "tokens_seen": 804651008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006110651955867603, + "loss": 1.423, + "theoretical_loss": 3.755454094397411, + "tokens_seen": 804716544 + }, + { + "epoch": 0.24, + "learning_rate": 0.000611049147442327, + "loss": 1.4931, + "theoretical_loss": 3.7554240393734073, + "tokens_seen": 804782080 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006110330992978937, + "loss": 1.4771, + "theoretical_loss": 3.755393987482011, + "tokens_seen": 804847616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006110170511534604, + "loss": 1.556, + "theoretical_loss": 3.755363938722642, + "tokens_seen": 804913152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006110010030090271, + "loss": 1.4867, + "theoretical_loss": 3.7553338930947167, + "tokens_seen": 804978688 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006109849548645938, + "loss": 1.4516, + "theoretical_loss": 3.7553038505976564, + "tokens_seen": 805044224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006109689067201605, + "loss": 1.3851, + "theoretical_loss": 3.7552738112308783, + "tokens_seen": 805109760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006109528585757272, + "loss": 1.4452, + "theoretical_loss": 3.7552437749938017, + "tokens_seen": 805175296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006109368104312939, + "loss": 1.4016, + "theoretical_loss": 3.755213741885847, + "tokens_seen": 805240832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006109207622868606, + "loss": 1.4594, + "theoretical_loss": 3.755183711906432, + "tokens_seen": 805306368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006109047141424273, + "loss": 1.407, + "theoretical_loss": 3.755153685054977, + "tokens_seen": 805371904 + }, + { + "epoch": 0.24, + "learning_rate": 0.000610888665997994, + "loss": 1.4916, + "theoretical_loss": 3.755123661330902, + "tokens_seen": 805437440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006108726178535607, + "loss": 1.4508, + "theoretical_loss": 3.755093640733626, + "tokens_seen": 805502976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006108565697091274, + "loss": 1.3616, + "theoretical_loss": 3.75506362326257, + "tokens_seen": 805568512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006108405215646941, + "loss": 1.5195, + "theoretical_loss": 3.755033608917153, + "tokens_seen": 805634048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006108244734202608, + "loss": 1.4473, + "theoretical_loss": 3.7550035976967964, + "tokens_seen": 805699584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006108084252758275, + "loss": 1.4535, + "theoretical_loss": 3.7549735896009206, + "tokens_seen": 805765120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006107923771313942, + "loss": 1.4389, + "theoretical_loss": 3.754943584628945, + "tokens_seen": 805830656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006107763289869609, + "loss": 1.4571, + "theoretical_loss": 3.7549135827802917, + "tokens_seen": 805896192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006107602808425276, + "loss": 1.5026, + "theoretical_loss": 3.7548835840543813, + "tokens_seen": 805961728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006107442326980943, + "loss": 1.3277, + "theoretical_loss": 3.7548535884506347, + "tokens_seen": 806027264 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 303663, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4312368631362915, + "objective/train/theoretical_loss": 3.754823595968473, + "objective/train/tokens_used": 826552800, + "theoretical_loss": 3.754823595968473, + "tokens_seen": 806092800 + }, + { + "epoch": 0.24, + "learning_rate": 0.000610728184553661, + "loss": 1.4312, + "theoretical_loss": 3.754823595968473, + "tokens_seen": 806092800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006107121364092277, + "loss": 1.4757, + "theoretical_loss": 3.7547936066073175, + "tokens_seen": 806158336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006106960882647944, + "loss": 1.4389, + "theoretical_loss": 3.7547636203665906, + "tokens_seen": 806223872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006106800401203611, + "loss": 1.5144, + "theoretical_loss": 3.7547336372457134, + "tokens_seen": 806289408 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006106639919759278, + "loss": 1.4485, + "theoretical_loss": 3.7547036572441073, + "tokens_seen": 806354944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006106479438314945, + "loss": 1.4746, + "theoretical_loss": 3.7546736803611953, + "tokens_seen": 806420480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006106318956870612, + "loss": 1.3978, + "theoretical_loss": 3.754643706596399, + "tokens_seen": 806486016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006106158475426279, + "loss": 1.3386, + "theoretical_loss": 3.7546137359491407, + "tokens_seen": 806551552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006105997993981946, + "loss": 1.4675, + "theoretical_loss": 3.7545837684188426, + "tokens_seen": 806617088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006105837512537613, + "loss": 1.4285, + "theoretical_loss": 3.7545538040049284, + "tokens_seen": 806682624 + }, + { + "epoch": 0.24, + "learning_rate": 0.000610567703109328, + "loss": 1.4335, + "theoretical_loss": 3.7545238427068197, + "tokens_seen": 806748160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006105516549648947, + "loss": 1.4562, + "theoretical_loss": 3.7544938845239404, + "tokens_seen": 806813696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006105356068204614, + "loss": 1.3776, + "theoretical_loss": 3.7544639294557127, + "tokens_seen": 806879232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006105195586760281, + "loss": 1.4826, + "theoretical_loss": 3.7544339775015603, + "tokens_seen": 806944768 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006105035105315948, + "loss": 1.3946, + "theoretical_loss": 3.7544040286609066, + "tokens_seen": 807010304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006104874623871615, + "loss": 1.3849, + "theoretical_loss": 3.7543740829331744, + "tokens_seen": 807075840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006104714142427282, + "loss": 1.3952, + "theoretical_loss": 3.754344140317789, + "tokens_seen": 807141376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006104553660982949, + "loss": 1.4137, + "theoretical_loss": 3.754314200814173, + "tokens_seen": 807206912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006104393179538616, + "loss": 1.5043, + "theoretical_loss": 3.754284264421751, + "tokens_seen": 807272448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006104232698094283, + "loss": 1.4024, + "theoretical_loss": 3.754254331139946, + "tokens_seen": 807337984 + }, + { + "epoch": 0.24, + "learning_rate": 0.000610407221664995, + "loss": 1.4397, + "theoretical_loss": 3.7542244009681847, + "tokens_seen": 807403520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006103911735205617, + "loss": 1.5229, + "theoretical_loss": 3.7541944739058892, + "tokens_seen": 807469056 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006103751253761284, + "loss": 1.4269, + "theoretical_loss": 3.754164549952485, + "tokens_seen": 807534592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006103590772316951, + "loss": 1.5104, + "theoretical_loss": 3.754134629107397, + "tokens_seen": 807600128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006103430290872618, + "loss": 1.4778, + "theoretical_loss": 3.7541047113700503, + "tokens_seen": 807665664 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006103269809428285, + "loss": 1.3901, + "theoretical_loss": 3.7540747967398698, + "tokens_seen": 807731200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006103109327983953, + "loss": 1.4706, + "theoretical_loss": 3.75404488521628, + "tokens_seen": 807796736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006102948846539619, + "loss": 1.4786, + "theoretical_loss": 3.7540149767987074, + "tokens_seen": 807862272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006102788365095287, + "loss": 1.3857, + "theoretical_loss": 3.753985071486577, + "tokens_seen": 807927808 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006102627883650953, + "loss": 1.4258, + "theoretical_loss": 3.7539551692793145, + "tokens_seen": 807993344 + }, + { + "epoch": 0.24, + "learning_rate": 0.000610246740220662, + "loss": 1.46, + "theoretical_loss": 3.753925270176346, + "tokens_seen": 808058880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006102306920762287, + "loss": 1.3652, + "theoretical_loss": 3.7538953741770973, + "tokens_seen": 808124416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006102146439317955, + "loss": 1.4604, + "theoretical_loss": 3.753865481280995, + "tokens_seen": 808189952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006101985957873622, + "loss": 1.4393, + "theoretical_loss": 3.753835591487464, + "tokens_seen": 808255488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006101825476429289, + "loss": 1.3751, + "theoretical_loss": 3.7538057047959326, + "tokens_seen": 808321024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006101664994984956, + "loss": 1.4404, + "theoretical_loss": 3.753775821205826, + "tokens_seen": 808386560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0006101504513540623, + "loss": 1.4447, + "theoretical_loss": 3.7537459407165716, + "tokens_seen": 808452096 + }, + { + "epoch": 0.25, + "learning_rate": 0.000610134403209629, + "loss": 1.4979, + "theoretical_loss": 3.7537160633275963, + "tokens_seen": 808517632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006101183550651957, + "loss": 1.4419, + "theoretical_loss": 3.7536861890383273, + "tokens_seen": 808583168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006101023069207623, + "loss": 1.4426, + "theoretical_loss": 3.7536563178481916, + "tokens_seen": 808648704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006100862587763291, + "loss": 1.4064, + "theoretical_loss": 3.753626449756616, + "tokens_seen": 808714240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006100702106318957, + "loss": 1.4199, + "theoretical_loss": 3.753596584763029, + "tokens_seen": 808779776 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006100541624874625, + "loss": 1.3277, + "theoretical_loss": 3.753566722866858, + "tokens_seen": 808845312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006100381143430291, + "loss": 1.3769, + "theoretical_loss": 3.753536864067531, + "tokens_seen": 808910848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006100220661985959, + "loss": 1.372, + "theoretical_loss": 3.7535070083644753, + "tokens_seen": 808976384 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006100060180541625, + "loss": 1.3814, + "theoretical_loss": 3.7534771557571194, + "tokens_seen": 809041920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006099899699097292, + "loss": 1.4068, + "theoretical_loss": 3.7534473062448916, + "tokens_seen": 809107456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006099739217652959, + "loss": 1.4483, + "theoretical_loss": 3.7534174598272205, + "tokens_seen": 809172992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006099578736208626, + "loss": 1.4229, + "theoretical_loss": 3.7533876165035345, + "tokens_seen": 809238528 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006099418254764293, + "loss": 1.4048, + "theoretical_loss": 3.7533577762732624, + "tokens_seen": 809304064 + }, + { + "epoch": 0.25, + "learning_rate": 0.000609925777331996, + "loss": 1.3686, + "theoretical_loss": 3.753327939135833, + "tokens_seen": 809369600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006099097291875627, + "loss": 1.5101, + "theoretical_loss": 3.7532981050906757, + "tokens_seen": 809435136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006098936810431294, + "loss": 1.4758, + "theoretical_loss": 3.753268274137219, + "tokens_seen": 809500672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006098776328986961, + "loss": 1.4439, + "theoretical_loss": 3.753238446274893, + "tokens_seen": 809566208 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006098615847542628, + "loss": 1.4046, + "theoretical_loss": 3.7532086215031266, + "tokens_seen": 809631744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006098455366098295, + "loss": 1.3121, + "theoretical_loss": 3.7531787998213497, + "tokens_seen": 809697280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006098294884653962, + "loss": 1.4517, + "theoretical_loss": 3.7531489812289927, + "tokens_seen": 809762816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006098134403209629, + "loss": 1.4309, + "theoretical_loss": 3.753119165725484, + "tokens_seen": 809828352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006097973921765296, + "loss": 1.3731, + "theoretical_loss": 3.753089353310255, + "tokens_seen": 809893888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006097813440320963, + "loss": 1.3393, + "theoretical_loss": 3.7530595439827357, + "tokens_seen": 809959424 + }, + { + "epoch": 0.25, + "learning_rate": 0.000609765295887663, + "loss": 1.3244, + "theoretical_loss": 3.7530297377423567, + "tokens_seen": 810024960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006097492477432297, + "loss": 1.4441, + "theoretical_loss": 3.752999934588548, + "tokens_seen": 810090496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006097331995987964, + "loss": 1.4039, + "theoretical_loss": 3.7529701345207402, + "tokens_seen": 810156032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006097171514543631, + "loss": 1.3765, + "theoretical_loss": 3.752940337538365, + "tokens_seen": 810221568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006097011033099298, + "loss": 1.4057, + "theoretical_loss": 3.752910543640853, + "tokens_seen": 810287104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006096850551654965, + "loss": 1.4546, + "theoretical_loss": 3.752880752827635, + "tokens_seen": 810352640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006096690070210632, + "loss": 1.4524, + "theoretical_loss": 3.752850965098143, + "tokens_seen": 810418176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006096529588766299, + "loss": 1.4754, + "theoretical_loss": 3.752821180451808, + "tokens_seen": 810483712 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006096369107321966, + "loss": 1.5441, + "theoretical_loss": 3.7527913988880615, + "tokens_seen": 810549248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006096208625877633, + "loss": 1.4721, + "theoretical_loss": 3.7527616204063357, + "tokens_seen": 810614784 + }, + { + "epoch": 0.25, + "learning_rate": 0.00060960481444333, + "loss": 1.5342, + "theoretical_loss": 3.7527318450060623, + "tokens_seen": 810680320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006095887662988967, + "loss": 1.5119, + "theoretical_loss": 3.752702072686673, + "tokens_seen": 810745856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006095727181544634, + "loss": 1.4874, + "theoretical_loss": 3.7526723034476004, + "tokens_seen": 810811392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006095566700100301, + "loss": 1.4281, + "theoretical_loss": 3.752642537288277, + "tokens_seen": 810876928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006095406218655968, + "loss": 1.5556, + "theoretical_loss": 3.752612774208135, + "tokens_seen": 810942464 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006095245737211635, + "loss": 1.4233, + "theoretical_loss": 3.752583014206608, + "tokens_seen": 811008000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006095085255767302, + "loss": 1.4071, + "theoretical_loss": 3.752553257283127, + "tokens_seen": 811073536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006094924774322969, + "loss": 1.4259, + "theoretical_loss": 3.7525235034371267, + "tokens_seen": 811139072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006094764292878636, + "loss": 1.5237, + "theoretical_loss": 3.7524937526680393, + "tokens_seen": 811204608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006094603811434303, + "loss": 1.4522, + "theoretical_loss": 3.752464004975298, + "tokens_seen": 811270144 + }, + { + "epoch": 0.25, + "learning_rate": 0.000609444332998997, + "loss": 1.4204, + "theoretical_loss": 3.7524342603583367, + "tokens_seen": 811335680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006094282848545637, + "loss": 1.3783, + "theoretical_loss": 3.752404518816588, + "tokens_seen": 811401216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006094122367101304, + "loss": 1.4046, + "theoretical_loss": 3.752374780349488, + "tokens_seen": 811466752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006093961885656971, + "loss": 1.3367, + "theoretical_loss": 3.752345044956468, + "tokens_seen": 811532288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006093801404212638, + "loss": 1.3766, + "theoretical_loss": 3.7523153126369633, + "tokens_seen": 811597824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006093640922768305, + "loss": 1.609, + "theoretical_loss": 3.7522855833904076, + "tokens_seen": 811663360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006093480441323972, + "loss": 1.4196, + "theoretical_loss": 3.7522558572162352, + "tokens_seen": 811728896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006093319959879639, + "loss": 1.5639, + "theoretical_loss": 3.752226134113881, + "tokens_seen": 811794432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006093159478435306, + "loss": 1.372, + "theoretical_loss": 3.7521964140827793, + "tokens_seen": 811859968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006092998996990973, + "loss": 1.4927, + "theoretical_loss": 3.7521666971223646, + "tokens_seen": 811925504 + }, + { + "epoch": 0.25, + "learning_rate": 0.000609283851554664, + "loss": 1.4662, + "theoretical_loss": 3.752136983232073, + "tokens_seen": 811991040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006092678034102307, + "loss": 1.5321, + "theoretical_loss": 3.7521072724113376, + "tokens_seen": 812056576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006092517552657974, + "loss": 1.4268, + "theoretical_loss": 3.7520775646595954, + "tokens_seen": 812122112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006092357071213641, + "loss": 1.3738, + "theoretical_loss": 3.752047859976281, + "tokens_seen": 812187648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006092196589769308, + "loss": 1.42, + "theoretical_loss": 3.75201815836083, + "tokens_seen": 812253184 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006092036108324975, + "loss": 1.3991, + "theoretical_loss": 3.751988459812678, + "tokens_seen": 812318720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006091875626880642, + "loss": 1.3719, + "theoretical_loss": 3.7519587643312606, + "tokens_seen": 812384256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006091715145436309, + "loss": 1.4029, + "theoretical_loss": 3.751929071916015, + "tokens_seen": 812449792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006091554663991976, + "loss": 1.3578, + "theoretical_loss": 3.751899382566376, + "tokens_seen": 812515328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006091394182547643, + "loss": 1.3346, + "theoretical_loss": 3.7518696962817795, + "tokens_seen": 812580864 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 306237, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4066722393035889, + "objective/train/theoretical_loss": 3.7518400130616634, + "objective/train/tokens_used": 833106400, + "theoretical_loss": 3.7518400130616634, + "tokens_seen": 812646400 + }, + { + "epoch": 0.25, + "learning_rate": 0.000609123370110331, + "loss": 1.4067, + "theoretical_loss": 3.7518400130616634, + "tokens_seen": 812646400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006091073219658977, + "loss": 1.4357, + "theoretical_loss": 3.751810332905463, + "tokens_seen": 812711936 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006090912738214644, + "loss": 1.4803, + "theoretical_loss": 3.751780655812616, + "tokens_seen": 812777472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006090752256770311, + "loss": 1.4539, + "theoretical_loss": 3.7517509817825583, + "tokens_seen": 812843008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006090591775325978, + "loss": 1.4915, + "theoretical_loss": 3.7517213108147276, + "tokens_seen": 812908544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006090431293881645, + "loss": 1.3757, + "theoretical_loss": 3.7516916429085603, + "tokens_seen": 812974080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006090270812437312, + "loss": 1.4024, + "theoretical_loss": 3.751661978063495, + "tokens_seen": 813039616 + }, + { + "epoch": 0.25, + "learning_rate": 0.000609011033099298, + "loss": 1.4954, + "theoretical_loss": 3.751632316278968, + "tokens_seen": 813105152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006089949849548647, + "loss": 1.4432, + "theoretical_loss": 3.751602657554418, + "tokens_seen": 813170688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006089789368104313, + "loss": 1.4967, + "theoretical_loss": 3.7515730018892812, + "tokens_seen": 813236224 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006089628886659981, + "loss": 1.541, + "theoretical_loss": 3.7515433492829966, + "tokens_seen": 813301760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006089468405215648, + "loss": 1.4113, + "theoretical_loss": 3.751513699735002, + "tokens_seen": 813367296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006089307923771315, + "loss": 1.5241, + "theoretical_loss": 3.7514840532447353, + "tokens_seen": 813432832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006089147442326982, + "loss": 1.3938, + "theoretical_loss": 3.7514544098116356, + "tokens_seen": 813498368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006088986960882649, + "loss": 1.4518, + "theoretical_loss": 3.7514247694351406, + "tokens_seen": 813563904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006088826479438316, + "loss": 1.4452, + "theoretical_loss": 3.7513951321146894, + "tokens_seen": 813629440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006088665997993983, + "loss": 1.4662, + "theoretical_loss": 3.7513654978497213, + "tokens_seen": 813694976 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608850551654965, + "loss": 1.2612, + "theoretical_loss": 3.751335866639674, + "tokens_seen": 813760512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006088345035105317, + "loss": 1.5427, + "theoretical_loss": 3.751306238483987, + "tokens_seen": 813826048 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006088184553660984, + "loss": 1.4326, + "theoretical_loss": 3.7512766133820996, + "tokens_seen": 813891584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006088024072216651, + "loss": 1.4346, + "theoretical_loss": 3.751246991333452, + "tokens_seen": 813957120 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006087863590772318, + "loss": 1.5025, + "theoretical_loss": 3.751217372337482, + "tokens_seen": 814022656 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006087703109327985, + "loss": 1.5074, + "theoretical_loss": 3.751187756393631, + "tokens_seen": 814088192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006087542627883652, + "loss": 1.3343, + "theoretical_loss": 3.751158143501338, + "tokens_seen": 814153728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006087382146439319, + "loss": 1.4288, + "theoretical_loss": 3.7511285336600433, + "tokens_seen": 814219264 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006087221664994986, + "loss": 1.4191, + "theoretical_loss": 3.7510989268691866, + "tokens_seen": 814284800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006087061183550652, + "loss": 1.5013, + "theoretical_loss": 3.751069323128209, + "tokens_seen": 814350336 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608690070210632, + "loss": 1.3999, + "theoretical_loss": 3.7510397224365493, + "tokens_seen": 814415872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006086740220661986, + "loss": 1.5202, + "theoretical_loss": 3.7510101247936496, + "tokens_seen": 814481408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006086579739217654, + "loss": 1.3912, + "theoretical_loss": 3.75098053019895, + "tokens_seen": 814546944 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608641925777332, + "loss": 1.4266, + "theoretical_loss": 3.7509509386518918, + "tokens_seen": 814612480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006086258776328988, + "loss": 1.42, + "theoretical_loss": 3.7509213501519154, + "tokens_seen": 814678016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006086098294884654, + "loss": 1.4885, + "theoretical_loss": 3.750891764698462, + "tokens_seen": 814743552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006085937813440322, + "loss": 1.4886, + "theoretical_loss": 3.750862182290973, + "tokens_seen": 814809088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006085777331995988, + "loss": 1.3532, + "theoretical_loss": 3.7508326029288908, + "tokens_seen": 814874624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006085616850551655, + "loss": 1.3854, + "theoretical_loss": 3.750803026611656, + "tokens_seen": 814940160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006085456369107322, + "loss": 1.3488, + "theoretical_loss": 3.7507734533387103, + "tokens_seen": 815005696 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006085295887662989, + "loss": 1.4773, + "theoretical_loss": 3.7507438831094957, + "tokens_seen": 815071232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006085135406218656, + "loss": 1.4157, + "theoretical_loss": 3.750714315923454, + "tokens_seen": 815136768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006084974924774323, + "loss": 1.3467, + "theoretical_loss": 3.7506847517800286, + "tokens_seen": 815202304 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608481444332999, + "loss": 1.4641, + "theoretical_loss": 3.7506551906786605, + "tokens_seen": 815267840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006084653961885657, + "loss": 1.5525, + "theoretical_loss": 3.7506256326187923, + "tokens_seen": 815333376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006084493480441324, + "loss": 1.4317, + "theoretical_loss": 3.7505960775998677, + "tokens_seen": 815398912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006084332998996991, + "loss": 1.4137, + "theoretical_loss": 3.7505665256213288, + "tokens_seen": 815464448 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006084172517552658, + "loss": 1.4844, + "theoretical_loss": 3.7505369766826178, + "tokens_seen": 815529984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006084012036108325, + "loss": 1.4832, + "theoretical_loss": 3.7505074307831787, + "tokens_seen": 815595520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006083851554663992, + "loss": 1.4327, + "theoretical_loss": 3.7504778879224547, + "tokens_seen": 815661056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006083691073219659, + "loss": 1.4491, + "theoretical_loss": 3.750448348099889, + "tokens_seen": 815726592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006083530591775326, + "loss": 1.3905, + "theoretical_loss": 3.7504188113149244, + "tokens_seen": 815792128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006083370110330993, + "loss": 1.4112, + "theoretical_loss": 3.750389277567006, + "tokens_seen": 815857664 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608320962888666, + "loss": 1.4819, + "theoretical_loss": 3.7503597468555765, + "tokens_seen": 815923200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006083049147442327, + "loss": 1.4018, + "theoretical_loss": 3.7503302191800802, + "tokens_seen": 815988736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006082888665997994, + "loss": 1.3573, + "theoretical_loss": 3.750300694539961, + "tokens_seen": 816054272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006082728184553661, + "loss": 1.3858, + "theoretical_loss": 3.750271172934663, + "tokens_seen": 816119808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006082567703109328, + "loss": 1.4455, + "theoretical_loss": 3.750241654363631, + "tokens_seen": 816185344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006082407221664995, + "loss": 1.4914, + "theoretical_loss": 3.75021213882631, + "tokens_seen": 816250880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006082246740220662, + "loss": 1.4416, + "theoretical_loss": 3.7501826263221436, + "tokens_seen": 816316416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006082086258776329, + "loss": 1.4527, + "theoretical_loss": 3.7501531168505773, + "tokens_seen": 816381952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006081925777331996, + "loss": 1.4257, + "theoretical_loss": 3.750123610411056, + "tokens_seen": 816447488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006081765295887663, + "loss": 1.45, + "theoretical_loss": 3.750094107003024, + "tokens_seen": 816513024 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608160481444333, + "loss": 1.4755, + "theoretical_loss": 3.7500646066259278, + "tokens_seen": 816578560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006081444332998997, + "loss": 1.3689, + "theoretical_loss": 3.750035109279212, + "tokens_seen": 816644096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006081283851554664, + "loss": 1.4886, + "theoretical_loss": 3.7500056149623227, + "tokens_seen": 816709632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006081123370110331, + "loss": 1.3616, + "theoretical_loss": 3.7499761236747053, + "tokens_seen": 816775168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006080962888665998, + "loss": 1.4156, + "theoretical_loss": 3.7499466354158058, + "tokens_seen": 816840704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006080802407221665, + "loss": 1.3829, + "theoretical_loss": 3.7499171501850697, + "tokens_seen": 816906240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006080641925777332, + "loss": 1.4137, + "theoretical_loss": 3.749887667981944, + "tokens_seen": 816971776 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006080481444332999, + "loss": 1.4017, + "theoretical_loss": 3.7498581888058746, + "tokens_seen": 817037312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006080320962888666, + "loss": 1.5097, + "theoretical_loss": 3.7498287126563072, + "tokens_seen": 817102848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006080160481444333, + "loss": 1.5489, + "theoretical_loss": 3.7497992395326896, + "tokens_seen": 817168384 + }, + { + "epoch": 0.25, + "learning_rate": 0.000608, + "loss": 1.4045, + "theoretical_loss": 3.749769769434468, + "tokens_seen": 817233920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006079839518555667, + "loss": 1.4504, + "theoretical_loss": 3.7497403023610882, + "tokens_seen": 817299456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006079679037111334, + "loss": 1.3361, + "theoretical_loss": 3.7497108383119997, + "tokens_seen": 817364992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006079518555667001, + "loss": 1.4563, + "theoretical_loss": 3.749681377286647, + "tokens_seen": 817430528 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006079358074222668, + "loss": 1.3859, + "theoretical_loss": 3.749651919284479, + "tokens_seen": 817496064 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006079197592778335, + "loss": 1.4602, + "theoretical_loss": 3.7496224643049425, + "tokens_seen": 817561600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006079037111334002, + "loss": 1.4299, + "theoretical_loss": 3.7495930123474857, + "tokens_seen": 817627136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006078876629889669, + "loss": 1.3425, + "theoretical_loss": 3.749563563411556, + "tokens_seen": 817692672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006078716148445336, + "loss": 1.4787, + "theoretical_loss": 3.7495341174966015, + "tokens_seen": 817758208 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006078555667001003, + "loss": 1.4433, + "theoretical_loss": 3.7495046746020693, + "tokens_seen": 817823744 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607839518555667, + "loss": 1.4776, + "theoretical_loss": 3.7494752347274085, + "tokens_seen": 817889280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006078234704112337, + "loss": 1.3676, + "theoretical_loss": 3.749445797872067, + "tokens_seen": 817954816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006078074222668004, + "loss": 1.4433, + "theoretical_loss": 3.7494163640354943, + "tokens_seen": 818020352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006077913741223672, + "loss": 1.5132, + "theoretical_loss": 3.7493869332171377, + "tokens_seen": 818085888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006077753259779338, + "loss": 1.4905, + "theoretical_loss": 3.7493575054164463, + "tokens_seen": 818151424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006077592778335006, + "loss": 1.43, + "theoretical_loss": 3.749328080632869, + "tokens_seen": 818216960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006077432296890672, + "loss": 1.3569, + "theoretical_loss": 3.749298658865855, + "tokens_seen": 818282496 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607727181544634, + "loss": 1.4341, + "theoretical_loss": 3.7492692401148537, + "tokens_seen": 818348032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006077111334002006, + "loss": 1.4348, + "theoretical_loss": 3.7492398243793144, + "tokens_seen": 818413568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006076950852557674, + "loss": 1.2429, + "theoretical_loss": 3.7492104116586855, + "tokens_seen": 818479104 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607679037111334, + "loss": 1.4691, + "theoretical_loss": 3.749181001952418, + "tokens_seen": 818544640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006076629889669008, + "loss": 1.4331, + "theoretical_loss": 3.7491515952599617, + "tokens_seen": 818610176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006076469408224674, + "loss": 1.3973, + "theoretical_loss": 3.7491221915807653, + "tokens_seen": 818675712 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006076308926780342, + "loss": 1.5303, + "theoretical_loss": 3.74909279091428, + "tokens_seen": 818741248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006076148445336008, + "loss": 1.3713, + "theoretical_loss": 3.749063393259955, + "tokens_seen": 818806784 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006075987963891676, + "loss": 1.5291, + "theoretical_loss": 3.749033998617241, + "tokens_seen": 818872320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006075827482447342, + "loss": 1.4732, + "theoretical_loss": 3.7490046069855896, + "tokens_seen": 818937856 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607566700100301, + "loss": 1.306, + "theoretical_loss": 3.74897521836445, + "tokens_seen": 819003392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006075506519558677, + "loss": 1.3514, + "theoretical_loss": 3.7489458327532734, + "tokens_seen": 819068928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006075346038114344, + "loss": 1.4694, + "theoretical_loss": 3.748916450151511, + "tokens_seen": 819134464 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 308621, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3499765396118164, + "objective/train/theoretical_loss": 3.7488870705586135, + "objective/train/tokens_used": 839660000, + "theoretical_loss": 3.7488870705586135, + "tokens_seen": 819200000 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607518555667001, + "loss": 1.35, + "theoretical_loss": 3.7488870705586135, + "tokens_seen": 819200000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006075025075225678, + "loss": 1.4294, + "theoretical_loss": 3.7488576939740326, + "tokens_seen": 819265536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006074864593781345, + "loss": 1.4777, + "theoretical_loss": 3.7488283203972195, + "tokens_seen": 819331072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006074704112337012, + "loss": 1.4329, + "theoretical_loss": 3.748798949827625, + "tokens_seen": 819396608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006074543630892679, + "loss": 1.3475, + "theoretical_loss": 3.7487695822647016, + "tokens_seen": 819462144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006074383149448346, + "loss": 1.3436, + "theoretical_loss": 3.7487402177079003, + "tokens_seen": 819527680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006074222668004013, + "loss": 1.5517, + "theoretical_loss": 3.7487108561566735, + "tokens_seen": 819593216 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607406218655968, + "loss": 1.4828, + "theoretical_loss": 3.748681497610474, + "tokens_seen": 819658752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006073901705115347, + "loss": 1.4936, + "theoretical_loss": 3.748652142068753, + "tokens_seen": 819724288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006073741223671014, + "loss": 1.4178, + "theoretical_loss": 3.748622789530963, + "tokens_seen": 819789824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006073580742226681, + "loss": 1.3499, + "theoretical_loss": 3.748593439996557, + "tokens_seen": 819855360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006073420260782348, + "loss": 1.4414, + "theoretical_loss": 3.748564093464987, + "tokens_seen": 819920896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006073259779338015, + "loss": 1.2947, + "theoretical_loss": 3.748534749935706, + "tokens_seen": 819986432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006073099297893681, + "loss": 1.4067, + "theoretical_loss": 3.748505409408167, + "tokens_seen": 820051968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006072938816449349, + "loss": 1.4022, + "theoretical_loss": 3.7484760718818233, + "tokens_seen": 820117504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006072778335005015, + "loss": 1.4439, + "theoretical_loss": 3.748446737356128, + "tokens_seen": 820183040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006072617853560683, + "loss": 1.4126, + "theoretical_loss": 3.7484174058305344, + "tokens_seen": 820248576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006072457372116349, + "loss": 1.4029, + "theoretical_loss": 3.748388077304496, + "tokens_seen": 820314112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006072296890672017, + "loss": 1.3831, + "theoretical_loss": 3.7483587517774666, + "tokens_seen": 820379648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006072136409227683, + "loss": 1.4019, + "theoretical_loss": 3.7483294292488996, + "tokens_seen": 820445184 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071975927783351, + "loss": 1.3678, + "theoretical_loss": 3.748300109718249, + "tokens_seen": 820510720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071815446339017, + "loss": 1.5468, + "theoretical_loss": 3.74827079318497, + "tokens_seen": 820576256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071654964894684, + "loss": 1.3841, + "theoretical_loss": 3.748241479648515, + "tokens_seen": 820641792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071494483450351, + "loss": 1.3931, + "theoretical_loss": 3.7482121691083394, + "tokens_seen": 820707328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071334002006018, + "loss": 1.3794, + "theoretical_loss": 3.748182861563898, + "tokens_seen": 820772864 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071173520561685, + "loss": 1.3401, + "theoretical_loss": 3.748153557014645, + "tokens_seen": 820838400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006071013039117352, + "loss": 1.2835, + "theoretical_loss": 3.748124255460035, + "tokens_seen": 820903936 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006070852557673019, + "loss": 1.3812, + "theoretical_loss": 3.748094956899523, + "tokens_seen": 820969472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006070692076228686, + "loss": 1.4267, + "theoretical_loss": 3.7480656613325642, + "tokens_seen": 821035008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006070531594784353, + "loss": 1.373, + "theoretical_loss": 3.7480363687586147, + "tokens_seen": 821100544 + }, + { + "epoch": 0.25, + "learning_rate": 0.000607037111334002, + "loss": 1.4263, + "theoretical_loss": 3.7480070791771283, + "tokens_seen": 821166080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006070210631895687, + "loss": 1.3746, + "theoretical_loss": 3.7479777925875615, + "tokens_seen": 821231616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006070050150451354, + "loss": 1.4983, + "theoretical_loss": 3.74794850898937, + "tokens_seen": 821297152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006069889669007021, + "loss": 1.4671, + "theoretical_loss": 3.7479192283820084, + "tokens_seen": 821362688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006069729187562688, + "loss": 1.4439, + "theoretical_loss": 3.747889950764934, + "tokens_seen": 821428224 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006069568706118355, + "loss": 1.367, + "theoretical_loss": 3.747860676137603, + "tokens_seen": 821493760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006069408224674022, + "loss": 1.3643, + "theoretical_loss": 3.74783140449947, + "tokens_seen": 821559296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006069247743229689, + "loss": 1.3967, + "theoretical_loss": 3.747802135849993, + "tokens_seen": 821624832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006069087261785356, + "loss": 1.3771, + "theoretical_loss": 3.747772870188628, + "tokens_seen": 821690368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006068926780341023, + "loss": 1.4804, + "theoretical_loss": 3.747743607514831, + "tokens_seen": 821755904 + }, + { + "epoch": 0.25, + "learning_rate": 0.000606876629889669, + "loss": 1.3764, + "theoretical_loss": 3.74771434782806, + "tokens_seen": 821821440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006068605817452357, + "loss": 1.4351, + "theoretical_loss": 3.747685091127771, + "tokens_seen": 821886976 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006068445336008024, + "loss": 1.4213, + "theoretical_loss": 3.7476558374134217, + "tokens_seen": 821952512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006068284854563691, + "loss": 1.5357, + "theoretical_loss": 3.7476265866844685, + "tokens_seen": 822018048 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006068124373119358, + "loss": 1.4203, + "theoretical_loss": 3.7475973389403694, + "tokens_seen": 822083584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006067963891675025, + "loss": 1.4555, + "theoretical_loss": 3.747568094180582, + "tokens_seen": 822149120 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006067803410230692, + "loss": 1.5752, + "theoretical_loss": 3.747538852404563, + "tokens_seen": 822214656 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006067642928786359, + "loss": 1.3263, + "theoretical_loss": 3.747509613611771, + "tokens_seen": 822280192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006067482447342026, + "loss": 1.3408, + "theoretical_loss": 3.747480377801664, + "tokens_seen": 822345728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006067321965897693, + "loss": 1.5384, + "theoretical_loss": 3.7474511449737, + "tokens_seen": 822411264 + }, + { + "epoch": 0.25, + "learning_rate": 0.000606716148445336, + "loss": 1.3759, + "theoretical_loss": 3.7474219151273367, + "tokens_seen": 822476800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006067001003009027, + "loss": 1.3916, + "theoretical_loss": 3.7473926882620328, + "tokens_seen": 822542336 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006066840521564694, + "loss": 1.4414, + "theoretical_loss": 3.7473634643772473, + "tokens_seen": 822607872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006066680040120361, + "loss": 1.4868, + "theoretical_loss": 3.747334243472438, + "tokens_seen": 822673408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006066519558676028, + "loss": 1.4075, + "theoretical_loss": 3.747305025547064, + "tokens_seen": 822738944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006066359077231695, + "loss": 1.4223, + "theoretical_loss": 3.747275810600584, + "tokens_seen": 822804480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006066198595787362, + "loss": 1.4335, + "theoretical_loss": 3.747246598632458, + "tokens_seen": 822870016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006066038114343029, + "loss": 1.5632, + "theoretical_loss": 3.7472173896421435, + "tokens_seen": 822935552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006065877632898697, + "loss": 1.4133, + "theoretical_loss": 3.7471881836291017, + "tokens_seen": 823001088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006065717151454363, + "loss": 1.4196, + "theoretical_loss": 3.7471589805927907, + "tokens_seen": 823066624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006065556670010031, + "loss": 1.4761, + "theoretical_loss": 3.7471297805326707, + "tokens_seen": 823132160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006065396188565697, + "loss": 1.37, + "theoretical_loss": 3.7471005834482014, + "tokens_seen": 823197696 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006065235707121365, + "loss": 1.5231, + "theoretical_loss": 3.7470713893388425, + "tokens_seen": 823263232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006065075225677031, + "loss": 1.4457, + "theoretical_loss": 3.7470421982040545, + "tokens_seen": 823328768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006064914744232699, + "loss": 1.4192, + "theoretical_loss": 3.7470130100432972, + "tokens_seen": 823394304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006064754262788365, + "loss": 1.4642, + "theoretical_loss": 3.746983824856031, + "tokens_seen": 823459840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006064593781344033, + "loss": 1.3173, + "theoretical_loss": 3.7469546426417164, + "tokens_seen": 823525376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006064433299899699, + "loss": 1.4535, + "theoretical_loss": 3.746925463399814, + "tokens_seen": 823590912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006064272818455367, + "loss": 1.5648, + "theoretical_loss": 3.746896287129785, + "tokens_seen": 823656448 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006064112337011033, + "loss": 1.3741, + "theoretical_loss": 3.7468671138310894, + "tokens_seen": 823721984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006063951855566701, + "loss": 1.4892, + "theoretical_loss": 3.7468379435031887, + "tokens_seen": 823787520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006063791374122367, + "loss": 1.4042, + "theoretical_loss": 3.7468087761455444, + "tokens_seen": 823853056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006063630892678035, + "loss": 1.4754, + "theoretical_loss": 3.746779611757617, + "tokens_seen": 823918592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006063470411233701, + "loss": 1.501, + "theoretical_loss": 3.746750450338869, + "tokens_seen": 823984128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006063309929789369, + "loss": 1.3964, + "theoretical_loss": 3.746721291888761, + "tokens_seen": 824049664 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006063149448345035, + "loss": 1.4674, + "theoretical_loss": 3.7466921364067556, + "tokens_seen": 824115200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006062988966900703, + "loss": 1.5132, + "theoretical_loss": 3.746662983892314, + "tokens_seen": 824180736 + }, + { + "epoch": 0.25, + "learning_rate": 0.000606282848545637, + "loss": 1.2766, + "theoretical_loss": 3.746633834344898, + "tokens_seen": 824246272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006062668004012037, + "loss": 1.4521, + "theoretical_loss": 3.7466046877639707, + "tokens_seen": 824311808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006062507522567703, + "loss": 1.3912, + "theoretical_loss": 3.7465755441489943, + "tokens_seen": 824377344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006062347041123371, + "loss": 1.3893, + "theoretical_loss": 3.74654640349943, + "tokens_seen": 824442880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006062186559679037, + "loss": 1.3537, + "theoretical_loss": 3.7465172658147416, + "tokens_seen": 824508416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006062026078234705, + "loss": 1.4195, + "theoretical_loss": 3.7464881310943916, + "tokens_seen": 824573952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006061865596790372, + "loss": 1.4517, + "theoretical_loss": 3.7464589993378423, + "tokens_seen": 824639488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006061705115346039, + "loss": 1.4378, + "theoretical_loss": 3.7464298705445573, + "tokens_seen": 824705024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006061544633901706, + "loss": 1.3334, + "theoretical_loss": 3.7464007447139993, + "tokens_seen": 824770560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006061384152457373, + "loss": 1.5353, + "theoretical_loss": 3.746371621845632, + "tokens_seen": 824836096 + }, + { + "epoch": 0.25, + "learning_rate": 0.000606122367101304, + "loss": 1.3238, + "theoretical_loss": 3.7463425019389187, + "tokens_seen": 824901632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006061063189568707, + "loss": 1.32, + "theoretical_loss": 3.746313384993323, + "tokens_seen": 824967168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006060902708124374, + "loss": 1.3704, + "theoretical_loss": 3.746284271008308, + "tokens_seen": 825032704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006060742226680041, + "loss": 1.3957, + "theoretical_loss": 3.7462551599833382, + "tokens_seen": 825098240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006060581745235708, + "loss": 1.4061, + "theoretical_loss": 3.746226051917877, + "tokens_seen": 825163776 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006060421263791375, + "loss": 1.4911, + "theoretical_loss": 3.7461969468113896, + "tokens_seen": 825229312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006060260782347042, + "loss": 1.4582, + "theoretical_loss": 3.746167844663339, + "tokens_seen": 825294848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006060100300902709, + "loss": 1.3998, + "theoretical_loss": 3.7461387454731905, + "tokens_seen": 825360384 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006059939819458376, + "loss": 1.3935, + "theoretical_loss": 3.746109649240408, + "tokens_seen": 825425920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006059779338014043, + "loss": 1.3014, + "theoretical_loss": 3.7460805559644568, + "tokens_seen": 825491456 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605961885656971, + "loss": 1.4562, + "theoretical_loss": 3.7460514656448005, + "tokens_seen": 825556992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006059458375125377, + "loss": 1.4584, + "theoretical_loss": 3.7460223782809057, + "tokens_seen": 825622528 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006059297893681044, + "loss": 1.4344, + "theoretical_loss": 3.7459932938722362, + "tokens_seen": 825688064 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 311019, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4440590143203735, + "objective/train/theoretical_loss": 3.745964212418258, + "objective/train/tokens_used": 846213600, + "theoretical_loss": 3.745964212418258, + "tokens_seen": 825753600 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605913741223671, + "loss": 1.4441, + "theoretical_loss": 3.745964212418258, + "tokens_seen": 825753600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006058976930792378, + "loss": 1.3985, + "theoretical_loss": 3.7459351339184366, + "tokens_seen": 825819136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006058816449348044, + "loss": 1.3844, + "theoretical_loss": 3.7459060583722366, + "tokens_seen": 825884672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006058655967903712, + "loss": 1.445, + "theoretical_loss": 3.745876985779124, + "tokens_seen": 825950208 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006058495486459378, + "loss": 1.5037, + "theoretical_loss": 3.745847916138565, + "tokens_seen": 826015744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006058335005015046, + "loss": 1.2946, + "theoretical_loss": 3.745818849450025, + "tokens_seen": 826081280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006058174523570712, + "loss": 1.4588, + "theoretical_loss": 3.7457897857129705, + "tokens_seen": 826146816 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605801404212638, + "loss": 1.3862, + "theoretical_loss": 3.7457607249268676, + "tokens_seen": 826212352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006057853560682046, + "loss": 1.4268, + "theoretical_loss": 3.7457316670911824, + "tokens_seen": 826277888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006057693079237713, + "loss": 1.4425, + "theoretical_loss": 3.7457026122053816, + "tokens_seen": 826343424 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605753259779338, + "loss": 1.2656, + "theoretical_loss": 3.7456735602689317, + "tokens_seen": 826408960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006057372116349047, + "loss": 1.4377, + "theoretical_loss": 3.7456445112812995, + "tokens_seen": 826474496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006057211634904714, + "loss": 1.3373, + "theoretical_loss": 3.745615465241952, + "tokens_seen": 826540032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006057051153460381, + "loss": 1.3355, + "theoretical_loss": 3.745586422150356, + "tokens_seen": 826605568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006056890672016048, + "loss": 1.3729, + "theoretical_loss": 3.745557382005979, + "tokens_seen": 826671104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006056730190571715, + "loss": 1.4642, + "theoretical_loss": 3.7455283448082883, + "tokens_seen": 826736640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006056569709127382, + "loss": 1.4211, + "theoretical_loss": 3.7454993105567507, + "tokens_seen": 826802176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006056409227683049, + "loss": 1.4358, + "theoretical_loss": 3.745470279250835, + "tokens_seen": 826867712 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006056248746238716, + "loss": 1.4468, + "theoretical_loss": 3.7454412508900075, + "tokens_seen": 826933248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006056088264794383, + "loss": 1.4576, + "theoretical_loss": 3.7454122254737374, + "tokens_seen": 826998784 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605592778335005, + "loss": 1.4403, + "theoretical_loss": 3.7453832030014915, + "tokens_seen": 827064320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006055767301905717, + "loss": 1.453, + "theoretical_loss": 3.745354183472739, + "tokens_seen": 827129856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006055606820461384, + "loss": 1.4031, + "theoretical_loss": 3.745325166886947, + "tokens_seen": 827195392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006055446339017051, + "loss": 1.4735, + "theoretical_loss": 3.7452961532435856, + "tokens_seen": 827260928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006055285857572718, + "loss": 1.4301, + "theoretical_loss": 3.7452671425421213, + "tokens_seen": 827326464 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006055125376128385, + "loss": 1.489, + "theoretical_loss": 3.7452381347820247, + "tokens_seen": 827392000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006054964894684052, + "loss": 1.3868, + "theoretical_loss": 3.7452091299627637, + "tokens_seen": 827457536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006054804413239719, + "loss": 1.4046, + "theoretical_loss": 3.745180128083807, + "tokens_seen": 827523072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006054643931795386, + "loss": 1.4216, + "theoretical_loss": 3.7451511291446242, + "tokens_seen": 827588608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006054483450351053, + "loss": 1.3861, + "theoretical_loss": 3.745122133144685, + "tokens_seen": 827654144 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605432296890672, + "loss": 1.4256, + "theoretical_loss": 3.7450931400834575, + "tokens_seen": 827719680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006054162487462387, + "loss": 1.3348, + "theoretical_loss": 3.7450641499604123, + "tokens_seen": 827785216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006054002006018054, + "loss": 1.4039, + "theoretical_loss": 3.7450351627750185, + "tokens_seen": 827850752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006053841524573722, + "loss": 1.4956, + "theoretical_loss": 3.7450061785267463, + "tokens_seen": 827916288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006053681043129388, + "loss": 1.37, + "theoretical_loss": 3.744977197215065, + "tokens_seen": 827981824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006053520561685056, + "loss": 1.4366, + "theoretical_loss": 3.744948218839446, + "tokens_seen": 828047360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006053360080240722, + "loss": 1.4127, + "theoretical_loss": 3.744919243399358, + "tokens_seen": 828112896 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605319959879639, + "loss": 1.4767, + "theoretical_loss": 3.744890270894272, + "tokens_seen": 828178432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006053039117352056, + "loss": 1.5057, + "theoretical_loss": 3.7448613013236587, + "tokens_seen": 828243968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006052878635907724, + "loss": 1.4402, + "theoretical_loss": 3.744832334686988, + "tokens_seen": 828309504 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605271815446339, + "loss": 1.4179, + "theoretical_loss": 3.744803370983732, + "tokens_seen": 828375040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006052557673019058, + "loss": 1.4624, + "theoretical_loss": 3.7447744102133598, + "tokens_seen": 828440576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006052397191574724, + "loss": 1.3564, + "theoretical_loss": 3.744745452375344, + "tokens_seen": 828506112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006052236710130392, + "loss": 1.5505, + "theoretical_loss": 3.7447164974691547, + "tokens_seen": 828571648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006052076228686058, + "loss": 1.3902, + "theoretical_loss": 3.744687545494264, + "tokens_seen": 828637184 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006051915747241726, + "loss": 1.4245, + "theoretical_loss": 3.744658596450143, + "tokens_seen": 828702720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006051755265797392, + "loss": 1.4262, + "theoretical_loss": 3.744629650336263, + "tokens_seen": 828768256 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605159478435306, + "loss": 1.4483, + "theoretical_loss": 3.744600707152096, + "tokens_seen": 828833792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006051434302908726, + "loss": 1.3817, + "theoretical_loss": 3.7445717668971144, + "tokens_seen": 828899328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006051273821464394, + "loss": 1.4675, + "theoretical_loss": 3.7445428295707894, + "tokens_seen": 828964864 + }, + { + "epoch": 0.25, + "learning_rate": 0.000605111334002006, + "loss": 1.386, + "theoretical_loss": 3.7445138951725934, + "tokens_seen": 829030400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006050952858575728, + "loss": 1.4086, + "theoretical_loss": 3.7444849637019986, + "tokens_seen": 829095936 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006050792377131394, + "loss": 1.4091, + "theoretical_loss": 3.7444560351584775, + "tokens_seen": 829161472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006050631895687062, + "loss": 1.4847, + "theoretical_loss": 3.744427109541503, + "tokens_seen": 829227008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006050471414242728, + "loss": 1.3044, + "theoretical_loss": 3.744398186850547, + "tokens_seen": 829292544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006050310932798396, + "loss": 1.4792, + "theoretical_loss": 3.7443692670850828, + "tokens_seen": 829358080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006050150451354062, + "loss": 1.4214, + "theoretical_loss": 3.744340350244583, + "tokens_seen": 829423616 + }, + { + "epoch": 0.25, + "learning_rate": 0.000604998996990973, + "loss": 1.3916, + "theoretical_loss": 3.7443114363285215, + "tokens_seen": 829489152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006049829488465396, + "loss": 1.4841, + "theoretical_loss": 3.74428252533637, + "tokens_seen": 829554688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006049669007021064, + "loss": 1.4892, + "theoretical_loss": 3.7442536172676038, + "tokens_seen": 829620224 + }, + { + "epoch": 0.25, + "learning_rate": 0.000604950852557673, + "loss": 1.3973, + "theoretical_loss": 3.744224712121695, + "tokens_seen": 829685760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006049348044132398, + "loss": 1.4762, + "theoretical_loss": 3.744195809898118, + "tokens_seen": 829751296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006049187562688064, + "loss": 1.3728, + "theoretical_loss": 3.7441669105963458, + "tokens_seen": 829816832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006049027081243732, + "loss": 1.5486, + "theoretical_loss": 3.744138014215853, + "tokens_seen": 829882368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006048866599799398, + "loss": 1.3795, + "theoretical_loss": 3.7441091207561135, + "tokens_seen": 829947904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006048706118355066, + "loss": 1.4176, + "theoretical_loss": 3.7440802302166016, + "tokens_seen": 830013440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006048545636910732, + "loss": 1.4577, + "theoretical_loss": 3.7440513425967907, + "tokens_seen": 830078976 + }, + { + "epoch": 0.25, + "learning_rate": 0.00060483851554664, + "loss": 1.4682, + "theoretical_loss": 3.7440224578961563, + "tokens_seen": 830144512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006048224674022067, + "loss": 1.3197, + "theoretical_loss": 3.743993576114173, + "tokens_seen": 830210048 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006048064192577734, + "loss": 1.347, + "theoretical_loss": 3.7439646972503144, + "tokens_seen": 830275584 + }, + { + "epoch": 0.25, + "learning_rate": 0.00060479037111334, + "loss": 1.5138, + "theoretical_loss": 3.7439358213040568, + "tokens_seen": 830341120 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006047743229689068, + "loss": 1.4191, + "theoretical_loss": 3.743906948274874, + "tokens_seen": 830406656 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006047582748244735, + "loss": 1.3718, + "theoretical_loss": 3.7438780781622425, + "tokens_seen": 830472192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006047422266800402, + "loss": 1.3985, + "theoretical_loss": 3.743849210965636, + "tokens_seen": 830537728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006047261785356069, + "loss": 1.4194, + "theoretical_loss": 3.7438203466845303, + "tokens_seen": 830603264 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006047101303911736, + "loss": 1.4559, + "theoretical_loss": 3.7437914853184022, + "tokens_seen": 830668800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006046940822467403, + "loss": 1.47, + "theoretical_loss": 3.743762626866726, + "tokens_seen": 830734336 + }, + { + "epoch": 0.25, + "learning_rate": 0.000604678034102307, + "loss": 1.4001, + "theoretical_loss": 3.7437337713289778, + "tokens_seen": 830799872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006046619859578737, + "loss": 1.4205, + "theoretical_loss": 3.743704918704634, + "tokens_seen": 830865408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006046459378134404, + "loss": 1.3873, + "theoretical_loss": 3.74367606899317, + "tokens_seen": 830930944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006046298896690071, + "loss": 1.4195, + "theoretical_loss": 3.743647222194063, + "tokens_seen": 830996480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006046138415245738, + "loss": 1.532, + "theoretical_loss": 3.7436183783067882, + "tokens_seen": 831062016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045977933801405, + "loss": 1.452, + "theoretical_loss": 3.7435895373308226, + "tokens_seen": 831127552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045817452357072, + "loss": 1.4707, + "theoretical_loss": 3.743560699265643, + "tokens_seen": 831193088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045656970912739, + "loss": 1.498, + "theoretical_loss": 3.743531864110726, + "tokens_seen": 831258624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045496489468406, + "loss": 1.4136, + "theoretical_loss": 3.743503031865549, + "tokens_seen": 831324160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045336008024073, + "loss": 1.5086, + "theoretical_loss": 3.743474202529588, + "tokens_seen": 831389696 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045175526579739, + "loss": 1.5085, + "theoretical_loss": 3.743445376102321, + "tokens_seen": 831455232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006045015045135407, + "loss": 1.4136, + "theoretical_loss": 3.7434165525832253, + "tokens_seen": 831520768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006044854563691073, + "loss": 1.4295, + "theoretical_loss": 3.7433877319717777, + "tokens_seen": 831586304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006044694082246741, + "loss": 1.4203, + "theoretical_loss": 3.743358914267456, + "tokens_seen": 831651840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006044533600802407, + "loss": 1.3911, + "theoretical_loss": 3.743330099469738, + "tokens_seen": 831717376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006044373119358075, + "loss": 1.3683, + "theoretical_loss": 3.7433012875781015, + "tokens_seen": 831782912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006044212637913741, + "loss": 1.5789, + "theoretical_loss": 3.743272478592025, + "tokens_seen": 831848448 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006044052156469409, + "loss": 1.4344, + "theoretical_loss": 3.743243672510986, + "tokens_seen": 831913984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006043891675025075, + "loss": 1.4347, + "theoretical_loss": 3.7432148693344627, + "tokens_seen": 831979520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006043731193580742, + "loss": 1.4038, + "theoretical_loss": 3.7431860690619336, + "tokens_seen": 832045056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006043570712136409, + "loss": 1.4194, + "theoretical_loss": 3.7431572716928776, + "tokens_seen": 832110592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006043410230692076, + "loss": 1.352, + "theoretical_loss": 3.743128477226773, + "tokens_seen": 832176128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006043249749247743, + "loss": 1.4525, + "theoretical_loss": 3.7430996856630987, + "tokens_seen": 832241664 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 313419, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4409335851669312, + "objective/train/theoretical_loss": 3.743070897001333, + "objective/train/tokens_used": 852767200, + "theoretical_loss": 3.743070897001333, + "tokens_seen": 832307200 + }, + { + "epoch": 0.25, + "learning_rate": 0.000604308926780341, + "loss": 1.4409, + "theoretical_loss": 3.743070897001333, + "tokens_seen": 832307200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006042928786359077, + "loss": 1.3578, + "theoretical_loss": 3.743042111240956, + "tokens_seen": 832372736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006042768304914744, + "loss": 1.5038, + "theoretical_loss": 3.7430133283814464, + "tokens_seen": 832438272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006042607823470411, + "loss": 1.506, + "theoretical_loss": 3.7429845484222835, + "tokens_seen": 832503808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006042447342026078, + "loss": 1.4693, + "theoretical_loss": 3.742955771362947, + "tokens_seen": 832569344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006042286860581745, + "loss": 1.3925, + "theoretical_loss": 3.7429269972029156, + "tokens_seen": 832634880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006042126379137412, + "loss": 1.5493, + "theoretical_loss": 3.74289822594167, + "tokens_seen": 832700416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041965897693079, + "loss": 1.4567, + "theoretical_loss": 3.7428694575786903, + "tokens_seen": 832765952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041805416248747, + "loss": 1.4147, + "theoretical_loss": 3.742840692113455, + "tokens_seen": 832831488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041644934804413, + "loss": 1.4731, + "theoretical_loss": 3.7428119295454456, + "tokens_seen": 832897024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041484453360081, + "loss": 1.5165, + "theoretical_loss": 3.7427831698741425, + "tokens_seen": 832962560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041323971915747, + "loss": 1.4041, + "theoretical_loss": 3.742754413099025, + "tokens_seen": 833028096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041163490471415, + "loss": 1.4396, + "theoretical_loss": 3.742725659219574, + "tokens_seen": 833093632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006041003009027081, + "loss": 1.3157, + "theoretical_loss": 3.7426969082352706, + "tokens_seen": 833159168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006040842527582749, + "loss": 1.3809, + "theoretical_loss": 3.7426681601455947, + "tokens_seen": 833224704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006040682046138415, + "loss": 1.3299, + "theoretical_loss": 3.7426394149500286, + "tokens_seen": 833290240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006040521564694083, + "loss": 1.3122, + "theoretical_loss": 3.742610672648053, + "tokens_seen": 833355776 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006040361083249749, + "loss": 1.4063, + "theoretical_loss": 3.7425819332391477, + "tokens_seen": 833421312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006040200601805417, + "loss": 1.4579, + "theoretical_loss": 3.7425531967227954, + "tokens_seen": 833486848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006040040120361083, + "loss": 1.4592, + "theoretical_loss": 3.7425244630984773, + "tokens_seen": 833552384 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006039879638916751, + "loss": 1.3673, + "theoretical_loss": 3.742495732365675, + "tokens_seen": 833617920 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006039719157472417, + "loss": 1.4323, + "theoretical_loss": 3.74246700452387, + "tokens_seen": 833683456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006039558676028085, + "loss": 1.3724, + "theoretical_loss": 3.7424382795725446, + "tokens_seen": 833748992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006039398194583751, + "loss": 1.427, + "theoretical_loss": 3.7424095575111807, + "tokens_seen": 833814528 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006039237713139419, + "loss": 1.4126, + "theoretical_loss": 3.74238083833926, + "tokens_seen": 833880064 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006039077231695085, + "loss": 1.3395, + "theoretical_loss": 3.742352122056265, + "tokens_seen": 833945600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006038916750250753, + "loss": 1.3877, + "theoretical_loss": 3.7423234086616777, + "tokens_seen": 834011136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006038756268806419, + "loss": 1.4156, + "theoretical_loss": 3.7422946981549816, + "tokens_seen": 834076672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006038595787362087, + "loss": 1.452, + "theoretical_loss": 3.7422659905356586, + "tokens_seen": 834142208 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006038435305917753, + "loss": 1.3603, + "theoretical_loss": 3.742237285803192, + "tokens_seen": 834207744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006038274824473421, + "loss": 1.4423, + "theoretical_loss": 3.7422085839570647, + "tokens_seen": 834273280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006038114343029087, + "loss": 1.4375, + "theoretical_loss": 3.7421798849967596, + "tokens_seen": 834338816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006037953861584755, + "loss": 1.4363, + "theoretical_loss": 3.742151188921759, + "tokens_seen": 834404352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006037793380140421, + "loss": 1.3552, + "theoretical_loss": 3.742122495731548, + "tokens_seen": 834469888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006037632898696089, + "loss": 1.5488, + "theoretical_loss": 3.742093805425609, + "tokens_seen": 834535424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006037472417251755, + "loss": 1.5095, + "theoretical_loss": 3.7420651180034255, + "tokens_seen": 834600960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006037311935807423, + "loss": 1.4127, + "theoretical_loss": 3.742036433464482, + "tokens_seen": 834666496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006037151454363089, + "loss": 1.4712, + "theoretical_loss": 3.742007751808261, + "tokens_seen": 834732032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036990972918757, + "loss": 1.4665, + "theoretical_loss": 3.741979073034248, + "tokens_seen": 834797568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036830491474423, + "loss": 1.3719, + "theoretical_loss": 3.7419503971419266, + "tokens_seen": 834863104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036670010030091, + "loss": 1.4695, + "theoretical_loss": 3.7419217241307807, + "tokens_seen": 834928640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036509528585757, + "loss": 1.4569, + "theoretical_loss": 3.741893054000295, + "tokens_seen": 834994176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036349047141425, + "loss": 1.4428, + "theoretical_loss": 3.7418643867499544, + "tokens_seen": 835059712 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036188565697091, + "loss": 1.3467, + "theoretical_loss": 3.7418357223792427, + "tokens_seen": 835125248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006036028084252759, + "loss": 1.4354, + "theoretical_loss": 3.7418070608876453, + "tokens_seen": 835190784 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006035867602808425, + "loss": 1.5043, + "theoretical_loss": 3.741778402274647, + "tokens_seen": 835256320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006035707121364093, + "loss": 1.4578, + "theoretical_loss": 3.7417497465397327, + "tokens_seen": 835321856 + }, + { + "epoch": 0.25, + "learning_rate": 0.000603554663991976, + "loss": 1.4423, + "theoretical_loss": 3.741721093682388, + "tokens_seen": 835387392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006035386158475427, + "loss": 1.4064, + "theoretical_loss": 3.7416924437020977, + "tokens_seen": 835452928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006035225677031093, + "loss": 1.3334, + "theoretical_loss": 3.741663796598348, + "tokens_seen": 835518464 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006035065195586761, + "loss": 1.5414, + "theoretical_loss": 3.741635152370624, + "tokens_seen": 835584000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006034904714142427, + "loss": 1.3631, + "theoretical_loss": 3.7416065110184116, + "tokens_seen": 835649536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006034744232698095, + "loss": 1.4814, + "theoretical_loss": 3.741577872541196, + "tokens_seen": 835715072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006034583751253762, + "loss": 1.4826, + "theoretical_loss": 3.741549236938464, + "tokens_seen": 835780608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006034423269809429, + "loss": 1.4465, + "theoretical_loss": 3.7415206042097013, + "tokens_seen": 835846144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006034262788365096, + "loss": 1.3102, + "theoretical_loss": 3.7414919743543944, + "tokens_seen": 835911680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006034102306920763, + "loss": 1.4875, + "theoretical_loss": 3.7414633473720293, + "tokens_seen": 835977216 + }, + { + "epoch": 0.25, + "learning_rate": 0.000603394182547643, + "loss": 1.3134, + "theoretical_loss": 3.741434723262093, + "tokens_seen": 836042752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006033781344032097, + "loss": 1.4311, + "theoretical_loss": 3.7414061020240723, + "tokens_seen": 836108288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006033620862587764, + "loss": 1.3919, + "theoretical_loss": 3.741377483657453, + "tokens_seen": 836173824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006033460381143431, + "loss": 1.4919, + "theoretical_loss": 3.741348868161723, + "tokens_seen": 836239360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006033299899699098, + "loss": 1.4254, + "theoretical_loss": 3.741320255536369, + "tokens_seen": 836304896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006033139418254765, + "loss": 1.376, + "theoretical_loss": 3.741291645780878, + "tokens_seen": 836370432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006032978936810432, + "loss": 1.4736, + "theoretical_loss": 3.741263038894737, + "tokens_seen": 836435968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006032818455366099, + "loss": 1.4685, + "theoretical_loss": 3.741234434877435, + "tokens_seen": 836501504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006032657973921766, + "loss": 1.4452, + "theoretical_loss": 3.7412058337284577, + "tokens_seen": 836567040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006032497492477433, + "loss": 1.481, + "theoretical_loss": 3.741177235447294, + "tokens_seen": 836632576 + }, + { + "epoch": 0.25, + "learning_rate": 0.00060323370110331, + "loss": 1.4173, + "theoretical_loss": 3.7411486400334315, + "tokens_seen": 836698112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006032176529588767, + "loss": 1.4179, + "theoretical_loss": 3.7411200474863575, + "tokens_seen": 836763648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006032016048144434, + "loss": 1.601, + "theoretical_loss": 3.7410914578055605, + "tokens_seen": 836829184 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006031855566700101, + "loss": 1.4515, + "theoretical_loss": 3.741062870990529, + "tokens_seen": 836894720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006031695085255768, + "loss": 1.4552, + "theoretical_loss": 3.741034287040751, + "tokens_seen": 836960256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006031534603811435, + "loss": 1.462, + "theoretical_loss": 3.7410057059557156, + "tokens_seen": 837025792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006031374122367102, + "loss": 1.4423, + "theoretical_loss": 3.7409771277349106, + "tokens_seen": 837091328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006031213640922769, + "loss": 1.4178, + "theoretical_loss": 3.7409485523778256, + "tokens_seen": 837156864 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006031053159478436, + "loss": 1.3826, + "theoretical_loss": 3.740919979883949, + "tokens_seen": 837222400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006030892678034102, + "loss": 1.3926, + "theoretical_loss": 3.740891410252769, + "tokens_seen": 837287936 + }, + { + "epoch": 0.25, + "learning_rate": 0.000603073219658977, + "loss": 1.5041, + "theoretical_loss": 3.740862843483777, + "tokens_seen": 837353472 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006030571715145436, + "loss": 1.5281, + "theoretical_loss": 3.7408342795764598, + "tokens_seen": 837419008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006030411233701104, + "loss": 1.4008, + "theoretical_loss": 3.740805718530308, + "tokens_seen": 837484544 + }, + { + "epoch": 0.25, + "learning_rate": 0.000603025075225677, + "loss": 1.4015, + "theoretical_loss": 3.7407771603448117, + "tokens_seen": 837550080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006030090270812438, + "loss": 1.4123, + "theoretical_loss": 3.7407486050194594, + "tokens_seen": 837615616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006029929789368104, + "loss": 1.4921, + "theoretical_loss": 3.7407200525537414, + "tokens_seen": 837681152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006029769307923772, + "loss": 1.4673, + "theoretical_loss": 3.740691502947148, + "tokens_seen": 837746688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006029608826479438, + "loss": 1.4354, + "theoretical_loss": 3.7406629561991687, + "tokens_seen": 837812224 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006029448345035106, + "loss": 1.4401, + "theoretical_loss": 3.740634412309294, + "tokens_seen": 837877760 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006029287863590772, + "loss": 1.5574, + "theoretical_loss": 3.740605871277014, + "tokens_seen": 837943296 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602912738214644, + "loss": 1.4752, + "theoretical_loss": 3.7405773331018195, + "tokens_seen": 838008832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006028966900702106, + "loss": 1.4675, + "theoretical_loss": 3.740548797783201, + "tokens_seen": 838074368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006028806419257774, + "loss": 1.4597, + "theoretical_loss": 3.740520265320649, + "tokens_seen": 838139904 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602864593781344, + "loss": 1.4582, + "theoretical_loss": 3.740491735713654, + "tokens_seen": 838205440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006028485456369108, + "loss": 1.3882, + "theoretical_loss": 3.740463208961708, + "tokens_seen": 838270976 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006028324974924774, + "loss": 1.3431, + "theoretical_loss": 3.7404346850643013, + "tokens_seen": 838336512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006028164493480442, + "loss": 1.4642, + "theoretical_loss": 3.740406164020926, + "tokens_seen": 838402048 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006028004012036108, + "loss": 1.3061, + "theoretical_loss": 3.740377645831072, + "tokens_seen": 838467584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006027843530591776, + "loss": 1.3809, + "theoretical_loss": 3.7403491304942325, + "tokens_seen": 838533120 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006027683049147442, + "loss": 1.4175, + "theoretical_loss": 3.740320618009898, + "tokens_seen": 838598656 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602752256770311, + "loss": 1.3319, + "theoretical_loss": 3.7402921083775604, + "tokens_seen": 838664192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006027362086258776, + "loss": 1.4919, + "theoretical_loss": 3.7402636015967117, + "tokens_seen": 838729728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006027201604814444, + "loss": 1.4921, + "theoretical_loss": 3.7402350976668446, + "tokens_seen": 838795264 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 315699, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4365313053131104, + "objective/train/theoretical_loss": 3.7402065965874502, + "objective/train/tokens_used": 859320800, + "theoretical_loss": 3.7402065965874502, + "tokens_seen": 838860800 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602704112337011, + "loss": 1.4365, + "theoretical_loss": 3.7402065965874502, + "tokens_seen": 838860800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006026880641925778, + "loss": 1.373, + "theoretical_loss": 3.7401780983580215, + "tokens_seen": 838926336 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006026720160481444, + "loss": 1.5032, + "theoretical_loss": 3.7401496029780503, + "tokens_seen": 838991872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006026559679037112, + "loss": 1.4986, + "theoretical_loss": 3.7401211104470296, + "tokens_seen": 839057408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006026399197592778, + "loss": 1.4864, + "theoretical_loss": 3.740092620764452, + "tokens_seen": 839122944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006026238716148446, + "loss": 1.4306, + "theoretical_loss": 3.74006413392981, + "tokens_seen": 839188480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006026078234704112, + "loss": 1.4272, + "theoretical_loss": 3.7400356499425973, + "tokens_seen": 839254016 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602591775325978, + "loss": 1.3683, + "theoretical_loss": 3.7400071688023067, + "tokens_seen": 839319552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006025757271815446, + "loss": 1.4921, + "theoretical_loss": 3.7399786905084307, + "tokens_seen": 839385088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006025596790371114, + "loss": 1.387, + "theoretical_loss": 3.739950215060463, + "tokens_seen": 839450624 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602543630892678, + "loss": 1.4373, + "theoretical_loss": 3.7399217424578977, + "tokens_seen": 839516160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006025275827482448, + "loss": 1.4514, + "theoretical_loss": 3.739893272700227, + "tokens_seen": 839581696 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006025115346038114, + "loss": 1.3528, + "theoretical_loss": 3.7398648057869464, + "tokens_seen": 839647232 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006024954864593782, + "loss": 1.469, + "theoretical_loss": 3.7398363417175484, + "tokens_seen": 839712768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006024794383149448, + "loss": 1.4787, + "theoretical_loss": 3.7398078804915276, + "tokens_seen": 839778304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006024633901705116, + "loss": 1.4924, + "theoretical_loss": 3.739779422108377, + "tokens_seen": 839843840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006024473420260782, + "loss": 1.4808, + "theoretical_loss": 3.7397509665675925, + "tokens_seen": 839909376 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602431293881645, + "loss": 1.4183, + "theoretical_loss": 3.7397225138686676, + "tokens_seen": 839974912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006024152457372116, + "loss": 1.4133, + "theoretical_loss": 3.739694064011097, + "tokens_seen": 840040448 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006023991975927784, + "loss": 1.4163, + "theoretical_loss": 3.7396656169943743, + "tokens_seen": 840105984 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602383149448345, + "loss": 1.4434, + "theoretical_loss": 3.739637172817996, + "tokens_seen": 840171520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006023671013039118, + "loss": 1.4403, + "theoretical_loss": 3.739608731481456, + "tokens_seen": 840237056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006023510531594784, + "loss": 1.3661, + "theoretical_loss": 3.7395802929842494, + "tokens_seen": 840302592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006023350050150452, + "loss": 1.4545, + "theoretical_loss": 3.739551857325871, + "tokens_seen": 840368128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006023189568706118, + "loss": 1.3586, + "theoretical_loss": 3.7395234245058164, + "tokens_seen": 840433664 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006023029087261786, + "loss": 1.424, + "theoretical_loss": 3.7394949945235814, + "tokens_seen": 840499200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006022868605817452, + "loss": 1.4287, + "theoretical_loss": 3.7394665673786607, + "tokens_seen": 840564736 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602270812437312, + "loss": 1.3748, + "theoretical_loss": 3.739438143070551, + "tokens_seen": 840630272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006022547642928786, + "loss": 1.4235, + "theoretical_loss": 3.739409721598747, + "tokens_seen": 840695808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006022387161484454, + "loss": 1.4556, + "theoretical_loss": 3.739381302962745, + "tokens_seen": 840761344 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602222668004012, + "loss": 1.4377, + "theoretical_loss": 3.739352887162041, + "tokens_seen": 840826880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006022066198595788, + "loss": 1.3962, + "theoretical_loss": 3.739324474196132, + "tokens_seen": 840892416 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006021905717151454, + "loss": 1.4822, + "theoretical_loss": 3.739296064064513, + "tokens_seen": 840957952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006021745235707122, + "loss": 1.4332, + "theoretical_loss": 3.739267656766681, + "tokens_seen": 841023488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006021584754262788, + "loss": 1.3353, + "theoretical_loss": 3.7392392523021325, + "tokens_seen": 841089024 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006021424272818456, + "loss": 1.3808, + "theoretical_loss": 3.7392108506703647, + "tokens_seen": 841154560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006021263791374122, + "loss": 1.3493, + "theoretical_loss": 3.7391824518708736, + "tokens_seen": 841220096 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602110330992979, + "loss": 1.3264, + "theoretical_loss": 3.739154055903157, + "tokens_seen": 841285632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006020942828485457, + "loss": 1.3855, + "theoretical_loss": 3.7391256627667104, + "tokens_seen": 841351168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0006020782347041124, + "loss": 1.3808, + "theoretical_loss": 3.7390972724610334, + "tokens_seen": 841416704 + }, + { + "epoch": 0.25, + "learning_rate": 0.000602062186559679, + "loss": 1.4555, + "theoretical_loss": 3.7390688849856213, + "tokens_seen": 841482240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006020461384152458, + "loss": 1.3791, + "theoretical_loss": 3.7390405003399723, + "tokens_seen": 841547776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006020300902708125, + "loss": 1.3616, + "theoretical_loss": 3.7390121185235836, + "tokens_seen": 841613312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006020140421263792, + "loss": 1.4384, + "theoretical_loss": 3.7389837395359535, + "tokens_seen": 841678848 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006019979939819459, + "loss": 1.3925, + "theoretical_loss": 3.7389553633765797, + "tokens_seen": 841744384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006019819458375126, + "loss": 1.3572, + "theoretical_loss": 3.73892699004496, + "tokens_seen": 841809920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006019658976930793, + "loss": 1.4158, + "theoretical_loss": 3.7388986195405924, + "tokens_seen": 841875456 + }, + { + "epoch": 0.26, + "learning_rate": 0.000601949849548646, + "loss": 1.4317, + "theoretical_loss": 3.7388702518629753, + "tokens_seen": 841940992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006019338014042127, + "loss": 1.4029, + "theoretical_loss": 3.738841887011607, + "tokens_seen": 842006528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006019177532597794, + "loss": 1.4048, + "theoretical_loss": 3.7388135249859866, + "tokens_seen": 842072064 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006019017051153461, + "loss": 1.3102, + "theoretical_loss": 3.738785165785611, + "tokens_seen": 842137600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006018856569709128, + "loss": 1.4518, + "theoretical_loss": 3.7387568094099812, + "tokens_seen": 842203136 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006018696088264795, + "loss": 1.5404, + "theoretical_loss": 3.738728455858594, + "tokens_seen": 842268672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006018535606820462, + "loss": 1.4931, + "theoretical_loss": 3.73870010513095, + "tokens_seen": 842334208 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006018375125376129, + "loss": 1.4461, + "theoretical_loss": 3.7386717572265473, + "tokens_seen": 842399744 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006018214643931796, + "loss": 1.5013, + "theoretical_loss": 3.738643412144885, + "tokens_seen": 842465280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006018054162487463, + "loss": 1.4497, + "theoretical_loss": 3.738615069885464, + "tokens_seen": 842530816 + }, + { + "epoch": 0.26, + "learning_rate": 0.000601789368104313, + "loss": 1.4072, + "theoretical_loss": 3.738586730447782, + "tokens_seen": 842596352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006017733199598797, + "loss": 1.3597, + "theoretical_loss": 3.7385583938313394, + "tokens_seen": 842661888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006017572718154464, + "loss": 1.4125, + "theoretical_loss": 3.738530060035636, + "tokens_seen": 842727424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006017412236710131, + "loss": 1.4177, + "theoretical_loss": 3.7385017290601716, + "tokens_seen": 842792960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006017251755265798, + "loss": 1.4837, + "theoretical_loss": 3.7384734009044465, + "tokens_seen": 842858496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006017091273821465, + "loss": 1.3946, + "theoretical_loss": 3.7384450755679604, + "tokens_seen": 842924032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006016930792377131, + "loss": 1.487, + "theoretical_loss": 3.7384167530502133, + "tokens_seen": 842989568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006016770310932799, + "loss": 1.3193, + "theoretical_loss": 3.738388433350707, + "tokens_seen": 843055104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006016609829488465, + "loss": 1.4664, + "theoretical_loss": 3.7383601164689404, + "tokens_seen": 843120640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006016449348044133, + "loss": 1.438, + "theoretical_loss": 3.7383318024044145, + "tokens_seen": 843186176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006016288866599799, + "loss": 1.4339, + "theoretical_loss": 3.738303491156631, + "tokens_seen": 843251712 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006016128385155467, + "loss": 1.5208, + "theoretical_loss": 3.73827518272509, + "tokens_seen": 843317248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015967903711133, + "loss": 1.3451, + "theoretical_loss": 3.738246877109293, + "tokens_seen": 843382784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015807422266801, + "loss": 1.427, + "theoretical_loss": 3.7382185743087404, + "tokens_seen": 843448320 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015646940822467, + "loss": 1.4517, + "theoretical_loss": 3.7381902743229345, + "tokens_seen": 843513856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015486459378135, + "loss": 1.4115, + "theoretical_loss": 3.7381619771513757, + "tokens_seen": 843579392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015325977933801, + "loss": 1.4419, + "theoretical_loss": 3.738133682793566, + "tokens_seen": 843644928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015165496489469, + "loss": 1.4163, + "theoretical_loss": 3.7381053912490074, + "tokens_seen": 843710464 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006015005015045135, + "loss": 1.4274, + "theoretical_loss": 3.7380771025172015, + "tokens_seen": 843776000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006014844533600803, + "loss": 1.4185, + "theoretical_loss": 3.73804881659765, + "tokens_seen": 843841536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006014684052156469, + "loss": 1.425, + "theoretical_loss": 3.738020533489855, + "tokens_seen": 843907072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006014523570712137, + "loss": 1.3562, + "theoretical_loss": 3.737992253193319, + "tokens_seen": 843972608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006014363089267803, + "loss": 1.3268, + "theoretical_loss": 3.737963975707544, + "tokens_seen": 844038144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006014202607823471, + "loss": 1.3786, + "theoretical_loss": 3.7379357010320318, + "tokens_seen": 844103680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006014042126379137, + "loss": 1.3363, + "theoretical_loss": 3.7379074291662864, + "tokens_seen": 844169216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006013881644934805, + "loss": 1.4821, + "theoretical_loss": 3.7378791601098094, + "tokens_seen": 844234752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006013721163490471, + "loss": 1.4359, + "theoretical_loss": 3.737850893862104, + "tokens_seen": 844300288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006013560682046139, + "loss": 1.415, + "theoretical_loss": 3.7378226304226727, + "tokens_seen": 844365824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006013400200601805, + "loss": 1.4997, + "theoretical_loss": 3.737794369791019, + "tokens_seen": 844431360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006013239719157473, + "loss": 1.3588, + "theoretical_loss": 3.7377661119666463, + "tokens_seen": 844496896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006013079237713139, + "loss": 1.3868, + "theoretical_loss": 3.7377378569490576, + "tokens_seen": 844562432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006012918756268807, + "loss": 1.4164, + "theoretical_loss": 3.7377096047377565, + "tokens_seen": 844627968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006012758274824473, + "loss": 1.4196, + "theoretical_loss": 3.7376813553322465, + "tokens_seen": 844693504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006012597793380141, + "loss": 1.4795, + "theoretical_loss": 3.737653108732031, + "tokens_seen": 844759040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006012437311935807, + "loss": 1.3718, + "theoretical_loss": 3.7376248649366137, + "tokens_seen": 844824576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006012276830491475, + "loss": 1.4303, + "theoretical_loss": 3.737596623945499, + "tokens_seen": 844890112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006012116349047141, + "loss": 1.3692, + "theoretical_loss": 3.737568385758191, + "tokens_seen": 844955648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006011955867602809, + "loss": 1.3566, + "theoretical_loss": 3.737540150374194, + "tokens_seen": 845021184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006011795386158475, + "loss": 1.3844, + "theoretical_loss": 3.737511917793012, + "tokens_seen": 845086720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006011634904714143, + "loss": 1.4246, + "theoretical_loss": 3.7374836880141493, + "tokens_seen": 845152256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006011474423269809, + "loss": 1.4294, + "theoretical_loss": 3.737455461037111, + "tokens_seen": 845217792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006011313941825477, + "loss": 1.4413, + "theoretical_loss": 3.737427236861401, + "tokens_seen": 845283328 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006011153460381143, + "loss": 1.4186, + "theoretical_loss": 3.7373990154865253, + "tokens_seen": 845348864 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 318483, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3942989110946655, + "objective/train/theoretical_loss": 3.7373707969119883, + "objective/train/tokens_used": 865874400, + "theoretical_loss": 3.7373707969119883, + "tokens_seen": 845414400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010992978936811, + "loss": 1.3943, + "theoretical_loss": 3.7373707969119883, + "tokens_seen": 845414400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010832497492477, + "loss": 1.3078, + "theoretical_loss": 3.7373425811372942, + "tokens_seen": 845479936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010672016048145, + "loss": 1.492, + "theoretical_loss": 3.7373143681619494, + "tokens_seen": 845545472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010511534603811, + "loss": 1.4471, + "theoretical_loss": 3.737286157985459, + "tokens_seen": 845611008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010351053159479, + "loss": 1.3349, + "theoretical_loss": 3.737257950607328, + "tokens_seen": 845676544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010190571715145, + "loss": 1.3998, + "theoretical_loss": 3.7372297460270625, + "tokens_seen": 845742080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006010030090270813, + "loss": 1.3584, + "theoretical_loss": 3.7372015442441677, + "tokens_seen": 845807616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006009869608826479, + "loss": 1.3802, + "theoretical_loss": 3.7371733452581504, + "tokens_seen": 845873152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006009709127382147, + "loss": 1.3709, + "theoretical_loss": 3.737145149068515, + "tokens_seen": 845938688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006009548645937813, + "loss": 1.3879, + "theoretical_loss": 3.737116955674769, + "tokens_seen": 846004224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006009388164493481, + "loss": 1.4528, + "theoretical_loss": 3.7370887650764177, + "tokens_seen": 846069760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006009227683049147, + "loss": 1.3697, + "theoretical_loss": 3.7370605772729677, + "tokens_seen": 846135296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006009067201604815, + "loss": 1.4119, + "theoretical_loss": 3.7370323922639264, + "tokens_seen": 846200832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006008906720160481, + "loss": 1.5364, + "theoretical_loss": 3.7370042100487986, + "tokens_seen": 846266368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006008746238716149, + "loss": 1.3602, + "theoretical_loss": 3.7369760306270923, + "tokens_seen": 846331904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006008585757271815, + "loss": 1.4338, + "theoretical_loss": 3.7369478539983145, + "tokens_seen": 846397440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006008425275827483, + "loss": 1.3963, + "theoretical_loss": 3.736919680161971, + "tokens_seen": 846462976 + }, + { + "epoch": 0.26, + "learning_rate": 0.000600826479438315, + "loss": 1.459, + "theoretical_loss": 3.7368915091175703, + "tokens_seen": 846528512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006008104312938817, + "loss": 1.4387, + "theoretical_loss": 3.736863340864618, + "tokens_seen": 846594048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006007943831494483, + "loss": 1.3399, + "theoretical_loss": 3.7368351754026228, + "tokens_seen": 846659584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006007783350050151, + "loss": 1.3299, + "theoretical_loss": 3.7368070127310915, + "tokens_seen": 846725120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006007622868605817, + "loss": 1.4539, + "theoretical_loss": 3.736778852849532, + "tokens_seen": 846790656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006007462387161485, + "loss": 1.4437, + "theoretical_loss": 3.736750695757452, + "tokens_seen": 846856192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006007301905717151, + "loss": 1.3006, + "theoretical_loss": 3.736722541454359, + "tokens_seen": 846921728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006007141424272819, + "loss": 1.3279, + "theoretical_loss": 3.7366943899397613, + "tokens_seen": 846987264 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006006980942828486, + "loss": 1.5343, + "theoretical_loss": 3.736666241213167, + "tokens_seen": 847052800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006006820461384153, + "loss": 1.451, + "theoretical_loss": 3.736638095274084, + "tokens_seen": 847118336 + }, + { + "epoch": 0.26, + "learning_rate": 0.000600665997993982, + "loss": 1.3482, + "theoretical_loss": 3.736609952122021, + "tokens_seen": 847183872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006006499498495487, + "loss": 1.4938, + "theoretical_loss": 3.736581811756486, + "tokens_seen": 847249408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006006339017051154, + "loss": 1.4229, + "theoretical_loss": 3.736553674176988, + "tokens_seen": 847314944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006006178535606821, + "loss": 1.3783, + "theoretical_loss": 3.736525539383036, + "tokens_seen": 847380480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006006018054162488, + "loss": 1.3859, + "theoretical_loss": 3.736497407374138, + "tokens_seen": 847446016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006005857572718155, + "loss": 1.3666, + "theoretical_loss": 3.7364692781498032, + "tokens_seen": 847511552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006005697091273823, + "loss": 1.2963, + "theoretical_loss": 3.7364411517095415, + "tokens_seen": 847577088 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006005536609829489, + "loss": 1.4159, + "theoretical_loss": 3.736413028052861, + "tokens_seen": 847642624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006005376128385157, + "loss": 1.3671, + "theoretical_loss": 3.736384907179272, + "tokens_seen": 847708160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006005215646940823, + "loss": 1.4265, + "theoretical_loss": 3.736356789088283, + "tokens_seen": 847773696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006005055165496491, + "loss": 1.3503, + "theoretical_loss": 3.7363286737794046, + "tokens_seen": 847839232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006004894684052157, + "loss": 1.3488, + "theoretical_loss": 3.7363005612521456, + "tokens_seen": 847904768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006004734202607825, + "loss": 1.3878, + "theoretical_loss": 3.736272451506016, + "tokens_seen": 847970304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006004573721163491, + "loss": 1.2443, + "theoretical_loss": 3.7362443445405265, + "tokens_seen": 848035840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006004413239719158, + "loss": 1.3704, + "theoretical_loss": 3.736216240355186, + "tokens_seen": 848101376 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006004252758274825, + "loss": 1.5809, + "theoretical_loss": 3.7361881389495064, + "tokens_seen": 848166912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006004092276830492, + "loss": 1.3379, + "theoretical_loss": 3.736160040322996, + "tokens_seen": 848232448 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006003931795386159, + "loss": 1.4392, + "theoretical_loss": 3.736131944475167, + "tokens_seen": 848297984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006003771313941826, + "loss": 1.4366, + "theoretical_loss": 3.7361038514055287, + "tokens_seen": 848363520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006003610832497493, + "loss": 1.3496, + "theoretical_loss": 3.7360757611135926, + "tokens_seen": 848429056 + }, + { + "epoch": 0.26, + "learning_rate": 0.000600345035105316, + "loss": 1.5124, + "theoretical_loss": 3.7360476735988692, + "tokens_seen": 848494592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006003289869608827, + "loss": 1.3552, + "theoretical_loss": 3.736019588860869, + "tokens_seen": 848560128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006003129388164494, + "loss": 1.4769, + "theoretical_loss": 3.7359915068991043, + "tokens_seen": 848625664 + }, + { + "epoch": 0.26, + "learning_rate": 0.000600296890672016, + "loss": 1.2432, + "theoretical_loss": 3.7359634277130853, + "tokens_seen": 848691200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006002808425275828, + "loss": 1.3953, + "theoretical_loss": 3.7359353513023237, + "tokens_seen": 848756736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006002647943831494, + "loss": 1.3855, + "theoretical_loss": 3.7359072776663305, + "tokens_seen": 848822272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006002487462387162, + "loss": 1.4316, + "theoretical_loss": 3.735879206804618, + "tokens_seen": 848887808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006002326980942828, + "loss": 1.3959, + "theoretical_loss": 3.735851138716697, + "tokens_seen": 848953344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006002166499498496, + "loss": 1.3504, + "theoretical_loss": 3.7358230734020803, + "tokens_seen": 849018880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006002006018054162, + "loss": 1.4852, + "theoretical_loss": 3.7357950108602793, + "tokens_seen": 849084416 + }, + { + "epoch": 0.26, + "learning_rate": 0.000600184553660983, + "loss": 1.3925, + "theoretical_loss": 3.735766951090806, + "tokens_seen": 849149952 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006001685055165496, + "loss": 1.4171, + "theoretical_loss": 3.735738894093173, + "tokens_seen": 849215488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006001524573721164, + "loss": 1.3646, + "theoretical_loss": 3.7357108398668917, + "tokens_seen": 849281024 + }, + { + "epoch": 0.26, + "learning_rate": 0.000600136409227683, + "loss": 1.3823, + "theoretical_loss": 3.7356827884114754, + "tokens_seen": 849346560 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006001203610832498, + "loss": 1.5112, + "theoretical_loss": 3.735654739726437, + "tokens_seen": 849412096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006001043129388164, + "loss": 1.4472, + "theoretical_loss": 3.735626693811288, + "tokens_seen": 849477632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006000882647943832, + "loss": 1.4307, + "theoretical_loss": 3.7355986506655414, + "tokens_seen": 849543168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006000722166499498, + "loss": 1.4708, + "theoretical_loss": 3.735570610288711, + "tokens_seen": 849608704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006000561685055166, + "loss": 1.4948, + "theoretical_loss": 3.73554257268031, + "tokens_seen": 849674240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006000401203610832, + "loss": 1.3339, + "theoretical_loss": 3.73551453783985, + "tokens_seen": 849739776 + }, + { + "epoch": 0.26, + "learning_rate": 0.00060002407221665, + "loss": 1.4042, + "theoretical_loss": 3.7354865057668456, + "tokens_seen": 849805312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0006000080240722166, + "loss": 1.4436, + "theoretical_loss": 3.73545847646081, + "tokens_seen": 849870848 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005999919759277834, + "loss": 1.4178, + "theoretical_loss": 3.735430449921256, + "tokens_seen": 849936384 + }, + { + "epoch": 0.26, + "learning_rate": 0.00059997592778335, + "loss": 1.3592, + "theoretical_loss": 3.735402426147698, + "tokens_seen": 850001920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005999598796389168, + "loss": 1.4526, + "theoretical_loss": 3.7353744051396505, + "tokens_seen": 850067456 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005999438314944834, + "loss": 1.4274, + "theoretical_loss": 3.7353463868966257, + "tokens_seen": 850132992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005999277833500502, + "loss": 1.5053, + "theoretical_loss": 3.7353183714181393, + "tokens_seen": 850198528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005999117352056168, + "loss": 1.4134, + "theoretical_loss": 3.735290358703704, + "tokens_seen": 850264064 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005998956870611836, + "loss": 1.4052, + "theoretical_loss": 3.735262348752835, + "tokens_seen": 850329600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005998796389167502, + "loss": 1.4566, + "theoretical_loss": 3.7352343415650466, + "tokens_seen": 850395136 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599863590772317, + "loss": 1.4314, + "theoretical_loss": 3.7352063371398527, + "tokens_seen": 850460672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005998475426278836, + "loss": 1.4106, + "theoretical_loss": 3.7351783354767685, + "tokens_seen": 850526208 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005998314944834504, + "loss": 1.3802, + "theoretical_loss": 3.735150336575309, + "tokens_seen": 850591744 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599815446339017, + "loss": 1.3993, + "theoretical_loss": 3.7351223404349887, + "tokens_seen": 850657280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005997993981945838, + "loss": 1.3846, + "theoretical_loss": 3.7350943470553224, + "tokens_seen": 850722816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005997833500501504, + "loss": 1.4889, + "theoretical_loss": 3.735066356435826, + "tokens_seen": 850788352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005997673019057172, + "loss": 1.438, + "theoretical_loss": 3.735038368576014, + "tokens_seen": 850853888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005997512537612838, + "loss": 1.4097, + "theoretical_loss": 3.7350103834754025, + "tokens_seen": 850919424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005997352056168506, + "loss": 1.4749, + "theoretical_loss": 3.734982401133506, + "tokens_seen": 850984960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005997191574724172, + "loss": 1.4439, + "theoretical_loss": 3.734954421549841, + "tokens_seen": 851050496 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599703109327984, + "loss": 1.3984, + "theoretical_loss": 3.734926444723923, + "tokens_seen": 851116032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005996870611835506, + "loss": 1.4486, + "theoretical_loss": 3.734898470655268, + "tokens_seen": 851181568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005996710130391174, + "loss": 1.3305, + "theoretical_loss": 3.7348704993433914, + "tokens_seen": 851247104 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599654964894684, + "loss": 1.394, + "theoretical_loss": 3.7348425307878097, + "tokens_seen": 851312640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005996389167502508, + "loss": 1.4528, + "theoretical_loss": 3.7348145649880395, + "tokens_seen": 851378176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005996228686058174, + "loss": 1.4744, + "theoretical_loss": 3.734786601943597, + "tokens_seen": 851443712 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005996068204613842, + "loss": 1.3265, + "theoretical_loss": 3.7347586416539986, + "tokens_seen": 851509248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005995907723169508, + "loss": 1.3484, + "theoretical_loss": 3.7347306841187606, + "tokens_seen": 851574784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005995747241725176, + "loss": 1.4123, + "theoretical_loss": 3.7347027293373998, + "tokens_seen": 851640320 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005995586760280842, + "loss": 1.3922, + "theoretical_loss": 3.7346747773094338, + "tokens_seen": 851705856 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599542627883651, + "loss": 1.4091, + "theoretical_loss": 3.7346468280343785, + "tokens_seen": 851771392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005995265797392176, + "loss": 1.4126, + "theoretical_loss": 3.734618881511752, + "tokens_seen": 851836928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005995105315947844, + "loss": 1.3989, + "theoretical_loss": 3.7345909377410704, + "tokens_seen": 851902464 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 320765, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5034074783325195, + "objective/train/theoretical_loss": 3.7345629967218517, + "objective/train/tokens_used": 872428000, + "theoretical_loss": 3.7345629967218517, + "tokens_seen": 851968000 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599494483450351, + "loss": 1.5034, + "theoretical_loss": 3.7345629967218517, + "tokens_seen": 851968000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005994784353059178, + "loss": 1.3603, + "theoretical_loss": 3.7345350584536137, + "tokens_seen": 852033536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005994623871614844, + "loss": 1.3971, + "theoretical_loss": 3.7345071229358733, + "tokens_seen": 852099072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005994463390170512, + "loss": 1.5526, + "theoretical_loss": 3.7344791901681487, + "tokens_seen": 852164608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005994302908726178, + "loss": 1.3887, + "theoretical_loss": 3.734451260149957, + "tokens_seen": 852230144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005994142427281846, + "loss": 1.411, + "theoretical_loss": 3.7344233328808176, + "tokens_seen": 852295680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005993981945837514, + "loss": 1.526, + "theoretical_loss": 3.7343954083602466, + "tokens_seen": 852361216 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599382146439318, + "loss": 1.4671, + "theoretical_loss": 3.734367486587764, + "tokens_seen": 852426752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005993660982948848, + "loss": 1.4575, + "theoretical_loss": 3.7343395675628868, + "tokens_seen": 852492288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005993500501504514, + "loss": 1.4345, + "theoretical_loss": 3.734311651285134, + "tokens_seen": 852557824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005993340020060182, + "loss": 1.4102, + "theoretical_loss": 3.7342837377540246, + "tokens_seen": 852623360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005993179538615848, + "loss": 1.3767, + "theoretical_loss": 3.7342558269690764, + "tokens_seen": 852688896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005993019057171516, + "loss": 1.4039, + "theoretical_loss": 3.7342279189298093, + "tokens_seen": 852754432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005992858575727182, + "loss": 1.4089, + "theoretical_loss": 3.7342000136357405, + "tokens_seen": 852819968 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599269809428285, + "loss": 1.415, + "theoretical_loss": 3.734172111086391, + "tokens_seen": 852885504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005992537612838516, + "loss": 1.4226, + "theoretical_loss": 3.7341442112812784, + "tokens_seen": 852951040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005992377131394184, + "loss": 1.343, + "theoretical_loss": 3.7341163142199223, + "tokens_seen": 853016576 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599221664994985, + "loss": 1.4195, + "theoretical_loss": 3.734088419901843, + "tokens_seen": 853082112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005992056168505518, + "loss": 1.4237, + "theoretical_loss": 3.734060528326559, + "tokens_seen": 853147648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005991895687061184, + "loss": 1.4015, + "theoretical_loss": 3.7340326394935905, + "tokens_seen": 853213184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005991735205616852, + "loss": 1.4356, + "theoretical_loss": 3.734004753402457, + "tokens_seen": 853278720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005991574724172518, + "loss": 1.403, + "theoretical_loss": 3.7339768700526785, + "tokens_seen": 853344256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005991414242728186, + "loss": 1.4051, + "theoretical_loss": 3.7339489894437756, + "tokens_seen": 853409792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005991253761283852, + "loss": 1.4272, + "theoretical_loss": 3.733921111575267, + "tokens_seen": 853475328 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599109327983952, + "loss": 1.3915, + "theoretical_loss": 3.7338932364466744, + "tokens_seen": 853540864 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005990932798395186, + "loss": 1.4761, + "theoretical_loss": 3.7338653640575172, + "tokens_seen": 853606400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005990772316950854, + "loss": 1.4398, + "theoretical_loss": 3.7338374944073163, + "tokens_seen": 853671936 + }, + { + "epoch": 0.26, + "learning_rate": 0.000599061183550652, + "loss": 1.4255, + "theoretical_loss": 3.7338096274955923, + "tokens_seen": 853737472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005990451354062187, + "loss": 1.3446, + "theoretical_loss": 3.733781763321866, + "tokens_seen": 853803008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005990290872617854, + "loss": 1.3075, + "theoretical_loss": 3.7337539018856587, + "tokens_seen": 853868544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005990130391173521, + "loss": 1.4593, + "theoretical_loss": 3.73372604318649, + "tokens_seen": 853934080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989969909729188, + "loss": 1.3969, + "theoretical_loss": 3.733698187223882, + "tokens_seen": 853999616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989809428284855, + "loss": 1.4267, + "theoretical_loss": 3.733670333997356, + "tokens_seen": 854065152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989648946840522, + "loss": 1.3974, + "theoretical_loss": 3.733642483506433, + "tokens_seen": 854130688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989488465396189, + "loss": 1.5286, + "theoretical_loss": 3.7336146357506337, + "tokens_seen": 854196224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989327983951856, + "loss": 1.4348, + "theoretical_loss": 3.7335867907294817, + "tokens_seen": 854261760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989167502507523, + "loss": 1.3914, + "theoretical_loss": 3.733558948442497, + "tokens_seen": 854327296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005989007021063189, + "loss": 1.4199, + "theoretical_loss": 3.733531108889202, + "tokens_seen": 854392832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005988846539618857, + "loss": 1.5527, + "theoretical_loss": 3.733503272069118, + "tokens_seen": 854458368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005988686058174523, + "loss": 1.4128, + "theoretical_loss": 3.7334754379817685, + "tokens_seen": 854523904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005988525576730191, + "loss": 1.4354, + "theoretical_loss": 3.733447606626674, + "tokens_seen": 854589440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005988365095285857, + "loss": 1.5357, + "theoretical_loss": 3.7334197780033582, + "tokens_seen": 854654976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005988204613841525, + "loss": 1.3664, + "theoretical_loss": 3.733391952111343, + "tokens_seen": 854720512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005988044132397191, + "loss": 1.4623, + "theoretical_loss": 3.73336412895015, + "tokens_seen": 854786048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005987883650952859, + "loss": 1.4568, + "theoretical_loss": 3.733336308519303, + "tokens_seen": 854851584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005987723169508525, + "loss": 1.3671, + "theoretical_loss": 3.7333084908183247, + "tokens_seen": 854917120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005987562688064193, + "loss": 1.3179, + "theoretical_loss": 3.7332806758467383, + "tokens_seen": 854982656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005987402206619859, + "loss": 1.3704, + "theoretical_loss": 3.7332528636040654, + "tokens_seen": 855048192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005987241725175527, + "loss": 1.4031, + "theoretical_loss": 3.7332250540898304, + "tokens_seen": 855113728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005987081243731193, + "loss": 1.4278, + "theoretical_loss": 3.733197247303556, + "tokens_seen": 855179264 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005986920762286861, + "loss": 1.4515, + "theoretical_loss": 3.733169443244766, + "tokens_seen": 855244800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005986760280842527, + "loss": 1.5505, + "theoretical_loss": 3.7331416419129835, + "tokens_seen": 855310336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005986599799398195, + "loss": 1.3758, + "theoretical_loss": 3.7331138433077324, + "tokens_seen": 855375872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005986439317953861, + "loss": 1.4887, + "theoretical_loss": 3.733086047428536, + "tokens_seen": 855441408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005986278836509529, + "loss": 1.4416, + "theoretical_loss": 3.733058254274919, + "tokens_seen": 855506944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005986118355065195, + "loss": 1.4767, + "theoretical_loss": 3.7330304638464042, + "tokens_seen": 855572480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005985957873620863, + "loss": 1.4259, + "theoretical_loss": 3.733002676142517, + "tokens_seen": 855638016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005985797392176529, + "loss": 1.5172, + "theoretical_loss": 3.7329748911627805, + "tokens_seen": 855703552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005985636910732197, + "loss": 1.4041, + "theoretical_loss": 3.7329471089067194, + "tokens_seen": 855769088 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005985476429287863, + "loss": 1.3092, + "theoretical_loss": 3.7329193293738587, + "tokens_seen": 855834624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005985315947843531, + "loss": 1.3647, + "theoretical_loss": 3.7328915525637227, + "tokens_seen": 855900160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005985155466399197, + "loss": 1.4647, + "theoretical_loss": 3.7328637784758354, + "tokens_seen": 855965696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984994984954865, + "loss": 1.3943, + "theoretical_loss": 3.7328360071097224, + "tokens_seen": 856031232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984834503510531, + "loss": 1.3581, + "theoretical_loss": 3.7328082384649086, + "tokens_seen": 856096768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984674022066199, + "loss": 1.2951, + "theoretical_loss": 3.7327804725409184, + "tokens_seen": 856162304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984513540621865, + "loss": 1.3794, + "theoretical_loss": 3.7327527093372774, + "tokens_seen": 856227840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984353059177533, + "loss": 1.4144, + "theoretical_loss": 3.7327249488535106, + "tokens_seen": 856293376 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984192577733199, + "loss": 1.3714, + "theoretical_loss": 3.7326971910891444, + "tokens_seen": 856358912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005984032096288867, + "loss": 1.3578, + "theoretical_loss": 3.732669436043703, + "tokens_seen": 856424448 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005983871614844533, + "loss": 1.4384, + "theoretical_loss": 3.7326416837167127, + "tokens_seen": 856489984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005983711133400201, + "loss": 1.3937, + "theoretical_loss": 3.7326139341076994, + "tokens_seen": 856555520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005983550651955867, + "loss": 1.3046, + "theoretical_loss": 3.7325861872161887, + "tokens_seen": 856621056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005983390170511535, + "loss": 1.4875, + "theoretical_loss": 3.732558443041707, + "tokens_seen": 856686592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005983229689067201, + "loss": 1.4506, + "theoretical_loss": 3.73253070158378, + "tokens_seen": 856752128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005983069207622869, + "loss": 1.4151, + "theoretical_loss": 3.732502962841934, + "tokens_seen": 856817664 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005982908726178535, + "loss": 1.4372, + "theoretical_loss": 3.732475226815695, + "tokens_seen": 856883200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005982748244734203, + "loss": 1.3573, + "theoretical_loss": 3.7324474935045906, + "tokens_seen": 856948736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005982587763289869, + "loss": 1.3637, + "theoretical_loss": 3.7324197629081466, + "tokens_seen": 857014272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005982427281845537, + "loss": 1.4742, + "theoretical_loss": 3.7323920350258897, + "tokens_seen": 857079808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005982266800401203, + "loss": 1.4568, + "theoretical_loss": 3.732364309857347, + "tokens_seen": 857145344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005982106318956871, + "loss": 1.3834, + "theoretical_loss": 3.7323365874020453, + "tokens_seen": 857210880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005981945837512539, + "loss": 1.4047, + "theoretical_loss": 3.732308867659511, + "tokens_seen": 857276416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005981785356068205, + "loss": 1.3738, + "theoretical_loss": 3.732281150629273, + "tokens_seen": 857341952 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005981624874623873, + "loss": 1.4846, + "theoretical_loss": 3.732253436310857, + "tokens_seen": 857407488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005981464393179539, + "loss": 1.4319, + "theoretical_loss": 3.732225724703791, + "tokens_seen": 857473024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005981303911735207, + "loss": 1.4941, + "theoretical_loss": 3.7321980158076027, + "tokens_seen": 857538560 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005981143430290873, + "loss": 1.4156, + "theoretical_loss": 3.7321703096218197, + "tokens_seen": 857604096 + }, + { + "epoch": 0.26, + "learning_rate": 0.000598098294884654, + "loss": 1.4441, + "theoretical_loss": 3.7321426061459704, + "tokens_seen": 857669632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005980822467402207, + "loss": 1.4606, + "theoretical_loss": 3.7321149053795812, + "tokens_seen": 857735168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005980661985957875, + "loss": 1.3809, + "theoretical_loss": 3.7320872073221807, + "tokens_seen": 857800704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005980501504513541, + "loss": 1.3391, + "theoretical_loss": 3.732059511973298, + "tokens_seen": 857866240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005980341023069209, + "loss": 1.2974, + "theoretical_loss": 3.7320318193324598, + "tokens_seen": 857931776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005980180541624875, + "loss": 1.3669, + "theoretical_loss": 3.732004129399196, + "tokens_seen": 857997312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005980020060180543, + "loss": 1.3902, + "theoretical_loss": 3.731976442173034, + "tokens_seen": 858062848 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005979859578736209, + "loss": 1.486, + "theoretical_loss": 3.731948757653503, + "tokens_seen": 858128384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005979699097291877, + "loss": 1.3762, + "theoretical_loss": 3.7319210758401318, + "tokens_seen": 858193920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005979538615847543, + "loss": 1.3303, + "theoretical_loss": 3.7318933967324486, + "tokens_seen": 858259456 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005979378134403211, + "loss": 1.4191, + "theoretical_loss": 3.731865720329983, + "tokens_seen": 858324992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005979217652958877, + "loss": 1.4683, + "theoretical_loss": 3.731838046632264, + "tokens_seen": 858390528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005979057171514545, + "loss": 1.451, + "theoretical_loss": 3.73181037563882, + "tokens_seen": 858456064 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 322986, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4144095182418823, + "objective/train/theoretical_loss": 3.7317827073491814, + "objective/train/tokens_used": 878981600, + "theoretical_loss": 3.7317827073491814, + "tokens_seen": 858521600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005978896690070211, + "loss": 1.4144, + "theoretical_loss": 3.7317827073491814, + "tokens_seen": 858521600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005978736208625879, + "loss": 1.3548, + "theoretical_loss": 3.7317550417628773, + "tokens_seen": 858587136 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005978575727181545, + "loss": 1.3528, + "theoretical_loss": 3.731727378879437, + "tokens_seen": 858652672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005978415245737213, + "loss": 1.5019, + "theoretical_loss": 3.7316997186983905, + "tokens_seen": 858718208 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005978254764292879, + "loss": 1.4287, + "theoretical_loss": 3.7316720612192666, + "tokens_seen": 858783744 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005978094282848547, + "loss": 1.3677, + "theoretical_loss": 3.731644406441597, + "tokens_seen": 858849280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005977933801404213, + "loss": 1.4482, + "theoretical_loss": 3.7316167543649104, + "tokens_seen": 858914816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005977773319959881, + "loss": 1.3677, + "theoretical_loss": 3.7315891049887373, + "tokens_seen": 858980352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005977612838515547, + "loss": 1.4032, + "theoretical_loss": 3.7315614583126084, + "tokens_seen": 859045888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005977452357071215, + "loss": 1.4383, + "theoretical_loss": 3.731533814336053, + "tokens_seen": 859111424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005977291875626881, + "loss": 1.3898, + "theoretical_loss": 3.7315061730586025, + "tokens_seen": 859176960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005977131394182549, + "loss": 1.3311, + "theoretical_loss": 3.7314785344797867, + "tokens_seen": 859242496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005976970912738215, + "loss": 1.5123, + "theoretical_loss": 3.731450898599138, + "tokens_seen": 859308032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005976810431293883, + "loss": 1.394, + "theoretical_loss": 3.731423265416185, + "tokens_seen": 859373568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005976649949849549, + "loss": 1.3531, + "theoretical_loss": 3.7313956349304602, + "tokens_seen": 859439104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005976489468405216, + "loss": 1.4374, + "theoretical_loss": 3.7313680071414943, + "tokens_seen": 859504640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005976328986960883, + "loss": 1.3206, + "theoretical_loss": 3.7313403820488187, + "tokens_seen": 859570176 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597616850551655, + "loss": 1.3265, + "theoretical_loss": 3.7313127596519644, + "tokens_seen": 859635712 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005976008024072217, + "loss": 1.4537, + "theoretical_loss": 3.7312851399504625, + "tokens_seen": 859701248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005975847542627884, + "loss": 1.4441, + "theoretical_loss": 3.731257522943846, + "tokens_seen": 859766784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005975687061183551, + "loss": 1.4642, + "theoretical_loss": 3.7312299086316445, + "tokens_seen": 859832320 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005975526579739218, + "loss": 1.5763, + "theoretical_loss": 3.731202297013391, + "tokens_seen": 859897856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005975366098294885, + "loss": 1.4581, + "theoretical_loss": 3.7311746880886174, + "tokens_seen": 859963392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005975205616850552, + "loss": 1.4535, + "theoretical_loss": 3.7311470818568555, + "tokens_seen": 860028928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005975045135406218, + "loss": 1.3697, + "theoretical_loss": 3.7311194783176376, + "tokens_seen": 860094464 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005974884653961886, + "loss": 1.4412, + "theoretical_loss": 3.7310918774704955, + "tokens_seen": 860160000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005974724172517552, + "loss": 1.4248, + "theoretical_loss": 3.731064279314962, + "tokens_seen": 860225536 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597456369107322, + "loss": 1.3192, + "theoretical_loss": 3.7310366838505704, + "tokens_seen": 860291072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005974403209628886, + "loss": 1.4749, + "theoretical_loss": 3.7310090910768516, + "tokens_seen": 860356608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005974242728184554, + "loss": 1.3932, + "theoretical_loss": 3.730981500993339, + "tokens_seen": 860422144 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597408224674022, + "loss": 1.3821, + "theoretical_loss": 3.7309539135995657, + "tokens_seen": 860487680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005973921765295888, + "loss": 1.3967, + "theoretical_loss": 3.7309263288950647, + "tokens_seen": 860553216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005973761283851554, + "loss": 1.3944, + "theoretical_loss": 3.730898746879369, + "tokens_seen": 860618752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005973600802407222, + "loss": 1.3535, + "theoretical_loss": 3.730871167552011, + "tokens_seen": 860684288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005973440320962888, + "loss": 1.3981, + "theoretical_loss": 3.7308435909125253, + "tokens_seen": 860749824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005973279839518556, + "loss": 1.4856, + "theoretical_loss": 3.7308160169604445, + "tokens_seen": 860815360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005973119358074222, + "loss": 1.4182, + "theoretical_loss": 3.7307884456953024, + "tokens_seen": 860880896 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597295887662989, + "loss": 1.2423, + "theoretical_loss": 3.7307608771166327, + "tokens_seen": 860946432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005972798395185556, + "loss": 1.4189, + "theoretical_loss": 3.730733311223969, + "tokens_seen": 861011968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005972637913741224, + "loss": 1.4311, + "theoretical_loss": 3.7307057480168453, + "tokens_seen": 861077504 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597247743229689, + "loss": 1.3973, + "theoretical_loss": 3.730678187494796, + "tokens_seen": 861143040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005972316950852558, + "loss": 1.4674, + "theoretical_loss": 3.7306506296573536, + "tokens_seen": 861208576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005972156469408224, + "loss": 1.4352, + "theoretical_loss": 3.7306230745040545, + "tokens_seen": 861274112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005971995987963892, + "loss": 1.3034, + "theoretical_loss": 3.730595522034432, + "tokens_seen": 861339648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005971835506519558, + "loss": 1.4862, + "theoretical_loss": 3.73056797224802, + "tokens_seen": 861405184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005971675025075226, + "loss": 1.3971, + "theoretical_loss": 3.730540425144354, + "tokens_seen": 861470720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005971514543630892, + "loss": 1.4213, + "theoretical_loss": 3.7305128807229684, + "tokens_seen": 861536256 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597135406218656, + "loss": 1.4791, + "theoretical_loss": 3.730485338983398, + "tokens_seen": 861601792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005971193580742226, + "loss": 1.3568, + "theoretical_loss": 3.730457799925178, + "tokens_seen": 861667328 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005971033099297894, + "loss": 1.3987, + "theoretical_loss": 3.7304302635478432, + "tokens_seen": 861732864 + }, + { + "epoch": 0.26, + "learning_rate": 0.000597087261785356, + "loss": 1.3911, + "theoretical_loss": 3.730402729850929, + "tokens_seen": 861798400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005970712136409228, + "loss": 1.467, + "theoretical_loss": 3.7303751988339697, + "tokens_seen": 861863936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005970551654964894, + "loss": 1.4217, + "theoretical_loss": 3.730347670496502, + "tokens_seen": 861929472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005970391173520562, + "loss": 1.4787, + "theoretical_loss": 3.7303201448380605, + "tokens_seen": 861995008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005970230692076228, + "loss": 1.4394, + "theoretical_loss": 3.7302926218581813, + "tokens_seen": 862060544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005970070210631896, + "loss": 1.4113, + "theoretical_loss": 3.730265101556401, + "tokens_seen": 862126080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005969909729187563, + "loss": 1.404, + "theoretical_loss": 3.7302375839322535, + "tokens_seen": 862191616 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596974924774323, + "loss": 1.5165, + "theoretical_loss": 3.7302100689852757, + "tokens_seen": 862257152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005969588766298897, + "loss": 1.333, + "theoretical_loss": 3.730182556715004, + "tokens_seen": 862322688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005969428284854564, + "loss": 1.4878, + "theoretical_loss": 3.7301550471209746, + "tokens_seen": 862388224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005969267803410231, + "loss": 1.3215, + "theoretical_loss": 3.730127540202723, + "tokens_seen": 862453760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005969107321965898, + "loss": 1.4762, + "theoretical_loss": 3.730100035959787, + "tokens_seen": 862519296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005968946840521565, + "loss": 1.4042, + "theoretical_loss": 3.730072534391702, + "tokens_seen": 862584832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005968786359077232, + "loss": 1.5453, + "theoretical_loss": 3.730045035498005, + "tokens_seen": 862650368 + }, + { + "epoch": 0.26, + "learning_rate": 0.00059686258776329, + "loss": 1.4539, + "theoretical_loss": 3.730017539278233, + "tokens_seen": 862715904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005968465396188566, + "loss": 1.4368, + "theoretical_loss": 3.729990045731922, + "tokens_seen": 862781440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005968304914744233, + "loss": 1.3772, + "theoretical_loss": 3.7299625548586106, + "tokens_seen": 862846976 + }, + { + "epoch": 0.26, + "learning_rate": 0.00059681444332999, + "loss": 1.4368, + "theoretical_loss": 3.729935066657835, + "tokens_seen": 862912512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005967983951855568, + "loss": 1.4279, + "theoretical_loss": 3.7299075811291322, + "tokens_seen": 862978048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005967823470411234, + "loss": 1.3733, + "theoretical_loss": 3.7298800982720404, + "tokens_seen": 863043584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005967662988966902, + "loss": 1.3412, + "theoretical_loss": 3.7298526180860962, + "tokens_seen": 863109120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005967502507522568, + "loss": 1.3876, + "theoretical_loss": 3.7298251405708376, + "tokens_seen": 863174656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005967342026078236, + "loss": 1.3429, + "theoretical_loss": 3.729797665725802, + "tokens_seen": 863240192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005967181544633902, + "loss": 1.4392, + "theoretical_loss": 3.729770193550528, + "tokens_seen": 863305728 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596702106318957, + "loss": 1.3594, + "theoretical_loss": 3.729742724044553, + "tokens_seen": 863371264 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005966860581745236, + "loss": 1.414, + "theoretical_loss": 3.7297152572074146, + "tokens_seen": 863436800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005966700100300904, + "loss": 1.4447, + "theoretical_loss": 3.729687793038652, + "tokens_seen": 863502336 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596653961885657, + "loss": 1.3322, + "theoretical_loss": 3.7296603315378025, + "tokens_seen": 863567872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005966379137412238, + "loss": 1.3669, + "theoretical_loss": 3.7296328727044052, + "tokens_seen": 863633408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005966218655967904, + "loss": 1.5763, + "theoretical_loss": 3.729605416537998, + "tokens_seen": 863698944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005966058174523572, + "loss": 1.491, + "theoretical_loss": 3.7295779630381203, + "tokens_seen": 863764480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005965897693079238, + "loss": 1.3041, + "theoretical_loss": 3.7295505122043098, + "tokens_seen": 863830016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005965737211634906, + "loss": 1.4548, + "theoretical_loss": 3.7295230640361066, + "tokens_seen": 863895552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005965576730190572, + "loss": 1.4378, + "theoretical_loss": 3.729495618533049, + "tokens_seen": 863961088 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596541624874624, + "loss": 1.4344, + "theoretical_loss": 3.7294681756946755, + "tokens_seen": 864026624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005965255767301906, + "loss": 1.4441, + "theoretical_loss": 3.7294407355205257, + "tokens_seen": 864092160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005965095285857574, + "loss": 1.422, + "theoretical_loss": 3.72941329801014, + "tokens_seen": 864157696 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596493480441324, + "loss": 1.5578, + "theoretical_loss": 3.7293858631630563, + "tokens_seen": 864223232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005964774322968908, + "loss": 1.4804, + "theoretical_loss": 3.7293584309788157, + "tokens_seen": 864288768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005964613841524574, + "loss": 1.3979, + "theoretical_loss": 3.729331001456956, + "tokens_seen": 864354304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005964453360080242, + "loss": 1.452, + "theoretical_loss": 3.7293035745970182, + "tokens_seen": 864419840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005964292878635908, + "loss": 1.4373, + "theoretical_loss": 3.729276150398542, + "tokens_seen": 864485376 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005964132397191576, + "loss": 1.3559, + "theoretical_loss": 3.7292487288610676, + "tokens_seen": 864550912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005963971915747242, + "loss": 1.4596, + "theoretical_loss": 3.7292213099841343, + "tokens_seen": 864616448 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596381143430291, + "loss": 1.4334, + "theoretical_loss": 3.7291938937672833, + "tokens_seen": 864681984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005963650952858576, + "loss": 1.3361, + "theoretical_loss": 3.729166480210054, + "tokens_seen": 864747520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005963490471414244, + "loss": 1.4686, + "theoretical_loss": 3.729139069311988, + "tokens_seen": 864813056 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596332998996991, + "loss": 1.3279, + "theoretical_loss": 3.729111661072625, + "tokens_seen": 864878592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005963169508525578, + "loss": 1.4462, + "theoretical_loss": 3.729084255491506, + "tokens_seen": 864944128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005963009027081244, + "loss": 1.4315, + "theoretical_loss": 3.7290568525681715, + "tokens_seen": 865009664 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 325229, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.391074776649475, + "objective/train/theoretical_loss": 3.7290294523021625, + "objective/train/tokens_used": 885535200, + "theoretical_loss": 3.7290294523021625, + "tokens_seen": 865075200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005962848545636912, + "loss": 1.3911, + "theoretical_loss": 3.7290294523021625, + "tokens_seen": 865075200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005962688064192578, + "loss": 1.5153, + "theoretical_loss": 3.729002054693021, + "tokens_seen": 865140736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005962527582748246, + "loss": 1.454, + "theoretical_loss": 3.7289746597402864, + "tokens_seen": 865206272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005962367101303912, + "loss": 1.4823, + "theoretical_loss": 3.7289472674435014, + "tokens_seen": 865271808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005962206619859579, + "loss": 1.4863, + "theoretical_loss": 3.728919877802207, + "tokens_seen": 865337344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005962046138415246, + "loss": 1.4393, + "theoretical_loss": 3.728892490815944, + "tokens_seen": 865402880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005961885656970913, + "loss": 1.3504, + "theoretical_loss": 3.7288651064842546, + "tokens_seen": 865468416 + }, + { + "epoch": 0.26, + "learning_rate": 0.000596172517552658, + "loss": 1.4408, + "theoretical_loss": 3.728837724806681, + "tokens_seen": 865533952 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005961564694082247, + "loss": 1.5147, + "theoretical_loss": 3.7288103457827635, + "tokens_seen": 865599488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005961404212637914, + "loss": 1.4479, + "theoretical_loss": 3.7287829694120456, + "tokens_seen": 865665024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005961243731193581, + "loss": 1.4261, + "theoretical_loss": 3.728755595694069, + "tokens_seen": 865730560 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005961083249749248, + "loss": 1.4748, + "theoretical_loss": 3.7287282246283757, + "tokens_seen": 865796096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005960922768304915, + "loss": 1.402, + "theoretical_loss": 3.728700856214507, + "tokens_seen": 865861632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005960762286860581, + "loss": 1.3777, + "theoretical_loss": 3.7286734904520076, + "tokens_seen": 865927168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005960601805416249, + "loss": 1.4653, + "theoretical_loss": 3.7286461273404177, + "tokens_seen": 865992704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005960441323971915, + "loss": 1.4553, + "theoretical_loss": 3.728618766879281, + "tokens_seen": 866058240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005960280842527583, + "loss": 1.441, + "theoretical_loss": 3.72859140906814, + "tokens_seen": 866123776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005960120361083249, + "loss": 1.3837, + "theoretical_loss": 3.7285640539065374, + "tokens_seen": 866189312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005959959879638917, + "loss": 1.3175, + "theoretical_loss": 3.7285367013940167, + "tokens_seen": 866254848 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005959799398194583, + "loss": 1.4744, + "theoretical_loss": 3.7285093515301213, + "tokens_seen": 866320384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005959638916750251, + "loss": 1.4098, + "theoretical_loss": 3.728482004314393, + "tokens_seen": 866385920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005959478435305917, + "loss": 1.473, + "theoretical_loss": 3.728454659746376, + "tokens_seen": 866451456 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005959317953861585, + "loss": 1.4394, + "theoretical_loss": 3.7284273178256133, + "tokens_seen": 866516992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005959157472417251, + "loss": 1.4154, + "theoretical_loss": 3.728399978551649, + "tokens_seen": 866582528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958996990972919, + "loss": 1.4281, + "theoretical_loss": 3.7283726419240266, + "tokens_seen": 866648064 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958836509528585, + "loss": 1.5484, + "theoretical_loss": 3.7283453079422895, + "tokens_seen": 866713600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958676028084253, + "loss": 1.4097, + "theoretical_loss": 3.7283179766059824, + "tokens_seen": 866779136 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958515546639919, + "loss": 1.3857, + "theoretical_loss": 3.7282906479146476, + "tokens_seen": 866844672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958355065195587, + "loss": 1.4763, + "theoretical_loss": 3.728263321867831, + "tokens_seen": 866910208 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958194583751253, + "loss": 1.4485, + "theoretical_loss": 3.7282359984650757, + "tokens_seen": 866975744 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005958034102306921, + "loss": 1.5658, + "theoretical_loss": 3.7282086777059265, + "tokens_seen": 867041280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005957873620862588, + "loss": 1.343, + "theoretical_loss": 3.728181359589928, + "tokens_seen": 867106816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005957713139418255, + "loss": 1.3968, + "theoretical_loss": 3.728154044116624, + "tokens_seen": 867172352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005957552657973922, + "loss": 1.3099, + "theoretical_loss": 3.72812673128556, + "tokens_seen": 867237888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005957392176529589, + "loss": 1.4087, + "theoretical_loss": 3.728099421096281, + "tokens_seen": 867303424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005957231695085256, + "loss": 1.3694, + "theoretical_loss": 3.72807211354833, + "tokens_seen": 867368960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005957071213640923, + "loss": 1.4411, + "theoretical_loss": 3.728044808641254, + "tokens_seen": 867434496 + }, + { + "epoch": 0.26, + "learning_rate": 0.000595691073219659, + "loss": 1.3473, + "theoretical_loss": 3.7280175063745973, + "tokens_seen": 867500032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005956750250752257, + "loss": 1.3821, + "theoretical_loss": 3.727990206747905, + "tokens_seen": 867565568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005956589769307924, + "loss": 1.536, + "theoretical_loss": 3.7279629097607234, + "tokens_seen": 867631104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005956429287863591, + "loss": 1.4954, + "theoretical_loss": 3.7279356154125964, + "tokens_seen": 867696640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005956268806419258, + "loss": 1.3823, + "theoretical_loss": 3.727908323703071, + "tokens_seen": 867762176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005956108324974925, + "loss": 1.4271, + "theoretical_loss": 3.7278810346316913, + "tokens_seen": 867827712 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005955947843530592, + "loss": 1.3917, + "theoretical_loss": 3.7278537481980045, + "tokens_seen": 867893248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005955787362086259, + "loss": 1.4586, + "theoretical_loss": 3.7278264644015566, + "tokens_seen": 867958784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005955626880641926, + "loss": 1.5422, + "theoretical_loss": 3.727799183241893, + "tokens_seen": 868024320 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005955466399197593, + "loss": 1.3969, + "theoretical_loss": 3.7277719047185593, + "tokens_seen": 868089856 + }, + { + "epoch": 0.26, + "learning_rate": 0.000595530591775326, + "loss": 1.4899, + "theoretical_loss": 3.7277446288311022, + "tokens_seen": 868155392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005955145436308927, + "loss": 1.4137, + "theoretical_loss": 3.7277173555790686, + "tokens_seen": 868220928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954984954864594, + "loss": 1.3836, + "theoretical_loss": 3.7276900849620045, + "tokens_seen": 868286464 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954824473420261, + "loss": 1.4994, + "theoretical_loss": 3.727662816979456, + "tokens_seen": 868352000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954663991975928, + "loss": 1.3978, + "theoretical_loss": 3.727635551630971, + "tokens_seen": 868417536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954503510531595, + "loss": 1.4269, + "theoretical_loss": 3.7276082889160955, + "tokens_seen": 868483072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954343029087263, + "loss": 1.4931, + "theoretical_loss": 3.727581028834376, + "tokens_seen": 868548608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954182547642929, + "loss": 1.4123, + "theoretical_loss": 3.7275537713853604, + "tokens_seen": 868614144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005954022066198597, + "loss": 1.3709, + "theoretical_loss": 3.727526516568595, + "tokens_seen": 868679680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005953861584754263, + "loss": 1.42, + "theoretical_loss": 3.727499264383628, + "tokens_seen": 868745216 + }, + { + "epoch": 0.26, + "learning_rate": 0.000595370110330993, + "loss": 1.4423, + "theoretical_loss": 3.7274720148300062, + "tokens_seen": 868810752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005953540621865597, + "loss": 1.4093, + "theoretical_loss": 3.727444767907277, + "tokens_seen": 868876288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005953380140421265, + "loss": 1.3647, + "theoretical_loss": 3.7274175236149887, + "tokens_seen": 868941824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005953219658976931, + "loss": 1.3933, + "theoretical_loss": 3.7273902819526876, + "tokens_seen": 869007360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005953059177532599, + "loss": 1.3523, + "theoretical_loss": 3.7273630429199223, + "tokens_seen": 869072896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005952898696088265, + "loss": 1.5056, + "theoretical_loss": 3.727335806516241, + "tokens_seen": 869138432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005952738214643933, + "loss": 1.4361, + "theoretical_loss": 3.727308572741191, + "tokens_seen": 869203968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005952577733199599, + "loss": 1.4701, + "theoretical_loss": 3.7272813415943222, + "tokens_seen": 869269504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005952417251755267, + "loss": 1.5216, + "theoretical_loss": 3.7272541130751806, + "tokens_seen": 869335040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005952256770310933, + "loss": 1.4525, + "theoretical_loss": 3.727226887183315, + "tokens_seen": 869400576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005952096288866601, + "loss": 1.4912, + "theoretical_loss": 3.7271996639182756, + "tokens_seen": 869466112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005951935807422267, + "loss": 1.3384, + "theoretical_loss": 3.727172443279609, + "tokens_seen": 869531648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005951775325977935, + "loss": 1.2992, + "theoretical_loss": 3.727145225266865, + "tokens_seen": 869597184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005951614844533601, + "loss": 1.4288, + "theoretical_loss": 3.727118009879592, + "tokens_seen": 869662720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005951454363089269, + "loss": 1.4238, + "theoretical_loss": 3.727090797117339, + "tokens_seen": 869728256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005951293881644935, + "loss": 1.2995, + "theoretical_loss": 3.7270635869796553, + "tokens_seen": 869793792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005951133400200603, + "loss": 1.3989, + "theoretical_loss": 3.7270363794660892, + "tokens_seen": 869859328 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950972918756269, + "loss": 1.4107, + "theoretical_loss": 3.727009174576191, + "tokens_seen": 869924864 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950812437311937, + "loss": 1.4168, + "theoretical_loss": 3.7269819723095097, + "tokens_seen": 869990400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950651955867603, + "loss": 1.384, + "theoretical_loss": 3.7269547726655947, + "tokens_seen": 870055936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950491474423271, + "loss": 1.4466, + "theoretical_loss": 3.7269275756439955, + "tokens_seen": 870121472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950330992978937, + "loss": 1.5258, + "theoretical_loss": 3.7269003812442616, + "tokens_seen": 870187008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950170511534605, + "loss": 1.3529, + "theoretical_loss": 3.7268731894659437, + "tokens_seen": 870252544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005950010030090271, + "loss": 1.4492, + "theoretical_loss": 3.726846000308591, + "tokens_seen": 870318080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005949849548645939, + "loss": 1.4904, + "theoretical_loss": 3.7268188137717537, + "tokens_seen": 870383616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005949689067201605, + "loss": 1.4114, + "theoretical_loss": 3.7267916298549815, + "tokens_seen": 870449152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005949528585757273, + "loss": 1.3952, + "theoretical_loss": 3.7267644485578257, + "tokens_seen": 870514688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005949368104312939, + "loss": 1.4683, + "theoretical_loss": 3.726737269879836, + "tokens_seen": 870580224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005949207622868607, + "loss": 1.3223, + "theoretical_loss": 3.7267100938205626, + "tokens_seen": 870645760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005949047141424273, + "loss": 1.4372, + "theoretical_loss": 3.726682920379556, + "tokens_seen": 870711296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005948886659979941, + "loss": 1.4773, + "theoretical_loss": 3.726655749556368, + "tokens_seen": 870776832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005948726178535607, + "loss": 1.5101, + "theoretical_loss": 3.726628581350549, + "tokens_seen": 870842368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005948565697091275, + "loss": 1.479, + "theoretical_loss": 3.7266014157616487, + "tokens_seen": 870907904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005948405215646941, + "loss": 1.4884, + "theoretical_loss": 3.72657425278922, + "tokens_seen": 870973440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005948244734202608, + "loss": 1.4523, + "theoretical_loss": 3.7265470924328126, + "tokens_seen": 871038976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005948084252758275, + "loss": 1.4609, + "theoretical_loss": 3.7265199346919786, + "tokens_seen": 871104512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005947923771313942, + "loss": 1.4599, + "theoretical_loss": 3.7264927795662692, + "tokens_seen": 871170048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005947763289869609, + "loss": 1.4732, + "theoretical_loss": 3.7264656270552354, + "tokens_seen": 871235584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005947602808425276, + "loss": 1.3494, + "theoretical_loss": 3.72643847715843, + "tokens_seen": 871301120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005947442326980943, + "loss": 1.3854, + "theoretical_loss": 3.726411329875403, + "tokens_seen": 871366656 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594728184553661, + "loss": 1.415, + "theoretical_loss": 3.726384185205707, + "tokens_seen": 871432192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005947121364092277, + "loss": 1.6194, + "theoretical_loss": 3.7263570431488944, + "tokens_seen": 871497728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005946960882647944, + "loss": 1.416, + "theoretical_loss": 3.7263299037045168, + "tokens_seen": 871563264 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 327729, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.386293649673462, + "objective/train/theoretical_loss": 3.7263027668721262, + "objective/train/tokens_used": 892088800, + "theoretical_loss": 3.7263027668721262, + "tokens_seen": 871628800 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594680040120361, + "loss": 1.3863, + "theoretical_loss": 3.7263027668721262, + "tokens_seen": 871628800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005946639919759278, + "loss": 1.3847, + "theoretical_loss": 3.7262756326512747, + "tokens_seen": 871694336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005946479438314944, + "loss": 1.4685, + "theoretical_loss": 3.7262485010415154, + "tokens_seen": 871759872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005946318956870612, + "loss": 1.4311, + "theoretical_loss": 3.7262213720424002, + "tokens_seen": 871825408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005946158475426278, + "loss": 1.4663, + "theoretical_loss": 3.726194245653482, + "tokens_seen": 871890944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005945997993981946, + "loss": 1.5144, + "theoretical_loss": 3.7261671218743135, + "tokens_seen": 871956480 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005945837512537613, + "loss": 1.3261, + "theoretical_loss": 3.7261400007044467, + "tokens_seen": 872022016 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594567703109328, + "loss": 1.3829, + "theoretical_loss": 3.7261128821434353, + "tokens_seen": 872087552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005945516549648947, + "loss": 1.3547, + "theoretical_loss": 3.7260857661908324, + "tokens_seen": 872153088 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005945356068204614, + "loss": 1.3749, + "theoretical_loss": 3.726058652846191, + "tokens_seen": 872218624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005945195586760281, + "loss": 1.4485, + "theoretical_loss": 3.726031542109064, + "tokens_seen": 872284160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005945035105315948, + "loss": 1.3022, + "theoretical_loss": 3.7260044339790053, + "tokens_seen": 872349696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005944874623871615, + "loss": 1.5032, + "theoretical_loss": 3.7259773284555675, + "tokens_seen": 872415232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005944714142427282, + "loss": 1.4002, + "theoretical_loss": 3.7259502255383055, + "tokens_seen": 872480768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005944553660982949, + "loss": 1.4508, + "theoretical_loss": 3.7259231252267715, + "tokens_seen": 872546304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005944393179538616, + "loss": 1.4182, + "theoretical_loss": 3.72589602752052, + "tokens_seen": 872611840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005944232698094283, + "loss": 1.4036, + "theoretical_loss": 3.7258689324191057, + "tokens_seen": 872677376 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594407221664995, + "loss": 1.5224, + "theoretical_loss": 3.725841839922081, + "tokens_seen": 872742912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005943911735205617, + "loss": 1.4715, + "theoretical_loss": 3.7258147500290013, + "tokens_seen": 872808448 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005943751253761284, + "loss": 1.5432, + "theoretical_loss": 3.72578766273942, + "tokens_seen": 872873984 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005943590772316951, + "loss": 1.602, + "theoretical_loss": 3.725760578052892, + "tokens_seen": 872939520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005943430290872618, + "loss": 1.2998, + "theoretical_loss": 3.725733495968972, + "tokens_seen": 873005056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005943269809428285, + "loss": 1.4331, + "theoretical_loss": 3.7257064164872142, + "tokens_seen": 873070592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005943109327983952, + "loss": 1.414, + "theoretical_loss": 3.7256793396071726, + "tokens_seen": 873136128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005942948846539619, + "loss": 1.5542, + "theoretical_loss": 3.725652265328403, + "tokens_seen": 873201664 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005942788365095286, + "loss": 1.3689, + "theoretical_loss": 3.7256251936504596, + "tokens_seen": 873267200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005942627883650953, + "loss": 1.4452, + "theoretical_loss": 3.725598124572898, + "tokens_seen": 873332736 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594246740220662, + "loss": 1.4048, + "theoretical_loss": 3.725571058095273, + "tokens_seen": 873398272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005942306920762287, + "loss": 1.4668, + "theoretical_loss": 3.7255439942171398, + "tokens_seen": 873463808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005942146439317954, + "loss": 1.4189, + "theoretical_loss": 3.7255169329380537, + "tokens_seen": 873529344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005941985957873621, + "loss": 1.3308, + "theoretical_loss": 3.72548987425757, + "tokens_seen": 873594880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005941825476429288, + "loss": 1.4799, + "theoretical_loss": 3.725462818175245, + "tokens_seen": 873660416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005941664994984955, + "loss": 1.4606, + "theoretical_loss": 3.7254357646906335, + "tokens_seen": 873725952 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005941504513540622, + "loss": 1.5929, + "theoretical_loss": 3.7254087138032914, + "tokens_seen": 873791488 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594134403209629, + "loss": 1.4375, + "theoretical_loss": 3.725381665512775, + "tokens_seen": 873857024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005941183550651956, + "loss": 1.4566, + "theoretical_loss": 3.7253546198186402, + "tokens_seen": 873922560 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005941023069207623, + "loss": 1.3591, + "theoretical_loss": 3.7253275767204426, + "tokens_seen": 873988096 + }, + { + "epoch": 0.26, + "learning_rate": 0.000594086258776329, + "loss": 1.5114, + "theoretical_loss": 3.7253005362177385, + "tokens_seen": 874053632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005940702106318958, + "loss": 1.5025, + "theoretical_loss": 3.7252734983100853, + "tokens_seen": 874119168 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005940541624874624, + "loss": 1.2964, + "theoretical_loss": 3.7252464629970383, + "tokens_seen": 874184704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005940381143430292, + "loss": 1.5178, + "theoretical_loss": 3.7252194302781536, + "tokens_seen": 874250240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005940220661985958, + "loss": 1.3876, + "theoretical_loss": 3.7251924001529892, + "tokens_seen": 874315776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005940060180541626, + "loss": 1.4029, + "theoretical_loss": 3.7251653726211016, + "tokens_seen": 874381312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0005939899699097292, + "loss": 1.3475, + "theoretical_loss": 3.7251383476820465, + "tokens_seen": 874446848 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593973921765296, + "loss": 1.4113, + "theoretical_loss": 3.725111325335382, + "tokens_seen": 874512384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005939578736208626, + "loss": 1.4513, + "theoretical_loss": 3.7250843055806646, + "tokens_seen": 874577920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005939418254764294, + "loss": 1.3819, + "theoretical_loss": 3.7250572884174518, + "tokens_seen": 874643456 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593925777331996, + "loss": 1.4076, + "theoretical_loss": 3.7250302738453005, + "tokens_seen": 874708992 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005939097291875628, + "loss": 1.4644, + "theoretical_loss": 3.725003261863769, + "tokens_seen": 874774528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005938936810431294, + "loss": 1.3927, + "theoretical_loss": 3.724976252472414, + "tokens_seen": 874840064 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005938776328986962, + "loss": 1.4843, + "theoretical_loss": 3.7249492456707936, + "tokens_seen": 874905600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005938615847542628, + "loss": 1.4091, + "theoretical_loss": 3.724922241458464, + "tokens_seen": 874971136 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005938455366098296, + "loss": 1.5056, + "theoretical_loss": 3.7248952398349857, + "tokens_seen": 875036672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005938294884653962, + "loss": 1.559, + "theoretical_loss": 3.7248682407999145, + "tokens_seen": 875102208 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593813440320963, + "loss": 1.4224, + "theoretical_loss": 3.7248412443528096, + "tokens_seen": 875167744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005937973921765296, + "loss": 1.4235, + "theoretical_loss": 3.7248142504932282, + "tokens_seen": 875233280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005937813440320964, + "loss": 1.454, + "theoretical_loss": 3.7247872592207303, + "tokens_seen": 875298816 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593765295887663, + "loss": 1.47, + "theoretical_loss": 3.7247602705348717, + "tokens_seen": 875364352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005937492477432298, + "loss": 1.4381, + "theoretical_loss": 3.7247332844352132, + "tokens_seen": 875429888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005937331995987964, + "loss": 1.3211, + "theoretical_loss": 3.724706300921312, + "tokens_seen": 875495424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005937171514543632, + "loss": 1.4481, + "theoretical_loss": 3.7246793199927275, + "tokens_seen": 875560960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005937011033099298, + "loss": 1.5432, + "theoretical_loss": 3.7246523416490187, + "tokens_seen": 875626496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005936850551654966, + "loss": 1.3327, + "theoretical_loss": 3.724625365889744, + "tokens_seen": 875692032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005936690070210632, + "loss": 1.4706, + "theoretical_loss": 3.724598392714462, + "tokens_seen": 875757568 + }, + { + "epoch": 0.27, + "learning_rate": 0.00059365295887663, + "loss": 1.4705, + "theoretical_loss": 3.724571422122733, + "tokens_seen": 875823104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005936369107321966, + "loss": 1.3292, + "theoretical_loss": 3.724544454114115, + "tokens_seen": 875888640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005936208625877634, + "loss": 1.4707, + "theoretical_loss": 3.7245174886881687, + "tokens_seen": 875954176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00059360481444333, + "loss": 1.5549, + "theoretical_loss": 3.7244905258444527, + "tokens_seen": 876019712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005935887662988968, + "loss": 1.364, + "theoretical_loss": 3.7244635655825267, + "tokens_seen": 876085248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005935727181544634, + "loss": 1.5082, + "theoretical_loss": 3.7244366079019504, + "tokens_seen": 876150784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005935566700100302, + "loss": 1.3491, + "theoretical_loss": 3.724409652802284, + "tokens_seen": 876216320 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005935406218655968, + "loss": 1.4024, + "theoretical_loss": 3.7243827002830865, + "tokens_seen": 876281856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005935245737211636, + "loss": 1.4883, + "theoretical_loss": 3.7243557503439186, + "tokens_seen": 876347392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005935085255767302, + "loss": 1.358, + "theoretical_loss": 3.72432880298434, + "tokens_seen": 876412928 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593492477432297, + "loss": 1.4297, + "theoretical_loss": 3.7243018582039116, + "tokens_seen": 876478464 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005934764292878636, + "loss": 1.3047, + "theoretical_loss": 3.724274916002193, + "tokens_seen": 876544000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005934603811434304, + "loss": 1.3425, + "theoretical_loss": 3.724247976378745, + "tokens_seen": 876609536 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593444332998997, + "loss": 1.3639, + "theoretical_loss": 3.724221039333128, + "tokens_seen": 876675072 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005934282848545637, + "loss": 1.4557, + "theoretical_loss": 3.724194104864903, + "tokens_seen": 876740608 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005934122367101304, + "loss": 1.3905, + "theoretical_loss": 3.72416717297363, + "tokens_seen": 876806144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005933961885656971, + "loss": 1.5184, + "theoretical_loss": 3.7241402436588706, + "tokens_seen": 876871680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005933801404212638, + "loss": 1.3343, + "theoretical_loss": 3.724113316920185, + "tokens_seen": 876937216 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005933640922768305, + "loss": 1.3943, + "theoretical_loss": 3.724086392757136, + "tokens_seen": 877002752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005933480441323972, + "loss": 1.4709, + "theoretical_loss": 3.7240594711692827, + "tokens_seen": 877068288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005933319959879639, + "loss": 1.3778, + "theoretical_loss": 3.7240325521561872, + "tokens_seen": 877133824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005933159478435306, + "loss": 1.37, + "theoretical_loss": 3.724005635717411, + "tokens_seen": 877199360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005932998996990973, + "loss": 1.3284, + "theoretical_loss": 3.7239787218525158, + "tokens_seen": 877264896 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593283851554664, + "loss": 1.4753, + "theoretical_loss": 3.723951810561063, + "tokens_seen": 877330432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005932678034102307, + "loss": 1.4382, + "theoretical_loss": 3.723924901842614, + "tokens_seen": 877395968 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005932517552657974, + "loss": 1.4069, + "theoretical_loss": 3.7238979956967317, + "tokens_seen": 877461504 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005932357071213641, + "loss": 1.5372, + "theoretical_loss": 3.7238710921229767, + "tokens_seen": 877527040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005932196589769308, + "loss": 1.3612, + "theoretical_loss": 3.7238441911209117, + "tokens_seen": 877592576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005932036108324975, + "loss": 1.4981, + "theoretical_loss": 3.723817292690099, + "tokens_seen": 877658112 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005931875626880642, + "loss": 1.4083, + "theoretical_loss": 3.723790396830101, + "tokens_seen": 877723648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005931715145436309, + "loss": 1.3853, + "theoretical_loss": 3.7237635035404795, + "tokens_seen": 877789184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005931554663991976, + "loss": 1.407, + "theoretical_loss": 3.723736612820797, + "tokens_seen": 877854720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005931394182547643, + "loss": 1.4535, + "theoretical_loss": 3.7237097246706163, + "tokens_seen": 877920256 + }, + { + "epoch": 0.27, + "learning_rate": 0.000593123370110331, + "loss": 1.3827, + "theoretical_loss": 3.7236828390895003, + "tokens_seen": 877985792 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005931073219658977, + "loss": 1.4715, + "theoretical_loss": 3.7236559560770113, + "tokens_seen": 878051328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005930912738214644, + "loss": 1.4503, + "theoretical_loss": 3.7236290756327133, + "tokens_seen": 878116864 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 330460, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3646528720855713, + "objective/train/theoretical_loss": 3.7236021977561675, + "objective/train/tokens_used": 898642400, + "theoretical_loss": 3.7236021977561675, + "tokens_seen": 878182400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005930752256770311, + "loss": 1.3647, + "theoretical_loss": 3.7236021977561675, + "tokens_seen": 878182400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005930591775325978, + "loss": 1.4684, + "theoretical_loss": 3.723575322446939, + "tokens_seen": 878247936 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005930431293881645, + "loss": 1.3717, + "theoretical_loss": 3.723548449704589, + "tokens_seen": 878313472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005930270812437312, + "loss": 1.4777, + "theoretical_loss": 3.7235215795286827, + "tokens_seen": 878379008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005930110330992979, + "loss": 1.5375, + "theoretical_loss": 3.7234947119187822, + "tokens_seen": 878444544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005929949849548646, + "loss": 1.4182, + "theoretical_loss": 3.723467846874452, + "tokens_seen": 878510080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005929789368104313, + "loss": 1.4305, + "theoretical_loss": 3.723440984395255, + "tokens_seen": 878575616 + }, + { + "epoch": 0.27, + "learning_rate": 0.000592962888665998, + "loss": 1.366, + "theoretical_loss": 3.7234141244807555, + "tokens_seen": 878641152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005929468405215647, + "loss": 1.4217, + "theoretical_loss": 3.7233872671305175, + "tokens_seen": 878706688 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005929307923771314, + "loss": 1.3649, + "theoretical_loss": 3.723360412344104, + "tokens_seen": 878772224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005929147442326981, + "loss": 1.419, + "theoretical_loss": 3.7233335601210804, + "tokens_seen": 878837760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005928986960882648, + "loss": 1.4868, + "theoretical_loss": 3.7233067104610096, + "tokens_seen": 878903296 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005928826479438315, + "loss": 1.3982, + "theoretical_loss": 3.723279863363457, + "tokens_seen": 878968832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005928665997993982, + "loss": 1.3755, + "theoretical_loss": 3.723253018827986, + "tokens_seen": 879034368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005928505516549649, + "loss": 1.3926, + "theoretical_loss": 3.723226176854162, + "tokens_seen": 879099904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005928345035105316, + "loss": 1.3103, + "theoretical_loss": 3.723199337441549, + "tokens_seen": 879165440 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005928184553660983, + "loss": 1.3478, + "theoretical_loss": 3.723172500589712, + "tokens_seen": 879230976 + }, + { + "epoch": 0.27, + "learning_rate": 0.000592802407221665, + "loss": 1.407, + "theoretical_loss": 3.7231456662982154, + "tokens_seen": 879296512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005927863590772317, + "loss": 1.4645, + "theoretical_loss": 3.723118834566625, + "tokens_seen": 879362048 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005927703109327984, + "loss": 1.351, + "theoretical_loss": 3.7230920053945056, + "tokens_seen": 879427584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005927542627883651, + "loss": 1.5514, + "theoretical_loss": 3.7230651787814217, + "tokens_seen": 879493120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005927382146439318, + "loss": 1.4461, + "theoretical_loss": 3.7230383547269392, + "tokens_seen": 879558656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005927221664994985, + "loss": 1.4321, + "theoretical_loss": 3.723011533230623, + "tokens_seen": 879624192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005927061183550653, + "loss": 1.3489, + "theoretical_loss": 3.7229847142920383, + "tokens_seen": 879689728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005926900702106319, + "loss": 1.4724, + "theoretical_loss": 3.7229578979107516, + "tokens_seen": 879755264 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005926740220661987, + "loss": 1.4659, + "theoretical_loss": 3.722931084086328, + "tokens_seen": 879820800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005926579739217653, + "loss": 1.364, + "theoretical_loss": 3.722904272818334, + "tokens_seen": 879886336 + }, + { + "epoch": 0.27, + "learning_rate": 0.000592641925777332, + "loss": 1.4882, + "theoretical_loss": 3.722877464106334, + "tokens_seen": 879951872 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005926258776328987, + "loss": 1.4219, + "theoretical_loss": 3.722850657949895, + "tokens_seen": 880017408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005926098294884655, + "loss": 1.4116, + "theoretical_loss": 3.722823854348583, + "tokens_seen": 880082944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005925937813440321, + "loss": 1.3917, + "theoretical_loss": 3.7227970533019645, + "tokens_seen": 880148480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005925777331995989, + "loss": 1.4318, + "theoretical_loss": 3.7227702548096056, + "tokens_seen": 880214016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005925616850551655, + "loss": 1.3439, + "theoretical_loss": 3.722743458871072, + "tokens_seen": 880279552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005925456369107323, + "loss": 1.3973, + "theoretical_loss": 3.7227166654859314, + "tokens_seen": 880345088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005925295887662989, + "loss": 1.448, + "theoretical_loss": 3.7226898746537493, + "tokens_seen": 880410624 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005925135406218657, + "loss": 1.431, + "theoretical_loss": 3.7226630863740935, + "tokens_seen": 880476160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924974924774323, + "loss": 1.3753, + "theoretical_loss": 3.7226363006465304, + "tokens_seen": 880541696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924814443329991, + "loss": 1.4673, + "theoretical_loss": 3.7226095174706266, + "tokens_seen": 880607232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924653961885657, + "loss": 1.4997, + "theoretical_loss": 3.7225827368459496, + "tokens_seen": 880672768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924493480441325, + "loss": 1.3984, + "theoretical_loss": 3.722555958772066, + "tokens_seen": 880738304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924332998996991, + "loss": 1.3711, + "theoretical_loss": 3.722529183248544, + "tokens_seen": 880803840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924172517552659, + "loss": 1.432, + "theoretical_loss": 3.7225024102749504, + "tokens_seen": 880869376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005924012036108325, + "loss": 1.43, + "theoretical_loss": 3.7224756398508525, + "tokens_seen": 880934912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005923851554663993, + "loss": 1.4573, + "theoretical_loss": 3.7224488719758178, + "tokens_seen": 881000448 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005923691073219659, + "loss": 1.3394, + "theoretical_loss": 3.7224221066494145, + "tokens_seen": 881065984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005923530591775327, + "loss": 1.2791, + "theoretical_loss": 3.7223953438712103, + "tokens_seen": 881131520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005923370110330993, + "loss": 1.5413, + "theoretical_loss": 3.7223685836407734, + "tokens_seen": 881197056 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005923209628886661, + "loss": 1.4522, + "theoretical_loss": 3.7223418259576704, + "tokens_seen": 881262592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005923049147442327, + "loss": 1.3979, + "theoretical_loss": 3.7223150708214714, + "tokens_seen": 881328128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005922888665997995, + "loss": 1.3372, + "theoretical_loss": 3.722288318231743, + "tokens_seen": 881393664 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005922728184553661, + "loss": 1.5759, + "theoretical_loss": 3.722261568188054, + "tokens_seen": 881459200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005922567703109329, + "loss": 1.3933, + "theoretical_loss": 3.722234820689973, + "tokens_seen": 881524736 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005922407221664995, + "loss": 1.4718, + "theoretical_loss": 3.722208075737069, + "tokens_seen": 881590272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005922246740220663, + "loss": 1.4376, + "theoretical_loss": 3.7221813333289093, + "tokens_seen": 881655808 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005922086258776329, + "loss": 1.4792, + "theoretical_loss": 3.722154593465064, + "tokens_seen": 881721344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005921925777331997, + "loss": 1.4549, + "theoretical_loss": 3.722127856145101, + "tokens_seen": 881786880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005921765295887663, + "loss": 1.4373, + "theoretical_loss": 3.72210112136859, + "tokens_seen": 881852416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005921604814443331, + "loss": 1.5002, + "theoretical_loss": 3.7220743891350994, + "tokens_seen": 881917952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005921444332998997, + "loss": 1.3755, + "theoretical_loss": 3.7220476594441987, + "tokens_seen": 881983488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005921283851554665, + "loss": 1.5154, + "theoretical_loss": 3.7220209322954574, + "tokens_seen": 882049024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005921123370110331, + "loss": 1.3574, + "theoretical_loss": 3.7219942076884447, + "tokens_seen": 882114560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005920962888665999, + "loss": 1.4133, + "theoretical_loss": 3.72196748562273, + "tokens_seen": 882180096 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005920802407221665, + "loss": 1.4383, + "theoretical_loss": 3.721940766097882, + "tokens_seen": 882245632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005920641925777333, + "loss": 1.341, + "theoretical_loss": 3.721914049113472, + "tokens_seen": 882311168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005920481444332999, + "loss": 1.3261, + "theoretical_loss": 3.721887334669069, + "tokens_seen": 882376704 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005920320962888666, + "loss": 1.3847, + "theoretical_loss": 3.7218606227642423, + "tokens_seen": 882442240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005920160481444333, + "loss": 1.5955, + "theoretical_loss": 3.721833913398563, + "tokens_seen": 882507776 + }, + { + "epoch": 0.27, + "learning_rate": 0.000592, + "loss": 1.3695, + "theoretical_loss": 3.7218072065716004, + "tokens_seen": 882573312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005919839518555667, + "loss": 1.3276, + "theoretical_loss": 3.7217805022829253, + "tokens_seen": 882638848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005919679037111334, + "loss": 1.4948, + "theoretical_loss": 3.721753800532108, + "tokens_seen": 882704384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005919518555667001, + "loss": 1.3518, + "theoretical_loss": 3.7217271013187183, + "tokens_seen": 882769920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005919358074222668, + "loss": 1.5082, + "theoretical_loss": 3.721700404642327, + "tokens_seen": 882835456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005919197592778335, + "loss": 1.4886, + "theoretical_loss": 3.721673710502505, + "tokens_seen": 882900992 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005919037111334002, + "loss": 1.3406, + "theoretical_loss": 3.7216470188988224, + "tokens_seen": 882966528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005918876629889669, + "loss": 1.3494, + "theoretical_loss": 3.721620329830851, + "tokens_seen": 883032064 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005918716148445336, + "loss": 1.5106, + "theoretical_loss": 3.721593643298161, + "tokens_seen": 883097600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005918555667001003, + "loss": 1.439, + "theoretical_loss": 3.7215669593003238, + "tokens_seen": 883163136 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591839518555667, + "loss": 1.4183, + "theoretical_loss": 3.72154027783691, + "tokens_seen": 883228672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005918234704112337, + "loss": 1.4902, + "theoretical_loss": 3.7215135989074914, + "tokens_seen": 883294208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005918074222668004, + "loss": 1.4449, + "theoretical_loss": 3.7214869225116396, + "tokens_seen": 883359744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005917913741223671, + "loss": 1.3331, + "theoretical_loss": 3.7214602486489254, + "tokens_seen": 883425280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005917753259779338, + "loss": 1.4227, + "theoretical_loss": 3.721433577318921, + "tokens_seen": 883490816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005917592778335005, + "loss": 1.4353, + "theoretical_loss": 3.721406908521197, + "tokens_seen": 883556352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005917432296890672, + "loss": 1.295, + "theoretical_loss": 3.7213802422553264, + "tokens_seen": 883621888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005917271815446339, + "loss": 1.4295, + "theoretical_loss": 3.7213535785208807, + "tokens_seen": 883687424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005917111334002006, + "loss": 1.4996, + "theoretical_loss": 3.7213269173174313, + "tokens_seen": 883752960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005916950852557673, + "loss": 1.4054, + "theoretical_loss": 3.7213002586445514, + "tokens_seen": 883818496 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591679037111334, + "loss": 1.4113, + "theoretical_loss": 3.721273602501812, + "tokens_seen": 883884032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005916629889669007, + "loss": 1.3824, + "theoretical_loss": 3.721246948888786, + "tokens_seen": 883949568 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005916469408224674, + "loss": 1.4706, + "theoretical_loss": 3.7212202978050457, + "tokens_seen": 884015104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005916308926780341, + "loss": 1.3955, + "theoretical_loss": 3.721193649250164, + "tokens_seen": 884080640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005916148445336008, + "loss": 1.5332, + "theoretical_loss": 3.7211670032237123, + "tokens_seen": 884146176 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005915987963891675, + "loss": 1.4182, + "theoretical_loss": 3.721140359725265, + "tokens_seen": 884211712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005915827482447342, + "loss": 1.3657, + "theoretical_loss": 3.7211137187543937, + "tokens_seen": 884277248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005915667001003009, + "loss": 1.4597, + "theoretical_loss": 3.7210870803106717, + "tokens_seen": 884342784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005915506519558676, + "loss": 1.2992, + "theoretical_loss": 3.721060444393672, + "tokens_seen": 884408320 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005915346038114343, + "loss": 1.2975, + "theoretical_loss": 3.721033811002968, + "tokens_seen": 884473856 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591518555667001, + "loss": 1.4225, + "theoretical_loss": 3.721007180138132, + "tokens_seen": 884539392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005915025075225677, + "loss": 1.4923, + "theoretical_loss": 3.720980551798739, + "tokens_seen": 884604928 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005914864593781344, + "loss": 1.5319, + "theoretical_loss": 3.720953925984361, + "tokens_seen": 884670464 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 332595, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3572536706924438, + "objective/train/theoretical_loss": 3.720927302694572, + "objective/train/tokens_used": 905196000, + "theoretical_loss": 3.720927302694572, + "tokens_seen": 884736000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005914704112337011, + "loss": 1.3573, + "theoretical_loss": 3.720927302694572, + "tokens_seen": 884736000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005914543630892678, + "loss": 1.4497, + "theoretical_loss": 3.7209006819289456, + "tokens_seen": 884801536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005914383149448345, + "loss": 1.4766, + "theoretical_loss": 3.7208740636870554, + "tokens_seen": 884867072 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005914222668004012, + "loss": 1.4117, + "theoretical_loss": 3.7208474479684757, + "tokens_seen": 884932608 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591406218655968, + "loss": 1.3195, + "theoretical_loss": 3.72082083477278, + "tokens_seen": 884998144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005913901705115346, + "loss": 1.3608, + "theoretical_loss": 3.7207942240995426, + "tokens_seen": 885063680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005913741223671013, + "loss": 1.4317, + "theoretical_loss": 3.7207676159483376, + "tokens_seen": 885129216 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591358074222668, + "loss": 1.4627, + "theoretical_loss": 3.720741010318739, + "tokens_seen": 885194752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005913420260782348, + "loss": 1.4532, + "theoretical_loss": 3.720714407210322, + "tokens_seen": 885260288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005913259779338014, + "loss": 1.3777, + "theoretical_loss": 3.72068780662266, + "tokens_seen": 885325824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005913099297893682, + "loss": 1.5278, + "theoretical_loss": 3.7206612085553283, + "tokens_seen": 885391360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005912938816449348, + "loss": 1.4619, + "theoretical_loss": 3.7206346130079018, + "tokens_seen": 885456896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005912778335005016, + "loss": 1.4097, + "theoretical_loss": 3.720608019979954, + "tokens_seen": 885522432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005912617853560682, + "loss": 1.4005, + "theoretical_loss": 3.7205814294710615, + "tokens_seen": 885587968 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591245737211635, + "loss": 1.475, + "theoretical_loss": 3.720554841480798, + "tokens_seen": 885653504 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005912296890672016, + "loss": 1.5318, + "theoretical_loss": 3.720528256008739, + "tokens_seen": 885719040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005912136409227684, + "loss": 1.449, + "theoretical_loss": 3.72050167305446, + "tokens_seen": 885784576 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591197592778335, + "loss": 1.3408, + "theoretical_loss": 3.7204750926175354, + "tokens_seen": 885850112 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005911815446339018, + "loss": 1.5116, + "theoretical_loss": 3.7204485146975417, + "tokens_seen": 885915648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005911654964894684, + "loss": 1.3694, + "theoretical_loss": 3.720421939294054, + "tokens_seen": 885981184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005911494483450352, + "loss": 1.3258, + "theoretical_loss": 3.7203953664066476, + "tokens_seen": 886046720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005911334002006018, + "loss": 1.3784, + "theoretical_loss": 3.7203687960348986, + "tokens_seen": 886112256 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005911173520561686, + "loss": 1.3871, + "theoretical_loss": 3.7203422281783824, + "tokens_seen": 886177792 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005911013039117352, + "loss": 1.5249, + "theoretical_loss": 3.7203156628366756, + "tokens_seen": 886243328 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591085255767302, + "loss": 1.3487, + "theoretical_loss": 3.720289100009353, + "tokens_seen": 886308864 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005910692076228686, + "loss": 1.5577, + "theoretical_loss": 3.7202625396959923, + "tokens_seen": 886374400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005910531594784354, + "loss": 1.4391, + "theoretical_loss": 3.7202359818961686, + "tokens_seen": 886439936 + }, + { + "epoch": 0.27, + "learning_rate": 0.000591037111334002, + "loss": 1.5048, + "theoretical_loss": 3.7202094266094585, + "tokens_seen": 886505472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005910210631895688, + "loss": 1.5327, + "theoretical_loss": 3.720182873835439, + "tokens_seen": 886571008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005910050150451354, + "loss": 1.3933, + "theoretical_loss": 3.7201563235736854, + "tokens_seen": 886636544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005909889669007022, + "loss": 1.4241, + "theoretical_loss": 3.7201297758237755, + "tokens_seen": 886702080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005909729187562689, + "loss": 1.4078, + "theoretical_loss": 3.720103230585286, + "tokens_seen": 886767616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005909568706118356, + "loss": 1.4294, + "theoretical_loss": 3.7200766878577927, + "tokens_seen": 886833152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005909408224674023, + "loss": 1.4615, + "theoretical_loss": 3.720050147640873, + "tokens_seen": 886898688 + }, + { + "epoch": 0.27, + "learning_rate": 0.000590924774322969, + "loss": 1.3713, + "theoretical_loss": 3.7200236099341044, + "tokens_seen": 886964224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005909087261785357, + "loss": 1.4213, + "theoretical_loss": 3.719997074737064, + "tokens_seen": 887029760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005908926780341024, + "loss": 1.3967, + "theoretical_loss": 3.7199705420493285, + "tokens_seen": 887095296 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005908766298896691, + "loss": 1.4807, + "theoretical_loss": 3.719944011870476, + "tokens_seen": 887160832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005908605817452358, + "loss": 1.3224, + "theoretical_loss": 3.7199174842000833, + "tokens_seen": 887226368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005908445336008025, + "loss": 1.4123, + "theoretical_loss": 3.719890959037728, + "tokens_seen": 887291904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005908284854563692, + "loss": 1.3911, + "theoretical_loss": 3.719864436382988, + "tokens_seen": 887357440 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005908124373119359, + "loss": 1.4284, + "theoretical_loss": 3.7198379162354414, + "tokens_seen": 887422976 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005907963891675026, + "loss": 1.3224, + "theoretical_loss": 3.7198113985946657, + "tokens_seen": 887488512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005907803410230693, + "loss": 1.3973, + "theoretical_loss": 3.719784883460238, + "tokens_seen": 887554048 + }, + { + "epoch": 0.27, + "learning_rate": 0.000590764292878636, + "loss": 1.442, + "theoretical_loss": 3.719758370831738, + "tokens_seen": 887619584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005907482447342026, + "loss": 1.4959, + "theoretical_loss": 3.719731860708743, + "tokens_seen": 887685120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005907321965897694, + "loss": 1.4613, + "theoretical_loss": 3.7197053530908315, + "tokens_seen": 887750656 + }, + { + "epoch": 0.27, + "learning_rate": 0.000590716148445336, + "loss": 1.4836, + "theoretical_loss": 3.7196788479775815, + "tokens_seen": 887816192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005907001003009028, + "loss": 1.4351, + "theoretical_loss": 3.7196523453685715, + "tokens_seen": 887881728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005906840521564694, + "loss": 1.4539, + "theoretical_loss": 3.719625845263381, + "tokens_seen": 887947264 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005906680040120362, + "loss": 1.3151, + "theoretical_loss": 3.719599347661587, + "tokens_seen": 888012800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005906519558676028, + "loss": 1.3832, + "theoretical_loss": 3.71957285256277, + "tokens_seen": 888078336 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005906359077231695, + "loss": 1.4835, + "theoretical_loss": 3.7195463599665075, + "tokens_seen": 888143872 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005906198595787362, + "loss": 1.453, + "theoretical_loss": 3.7195198698723795, + "tokens_seen": 888209408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005906038114343029, + "loss": 1.4426, + "theoretical_loss": 3.719493382279965, + "tokens_seen": 888274944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005905877632898696, + "loss": 1.3666, + "theoretical_loss": 3.719466897188843, + "tokens_seen": 888340480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005905717151454363, + "loss": 1.5264, + "theoretical_loss": 3.7194404145985924, + "tokens_seen": 888406016 + }, + { + "epoch": 0.27, + "learning_rate": 0.000590555667001003, + "loss": 1.407, + "theoretical_loss": 3.719413934508793, + "tokens_seen": 888471552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005905396188565697, + "loss": 1.4457, + "theoretical_loss": 3.719387456919024, + "tokens_seen": 888537088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005905235707121364, + "loss": 1.351, + "theoretical_loss": 3.7193609818288653, + "tokens_seen": 888602624 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005905075225677031, + "loss": 1.4096, + "theoretical_loss": 3.7193345092378967, + "tokens_seen": 888668160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005904914744232698, + "loss": 1.507, + "theoretical_loss": 3.719308039145698, + "tokens_seen": 888733696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005904754262788365, + "loss": 1.4146, + "theoretical_loss": 3.7192815715518486, + "tokens_seen": 888799232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005904593781344032, + "loss": 1.4085, + "theoretical_loss": 3.719255106455929, + "tokens_seen": 888864768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005904433299899699, + "loss": 1.4209, + "theoretical_loss": 3.7192286438575186, + "tokens_seen": 888930304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005904272818455366, + "loss": 1.4254, + "theoretical_loss": 3.7192021837561984, + "tokens_seen": 888995840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005904112337011033, + "loss": 1.409, + "theoretical_loss": 3.7191757261515486, + "tokens_seen": 889061376 + }, + { + "epoch": 0.27, + "learning_rate": 0.00059039518555667, + "loss": 1.451, + "theoretical_loss": 3.7191492710431495, + "tokens_seen": 889126912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005903791374122367, + "loss": 1.4954, + "theoretical_loss": 3.7191228184305816, + "tokens_seen": 889192448 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005903630892678034, + "loss": 1.4882, + "theoretical_loss": 3.719096368313425, + "tokens_seen": 889257984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005903470411233701, + "loss": 1.3922, + "theoretical_loss": 3.719069920691261, + "tokens_seen": 889323520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005903309929789368, + "loss": 1.4553, + "theoretical_loss": 3.7190434755636708, + "tokens_seen": 889389056 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005903149448345035, + "loss": 1.3723, + "theoretical_loss": 3.7190170329302346, + "tokens_seen": 889454592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005902988966900702, + "loss": 1.4084, + "theoretical_loss": 3.718990592790534, + "tokens_seen": 889520128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005902828485456369, + "loss": 1.3343, + "theoretical_loss": 3.718964155144149, + "tokens_seen": 889585664 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005902668004012036, + "loss": 1.4257, + "theoretical_loss": 3.718937719990662, + "tokens_seen": 889651200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005902507522567703, + "loss": 1.4705, + "theoretical_loss": 3.7189112873296537, + "tokens_seen": 889716736 + }, + { + "epoch": 0.27, + "learning_rate": 0.000590234704112337, + "loss": 1.4725, + "theoretical_loss": 3.718884857160706, + "tokens_seen": 889782272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005902186559679037, + "loss": 1.3402, + "theoretical_loss": 3.7188584294834004, + "tokens_seen": 889847808 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005902026078234704, + "loss": 1.4678, + "theoretical_loss": 3.718832004297318, + "tokens_seen": 889913344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005901865596790371, + "loss": 1.4842, + "theoretical_loss": 3.718805581602041, + "tokens_seen": 889978880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005901705115346038, + "loss": 1.4723, + "theoretical_loss": 3.7187791613971504, + "tokens_seen": 890044416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005901544633901705, + "loss": 1.3906, + "theoretical_loss": 3.7187527436822294, + "tokens_seen": 890109952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005901384152457372, + "loss": 1.3922, + "theoretical_loss": 3.7187263284568592, + "tokens_seen": 890175488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005901223671013039, + "loss": 1.4085, + "theoretical_loss": 3.7186999157206224, + "tokens_seen": 890241024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005901063189568706, + "loss": 1.5706, + "theoretical_loss": 3.7186735054731006, + "tokens_seen": 890306560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005900902708124373, + "loss": 1.4488, + "theoretical_loss": 3.718647097713877, + "tokens_seen": 890372096 + }, + { + "epoch": 0.27, + "learning_rate": 0.000590074222668004, + "loss": 1.4839, + "theoretical_loss": 3.718620692442533, + "tokens_seen": 890437632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005900581745235707, + "loss": 1.4747, + "theoretical_loss": 3.718594289658652, + "tokens_seen": 890503168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005900421263791374, + "loss": 1.4548, + "theoretical_loss": 3.7185678893618164, + "tokens_seen": 890568704 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005900260782347041, + "loss": 1.3496, + "theoretical_loss": 3.7185414915516084, + "tokens_seen": 890634240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005900100300902708, + "loss": 1.4301, + "theoretical_loss": 3.7185150962276117, + "tokens_seen": 890699776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005899939819458375, + "loss": 1.545, + "theoretical_loss": 3.7184887033894087, + "tokens_seen": 890765312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005899779338014042, + "loss": 1.438, + "theoretical_loss": 3.718462313036582, + "tokens_seen": 890830848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005899618856569709, + "loss": 1.3928, + "theoretical_loss": 3.7184359251687162, + "tokens_seen": 890896384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005899458375125377, + "loss": 1.4958, + "theoretical_loss": 3.718409539785393, + "tokens_seen": 890961920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005899297893681043, + "loss": 1.57, + "theoretical_loss": 3.7183831568861967, + "tokens_seen": 891027456 + }, + { + "epoch": 0.27, + "learning_rate": 0.000589913741223671, + "loss": 1.5698, + "theoretical_loss": 3.718356776470711, + "tokens_seen": 891092992 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898976930792377, + "loss": 1.3993, + "theoretical_loss": 3.7183303985385177, + "tokens_seen": 891158528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898816449348045, + "loss": 1.4317, + "theoretical_loss": 3.7183040230892024, + "tokens_seen": 891224064 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 334906, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3892271518707275, + "objective/train/theoretical_loss": 3.7182776501223476, + "objective/train/tokens_used": 911749600, + "theoretical_loss": 3.7182776501223476, + "tokens_seen": 891289600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898655967903711, + "loss": 1.3892, + "theoretical_loss": 3.7182776501223476, + "tokens_seen": 891289600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898495486459379, + "loss": 1.4056, + "theoretical_loss": 3.7182512796375375, + "tokens_seen": 891355136 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898335005015045, + "loss": 1.3511, + "theoretical_loss": 3.7182249116343566, + "tokens_seen": 891420672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898174523570713, + "loss": 1.3773, + "theoretical_loss": 3.7181985461123883, + "tokens_seen": 891486208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005898014042126379, + "loss": 1.501, + "theoretical_loss": 3.718172183071217, + "tokens_seen": 891551744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005897853560682047, + "loss": 1.4295, + "theoretical_loss": 3.7181458225104267, + "tokens_seen": 891617280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005897693079237714, + "loss": 1.4352, + "theoretical_loss": 3.718119464429602, + "tokens_seen": 891682816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005897532597793381, + "loss": 1.3797, + "theoretical_loss": 3.718093108828327, + "tokens_seen": 891748352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005897372116349048, + "loss": 1.4326, + "theoretical_loss": 3.718066755706187, + "tokens_seen": 891813888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005897211634904715, + "loss": 1.3532, + "theoretical_loss": 3.718040405062766, + "tokens_seen": 891879424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005897051153460382, + "loss": 1.3705, + "theoretical_loss": 3.7180140568976485, + "tokens_seen": 891944960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005896890672016049, + "loss": 1.5464, + "theoretical_loss": 3.7179877112104194, + "tokens_seen": 892010496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005896730190571716, + "loss": 1.4503, + "theoretical_loss": 3.7179613680006645, + "tokens_seen": 892076032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005896569709127383, + "loss": 1.5694, + "theoretical_loss": 3.7179350272679685, + "tokens_seen": 892141568 + }, + { + "epoch": 0.27, + "learning_rate": 0.000589640922768305, + "loss": 1.4364, + "theoretical_loss": 3.7179086890119164, + "tokens_seen": 892207104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005896248746238717, + "loss": 1.4681, + "theoretical_loss": 3.717882353232093, + "tokens_seen": 892272640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005896088264794384, + "loss": 1.6059, + "theoretical_loss": 3.7178560199280843, + "tokens_seen": 892338176 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005895927783350051, + "loss": 1.4096, + "theoretical_loss": 3.7178296890994753, + "tokens_seen": 892403712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005895767301905718, + "loss": 1.4227, + "theoretical_loss": 3.717803360745852, + "tokens_seen": 892469248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005895767301905718, + "loss": 1.4654, + "theoretical_loss": 3.717777034866799, + "tokens_seen": 892534784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005895606820461385, + "loss": 1.5339, + "theoretical_loss": 3.7177507114619037, + "tokens_seen": 892600320 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005895446339017052, + "loss": 1.4875, + "theoretical_loss": 3.7177243905307504, + "tokens_seen": 892665856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005895285857572719, + "loss": 1.3566, + "theoretical_loss": 3.7176980720729267, + "tokens_seen": 892731392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005895125376128386, + "loss": 1.4796, + "theoretical_loss": 3.717671756088017, + "tokens_seen": 892796928 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005894964894684053, + "loss": 1.4947, + "theoretical_loss": 3.717645442575608, + "tokens_seen": 892862464 + }, + { + "epoch": 0.27, + "learning_rate": 0.000589480441323972, + "loss": 1.4873, + "theoretical_loss": 3.717619131535286, + "tokens_seen": 892928000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005894643931795387, + "loss": 1.5414, + "theoretical_loss": 3.7175928229666377, + "tokens_seen": 892993536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005894483450351054, + "loss": 1.4078, + "theoretical_loss": 3.717566516869249, + "tokens_seen": 893059072 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005894322968906721, + "loss": 1.4999, + "theoretical_loss": 3.7175402132427067, + "tokens_seen": 893124608 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005894162487462388, + "loss": 1.4914, + "theoretical_loss": 3.7175139120865976, + "tokens_seen": 893190144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005894002006018055, + "loss": 1.4484, + "theoretical_loss": 3.7174876134005075, + "tokens_seen": 893255680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005893841524573722, + "loss": 1.4566, + "theoretical_loss": 3.717461317184025, + "tokens_seen": 893321216 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005893681043129389, + "loss": 1.5516, + "theoretical_loss": 3.7174350234367353, + "tokens_seen": 893386752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005893520561685055, + "loss": 1.4857, + "theoretical_loss": 3.7174087321582263, + "tokens_seen": 893452288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005893360080240723, + "loss": 1.4414, + "theoretical_loss": 3.717382443348085, + "tokens_seen": 893517824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005893199598796389, + "loss": 1.4243, + "theoretical_loss": 3.717356157005899, + "tokens_seen": 893583360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005893039117352057, + "loss": 1.375, + "theoretical_loss": 3.717329873131255, + "tokens_seen": 893648896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005892878635907723, + "loss": 1.4254, + "theoretical_loss": 3.7173035917237405, + "tokens_seen": 893714432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005892718154463391, + "loss": 1.3763, + "theoretical_loss": 3.7172773127829437, + "tokens_seen": 893779968 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005892557673019057, + "loss": 1.291, + "theoretical_loss": 3.7172510363084514, + "tokens_seen": 893845504 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005892397191574725, + "loss": 1.3637, + "theoretical_loss": 3.717224762299852, + "tokens_seen": 893911040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005892236710130391, + "loss": 1.4245, + "theoretical_loss": 3.717198490756733, + "tokens_seen": 893976576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005892076228686058, + "loss": 1.463, + "theoretical_loss": 3.717172221678682, + "tokens_seen": 894042112 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005891915747241725, + "loss": 1.4785, + "theoretical_loss": 3.7171459550652877, + "tokens_seen": 894107648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005891755265797392, + "loss": 1.4146, + "theoretical_loss": 3.717119690916138, + "tokens_seen": 894173184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005891594784353059, + "loss": 1.4427, + "theoretical_loss": 3.717093429230821, + "tokens_seen": 894238720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005891434302908726, + "loss": 1.3921, + "theoretical_loss": 3.7170671700089244, + "tokens_seen": 894304256 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005891273821464393, + "loss": 1.5263, + "theoretical_loss": 3.717040913250038, + "tokens_seen": 894369792 + }, + { + "epoch": 0.27, + "learning_rate": 0.000589111334002006, + "loss": 1.4411, + "theoretical_loss": 3.7170146589537496, + "tokens_seen": 894435328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005890952858575727, + "loss": 1.481, + "theoretical_loss": 3.716988407119648, + "tokens_seen": 894500864 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005890792377131394, + "loss": 1.4609, + "theoretical_loss": 3.7169621577473215, + "tokens_seen": 894566400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005890631895687061, + "loss": 1.4373, + "theoretical_loss": 3.716935910836359, + "tokens_seen": 894631936 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005890471414242728, + "loss": 1.4397, + "theoretical_loss": 3.71690966638635, + "tokens_seen": 894697472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005890310932798395, + "loss": 1.4539, + "theoretical_loss": 3.716883424396883, + "tokens_seen": 894763008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005890150451354062, + "loss": 1.4635, + "theoretical_loss": 3.716857184867547, + "tokens_seen": 894828544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005889989969909729, + "loss": 1.3792, + "theoretical_loss": 3.716830947797932, + "tokens_seen": 894894080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005889829488465396, + "loss": 1.3937, + "theoretical_loss": 3.7168047131876265, + "tokens_seen": 894959616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005889669007021063, + "loss": 1.5335, + "theoretical_loss": 3.7167784810362203, + "tokens_seen": 895025152 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588950852557673, + "loss": 1.432, + "theoretical_loss": 3.7167522513433022, + "tokens_seen": 895090688 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005889348044132397, + "loss": 1.4095, + "theoretical_loss": 3.716726024108463, + "tokens_seen": 895156224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005889187562688064, + "loss": 1.4244, + "theoretical_loss": 3.716699799331292, + "tokens_seen": 895221760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005889027081243731, + "loss": 1.5176, + "theoretical_loss": 3.716673577011379, + "tokens_seen": 895287296 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005888866599799398, + "loss": 1.4181, + "theoretical_loss": 3.716647357148313, + "tokens_seen": 895352832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005888706118355065, + "loss": 1.468, + "theoretical_loss": 3.7166211397416853, + "tokens_seen": 895418368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005888545636910732, + "loss": 1.4139, + "theoretical_loss": 3.716594924791085, + "tokens_seen": 895483904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005888385155466399, + "loss": 1.504, + "theoretical_loss": 3.7165687122961035, + "tokens_seen": 895549440 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005888224674022066, + "loss": 1.3529, + "theoretical_loss": 3.71654250225633, + "tokens_seen": 895614976 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005888064192577733, + "loss": 1.3496, + "theoretical_loss": 3.7165162946713552, + "tokens_seen": 895680512 + }, + { + "epoch": 0.27, + "learning_rate": 0.00058879037111334, + "loss": 1.4915, + "theoretical_loss": 3.7164900895407698, + "tokens_seen": 895746048 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005887743229689067, + "loss": 1.4306, + "theoretical_loss": 3.7164638868641635, + "tokens_seen": 895811584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005887582748244734, + "loss": 1.4519, + "theoretical_loss": 3.716437686641129, + "tokens_seen": 895877120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005887422266800401, + "loss": 1.4976, + "theoretical_loss": 3.7164114888712545, + "tokens_seen": 895942656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005887261785356068, + "loss": 1.3837, + "theoretical_loss": 3.7163852935541333, + "tokens_seen": 896008192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005887101303911735, + "loss": 1.3312, + "theoretical_loss": 3.716359100689355, + "tokens_seen": 896073728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005886940822467402, + "loss": 1.4089, + "theoretical_loss": 3.7163329102765106, + "tokens_seen": 896139264 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588678034102307, + "loss": 1.3591, + "theoretical_loss": 3.7163067223151924, + "tokens_seen": 896204800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005886619859578736, + "loss": 1.4139, + "theoretical_loss": 3.7162805368049905, + "tokens_seen": 896270336 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005886459378134403, + "loss": 1.457, + "theoretical_loss": 3.716254353745497, + "tokens_seen": 896335872 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588629889669007, + "loss": 1.4351, + "theoretical_loss": 3.7162281731363027, + "tokens_seen": 896401408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005886138415245737, + "loss": 1.5262, + "theoretical_loss": 3.7162019949769998, + "tokens_seen": 896466944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005885977933801404, + "loss": 1.3452, + "theoretical_loss": 3.71617581926718, + "tokens_seen": 896532480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005885817452357072, + "loss": 1.3393, + "theoretical_loss": 3.7161496460064347, + "tokens_seen": 896598016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005885656970912739, + "loss": 1.4376, + "theoretical_loss": 3.716123475194356, + "tokens_seen": 896663552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005885496489468406, + "loss": 1.3771, + "theoretical_loss": 3.7160973068305356, + "tokens_seen": 896729088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005885336008024073, + "loss": 1.3794, + "theoretical_loss": 3.7160711409145657, + "tokens_seen": 896794624 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588517552657974, + "loss": 1.362, + "theoretical_loss": 3.716044977446039, + "tokens_seen": 896860160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005885015045135407, + "loss": 1.4612, + "theoretical_loss": 3.7160188164245467, + "tokens_seen": 896925696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005884854563691074, + "loss": 1.5023, + "theoretical_loss": 3.7159926578496822, + "tokens_seen": 896991232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005884694082246741, + "loss": 1.4721, + "theoretical_loss": 3.715966501721037, + "tokens_seen": 897056768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005884533600802408, + "loss": 1.3598, + "theoretical_loss": 3.7159403480382043, + "tokens_seen": 897122304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005884373119358075, + "loss": 1.5259, + "theoretical_loss": 3.715914196800777, + "tokens_seen": 897187840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005884212637913742, + "loss": 1.3475, + "theoretical_loss": 3.715888048008347, + "tokens_seen": 897253376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005884052156469409, + "loss": 1.4317, + "theoretical_loss": 3.7158619016605074, + "tokens_seen": 897318912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005883891675025076, + "loss": 1.4997, + "theoretical_loss": 3.715835757756852, + "tokens_seen": 897384448 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005883731193580743, + "loss": 1.496, + "theoretical_loss": 3.7158096162969727, + "tokens_seen": 897449984 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588357071213641, + "loss": 1.4427, + "theoretical_loss": 3.715783477280463, + "tokens_seen": 897515520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005883410230692077, + "loss": 1.4778, + "theoretical_loss": 3.715757340706917, + "tokens_seen": 897581056 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005883249749247744, + "loss": 1.3038, + "theoretical_loss": 3.715731206575927, + "tokens_seen": 897646592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005883089267803411, + "loss": 1.4527, + "theoretical_loss": 3.7157050748870866, + "tokens_seen": 897712128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005882928786359078, + "loss": 1.4538, + "theoretical_loss": 3.715678945639989, + "tokens_seen": 897777664 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 337352, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4420275688171387, + "objective/train/theoretical_loss": 3.7156528188342284, + "objective/train/tokens_used": 918303200, + "theoretical_loss": 3.7156528188342284, + "tokens_seen": 897843200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005882768304914745, + "loss": 1.442, + "theoretical_loss": 3.7156528188342284, + "tokens_seen": 897843200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005882607823470412, + "loss": 1.4596, + "theoretical_loss": 3.7156266944693987, + "tokens_seen": 897908736 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005882447342026079, + "loss": 1.5098, + "theoretical_loss": 3.7156005725450933, + "tokens_seen": 897974272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005882286860581746, + "loss": 1.412, + "theoretical_loss": 3.7155744530609063, + "tokens_seen": 898039808 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005882126379137413, + "loss": 1.5341, + "theoretical_loss": 3.7155483360164316, + "tokens_seen": 898105344 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588196589769308, + "loss": 1.4718, + "theoretical_loss": 3.7155222214112635, + "tokens_seen": 898170880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005881805416248747, + "loss": 1.4674, + "theoretical_loss": 3.715496109244996, + "tokens_seen": 898236416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005881644934804414, + "loss": 1.409, + "theoretical_loss": 3.715469999517224, + "tokens_seen": 898301952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005881484453360081, + "loss": 1.5199, + "theoretical_loss": 3.7154438922275403, + "tokens_seen": 898367488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005881323971915748, + "loss": 1.3449, + "theoretical_loss": 3.7154177873755416, + "tokens_seen": 898433024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005881163490471415, + "loss": 1.4268, + "theoretical_loss": 3.715391684960821, + "tokens_seen": 898498560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005881003009027082, + "loss": 1.3927, + "theoretical_loss": 3.715365584982973, + "tokens_seen": 898564096 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005880842527582749, + "loss": 1.3631, + "theoretical_loss": 3.7153394874415935, + "tokens_seen": 898629632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005880682046138416, + "loss": 1.397, + "theoretical_loss": 3.7153133923362764, + "tokens_seen": 898695168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005880521564694083, + "loss": 1.4703, + "theoretical_loss": 3.7152872996666177, + "tokens_seen": 898760704 + }, + { + "epoch": 0.27, + "learning_rate": 0.000588036108324975, + "loss": 1.3496, + "theoretical_loss": 3.715261209432212, + "tokens_seen": 898826240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005880200601805417, + "loss": 1.393, + "theoretical_loss": 3.715235121632654, + "tokens_seen": 898891776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005880040120361084, + "loss": 1.5046, + "theoretical_loss": 3.7152090362675394, + "tokens_seen": 898957312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005879879638916751, + "loss": 1.5047, + "theoretical_loss": 3.7151829533364635, + "tokens_seen": 899022848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005879719157472418, + "loss": 1.392, + "theoretical_loss": 3.715156872839022, + "tokens_seen": 899088384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005879558676028084, + "loss": 1.4361, + "theoretical_loss": 3.7151307947748107, + "tokens_seen": 899153920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005879398194583752, + "loss": 1.5738, + "theoretical_loss": 3.715104719143424, + "tokens_seen": 899219456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005879237713139418, + "loss": 1.4489, + "theoretical_loss": 3.715078645944459, + "tokens_seen": 899284992 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005879077231695086, + "loss": 1.3176, + "theoretical_loss": 3.715052575177511, + "tokens_seen": 899350528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005878916750250752, + "loss": 1.5694, + "theoretical_loss": 3.715026506842176, + "tokens_seen": 899416064 + }, + { + "epoch": 0.27, + "learning_rate": 0.000587875626880642, + "loss": 1.4045, + "theoretical_loss": 3.7150004409380504, + "tokens_seen": 899481600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005878595787362086, + "loss": 1.5224, + "theoretical_loss": 3.7149743774647295, + "tokens_seen": 899547136 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005878435305917754, + "loss": 1.4402, + "theoretical_loss": 3.7149483164218102, + "tokens_seen": 899612672 + }, + { + "epoch": 0.27, + "learning_rate": 0.000587827482447342, + "loss": 1.4106, + "theoretical_loss": 3.714922257808889, + "tokens_seen": 899678208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005878114343029087, + "loss": 1.4765, + "theoretical_loss": 3.7148962016255616, + "tokens_seen": 899743744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005877953861584754, + "loss": 1.3844, + "theoretical_loss": 3.714870147871425, + "tokens_seen": 899809280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005877793380140421, + "loss": 1.4133, + "theoretical_loss": 3.7148440965460763, + "tokens_seen": 899874816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005877632898696088, + "loss": 1.4816, + "theoretical_loss": 3.7148180476491115, + "tokens_seen": 899940352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005877472417251755, + "loss": 1.3468, + "theoretical_loss": 3.714792001180127, + "tokens_seen": 900005888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005877311935807422, + "loss": 1.4092, + "theoretical_loss": 3.714765957138721, + "tokens_seen": 900071424 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005877151454363089, + "loss": 1.526, + "theoretical_loss": 3.71473991552449, + "tokens_seen": 900136960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005876990972918756, + "loss": 1.4664, + "theoretical_loss": 3.71471387633703, + "tokens_seen": 900202496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005876830491474423, + "loss": 1.5121, + "theoretical_loss": 3.7146878395759404, + "tokens_seen": 900268032 + }, + { + "epoch": 0.27, + "learning_rate": 0.000587667001003009, + "loss": 1.4134, + "theoretical_loss": 3.7146618052408167, + "tokens_seen": 900333568 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005876509528585757, + "loss": 1.3915, + "theoretical_loss": 3.714635773331257, + "tokens_seen": 900399104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005876349047141424, + "loss": 1.407, + "theoretical_loss": 3.7146097438468586, + "tokens_seen": 900464640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005876188565697091, + "loss": 1.486, + "theoretical_loss": 3.714583716787219, + "tokens_seen": 900530176 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005876028084252758, + "loss": 1.3973, + "theoretical_loss": 3.7145576921519363, + "tokens_seen": 900595712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005875867602808425, + "loss": 1.4092, + "theoretical_loss": 3.714531669940608, + "tokens_seen": 900661248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005875707121364092, + "loss": 1.3065, + "theoretical_loss": 3.714505650152832, + "tokens_seen": 900726784 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005875546639919759, + "loss": 1.408, + "theoretical_loss": 3.7144796327882066, + "tokens_seen": 900792320 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005875386158475426, + "loss": 1.4195, + "theoretical_loss": 3.714453617846329, + "tokens_seen": 900857856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005875225677031093, + "loss": 1.3938, + "theoretical_loss": 3.714427605326798, + "tokens_seen": 900923392 + }, + { + "epoch": 0.27, + "learning_rate": 0.000587506519558676, + "loss": 1.4178, + "theoretical_loss": 3.714401595229212, + "tokens_seen": 900988928 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005874904714142427, + "loss": 1.3759, + "theoretical_loss": 3.714375587553169, + "tokens_seen": 901054464 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005874744232698094, + "loss": 1.4326, + "theoretical_loss": 3.714349582298268, + "tokens_seen": 901120000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005874583751253761, + "loss": 1.3705, + "theoretical_loss": 3.7143235794641067, + "tokens_seen": 901185536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005874423269809428, + "loss": 1.4297, + "theoretical_loss": 3.7142975790502843, + "tokens_seen": 901251072 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005874262788365095, + "loss": 1.4966, + "theoretical_loss": 3.7142715810563995, + "tokens_seen": 901316608 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005874102306920762, + "loss": 1.3466, + "theoretical_loss": 3.7142455854820513, + "tokens_seen": 901382144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005873941825476429, + "loss": 1.3671, + "theoretical_loss": 3.714219592326838, + "tokens_seen": 901447680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005873781344032096, + "loss": 1.3861, + "theoretical_loss": 3.714193601590359, + "tokens_seen": 901513216 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005873620862587764, + "loss": 1.4625, + "theoretical_loss": 3.7141676132722137, + "tokens_seen": 901578752 + }, + { + "epoch": 0.27, + "learning_rate": 0.000587346038114343, + "loss": 1.4021, + "theoretical_loss": 3.714141627372001, + "tokens_seen": 901644288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005873299899699098, + "loss": 1.4606, + "theoretical_loss": 3.7141156438893206, + "tokens_seen": 901709824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005873139418254764, + "loss": 1.44, + "theoretical_loss": 3.7140896628237714, + "tokens_seen": 901775360 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005872978936810432, + "loss": 1.4641, + "theoretical_loss": 3.7140636841749526, + "tokens_seen": 901840896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005872818455366098, + "loss": 1.3897, + "theoretical_loss": 3.714037707942465, + "tokens_seen": 901906432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005872657973921766, + "loss": 1.4289, + "theoretical_loss": 3.714011734125907, + "tokens_seen": 901971968 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005872497492477432, + "loss": 1.3001, + "theoretical_loss": 3.7139857627248793, + "tokens_seen": 902037504 + }, + { + "epoch": 0.27, + "learning_rate": 0.00058723370110331, + "loss": 1.4677, + "theoretical_loss": 3.7139597937389817, + "tokens_seen": 902103040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005872176529588767, + "loss": 1.4081, + "theoretical_loss": 3.7139338271678133, + "tokens_seen": 902168576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005872016048144434, + "loss": 1.4364, + "theoretical_loss": 3.7139078630109754, + "tokens_seen": 902234112 + }, + { + "epoch": 0.27, + "learning_rate": 0.00058718555667001, + "loss": 1.411, + "theoretical_loss": 3.713881901268068, + "tokens_seen": 902299648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005871695085255768, + "loss": 1.5031, + "theoretical_loss": 3.7138559419386903, + "tokens_seen": 902365184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005871534603811435, + "loss": 1.4337, + "theoretical_loss": 3.713829985022443, + "tokens_seen": 902430720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005871374122367102, + "loss": 1.5035, + "theoretical_loss": 3.7138040305189275, + "tokens_seen": 902496256 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005871213640922769, + "loss": 1.3823, + "theoretical_loss": 3.713778078427744, + "tokens_seen": 902561792 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005871053159478436, + "loss": 1.4074, + "theoretical_loss": 3.7137521287484923, + "tokens_seen": 902627328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005870892678034103, + "loss": 1.3689, + "theoretical_loss": 3.713726181480774, + "tokens_seen": 902692864 + }, + { + "epoch": 0.27, + "learning_rate": 0.000587073219658977, + "loss": 1.3905, + "theoretical_loss": 3.71370023662419, + "tokens_seen": 902758400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005870571715145437, + "loss": 1.4546, + "theoretical_loss": 3.7136742941783405, + "tokens_seen": 902823936 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005870411233701104, + "loss": 1.4917, + "theoretical_loss": 3.713648354142827, + "tokens_seen": 902889472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005870250752256771, + "loss": 1.4162, + "theoretical_loss": 3.713622416517251, + "tokens_seen": 902955008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005870090270812438, + "loss": 1.4075, + "theoretical_loss": 3.7135964813012126, + "tokens_seen": 903020544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005869929789368105, + "loss": 1.5102, + "theoretical_loss": 3.7135705484943142, + "tokens_seen": 903086080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005869769307923772, + "loss": 1.3908, + "theoretical_loss": 3.713544618096157, + "tokens_seen": 903151616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005869608826479439, + "loss": 1.294, + "theoretical_loss": 3.713518690106342, + "tokens_seen": 903217152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005869448345035106, + "loss": 1.4214, + "theoretical_loss": 3.7134927645244713, + "tokens_seen": 903282688 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005869287863590773, + "loss": 1.3453, + "theoretical_loss": 3.7134668413501464, + "tokens_seen": 903348224 + }, + { + "epoch": 0.27, + "learning_rate": 0.000586912738214644, + "loss": 1.5913, + "theoretical_loss": 3.713440920582969, + "tokens_seen": 903413760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868966900702107, + "loss": 1.3498, + "theoretical_loss": 3.713415002222541, + "tokens_seen": 903479296 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868806419257774, + "loss": 1.419, + "theoretical_loss": 3.7133890862684646, + "tokens_seen": 903544832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868645937813441, + "loss": 1.3939, + "theoretical_loss": 3.7133631727203418, + "tokens_seen": 903610368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868485456369108, + "loss": 1.4015, + "theoretical_loss": 3.7133372615777747, + "tokens_seen": 903675904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868324974924775, + "loss": 1.4643, + "theoretical_loss": 3.713311352840366, + "tokens_seen": 903741440 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868164493480442, + "loss": 1.4515, + "theoretical_loss": 3.713285446507717, + "tokens_seen": 903806976 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005868004012036109, + "loss": 1.4899, + "theoretical_loss": 3.7132595425794306, + "tokens_seen": 903872512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005867843530591776, + "loss": 1.3885, + "theoretical_loss": 3.7132336410551097, + "tokens_seen": 903938048 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005867683049147443, + "loss": 1.5006, + "theoretical_loss": 3.713207741934357, + "tokens_seen": 904003584 + }, + { + "epoch": 0.27, + "learning_rate": 0.000586752256770311, + "loss": 1.4099, + "theoretical_loss": 3.713181845216775, + "tokens_seen": 904069120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005867362086258777, + "loss": 1.4015, + "theoretical_loss": 3.7131559509019665, + "tokens_seen": 904134656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005867201604814444, + "loss": 1.5637, + "theoretical_loss": 3.713130058989534, + "tokens_seen": 904200192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005867041123370111, + "loss": 1.3218, + "theoretical_loss": 3.713104169479082, + "tokens_seen": 904265728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005866880641925778, + "loss": 1.492, + "theoretical_loss": 3.713078282370211, + "tokens_seen": 904331264 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 340260, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4158051013946533, + "objective/train/theoretical_loss": 3.713052397662527, + "objective/train/tokens_used": 924856800, + "theoretical_loss": 3.713052397662527, + "tokens_seen": 904396800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005866720160481445, + "loss": 1.4158, + "theoretical_loss": 3.713052397662527, + "tokens_seen": 904396800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005866559679037112, + "loss": 1.4448, + "theoretical_loss": 3.713026515355632, + "tokens_seen": 904462336 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005866399197592779, + "loss": 1.4845, + "theoretical_loss": 3.713000635449129, + "tokens_seen": 904527872 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005866238716148446, + "loss": 1.4704, + "theoretical_loss": 3.7129747579426224, + "tokens_seen": 904593408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005866078234704113, + "loss": 1.5373, + "theoretical_loss": 3.712948882835715, + "tokens_seen": 904658944 + }, + { + "epoch": 0.27, + "learning_rate": 0.000586591775325978, + "loss": 1.3911, + "theoretical_loss": 3.7129230101280113, + "tokens_seen": 904724480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005865757271815447, + "loss": 1.3435, + "theoretical_loss": 3.7128971398191144, + "tokens_seen": 904790016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005865596790371113, + "loss": 1.4532, + "theoretical_loss": 3.712871271908628, + "tokens_seen": 904855552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005865436308926781, + "loss": 1.3434, + "theoretical_loss": 3.7128454063961565, + "tokens_seen": 904921088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005865275827482447, + "loss": 1.402, + "theoretical_loss": 3.712819543281304, + "tokens_seen": 904986624 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005865115346038115, + "loss": 1.4889, + "theoretical_loss": 3.712793682563675, + "tokens_seen": 905052160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005864954864593781, + "loss": 1.4596, + "theoretical_loss": 3.712767824242873, + "tokens_seen": 905117696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005864794383149449, + "loss": 1.4699, + "theoretical_loss": 3.712741968318502, + "tokens_seen": 905183232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005864633901705115, + "loss": 1.5486, + "theoretical_loss": 3.712716114790168, + "tokens_seen": 905248768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005864473420260783, + "loss": 1.5787, + "theoretical_loss": 3.712690263657474, + "tokens_seen": 905314304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005864312938816449, + "loss": 1.5319, + "theoretical_loss": 3.7126644149200247, + "tokens_seen": 905379840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005864312938816449, + "loss": 1.4895, + "theoretical_loss": 3.712638568577426, + "tokens_seen": 905445376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005864152457372116, + "loss": 1.4077, + "theoretical_loss": 3.712612724629282, + "tokens_seen": 905510912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005863991975927783, + "loss": 1.4833, + "theoretical_loss": 3.712586883075198, + "tokens_seen": 905576448 + }, + { + "epoch": 0.27, + "learning_rate": 0.000586383149448345, + "loss": 1.4978, + "theoretical_loss": 3.7125610439147776, + "tokens_seen": 905641984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005863671013039117, + "loss": 1.4448, + "theoretical_loss": 3.7125352071476274, + "tokens_seen": 905707520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005863510531594784, + "loss": 1.9204, + "theoretical_loss": 3.7125093727733516, + "tokens_seen": 905773056 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005863350050150451, + "loss": 1.5827, + "theoretical_loss": 3.7124835407915566, + "tokens_seen": 905838592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005863189568706118, + "loss": 1.4913, + "theoretical_loss": 3.712457711201847, + "tokens_seen": 905904128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005863029087261785, + "loss": 1.6359, + "theoretical_loss": 3.712431884003828, + "tokens_seen": 905969664 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005862868605817452, + "loss": 1.5105, + "theoretical_loss": 3.7124060591971055, + "tokens_seen": 906035200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005862708124373119, + "loss": 1.5142, + "theoretical_loss": 3.7123802367812857, + "tokens_seen": 906100736 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005862547642928786, + "loss": 1.5299, + "theoretical_loss": 3.712354416755973, + "tokens_seen": 906166272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005862387161484453, + "loss": 1.5491, + "theoretical_loss": 3.7123285991207746, + "tokens_seen": 906231808 + }, + { + "epoch": 0.27, + "learning_rate": 0.000586222668004012, + "loss": 1.4826, + "theoretical_loss": 3.712302783875296, + "tokens_seen": 906297344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005862066198595787, + "loss": 1.459, + "theoretical_loss": 3.7122769710191426, + "tokens_seen": 906362880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005861905717151454, + "loss": 1.3895, + "theoretical_loss": 3.7122511605519217, + "tokens_seen": 906428416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005861745235707121, + "loss": 1.4492, + "theoretical_loss": 3.7122253524732383, + "tokens_seen": 906493952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005861584754262789, + "loss": 1.4772, + "theoretical_loss": 3.712199546782699, + "tokens_seen": 906559488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005861424272818455, + "loss": 1.4189, + "theoretical_loss": 3.7121737434799114, + "tokens_seen": 906625024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005861263791374123, + "loss": 1.5071, + "theoretical_loss": 3.71214794256448, + "tokens_seen": 906690560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005861103309929789, + "loss": 1.4578, + "theoretical_loss": 3.712122144036013, + "tokens_seen": 906756096 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005860942828485457, + "loss": 1.4639, + "theoretical_loss": 3.712096347894116, + "tokens_seen": 906821632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005860782347041123, + "loss": 1.5129, + "theoretical_loss": 3.7120705541383967, + "tokens_seen": 906887168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005860621865596791, + "loss": 1.5095, + "theoretical_loss": 3.7120447627684614, + "tokens_seen": 906952704 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005860461384152457, + "loss": 1.5151, + "theoretical_loss": 3.712018973783917, + "tokens_seen": 907018240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005860300902708125, + "loss": 1.453, + "theoretical_loss": 3.7119931871843708, + "tokens_seen": 907083776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005860140421263791, + "loss": 1.4616, + "theoretical_loss": 3.71196740296943, + "tokens_seen": 907149312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005859979939819459, + "loss": 1.5029, + "theoretical_loss": 3.7119416211387013, + "tokens_seen": 907214848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005859819458375125, + "loss": 1.4447, + "theoretical_loss": 3.7119158416917926, + "tokens_seen": 907280384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005859658976930793, + "loss": 1.4481, + "theoretical_loss": 3.711890064628311, + "tokens_seen": 907345920 + }, + { + "epoch": 0.27, + "learning_rate": 0.000585949849548646, + "loss": 1.4341, + "theoretical_loss": 3.711864289947864, + "tokens_seen": 907411456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0005859338014042127, + "loss": 1.5978, + "theoretical_loss": 3.71183851765006, + "tokens_seen": 907476992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005859177532597793, + "loss": 1.5085, + "theoretical_loss": 3.7118127477345055, + "tokens_seen": 907542528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005859017051153461, + "loss": 1.4459, + "theoretical_loss": 3.7117869802008094, + "tokens_seen": 907608064 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005858856569709127, + "loss": 1.4197, + "theoretical_loss": 3.7117612150485786, + "tokens_seen": 907673600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005858696088264795, + "loss": 1.4134, + "theoretical_loss": 3.7117354522774217, + "tokens_seen": 907739136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005858535606820462, + "loss": 1.3715, + "theoretical_loss": 3.7117096918869468, + "tokens_seen": 907804672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005858375125376129, + "loss": 1.4087, + "theoretical_loss": 3.7116839338767615, + "tokens_seen": 907870208 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005858214643931796, + "loss": 1.4342, + "theoretical_loss": 3.7116581782464744, + "tokens_seen": 907935744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005858054162487463, + "loss": 1.4538, + "theoretical_loss": 3.711632424995694, + "tokens_seen": 908001280 + }, + { + "epoch": 0.28, + "learning_rate": 0.000585789368104313, + "loss": 1.4327, + "theoretical_loss": 3.7116066741240283, + "tokens_seen": 908066816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005857733199598797, + "loss": 1.3409, + "theoretical_loss": 3.7115809256310865, + "tokens_seen": 908132352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005857572718154464, + "loss": 1.3816, + "theoretical_loss": 3.711555179516477, + "tokens_seen": 908197888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005857412236710131, + "loss": 1.509, + "theoretical_loss": 3.7115294357798083, + "tokens_seen": 908263424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005857251755265798, + "loss": 1.4617, + "theoretical_loss": 3.7115036944206885, + "tokens_seen": 908328960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005857091273821465, + "loss": 1.4597, + "theoretical_loss": 3.7114779554387285, + "tokens_seen": 908394496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005856930792377132, + "loss": 1.3199, + "theoretical_loss": 3.7114522188335357, + "tokens_seen": 908460032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005856770310932799, + "loss": 1.4122, + "theoretical_loss": 3.7114264846047194, + "tokens_seen": 908525568 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005856609829488466, + "loss": 1.5221, + "theoretical_loss": 3.7114007527518886, + "tokens_seen": 908591104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005856449348044133, + "loss": 1.4851, + "theoretical_loss": 3.7113750232746536, + "tokens_seen": 908656640 + }, + { + "epoch": 0.28, + "learning_rate": 0.00058562888665998, + "loss": 1.4561, + "theoretical_loss": 3.711349296172623, + "tokens_seen": 908722176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005856128385155467, + "loss": 1.4785, + "theoretical_loss": 3.7113235714454063, + "tokens_seen": 908787712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855967903711134, + "loss": 1.5034, + "theoretical_loss": 3.711297849092613, + "tokens_seen": 908853248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855807422266801, + "loss": 1.4363, + "theoretical_loss": 3.711272129113853, + "tokens_seen": 908918784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855646940822468, + "loss": 1.4938, + "theoretical_loss": 3.7112464115087365, + "tokens_seen": 908984320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855486459378135, + "loss": 1.4059, + "theoretical_loss": 3.7112206962768717, + "tokens_seen": 909049856 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855325977933802, + "loss": 1.391, + "theoretical_loss": 3.7111949834178706, + "tokens_seen": 909115392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855165496489469, + "loss": 1.4414, + "theoretical_loss": 3.7111692729313415, + "tokens_seen": 909180928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005855005015045136, + "loss": 1.4447, + "theoretical_loss": 3.7111435648168953, + "tokens_seen": 909246464 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005854844533600803, + "loss": 1.4363, + "theoretical_loss": 3.7111178590741423, + "tokens_seen": 909312000 + }, + { + "epoch": 0.28, + "learning_rate": 0.000585468405215647, + "loss": 1.5004, + "theoretical_loss": 3.7110921557026924, + "tokens_seen": 909377536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005854523570712137, + "loss": 1.3981, + "theoretical_loss": 3.7110664547021566, + "tokens_seen": 909443072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005854363089267804, + "loss": 1.2734, + "theoretical_loss": 3.711040756072144, + "tokens_seen": 909508608 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005854202607823471, + "loss": 1.3675, + "theoretical_loss": 3.711015059812267, + "tokens_seen": 909574144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005854042126379138, + "loss": 1.4411, + "theoretical_loss": 3.710989365922135, + "tokens_seen": 909639680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005853881644934805, + "loss": 1.3995, + "theoretical_loss": 3.710963674401359, + "tokens_seen": 909705216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005853721163490472, + "loss": 1.4839, + "theoretical_loss": 3.7109379852495503, + "tokens_seen": 909770752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005853560682046139, + "loss": 1.4808, + "theoretical_loss": 3.7109122984663196, + "tokens_seen": 909836288 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005853400200601806, + "loss": 1.4328, + "theoretical_loss": 3.7108866140512777, + "tokens_seen": 909901824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005853239719157473, + "loss": 1.4845, + "theoretical_loss": 3.7108609320040356, + "tokens_seen": 909967360 + }, + { + "epoch": 0.28, + "learning_rate": 0.000585307923771314, + "loss": 1.3751, + "theoretical_loss": 3.7108352523242045, + "tokens_seen": 910032896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005852918756268807, + "loss": 1.4113, + "theoretical_loss": 3.710809575011397, + "tokens_seen": 910098432 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005852758274824474, + "loss": 1.4076, + "theoretical_loss": 3.7107839000652225, + "tokens_seen": 910163968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005852597793380141, + "loss": 1.4839, + "theoretical_loss": 3.710758227485294, + "tokens_seen": 910229504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005852437311935808, + "loss": 1.4962, + "theoretical_loss": 3.710732557271222, + "tokens_seen": 910295040 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005852276830491475, + "loss": 1.4796, + "theoretical_loss": 3.710706889422619, + "tokens_seen": 910360576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005852116349047142, + "loss": 1.492, + "theoretical_loss": 3.7106812239390967, + "tokens_seen": 910426112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005851955867602809, + "loss": 1.4217, + "theoretical_loss": 3.7106555608202667, + "tokens_seen": 910491648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005851795386158476, + "loss": 1.3121, + "theoretical_loss": 3.71062990006574, + "tokens_seen": 910557184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005851634904714142, + "loss": 1.373, + "theoretical_loss": 3.7106042416751306, + "tokens_seen": 910622720 + }, + { + "epoch": 0.28, + "learning_rate": 0.000585147442326981, + "loss": 1.3999, + "theoretical_loss": 3.7105785856480495, + "tokens_seen": 910688256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005851313941825476, + "loss": 1.4912, + "theoretical_loss": 3.710552931984109, + "tokens_seen": 910753792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005851153460381144, + "loss": 1.4605, + "theoretical_loss": 3.7105272806829213, + "tokens_seen": 910819328 + }, + { + "epoch": 0.28, + "learning_rate": 0.000585099297893681, + "loss": 1.422, + "theoretical_loss": 3.7105016317440986, + "tokens_seen": 910884864 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 342389, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3579729795455933, + "objective/train/theoretical_loss": 3.710475985167254, + "objective/train/tokens_used": 931410400, + "theoretical_loss": 3.710475985167254, + "tokens_seen": 910950400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005850832497492478, + "loss": 1.358, + "theoretical_loss": 3.710475985167254, + "tokens_seen": 910950400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005850672016048144, + "loss": 1.4015, + "theoretical_loss": 3.710450340952, + "tokens_seen": 911015936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005850511534603812, + "loss": 1.3766, + "theoretical_loss": 3.710424699097949, + "tokens_seen": 911081472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005850351053159478, + "loss": 1.327, + "theoretical_loss": 3.710399059604714, + "tokens_seen": 911147008 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005850190571715145, + "loss": 1.4459, + "theoretical_loss": 3.710373422471908, + "tokens_seen": 911212544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005850030090270812, + "loss": 1.5645, + "theoretical_loss": 3.710347787699143, + "tokens_seen": 911278080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005849869608826479, + "loss": 1.3011, + "theoretical_loss": 3.710322155286033, + "tokens_seen": 911343616 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005849709127382146, + "loss": 1.5364, + "theoretical_loss": 3.710296525232191, + "tokens_seen": 911409152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005849548645937814, + "loss": 1.4016, + "theoretical_loss": 3.71027089753723, + "tokens_seen": 911474688 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584938816449348, + "loss": 1.3559, + "theoretical_loss": 3.7102452722007637, + "tokens_seen": 911540224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005849227683049148, + "loss": 1.4852, + "theoretical_loss": 3.710219649222404, + "tokens_seen": 911605760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005849067201604814, + "loss": 1.3596, + "theoretical_loss": 3.710194028601767, + "tokens_seen": 911671296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005848906720160482, + "loss": 1.4727, + "theoretical_loss": 3.7101684103384645, + "tokens_seen": 911736832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005848746238716148, + "loss": 1.4022, + "theoretical_loss": 3.7101427944321106, + "tokens_seen": 911802368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005848585757271816, + "loss": 1.3588, + "theoretical_loss": 3.7101171808823192, + "tokens_seen": 911867904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005848425275827482, + "loss": 1.3211, + "theoretical_loss": 3.7100915696887036, + "tokens_seen": 911933440 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584826479438315, + "loss": 1.4235, + "theoretical_loss": 3.710065960850878, + "tokens_seen": 911998976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005848104312938816, + "loss": 1.3115, + "theoretical_loss": 3.710040354368457, + "tokens_seen": 912064512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005847943831494484, + "loss": 1.4554, + "theoretical_loss": 3.7100147502410543, + "tokens_seen": 912130048 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584778335005015, + "loss": 1.4126, + "theoretical_loss": 3.709989148468284, + "tokens_seen": 912195584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005847622868605818, + "loss": 1.471, + "theoretical_loss": 3.7099635490497604, + "tokens_seen": 912261120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005847462387161484, + "loss": 1.4272, + "theoretical_loss": 3.709937951985099, + "tokens_seen": 912326656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005847301905717152, + "loss": 1.3471, + "theoretical_loss": 3.7099123572739128, + "tokens_seen": 912392192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005847141424272818, + "loss": 1.4982, + "theoretical_loss": 3.709886764915817, + "tokens_seen": 912457728 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005846980942828486, + "loss": 1.3977, + "theoretical_loss": 3.7098611749104258, + "tokens_seen": 912523264 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005846820461384152, + "loss": 1.3692, + "theoretical_loss": 3.709835587257355, + "tokens_seen": 912588800 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584665997993982, + "loss": 1.3256, + "theoretical_loss": 3.7098100019562192, + "tokens_seen": 912654336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005846499498495486, + "loss": 1.4497, + "theoretical_loss": 3.7097844190066325, + "tokens_seen": 912719872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005846339017051154, + "loss": 1.3007, + "theoretical_loss": 3.7097588384082107, + "tokens_seen": 912785408 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584617853560682, + "loss": 1.4033, + "theoretical_loss": 3.7097332601605686, + "tokens_seen": 912850944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005846018054162488, + "loss": 1.4387, + "theoretical_loss": 3.7097076842633214, + "tokens_seen": 912916480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005845857572718154, + "loss": 1.3877, + "theoretical_loss": 3.709682110716085, + "tokens_seen": 912982016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005845697091273822, + "loss": 1.5047, + "theoretical_loss": 3.709656539518474, + "tokens_seen": 913047552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005845536609829488, + "loss": 1.3375, + "theoretical_loss": 3.709630970670105, + "tokens_seen": 913113088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005845376128385156, + "loss": 1.4063, + "theoretical_loss": 3.709605404170592, + "tokens_seen": 913178624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005845215646940822, + "loss": 1.5177, + "theoretical_loss": 3.7095798400195523, + "tokens_seen": 913244160 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584505516549649, + "loss": 1.3812, + "theoretical_loss": 3.7095542782166, + "tokens_seen": 913309696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005844894684052156, + "loss": 1.3776, + "theoretical_loss": 3.7095287187613524, + "tokens_seen": 913375232 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005844734202607824, + "loss": 1.4071, + "theoretical_loss": 3.709503161653425, + "tokens_seen": 913440768 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584457372116349, + "loss": 1.4093, + "theoretical_loss": 3.7094776068924338, + "tokens_seen": 913506304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005844413239719158, + "loss": 1.3857, + "theoretical_loss": 3.7094520544779948, + "tokens_seen": 913571840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005844252758274825, + "loss": 1.4057, + "theoretical_loss": 3.7094265044097243, + "tokens_seen": 913637376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005844092276830492, + "loss": 1.3763, + "theoretical_loss": 3.7094009566872383, + "tokens_seen": 913702912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005843931795386159, + "loss": 1.5178, + "theoretical_loss": 3.7093754113101536, + "tokens_seen": 913768448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005843771313941826, + "loss": 1.4679, + "theoretical_loss": 3.709349868278087, + "tokens_seen": 913833984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005843610832497493, + "loss": 1.4622, + "theoretical_loss": 3.7093243275906547, + "tokens_seen": 913899520 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584345035105316, + "loss": 1.2789, + "theoretical_loss": 3.7092987892474727, + "tokens_seen": 913965056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005843289869608827, + "loss": 1.322, + "theoretical_loss": 3.709273253248159, + "tokens_seen": 914030592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005843129388164494, + "loss": 1.2889, + "theoretical_loss": 3.7092477195923292, + "tokens_seen": 914096128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842968906720161, + "loss": 1.4135, + "theoretical_loss": 3.709222188279602, + "tokens_seen": 914161664 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842808425275828, + "loss": 1.4098, + "theoretical_loss": 3.709196659309592, + "tokens_seen": 914227200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842647943831495, + "loss": 1.5022, + "theoretical_loss": 3.7091711326819192, + "tokens_seen": 914292736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842487462387162, + "loss": 1.392, + "theoretical_loss": 3.7091456083961982, + "tokens_seen": 914358272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842326980942829, + "loss": 1.4863, + "theoretical_loss": 3.7091200864520477, + "tokens_seen": 914423808 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842166499498496, + "loss": 1.3945, + "theoretical_loss": 3.709094566849085, + "tokens_seen": 914489344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005842006018054163, + "loss": 1.4679, + "theoretical_loss": 3.709069049586927, + "tokens_seen": 914554880 + }, + { + "epoch": 0.28, + "learning_rate": 0.000584184553660983, + "loss": 1.4828, + "theoretical_loss": 3.7090435346651915, + "tokens_seen": 914620416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005841685055165497, + "loss": 1.3706, + "theoretical_loss": 3.7090180220834967, + "tokens_seen": 914685952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005841524573721164, + "loss": 1.3316, + "theoretical_loss": 3.7089925118414597, + "tokens_seen": 914751488 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005841364092276831, + "loss": 1.4007, + "theoretical_loss": 3.7089670039386986, + "tokens_seen": 914817024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005841203610832498, + "loss": 1.3527, + "theoretical_loss": 3.7089414983748314, + "tokens_seen": 914882560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005841043129388165, + "loss": 1.4192, + "theoretical_loss": 3.708915995149476, + "tokens_seen": 914948096 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005840882647943832, + "loss": 1.327, + "theoretical_loss": 3.70889049426225, + "tokens_seen": 915013632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005840722166499499, + "loss": 1.4063, + "theoretical_loss": 3.708864995712773, + "tokens_seen": 915079168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005840561685055166, + "loss": 1.3538, + "theoretical_loss": 3.708839499500662, + "tokens_seen": 915144704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005840401203610833, + "loss": 1.3833, + "theoretical_loss": 3.708814005625536, + "tokens_seen": 915210240 + }, + { + "epoch": 0.28, + "learning_rate": 0.00058402407221665, + "loss": 1.3177, + "theoretical_loss": 3.708788514087013, + "tokens_seen": 915275776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005840080240722167, + "loss": 1.438, + "theoretical_loss": 3.7087630248847123, + "tokens_seen": 915341312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005839919759277834, + "loss": 1.5053, + "theoretical_loss": 3.7087375380182515, + "tokens_seen": 915406848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005839759277833501, + "loss": 1.3286, + "theoretical_loss": 3.7087120534872504, + "tokens_seen": 915472384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005839598796389168, + "loss": 1.4418, + "theoretical_loss": 3.708686571291327, + "tokens_seen": 915537920 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005839438314944835, + "loss": 1.4184, + "theoretical_loss": 3.7086610914301006, + "tokens_seen": 915603456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005839277833500502, + "loss": 1.3862, + "theoretical_loss": 3.70863561390319, + "tokens_seen": 915668992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005839117352056169, + "loss": 1.3193, + "theoretical_loss": 3.708610138710215, + "tokens_seen": 915734528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005838956870611836, + "loss": 1.4112, + "theoretical_loss": 3.708584665850794, + "tokens_seen": 915800064 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005838796389167503, + "loss": 1.435, + "theoretical_loss": 3.7085591953245465, + "tokens_seen": 915865600 + }, + { + "epoch": 0.28, + "learning_rate": 0.000583863590772317, + "loss": 1.3785, + "theoretical_loss": 3.708533727131092, + "tokens_seen": 915931136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005838475426278837, + "loss": 1.3101, + "theoretical_loss": 3.7085082612700493, + "tokens_seen": 915996672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005838314944834504, + "loss": 1.3677, + "theoretical_loss": 3.7084827977410395, + "tokens_seen": 916062208 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005838154463390171, + "loss": 1.4688, + "theoretical_loss": 3.70845733654368, + "tokens_seen": 916127744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837993981945838, + "loss": 1.384, + "theoretical_loss": 3.708431877677593, + "tokens_seen": 916193280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837833500501505, + "loss": 1.3664, + "theoretical_loss": 3.708406421142396, + "tokens_seen": 916258816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837673019057172, + "loss": 1.4109, + "theoretical_loss": 3.7083809669377104, + "tokens_seen": 916324352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837512537612839, + "loss": 1.4267, + "theoretical_loss": 3.7083555150631557, + "tokens_seen": 916389888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837352056168505, + "loss": 1.3817, + "theoretical_loss": 3.708330065518352, + "tokens_seen": 916455424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837191574724173, + "loss": 1.3297, + "theoretical_loss": 3.70830461830292, + "tokens_seen": 916520960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005837031093279839, + "loss": 1.432, + "theoretical_loss": 3.7082791734164786, + "tokens_seen": 916586496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005836870611835507, + "loss": 1.3497, + "theoretical_loss": 3.708253730858649, + "tokens_seen": 916652032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005836710130391173, + "loss": 1.5194, + "theoretical_loss": 3.7082282906290525, + "tokens_seen": 916717568 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005836549648946841, + "loss": 1.4692, + "theoretical_loss": 3.7082028527273083, + "tokens_seen": 916783104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005836389167502507, + "loss": 1.4706, + "theoretical_loss": 3.708177417153037, + "tokens_seen": 916848640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005836228686058175, + "loss": 1.4163, + "theoretical_loss": 3.70815198390586, + "tokens_seen": 916914176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005836068204613841, + "loss": 1.3586, + "theoretical_loss": 3.708126552985398, + "tokens_seen": 916979712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005835907723169509, + "loss": 1.3829, + "theoretical_loss": 3.7081011243912716, + "tokens_seen": 917045248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005835747241725175, + "loss": 1.3363, + "theoretical_loss": 3.7080756981231016, + "tokens_seen": 917110784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005835586760280843, + "loss": 1.5672, + "theoretical_loss": 3.7080502741805104, + "tokens_seen": 917176320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005835426278836509, + "loss": 1.4554, + "theoretical_loss": 3.708024852563117, + "tokens_seen": 917241856 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005835265797392177, + "loss": 1.3801, + "theoretical_loss": 3.707999433270544, + "tokens_seen": 917307392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005835105315947843, + "loss": 1.3608, + "theoretical_loss": 3.7079740163024124, + "tokens_seen": 917372928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005834944834503511, + "loss": 1.4784, + "theoretical_loss": 3.7079486016583436, + "tokens_seen": 917438464 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 344529, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4405187368392944, + "objective/train/theoretical_loss": 3.7079231893379596, + "objective/train/tokens_used": 937964000, + "theoretical_loss": 3.7079231893379596, + "tokens_seen": 917504000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005834784353059177, + "loss": 1.4405, + "theoretical_loss": 3.7079231893379596, + "tokens_seen": 917504000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005834623871614845, + "loss": 1.4344, + "theoretical_loss": 3.707897779340881, + "tokens_seen": 917569536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005834463390170511, + "loss": 1.4389, + "theoretical_loss": 3.7078723716667303, + "tokens_seen": 917635072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005834302908726179, + "loss": 1.3954, + "theoretical_loss": 3.7078469663151292, + "tokens_seen": 917700608 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005834142427281845, + "loss": 1.4981, + "theoretical_loss": 3.707821563285699, + "tokens_seen": 917766144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833981945837513, + "loss": 1.4477, + "theoretical_loss": 3.707796162578062, + "tokens_seen": 917831680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833821464393179, + "loss": 1.4773, + "theoretical_loss": 3.7077707641918405, + "tokens_seen": 917897216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833660982948847, + "loss": 1.5092, + "theoretical_loss": 3.7077453681266563, + "tokens_seen": 917962752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833500501504513, + "loss": 1.3924, + "theoretical_loss": 3.7077199743821314, + "tokens_seen": 918028288 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833340020060181, + "loss": 1.4685, + "theoretical_loss": 3.7076945829578882, + "tokens_seen": 918093824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833179538615847, + "loss": 1.4359, + "theoretical_loss": 3.7076691938535493, + "tokens_seen": 918159360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005833019057171515, + "loss": 1.5738, + "theoretical_loss": 3.707643807068737, + "tokens_seen": 918224896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005832858575727181, + "loss": 1.4191, + "theoretical_loss": 3.7076184226030744, + "tokens_seen": 918290432 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005832698094282849, + "loss": 1.4502, + "theoretical_loss": 3.707593040456184, + "tokens_seen": 918355968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005832537612838515, + "loss": 1.3161, + "theoretical_loss": 3.7075676606276877, + "tokens_seen": 918421504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005832377131394183, + "loss": 1.482, + "theoretical_loss": 3.707542283117209, + "tokens_seen": 918487040 + }, + { + "epoch": 0.28, + "learning_rate": 0.000583221664994985, + "loss": 1.5384, + "theoretical_loss": 3.7075169079243704, + "tokens_seen": 918552576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005832056168505517, + "loss": 1.3959, + "theoretical_loss": 3.7074915350487956, + "tokens_seen": 918618112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005831895687061183, + "loss": 1.4162, + "theoretical_loss": 3.707466164490107, + "tokens_seen": 918683648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005831735205616851, + "loss": 1.4605, + "theoretical_loss": 3.707440796247928, + "tokens_seen": 918749184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005831574724172517, + "loss": 1.4463, + "theoretical_loss": 3.7074154303218823, + "tokens_seen": 918814720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005831414242728185, + "loss": 1.3286, + "theoretical_loss": 3.707390066711593, + "tokens_seen": 918880256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005831253761283851, + "loss": 1.5064, + "theoretical_loss": 3.7073647054166834, + "tokens_seen": 918945792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005831093279839519, + "loss": 1.4229, + "theoretical_loss": 3.7073393464367768, + "tokens_seen": 919011328 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005830932798395186, + "loss": 1.3961, + "theoretical_loss": 3.707313989771497, + "tokens_seen": 919076864 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005830772316950853, + "loss": 1.4721, + "theoretical_loss": 3.707288635420468, + "tokens_seen": 919142400 + }, + { + "epoch": 0.28, + "learning_rate": 0.000583061183550652, + "loss": 1.3804, + "theoretical_loss": 3.7072632833833135, + "tokens_seen": 919207936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005830451354062187, + "loss": 1.3278, + "theoretical_loss": 3.707237933659657, + "tokens_seen": 919273472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005830290872617854, + "loss": 1.4506, + "theoretical_loss": 3.7072125862491228, + "tokens_seen": 919339008 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005830130391173521, + "loss": 1.3665, + "theoretical_loss": 3.7071872411513347, + "tokens_seen": 919404544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005829969909729188, + "loss": 1.4745, + "theoretical_loss": 3.707161898365918, + "tokens_seen": 919470080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005829809428284855, + "loss": 1.3823, + "theoretical_loss": 3.707136557892495, + "tokens_seen": 919535616 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005829648946840522, + "loss": 1.2606, + "theoretical_loss": 3.7071112197306917, + "tokens_seen": 919601152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005829488465396189, + "loss": 1.5055, + "theoretical_loss": 3.7070858838801315, + "tokens_seen": 919666688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005829327983951856, + "loss": 1.4656, + "theoretical_loss": 3.7070605503404392, + "tokens_seen": 919732224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005829167502507523, + "loss": 1.3869, + "theoretical_loss": 3.7070352191112397, + "tokens_seen": 919797760 + }, + { + "epoch": 0.28, + "learning_rate": 0.000582900702106319, + "loss": 1.4022, + "theoretical_loss": 3.707009890192157, + "tokens_seen": 919863296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005828846539618857, + "loss": 1.352, + "theoretical_loss": 3.7069845635828167, + "tokens_seen": 919928832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005828686058174524, + "loss": 1.4296, + "theoretical_loss": 3.7069592392828428, + "tokens_seen": 919994368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005828525576730191, + "loss": 1.4631, + "theoretical_loss": 3.706933917291861, + "tokens_seen": 920059904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005828365095285858, + "loss": 1.4363, + "theoretical_loss": 3.7069085976094955, + "tokens_seen": 920125440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005828204613841525, + "loss": 1.404, + "theoretical_loss": 3.7068832802353726, + "tokens_seen": 920190976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005828044132397192, + "loss": 1.5223, + "theoretical_loss": 3.7068579651691165, + "tokens_seen": 920256512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005827883650952859, + "loss": 1.4225, + "theoretical_loss": 3.7068326524103528, + "tokens_seen": 920322048 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005827723169508526, + "loss": 1.369, + "theoretical_loss": 3.706807341958706, + "tokens_seen": 920387584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005827562688064193, + "loss": 1.4124, + "theoretical_loss": 3.706782033813804, + "tokens_seen": 920453120 + }, + { + "epoch": 0.28, + "learning_rate": 0.000582740220661986, + "loss": 1.293, + "theoretical_loss": 3.70675672797527, + "tokens_seen": 920518656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005827241725175527, + "loss": 1.3029, + "theoretical_loss": 3.7067314244427303, + "tokens_seen": 920584192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005827081243731194, + "loss": 1.3695, + "theoretical_loss": 3.7067061232158105, + "tokens_seen": 920649728 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005826920762286861, + "loss": 1.3429, + "theoretical_loss": 3.7066808242941374, + "tokens_seen": 920715264 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005826760280842528, + "loss": 1.3986, + "theoretical_loss": 3.706655527677335, + "tokens_seen": 920780800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005826599799398195, + "loss": 1.312, + "theoretical_loss": 3.706630233365032, + "tokens_seen": 920846336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005826439317953862, + "loss": 1.3686, + "theoretical_loss": 3.7066049413568516, + "tokens_seen": 920911872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005826278836509529, + "loss": 1.4666, + "theoretical_loss": 3.7065796516524214, + "tokens_seen": 920977408 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005826118355065196, + "loss": 1.4694, + "theoretical_loss": 3.7065543642513683, + "tokens_seen": 921042944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005825957873620863, + "loss": 1.3795, + "theoretical_loss": 3.7065290791533174, + "tokens_seen": 921108480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005825797392176531, + "loss": 1.3052, + "theoretical_loss": 3.7065037963578957, + "tokens_seen": 921174016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005825636910732197, + "loss": 1.432, + "theoretical_loss": 3.7064785158647293, + "tokens_seen": 921239552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005825476429287865, + "loss": 1.4279, + "theoretical_loss": 3.7064532376734456, + "tokens_seen": 921305088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005825315947843531, + "loss": 1.4449, + "theoretical_loss": 3.70642796178367, + "tokens_seen": 921370624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005825155466399199, + "loss": 1.5494, + "theoretical_loss": 3.706402688195031, + "tokens_seen": 921436160 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005824994984954865, + "loss": 1.3658, + "theoretical_loss": 3.706377416907154, + "tokens_seen": 921501696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005824834503510532, + "loss": 1.5619, + "theoretical_loss": 3.7063521479196666, + "tokens_seen": 921567232 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005824674022066199, + "loss": 1.3485, + "theoretical_loss": 3.7063268812321954, + "tokens_seen": 921632768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005824513540621866, + "loss": 1.4749, + "theoretical_loss": 3.7063016168443683, + "tokens_seen": 921698304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005824353059177533, + "loss": 1.3971, + "theoretical_loss": 3.706276354755812, + "tokens_seen": 921763840 + }, + { + "epoch": 0.28, + "learning_rate": 0.00058241925777332, + "loss": 1.3621, + "theoretical_loss": 3.706251094966153, + "tokens_seen": 921829376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005824032096288867, + "loss": 1.4069, + "theoretical_loss": 3.7062258374750208, + "tokens_seen": 921894912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005823871614844534, + "loss": 1.3546, + "theoretical_loss": 3.7062005822820403, + "tokens_seen": 921960448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005823711133400201, + "loss": 1.3108, + "theoretical_loss": 3.7061753293868414, + "tokens_seen": 922025984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005823550651955868, + "loss": 1.4591, + "theoretical_loss": 3.70615007878905, + "tokens_seen": 922091520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005823390170511534, + "loss": 1.385, + "theoretical_loss": 3.706124830488295, + "tokens_seen": 922157056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005823229689067202, + "loss": 1.3332, + "theoretical_loss": 3.7060995844842033, + "tokens_seen": 922222592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005823069207622868, + "loss": 1.4539, + "theoretical_loss": 3.706074340776403, + "tokens_seen": 922288128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005822908726178536, + "loss": 1.35, + "theoretical_loss": 3.7060490993645225, + "tokens_seen": 922353664 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005822748244734202, + "loss": 1.5279, + "theoretical_loss": 3.70602386024819, + "tokens_seen": 922419200 + }, + { + "epoch": 0.28, + "learning_rate": 0.000582258776328987, + "loss": 1.3307, + "theoretical_loss": 3.7059986234270332, + "tokens_seen": 922484736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005822427281845536, + "loss": 1.4376, + "theoretical_loss": 3.7059733889006807, + "tokens_seen": 922550272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005822266800401204, + "loss": 1.4922, + "theoretical_loss": 3.70594815666876, + "tokens_seen": 922615808 + }, + { + "epoch": 0.28, + "learning_rate": 0.000582210631895687, + "loss": 1.5131, + "theoretical_loss": 3.7059229267309006, + "tokens_seen": 922681344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005821945837512538, + "loss": 1.4265, + "theoretical_loss": 3.7058976990867305, + "tokens_seen": 922746880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005821785356068204, + "loss": 1.3736, + "theoretical_loss": 3.7058724737358784, + "tokens_seen": 922812416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005821624874623872, + "loss": 1.4566, + "theoretical_loss": 3.705847250677973, + "tokens_seen": 922877952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005821464393179538, + "loss": 1.4498, + "theoretical_loss": 3.7058220299126425, + "tokens_seen": 922943488 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005821303911735206, + "loss": 1.3649, + "theoretical_loss": 3.705796811439517, + "tokens_seen": 923009024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005821143430290872, + "loss": 1.408, + "theoretical_loss": 3.7057715952582244, + "tokens_seen": 923074560 + }, + { + "epoch": 0.28, + "learning_rate": 0.000582098294884654, + "loss": 1.3563, + "theoretical_loss": 3.705746381368394, + "tokens_seen": 923140096 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005820822467402206, + "loss": 1.4277, + "theoretical_loss": 3.705721169769655, + "tokens_seen": 923205632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005820661985957874, + "loss": 1.3774, + "theoretical_loss": 3.705695960461636, + "tokens_seen": 923271168 + }, + { + "epoch": 0.28, + "learning_rate": 0.000582050150451354, + "loss": 1.4267, + "theoretical_loss": 3.7056707534439672, + "tokens_seen": 923336704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005820341023069208, + "loss": 1.3839, + "theoretical_loss": 3.7056455487162783, + "tokens_seen": 923402240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005820180541624874, + "loss": 1.4488, + "theoretical_loss": 3.705620346278198, + "tokens_seen": 923467776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005820020060180542, + "loss": 1.4642, + "theoretical_loss": 3.7055951461293546, + "tokens_seen": 923533312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005819859578736208, + "loss": 1.5054, + "theoretical_loss": 3.70556994826938, + "tokens_seen": 923598848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005819699097291876, + "loss": 1.359, + "theoretical_loss": 3.705544752697903, + "tokens_seen": 923664384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005819538615847542, + "loss": 1.3762, + "theoretical_loss": 3.705519559414553, + "tokens_seen": 923729920 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581937813440321, + "loss": 1.4071, + "theoretical_loss": 3.7054943684189605, + "tokens_seen": 923795456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005819217652958876, + "loss": 1.4652, + "theoretical_loss": 3.705469179710755, + "tokens_seen": 923860992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005819057171514544, + "loss": 1.4613, + "theoretical_loss": 3.705443993289567, + "tokens_seen": 923926528 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581889669007021, + "loss": 1.3496, + "theoretical_loss": 3.7054188091550264, + "tokens_seen": 923992064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 346717, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4494831562042236, + "objective/train/theoretical_loss": 3.705393627306764, + "objective/train/tokens_used": 944517600, + "theoretical_loss": 3.705393627306764, + "tokens_seen": 924057600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005818736208625878, + "loss": 1.4495, + "theoretical_loss": 3.705393627306764, + "tokens_seen": 924057600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005818575727181544, + "loss": 1.3853, + "theoretical_loss": 3.7053684477444087, + "tokens_seen": 924123136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005818415245737212, + "loss": 1.3935, + "theoretical_loss": 3.7053432704675924, + "tokens_seen": 924188672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005818254764292878, + "loss": 1.3553, + "theoretical_loss": 3.7053180954759446, + "tokens_seen": 924254208 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005818094282848546, + "loss": 1.4429, + "theoretical_loss": 3.7052929227690967, + "tokens_seen": 924319744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005817933801404212, + "loss": 1.4473, + "theoretical_loss": 3.7052677523466784, + "tokens_seen": 924385280 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581777331995988, + "loss": 1.4736, + "theoretical_loss": 3.7052425842083214, + "tokens_seen": 924450816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005817612838515546, + "loss": 1.3846, + "theoretical_loss": 3.705217418353656, + "tokens_seen": 924516352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005817452357071214, + "loss": 1.4399, + "theoretical_loss": 3.7051922547823137, + "tokens_seen": 924581888 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581729187562688, + "loss": 1.3206, + "theoretical_loss": 3.7051670934939245, + "tokens_seen": 924647424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005817131394182548, + "loss": 1.343, + "theoretical_loss": 3.70514193448812, + "tokens_seen": 924712960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005816970912738215, + "loss": 1.478, + "theoretical_loss": 3.705116777764532, + "tokens_seen": 924778496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005816810431293882, + "loss": 1.3627, + "theoretical_loss": 3.7050916233227906, + "tokens_seen": 924844032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005816649949849549, + "loss": 1.4725, + "theoretical_loss": 3.705066471162528, + "tokens_seen": 924909568 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005816489468405216, + "loss": 1.4006, + "theoretical_loss": 3.7050413212833755, + "tokens_seen": 924975104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005816328986960883, + "loss": 1.4975, + "theoretical_loss": 3.7050161736849647, + "tokens_seen": 925040640 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581616850551655, + "loss": 1.3911, + "theoretical_loss": 3.7049910283669263, + "tokens_seen": 925106176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005816008024072217, + "loss": 1.3895, + "theoretical_loss": 3.7049658853288934, + "tokens_seen": 925171712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005815847542627884, + "loss": 1.4423, + "theoretical_loss": 3.7049407445704965, + "tokens_seen": 925237248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005815687061183551, + "loss": 1.4088, + "theoretical_loss": 3.704915606091369, + "tokens_seen": 925302784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005815526579739218, + "loss": 1.4134, + "theoretical_loss": 3.704890469891141, + "tokens_seen": 925368320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005815366098294885, + "loss": 1.4021, + "theoretical_loss": 3.704865335969446, + "tokens_seen": 925433856 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005815205616850552, + "loss": 1.4589, + "theoretical_loss": 3.7048402043259148, + "tokens_seen": 925499392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005815045135406219, + "loss": 1.423, + "theoretical_loss": 3.704815074960181, + "tokens_seen": 925564928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005814884653961886, + "loss": 1.3393, + "theoretical_loss": 3.704789947871876, + "tokens_seen": 925630464 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005814724172517553, + "loss": 1.4231, + "theoretical_loss": 3.7047648230606325, + "tokens_seen": 925696000 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581456369107322, + "loss": 1.4978, + "theoretical_loss": 3.7047397005260834, + "tokens_seen": 925761536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005814403209628887, + "loss": 1.3746, + "theoretical_loss": 3.70471458026786, + "tokens_seen": 925827072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005814242728184554, + "loss": 1.4906, + "theoretical_loss": 3.7046894622855957, + "tokens_seen": 925892608 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005814082246740221, + "loss": 1.405, + "theoretical_loss": 3.704664346578924, + "tokens_seen": 925958144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005813921765295888, + "loss": 1.4886, + "theoretical_loss": 3.704639233147476, + "tokens_seen": 926023680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005813761283851556, + "loss": 1.5471, + "theoretical_loss": 3.704614121990886, + "tokens_seen": 926089216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005813600802407222, + "loss": 1.4919, + "theoretical_loss": 3.7045890131087864, + "tokens_seen": 926154752 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581344032096289, + "loss": 1.4107, + "theoretical_loss": 3.70456390650081, + "tokens_seen": 926220288 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005813279839518556, + "loss": 1.4862, + "theoretical_loss": 3.7045388021665904, + "tokens_seen": 926285824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005813119358074224, + "loss": 1.3755, + "theoretical_loss": 3.7045137001057613, + "tokens_seen": 926351360 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581295887662989, + "loss": 1.4468, + "theoretical_loss": 3.704488600317955, + "tokens_seen": 926416896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005812798395185558, + "loss": 1.4702, + "theoretical_loss": 3.7044635028028052, + "tokens_seen": 926482432 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005812637913741224, + "loss": 1.4277, + "theoretical_loss": 3.7044384075599455, + "tokens_seen": 926547968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005812477432296892, + "loss": 1.3141, + "theoretical_loss": 3.7044133145890097, + "tokens_seen": 926613504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005812316950852558, + "loss": 1.3942, + "theoretical_loss": 3.7043882238896306, + "tokens_seen": 926679040 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005812156469408226, + "loss": 1.3326, + "theoretical_loss": 3.704363135461443, + "tokens_seen": 926744576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005811995987963892, + "loss": 1.3538, + "theoretical_loss": 3.7043380493040807, + "tokens_seen": 926810112 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581183550651956, + "loss": 1.4066, + "theoretical_loss": 3.7043129654171767, + "tokens_seen": 926875648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005811675025075226, + "loss": 1.4304, + "theoretical_loss": 3.704287883800365, + "tokens_seen": 926941184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005811514543630894, + "loss": 1.4597, + "theoretical_loss": 3.7042628044532813, + "tokens_seen": 927006720 + }, + { + "epoch": 0.28, + "learning_rate": 0.000581135406218656, + "loss": 1.4382, + "theoretical_loss": 3.704237727375558, + "tokens_seen": 927072256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005811193580742228, + "loss": 1.3543, + "theoretical_loss": 3.70421265256683, + "tokens_seen": 927137792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005811033099297894, + "loss": 1.4389, + "theoretical_loss": 3.7041875800267317, + "tokens_seen": 927203328 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005810872617853561, + "loss": 1.4045, + "theoretical_loss": 3.704162509754898, + "tokens_seen": 927268864 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005810712136409228, + "loss": 1.3522, + "theoretical_loss": 3.7041374417509623, + "tokens_seen": 927334400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005810551654964895, + "loss": 1.3666, + "theoretical_loss": 3.7041123760145593, + "tokens_seen": 927399936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005810391173520562, + "loss": 1.3982, + "theoretical_loss": 3.704087312545325, + "tokens_seen": 927465472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005810230692076229, + "loss": 1.5163, + "theoretical_loss": 3.7040622513428927, + "tokens_seen": 927531008 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005810070210631896, + "loss": 1.4558, + "theoretical_loss": 3.704037192406898, + "tokens_seen": 927596544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005809909729187563, + "loss": 1.3991, + "theoretical_loss": 3.704012135736975, + "tokens_seen": 927662080 + }, + { + "epoch": 0.28, + "learning_rate": 0.000580974924774323, + "loss": 1.4797, + "theoretical_loss": 3.7039870813327602, + "tokens_seen": 927727616 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005809588766298897, + "loss": 1.4775, + "theoretical_loss": 3.703962029193888, + "tokens_seen": 927793152 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005809428284854563, + "loss": 1.4044, + "theoretical_loss": 3.7039369793199928, + "tokens_seen": 927858688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005809267803410231, + "loss": 1.4844, + "theoretical_loss": 3.7039119317107105, + "tokens_seen": 927924224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005809107321965897, + "loss": 1.4437, + "theoretical_loss": 3.7038868863656766, + "tokens_seen": 927989760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005808946840521565, + "loss": 1.4941, + "theoretical_loss": 3.7038618432845265, + "tokens_seen": 928055296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005808786359077231, + "loss": 1.4497, + "theoretical_loss": 3.7038368024668955, + "tokens_seen": 928120832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005808625877632899, + "loss": 1.4755, + "theoretical_loss": 3.703811763912419, + "tokens_seen": 928186368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005808465396188565, + "loss": 1.5232, + "theoretical_loss": 3.7037867276207335, + "tokens_seen": 928251904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005808304914744233, + "loss": 1.5279, + "theoretical_loss": 3.703761693591474, + "tokens_seen": 928317440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005808144433299899, + "loss": 1.4612, + "theoretical_loss": 3.7037366618242764, + "tokens_seen": 928382976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807983951855567, + "loss": 1.3811, + "theoretical_loss": 3.703711632318777, + "tokens_seen": 928448512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807823470411233, + "loss": 1.3934, + "theoretical_loss": 3.703686605074612, + "tokens_seen": 928514048 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807662988966901, + "loss": 1.4061, + "theoretical_loss": 3.7036615800914165, + "tokens_seen": 928579584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807502507522567, + "loss": 1.424, + "theoretical_loss": 3.703636557368828, + "tokens_seen": 928645120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807342026078235, + "loss": 1.3388, + "theoretical_loss": 3.7036115369064815, + "tokens_seen": 928710656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807181544633901, + "loss": 1.4321, + "theoretical_loss": 3.7035865187040145, + "tokens_seen": 928776192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005807021063189569, + "loss": 1.4153, + "theoretical_loss": 3.703561502761063, + "tokens_seen": 928841728 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005806860581745235, + "loss": 1.4261, + "theoretical_loss": 3.7035364890772637, + "tokens_seen": 928907264 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005806700100300903, + "loss": 1.4093, + "theoretical_loss": 3.7035114776522526, + "tokens_seen": 928972800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005806539618856569, + "loss": 1.4629, + "theoretical_loss": 3.703486468485667, + "tokens_seen": 929038336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005806379137412237, + "loss": 1.3992, + "theoretical_loss": 3.7034614615771435, + "tokens_seen": 929103872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005806218655967903, + "loss": 1.4066, + "theoretical_loss": 3.7034364569263185, + "tokens_seen": 929169408 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005806058174523571, + "loss": 1.398, + "theoretical_loss": 3.70341145453283, + "tokens_seen": 929234944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005805897693079237, + "loss": 1.4142, + "theoretical_loss": 3.703386454396314, + "tokens_seen": 929300480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005805737211634905, + "loss": 1.476, + "theoretical_loss": 3.703361456516408, + "tokens_seen": 929366016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005805576730190571, + "loss": 1.4579, + "theoretical_loss": 3.7033364608927495, + "tokens_seen": 929431552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005805416248746239, + "loss": 1.4819, + "theoretical_loss": 3.7033114675249754, + "tokens_seen": 929497088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005805255767301905, + "loss": 1.3487, + "theoretical_loss": 3.703286476412724, + "tokens_seen": 929562624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005805095285857573, + "loss": 1.3872, + "theoretical_loss": 3.703261487555631, + "tokens_seen": 929628160 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005804934804413239, + "loss": 1.4298, + "theoretical_loss": 3.703236500953335, + "tokens_seen": 929693696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005804774322968907, + "loss": 1.3265, + "theoretical_loss": 3.7032115166054735, + "tokens_seen": 929759232 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005804613841524573, + "loss": 1.4658, + "theoretical_loss": 3.7031865345116843, + "tokens_seen": 929824768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005804453360080241, + "loss": 1.4216, + "theoretical_loss": 3.7031615546716052, + "tokens_seen": 929890304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005804292878635907, + "loss": 1.4168, + "theoretical_loss": 3.7031365770848743, + "tokens_seen": 929955840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005804132397191575, + "loss": 1.3734, + "theoretical_loss": 3.7031116017511287, + "tokens_seen": 930021376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005803971915747241, + "loss": 1.4091, + "theoretical_loss": 3.7030866286700075, + "tokens_seen": 930086912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005803811434302909, + "loss": 1.401, + "theoretical_loss": 3.703061657841148, + "tokens_seen": 930152448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005803650952858576, + "loss": 1.3482, + "theoretical_loss": 3.703036689264189, + "tokens_seen": 930217984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005803490471414243, + "loss": 1.4256, + "theoretical_loss": 3.703011722938768, + "tokens_seen": 930283520 + }, + { + "epoch": 0.28, + "learning_rate": 0.000580332998996991, + "loss": 1.4993, + "theoretical_loss": 3.702986758864524, + "tokens_seen": 930349056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005803169508525577, + "loss": 1.3859, + "theoretical_loss": 3.7029617970410955, + "tokens_seen": 930414592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005803009027081244, + "loss": 1.5061, + "theoretical_loss": 3.702936837468121, + "tokens_seen": 930480128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005802848545636911, + "loss": 1.4252, + "theoretical_loss": 3.7029118801452388, + "tokens_seen": 930545664 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 349159, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3531581163406372, + "objective/train/theoretical_loss": 3.7028869250720877, + "objective/train/tokens_used": 951071200, + "theoretical_loss": 3.7028869250720877, + "tokens_seen": 930611200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005802688064192578, + "loss": 1.3532, + "theoretical_loss": 3.7028869250720877, + "tokens_seen": 930611200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005802527582748245, + "loss": 1.3383, + "theoretical_loss": 3.702861972248307, + "tokens_seen": 930676736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005802367101303912, + "loss": 1.3107, + "theoretical_loss": 3.702837021673535, + "tokens_seen": 930742272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005802206619859579, + "loss": 1.357, + "theoretical_loss": 3.7028120733474106, + "tokens_seen": 930807808 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005802046138415246, + "loss": 1.3068, + "theoretical_loss": 3.7027871272695734, + "tokens_seen": 930873344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005801885656970913, + "loss": 1.4253, + "theoretical_loss": 3.7027621834396625, + "tokens_seen": 930938880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005801725175526581, + "loss": 1.2299, + "theoretical_loss": 3.7027372418573163, + "tokens_seen": 931004416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005801564694082247, + "loss": 1.3994, + "theoretical_loss": 3.7027123025221744, + "tokens_seen": 931069952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005801404212637915, + "loss": 1.452, + "theoretical_loss": 3.702687365433877, + "tokens_seen": 931135488 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005801243731193581, + "loss": 1.4024, + "theoretical_loss": 3.7026624305920626, + "tokens_seen": 931201024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005801083249749249, + "loss": 1.4352, + "theoretical_loss": 3.702637497996371, + "tokens_seen": 931266560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005800922768304915, + "loss": 1.453, + "theoretical_loss": 3.702612567646442, + "tokens_seen": 931332096 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005800762286860583, + "loss": 1.2798, + "theoretical_loss": 3.7025876395419153, + "tokens_seen": 931397632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005800601805416249, + "loss": 1.546, + "theoretical_loss": 3.702562713682431, + "tokens_seen": 931463168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005800441323971917, + "loss": 1.4685, + "theoretical_loss": 3.7025377900676277, + "tokens_seen": 931528704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005800280842527583, + "loss": 1.4398, + "theoretical_loss": 3.7025128686971467, + "tokens_seen": 931594240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005800120361083251, + "loss": 1.3525, + "theoretical_loss": 3.7024879495706275, + "tokens_seen": 931659776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005799959879638917, + "loss": 1.3952, + "theoretical_loss": 3.70246303268771, + "tokens_seen": 931725312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005799799398194585, + "loss": 1.4192, + "theoretical_loss": 3.7024381180480352, + "tokens_seen": 931790848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005799638916750251, + "loss": 1.2788, + "theoretical_loss": 3.702413205651243, + "tokens_seen": 931856384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005799478435305919, + "loss": 1.4378, + "theoretical_loss": 3.7023882954969736, + "tokens_seen": 931921920 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005799317953861585, + "loss": 1.3374, + "theoretical_loss": 3.7023633875848674, + "tokens_seen": 931987456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005799157472417253, + "loss": 1.3925, + "theoretical_loss": 3.7023384819145644, + "tokens_seen": 932052992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798996990972919, + "loss": 1.4258, + "theoretical_loss": 3.7023135784857066, + "tokens_seen": 932118528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798836509528587, + "loss": 1.4005, + "theoretical_loss": 3.7022886772979344, + "tokens_seen": 932184064 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798676028084253, + "loss": 1.3904, + "theoretical_loss": 3.702263778350887, + "tokens_seen": 932249600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798515546639921, + "loss": 1.4073, + "theoretical_loss": 3.702238881644208, + "tokens_seen": 932315136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798355065195587, + "loss": 1.4581, + "theoretical_loss": 3.7022139871775357, + "tokens_seen": 932380672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798194583751255, + "loss": 1.4216, + "theoretical_loss": 3.702189094950512, + "tokens_seen": 932446208 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005798034102306921, + "loss": 1.4593, + "theoretical_loss": 3.702164204962779, + "tokens_seen": 932511744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005797873620862589, + "loss": 1.3873, + "theoretical_loss": 3.7021393172139767, + "tokens_seen": 932577280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005797713139418255, + "loss": 1.4447, + "theoretical_loss": 3.7021144317037473, + "tokens_seen": 932642816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005797552657973923, + "loss": 1.3246, + "theoretical_loss": 3.702089548431731, + "tokens_seen": 932708352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005797392176529589, + "loss": 1.365, + "theoretical_loss": 3.7020646673975706, + "tokens_seen": 932773888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005797231695085257, + "loss": 1.3676, + "theoretical_loss": 3.7020397886009064, + "tokens_seen": 932839424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005797071213640923, + "loss": 1.5935, + "theoretical_loss": 3.7020149120413812, + "tokens_seen": 932904960 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579691073219659, + "loss": 1.3946, + "theoretical_loss": 3.7019900377186357, + "tokens_seen": 932970496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005796750250752257, + "loss": 1.4193, + "theoretical_loss": 3.7019651656323127, + "tokens_seen": 933036032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005796589769307924, + "loss": 1.4365, + "theoretical_loss": 3.7019402957820526, + "tokens_seen": 933101568 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005796429287863591, + "loss": 1.4782, + "theoretical_loss": 3.7019154281674984, + "tokens_seen": 933167104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005796268806419258, + "loss": 1.3736, + "theoretical_loss": 3.701890562788292, + "tokens_seen": 933232640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005796108324974925, + "loss": 1.3358, + "theoretical_loss": 3.7018656996440757, + "tokens_seen": 933298176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005795947843530592, + "loss": 1.389, + "theoretical_loss": 3.701840838734491, + "tokens_seen": 933363712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005795787362086259, + "loss": 1.4262, + "theoretical_loss": 3.701815980059181, + "tokens_seen": 933429248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005795626880641926, + "loss": 1.5161, + "theoretical_loss": 3.701791123617787, + "tokens_seen": 933494784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005795466399197592, + "loss": 1.3314, + "theoretical_loss": 3.7017662694099527, + "tokens_seen": 933560320 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579530591775326, + "loss": 1.3859, + "theoretical_loss": 3.70174141743532, + "tokens_seen": 933625856 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005795145436308926, + "loss": 1.3723, + "theoretical_loss": 3.7017165676935315, + "tokens_seen": 933691392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005794984954864594, + "loss": 1.4651, + "theoretical_loss": 3.701691720184229, + "tokens_seen": 933756928 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579482447342026, + "loss": 1.4213, + "theoretical_loss": 3.7016668749070574, + "tokens_seen": 933822464 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005794663991975928, + "loss": 1.3878, + "theoretical_loss": 3.7016420318616574, + "tokens_seen": 933888000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005794503510531594, + "loss": 1.4056, + "theoretical_loss": 3.7016171910476734, + "tokens_seen": 933953536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005794343029087262, + "loss": 1.434, + "theoretical_loss": 3.701592352464748, + "tokens_seen": 934019072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005794182547642928, + "loss": 1.408, + "theoretical_loss": 3.701567516112523, + "tokens_seen": 934084608 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005794022066198596, + "loss": 1.3813, + "theoretical_loss": 3.701542681990644, + "tokens_seen": 934150144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005793861584754262, + "loss": 1.4556, + "theoretical_loss": 3.7015178500987522, + "tokens_seen": 934215680 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579370110330993, + "loss": 1.3713, + "theoretical_loss": 3.701493020436492, + "tokens_seen": 934281216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005793540621865596, + "loss": 1.338, + "theoretical_loss": 3.7014681930035067, + "tokens_seen": 934346752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005793380140421264, + "loss": 1.4204, + "theoretical_loss": 3.7014433677994396, + "tokens_seen": 934412288 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579321965897693, + "loss": 1.4246, + "theoretical_loss": 3.7014185448239343, + "tokens_seen": 934477824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005793059177532598, + "loss": 1.3917, + "theoretical_loss": 3.7013937240766346, + "tokens_seen": 934543360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005792898696088264, + "loss": 1.4112, + "theoretical_loss": 3.7013689055571843, + "tokens_seen": 934608896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005792738214643932, + "loss": 1.3948, + "theoretical_loss": 3.701344089265227, + "tokens_seen": 934674432 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005792577733199598, + "loss": 1.357, + "theoretical_loss": 3.7013192752004063, + "tokens_seen": 934739968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005792417251755266, + "loss": 1.5222, + "theoretical_loss": 3.701294463362367, + "tokens_seen": 934805504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005792256770310932, + "loss": 1.3849, + "theoretical_loss": 3.701269653750753, + "tokens_seen": 934871040 + }, + { + "epoch": 0.28, + "learning_rate": 0.00057920962888666, + "loss": 1.3657, + "theoretical_loss": 3.701244846365208, + "tokens_seen": 934936576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005791935807422266, + "loss": 1.5429, + "theoretical_loss": 3.701220041205377, + "tokens_seen": 935002112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005791775325977934, + "loss": 1.4101, + "theoretical_loss": 3.701195238270903, + "tokens_seen": 935067648 + }, + { + "epoch": 0.28, + "learning_rate": 0.00057916148445336, + "loss": 1.3516, + "theoretical_loss": 3.701170437561432, + "tokens_seen": 935133184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005791454363089268, + "loss": 1.3723, + "theoretical_loss": 3.701145639076607, + "tokens_seen": 935198720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005791293881644934, + "loss": 1.4133, + "theoretical_loss": 3.7011208428160742, + "tokens_seen": 935264256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005791133400200602, + "loss": 1.3756, + "theoretical_loss": 3.701096048779477, + "tokens_seen": 935329792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005790972918756268, + "loss": 1.44, + "theoretical_loss": 3.7010712569664603, + "tokens_seen": 935395328 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005790812437311936, + "loss": 1.4023, + "theoretical_loss": 3.701046467376669, + "tokens_seen": 935460864 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005790651955867602, + "loss": 1.329, + "theoretical_loss": 3.701021680009749, + "tokens_seen": 935526400 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579049147442327, + "loss": 1.2647, + "theoretical_loss": 3.7009968948653436, + "tokens_seen": 935591936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005790330992978936, + "loss": 1.3254, + "theoretical_loss": 3.7009721119430985, + "tokens_seen": 935657472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005790170511534604, + "loss": 1.346, + "theoretical_loss": 3.7009473312426593, + "tokens_seen": 935723008 + }, + { + "epoch": 0.28, + "learning_rate": 0.000579001003009027, + "loss": 1.3243, + "theoretical_loss": 3.700922552763671, + "tokens_seen": 935788544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005789849548645938, + "loss": 1.3732, + "theoretical_loss": 3.700897776505779, + "tokens_seen": 935854080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005789689067201606, + "loss": 1.4536, + "theoretical_loss": 3.7008730024686285, + "tokens_seen": 935919616 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005789528585757272, + "loss": 1.3978, + "theoretical_loss": 3.7008482306518644, + "tokens_seen": 935985152 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578936810431294, + "loss": 1.4803, + "theoretical_loss": 3.7008234610551334, + "tokens_seen": 936050688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005789207622868606, + "loss": 1.331, + "theoretical_loss": 3.70079869367808, + "tokens_seen": 936116224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005789047141424274, + "loss": 1.4003, + "theoretical_loss": 3.700773928520351, + "tokens_seen": 936181760 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578888665997994, + "loss": 1.4682, + "theoretical_loss": 3.7007491655815916, + "tokens_seen": 936247296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005788726178535608, + "loss": 1.4097, + "theoretical_loss": 3.7007244048614476, + "tokens_seen": 936312832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005788565697091274, + "loss": 1.3408, + "theoretical_loss": 3.700699646359565, + "tokens_seen": 936378368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005788405215646942, + "loss": 1.4695, + "theoretical_loss": 3.7006748900755904, + "tokens_seen": 936443904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005788244734202608, + "loss": 1.3516, + "theoretical_loss": 3.700650136009169, + "tokens_seen": 936509440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005788084252758276, + "loss": 1.3351, + "theoretical_loss": 3.700625384159948, + "tokens_seen": 936574976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005787923771313942, + "loss": 1.443, + "theoretical_loss": 3.7006006345275724, + "tokens_seen": 936640512 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578776328986961, + "loss": 1.4035, + "theoretical_loss": 3.70057588711169, + "tokens_seen": 936706048 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005787602808425276, + "loss": 1.5509, + "theoretical_loss": 3.700551141911946, + "tokens_seen": 936771584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005787442326980944, + "loss": 1.4191, + "theoretical_loss": 3.700526398927988, + "tokens_seen": 936837120 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578728184553661, + "loss": 1.3613, + "theoretical_loss": 3.700501658159462, + "tokens_seen": 936902656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005787121364092278, + "loss": 1.4384, + "theoretical_loss": 3.7004769196060145, + "tokens_seen": 936968192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005786960882647944, + "loss": 1.3624, + "theoretical_loss": 3.7004521832672923, + "tokens_seen": 937033728 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005786800401203612, + "loss": 1.4908, + "theoretical_loss": 3.700427449142943, + "tokens_seen": 937099264 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 351493, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4419939517974854, + "objective/train/theoretical_loss": 3.700402717232613, + "objective/train/tokens_used": 957624800, + "theoretical_loss": 3.700402717232613, + "tokens_seen": 937164800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005786639919759278, + "loss": 1.442, + "theoretical_loss": 3.700402717232613, + "tokens_seen": 937164800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005786479438314946, + "loss": 1.4337, + "theoretical_loss": 3.700377987535949, + "tokens_seen": 937230336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005786318956870612, + "loss": 1.3445, + "theoretical_loss": 3.700353260052599, + "tokens_seen": 937295872 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578615847542628, + "loss": 1.3484, + "theoretical_loss": 3.7003285347822086, + "tokens_seen": 937361408 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005785997993981946, + "loss": 1.3562, + "theoretical_loss": 3.7003038117244267, + "tokens_seen": 937426944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005785837512537614, + "loss": 1.4071, + "theoretical_loss": 3.7002790908789, + "tokens_seen": 937492480 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578567703109328, + "loss": 1.3862, + "theoretical_loss": 3.7002543722452756, + "tokens_seen": 937558016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005785516549648948, + "loss": 1.5795, + "theoretical_loss": 3.7002296558232013, + "tokens_seen": 937623552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005785356068204614, + "loss": 1.4587, + "theoretical_loss": 3.7002049416123253, + "tokens_seen": 937689088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005785195586760282, + "loss": 1.4159, + "theoretical_loss": 3.700180229612294, + "tokens_seen": 937754624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005785035105315948, + "loss": 1.4384, + "theoretical_loss": 3.700155519822756, + "tokens_seen": 937820160 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005784874623871616, + "loss": 1.4076, + "theoretical_loss": 3.700130812243359, + "tokens_seen": 937885696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005784714142427282, + "loss": 1.4113, + "theoretical_loss": 3.7001061068737506, + "tokens_seen": 937951232 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578455366098295, + "loss": 1.4094, + "theoretical_loss": 3.700081403713579, + "tokens_seen": 938016768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005784393179538616, + "loss": 1.4288, + "theoretical_loss": 3.700056702762492, + "tokens_seen": 938082304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005784232698094284, + "loss": 1.3465, + "theoretical_loss": 3.700032004020138, + "tokens_seen": 938147840 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578407221664995, + "loss": 1.4444, + "theoretical_loss": 3.7000073074861657, + "tokens_seen": 938213376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005783911735205618, + "loss": 1.3574, + "theoretical_loss": 3.699982613160223, + "tokens_seen": 938278912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005783751253761284, + "loss": 1.4329, + "theoretical_loss": 3.6999579210419578, + "tokens_seen": 938344448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005783590772316952, + "loss": 1.4308, + "theoretical_loss": 3.699933231131019, + "tokens_seen": 938409984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005783430290872618, + "loss": 1.3297, + "theoretical_loss": 3.6999085434270547, + "tokens_seen": 938475520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005783269809428286, + "loss": 1.4212, + "theoretical_loss": 3.6998838579297146, + "tokens_seen": 938541056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005783109327983952, + "loss": 1.4809, + "theoretical_loss": 3.699859174638646, + "tokens_seen": 938606592 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578294884653962, + "loss": 1.3501, + "theoretical_loss": 3.6998344935534995, + "tokens_seen": 938672128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005782788365095286, + "loss": 1.3754, + "theoretical_loss": 3.699809814673922, + "tokens_seen": 938737664 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005782627883650953, + "loss": 1.3572, + "theoretical_loss": 3.699785137999563, + "tokens_seen": 938803200 + }, + { + "epoch": 0.28, + "learning_rate": 0.000578246740220662, + "loss": 1.4691, + "theoretical_loss": 3.6997604635300725, + "tokens_seen": 938868736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005782306920762287, + "loss": 1.3218, + "theoretical_loss": 3.699735791265099, + "tokens_seen": 938934272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005782146439317954, + "loss": 1.3947, + "theoretical_loss": 3.6997111212042917, + "tokens_seen": 938999808 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781985957873621, + "loss": 1.365, + "theoretical_loss": 3.6996864533473, + "tokens_seen": 939065344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781825476429288, + "loss": 1.4241, + "theoretical_loss": 3.6996617876937723, + "tokens_seen": 939130880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781664994984955, + "loss": 1.4363, + "theoretical_loss": 3.6996371242433597, + "tokens_seen": 939196416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781504513540621, + "loss": 1.4572, + "theoretical_loss": 3.69961246299571, + "tokens_seen": 939261952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781344032096289, + "loss": 1.3737, + "theoretical_loss": 3.699587803950474, + "tokens_seen": 939327488 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781183550651955, + "loss": 1.371, + "theoretical_loss": 3.699563147107301, + "tokens_seen": 939393024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005781023069207623, + "loss": 1.3812, + "theoretical_loss": 3.6995384924658405, + "tokens_seen": 939458560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005780862587763289, + "loss": 1.4477, + "theoretical_loss": 3.699513840025743, + "tokens_seen": 939524096 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005780702106318957, + "loss": 1.4081, + "theoretical_loss": 3.6994891897866573, + "tokens_seen": 939589632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005780541624874623, + "loss": 1.3504, + "theoretical_loss": 3.6994645417482346, + "tokens_seen": 939655168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005780381143430291, + "loss": 1.3967, + "theoretical_loss": 3.6994398959101242, + "tokens_seen": 939720704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005780220661985957, + "loss": 1.4073, + "theoretical_loss": 3.6994152522719768, + "tokens_seen": 939786240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005780060180541625, + "loss": 1.3391, + "theoretical_loss": 3.699390610833442, + "tokens_seen": 939851776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005779899699097291, + "loss": 1.5003, + "theoretical_loss": 3.6993659715941702, + "tokens_seen": 939917312 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005779739217652959, + "loss": 1.4747, + "theoretical_loss": 3.699341334553812, + "tokens_seen": 939982848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005779578736208625, + "loss": 1.4108, + "theoretical_loss": 3.6993166997120186, + "tokens_seen": 940048384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005779418254764293, + "loss": 1.4323, + "theoretical_loss": 3.6992920670684395, + "tokens_seen": 940113920 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005779257773319959, + "loss": 1.3137, + "theoretical_loss": 3.6992674366227254, + "tokens_seen": 940179456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005779097291875627, + "loss": 1.3306, + "theoretical_loss": 3.6992428083745272, + "tokens_seen": 940244992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005778936810431293, + "loss": 1.374, + "theoretical_loss": 3.699218182323496, + "tokens_seen": 940310528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005778776328986961, + "loss": 1.3893, + "theoretical_loss": 3.6991935584692826, + "tokens_seen": 940376064 + }, + { + "epoch": 0.28, + "learning_rate": 0.0005778615847542627, + "loss": 1.3263, + "theoretical_loss": 3.6991689368115375, + "tokens_seen": 940441600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005778455366098295, + "loss": 1.4544, + "theoretical_loss": 3.699144317349912, + "tokens_seen": 940507136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005778294884653961, + "loss": 1.4355, + "theoretical_loss": 3.6991197000840574, + "tokens_seen": 940572672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005778134403209629, + "loss": 1.2751, + "theoretical_loss": 3.6990950850136244, + "tokens_seen": 940638208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005777973921765295, + "loss": 1.4229, + "theoretical_loss": 3.6990704721382652, + "tokens_seen": 940703744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005777813440320963, + "loss": 1.4011, + "theoretical_loss": 3.69904586145763, + "tokens_seen": 940769280 + }, + { + "epoch": 0.29, + "learning_rate": 0.000577765295887663, + "loss": 1.3827, + "theoretical_loss": 3.699021252971371, + "tokens_seen": 940834816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005777492477432297, + "loss": 1.3972, + "theoretical_loss": 3.698996646679139, + "tokens_seen": 940900352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005777331995987965, + "loss": 1.4443, + "theoretical_loss": 3.6989720425805865, + "tokens_seen": 940965888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005777171514543631, + "loss": 1.4798, + "theoretical_loss": 3.6989474406753646, + "tokens_seen": 941031424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005777011033099299, + "loss": 1.4488, + "theoretical_loss": 3.6989228409631254, + "tokens_seen": 941096960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005776850551654965, + "loss": 1.3743, + "theoretical_loss": 3.6988982434435203, + "tokens_seen": 941162496 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005776690070210633, + "loss": 1.406, + "theoretical_loss": 3.6988736481162015, + "tokens_seen": 941228032 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005776529588766299, + "loss": 1.3428, + "theoretical_loss": 3.698849054980821, + "tokens_seen": 941293568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005776369107321967, + "loss": 1.3846, + "theoretical_loss": 3.6988244640370302, + "tokens_seen": 941359104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005776208625877633, + "loss": 1.3521, + "theoretical_loss": 3.6987998752844815, + "tokens_seen": 941424640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005776048144433301, + "loss": 1.2881, + "theoretical_loss": 3.6987752887228282, + "tokens_seen": 941490176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005775887662988967, + "loss": 1.4789, + "theoretical_loss": 3.698750704351722, + "tokens_seen": 941555712 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005775727181544635, + "loss": 1.3514, + "theoretical_loss": 3.6987261221708145, + "tokens_seen": 941621248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005775566700100301, + "loss": 1.4809, + "theoretical_loss": 3.6987015421797587, + "tokens_seen": 941686784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005775406218655969, + "loss": 1.4398, + "theoretical_loss": 3.698676964378208, + "tokens_seen": 941752320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005775245737211635, + "loss": 1.3154, + "theoretical_loss": 3.6986523887658134, + "tokens_seen": 941817856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005775085255767303, + "loss": 1.3834, + "theoretical_loss": 3.6986278153422285, + "tokens_seen": 941883392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005774924774322969, + "loss": 1.389, + "theoretical_loss": 3.6986032441071064, + "tokens_seen": 941948928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005774764292878637, + "loss": 1.3931, + "theoretical_loss": 3.6985786750600993, + "tokens_seen": 942014464 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005774603811434303, + "loss": 1.3816, + "theoretical_loss": 3.6985541082008604, + "tokens_seen": 942080000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005774443329989971, + "loss": 1.3242, + "theoretical_loss": 3.698529543529043, + "tokens_seen": 942145536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005774282848545637, + "loss": 1.3913, + "theoretical_loss": 3.6985049810443, + "tokens_seen": 942211072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005774122367101305, + "loss": 1.3625, + "theoretical_loss": 3.6984804207462836, + "tokens_seen": 942276608 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005773961885656971, + "loss": 1.4777, + "theoretical_loss": 3.698455862634648, + "tokens_seen": 942342144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005773801404212639, + "loss": 1.3869, + "theoretical_loss": 3.6984313067090477, + "tokens_seen": 942407680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005773640922768305, + "loss": 1.4349, + "theoretical_loss": 3.698406752969134, + "tokens_seen": 942473216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005773480441323973, + "loss": 1.3081, + "theoretical_loss": 3.698382201414561, + "tokens_seen": 942538752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005773319959879639, + "loss": 1.3646, + "theoretical_loss": 3.6983576520449835, + "tokens_seen": 942604288 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005773159478435307, + "loss": 1.3715, + "theoretical_loss": 3.6983331048600534, + "tokens_seen": 942669824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772998996990973, + "loss": 1.4284, + "theoretical_loss": 3.6983085598594254, + "tokens_seen": 942735360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772838515546641, + "loss": 1.3866, + "theoretical_loss": 3.698284017042753, + "tokens_seen": 942800896 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772678034102307, + "loss": 1.5589, + "theoretical_loss": 3.6982594764096905, + "tokens_seen": 942866432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772517552657975, + "loss": 1.447, + "theoretical_loss": 3.6982349379598913, + "tokens_seen": 942931968 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772357071213641, + "loss": 1.4586, + "theoretical_loss": 3.69821040169301, + "tokens_seen": 942997504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772196589769309, + "loss": 1.4774, + "theoretical_loss": 3.6981858676087005, + "tokens_seen": 943063040 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005772036108324975, + "loss": 1.4619, + "theoretical_loss": 3.698161335706617, + "tokens_seen": 943128576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005771875626880643, + "loss": 1.4147, + "theoretical_loss": 3.6981368059864135, + "tokens_seen": 943194112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005771715145436309, + "loss": 1.4092, + "theoretical_loss": 3.698112278447744, + "tokens_seen": 943259648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005771554663991977, + "loss": 1.388, + "theoretical_loss": 3.6980877530902645, + "tokens_seen": 943325184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005771394182547643, + "loss": 1.4452, + "theoretical_loss": 3.698063229913628, + "tokens_seen": 943390720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005771233701103311, + "loss": 1.3872, + "theoretical_loss": 3.6980387089174895, + "tokens_seen": 943456256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005771073219658977, + "loss": 1.4357, + "theoretical_loss": 3.6980141901015045, + "tokens_seen": 943521792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005770912738214645, + "loss": 1.4715, + "theoretical_loss": 3.697989673465327, + "tokens_seen": 943587328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005770752256770311, + "loss": 1.4312, + "theoretical_loss": 3.6979651590086107, + "tokens_seen": 943652864 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 354246, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3995997905731201, + "objective/train/theoretical_loss": 3.6979406467310127, + "objective/train/tokens_used": 964178400, + "theoretical_loss": 3.6979406467310127, + "tokens_seen": 943718400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005770591775325979, + "loss": 1.3996, + "theoretical_loss": 3.6979406467310127, + "tokens_seen": 943718400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005770431293881645, + "loss": 1.3925, + "theoretical_loss": 3.6979161366321867, + "tokens_seen": 943783936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005770270812437313, + "loss": 1.3854, + "theoretical_loss": 3.6978916287117882, + "tokens_seen": 943849472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005770110330992979, + "loss": 1.3623, + "theoretical_loss": 3.6978671229694724, + "tokens_seen": 943915008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005769949849548647, + "loss": 1.4437, + "theoretical_loss": 3.697842619404894, + "tokens_seen": 943980544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005769789368104313, + "loss": 1.4226, + "theoretical_loss": 3.6978181180177083, + "tokens_seen": 944046080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005769628886659981, + "loss": 1.378, + "theoretical_loss": 3.6977936188075717, + "tokens_seen": 944111616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005769468405215647, + "loss": 1.4286, + "theoretical_loss": 3.6977691217741384, + "tokens_seen": 944177152 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005769307923771315, + "loss": 1.4393, + "theoretical_loss": 3.697744626917065, + "tokens_seen": 944242688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005769147442326981, + "loss": 1.4236, + "theoretical_loss": 3.6977201342360067, + "tokens_seen": 944308224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005768986960882649, + "loss": 1.3499, + "theoretical_loss": 3.6976956437306185, + "tokens_seen": 944373760 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005768826479438315, + "loss": 1.345, + "theoretical_loss": 3.697671155400557, + "tokens_seen": 944439296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005768665997993982, + "loss": 1.3403, + "theoretical_loss": 3.6976466692454784, + "tokens_seen": 944504832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005768505516549649, + "loss": 1.4403, + "theoretical_loss": 3.697622185265037, + "tokens_seen": 944570368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005768345035105316, + "loss": 1.4356, + "theoretical_loss": 3.6975977034588907, + "tokens_seen": 944635904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005768184553660983, + "loss": 1.3821, + "theoretical_loss": 3.6975732238266943, + "tokens_seen": 944701440 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576802407221665, + "loss": 1.4542, + "theoretical_loss": 3.697548746368105, + "tokens_seen": 944766976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005767863590772317, + "loss": 1.4211, + "theoretical_loss": 3.697524271082778, + "tokens_seen": 944832512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005767703109327984, + "loss": 1.3796, + "theoretical_loss": 3.6974997979703703, + "tokens_seen": 944898048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005767542627883651, + "loss": 1.35, + "theoretical_loss": 3.697475327030538, + "tokens_seen": 944963584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005767382146439318, + "loss": 1.4659, + "theoretical_loss": 3.697450858262938, + "tokens_seen": 945029120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005767221664994984, + "loss": 1.4403, + "theoretical_loss": 3.697426391667226, + "tokens_seen": 945094656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005767061183550652, + "loss": 1.3908, + "theoretical_loss": 3.6974019272430594, + "tokens_seen": 945160192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005766900702106318, + "loss": 1.3907, + "theoretical_loss": 3.697377464990095, + "tokens_seen": 945225728 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005766740220661986, + "loss": 1.3631, + "theoretical_loss": 3.697353004907989, + "tokens_seen": 945291264 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005766579739217652, + "loss": 1.3298, + "theoretical_loss": 3.697328546996398, + "tokens_seen": 945356800 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576641925777332, + "loss": 1.3958, + "theoretical_loss": 3.69730409125498, + "tokens_seen": 945422336 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005766258776328986, + "loss": 1.3629, + "theoretical_loss": 3.6972796376833914, + "tokens_seen": 945487872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005766098294884654, + "loss": 1.514, + "theoretical_loss": 3.6972551862812892, + "tokens_seen": 945553408 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576593781344032, + "loss": 1.2594, + "theoretical_loss": 3.697230737048331, + "tokens_seen": 945618944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005765777331995988, + "loss": 1.3979, + "theoretical_loss": 3.697206289984174, + "tokens_seen": 945684480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005765616850551655, + "loss": 1.3865, + "theoretical_loss": 3.6971818450884752, + "tokens_seen": 945750016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005765456369107322, + "loss": 1.3536, + "theoretical_loss": 3.6971574023608924, + "tokens_seen": 945815552 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576529588766299, + "loss": 1.2693, + "theoretical_loss": 3.6971329618010826, + "tokens_seen": 945881088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005765135406218656, + "loss": 1.5327, + "theoretical_loss": 3.697108523408704, + "tokens_seen": 945946624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005764974924774323, + "loss": 1.4226, + "theoretical_loss": 3.6970840871834136, + "tokens_seen": 946012160 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576481444332999, + "loss": 1.4065, + "theoretical_loss": 3.697059653124869, + "tokens_seen": 946077696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005764653961885658, + "loss": 1.3847, + "theoretical_loss": 3.697035221232729, + "tokens_seen": 946143232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005764493480441324, + "loss": 1.2973, + "theoretical_loss": 3.697010791506651, + "tokens_seen": 946208768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005764332998996992, + "loss": 1.3564, + "theoretical_loss": 3.6969863639462925, + "tokens_seen": 946274304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005764172517552658, + "loss": 1.4382, + "theoretical_loss": 3.696961938551312, + "tokens_seen": 946339840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005764012036108326, + "loss": 1.3029, + "theoretical_loss": 3.696937515321368, + "tokens_seen": 946405376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005763851554663992, + "loss": 1.4939, + "theoretical_loss": 3.696913094256117, + "tokens_seen": 946470912 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576369107321966, + "loss": 1.471, + "theoretical_loss": 3.6968886753552193, + "tokens_seen": 946536448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005763530591775326, + "loss": 1.506, + "theoretical_loss": 3.6968642586183327, + "tokens_seen": 946601984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005763370110330994, + "loss": 1.4216, + "theoretical_loss": 3.6968398440451145, + "tokens_seen": 946667520 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576320962888666, + "loss": 1.3486, + "theoretical_loss": 3.696815431635225, + "tokens_seen": 946733056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005763049147442328, + "loss": 1.4111, + "theoretical_loss": 3.696791021388321, + "tokens_seen": 946798592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005762888665997994, + "loss": 1.4463, + "theoretical_loss": 3.6967666133040624, + "tokens_seen": 946864128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005762728184553662, + "loss": 1.3698, + "theoretical_loss": 3.6967422073821075, + "tokens_seen": 946929664 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005762567703109328, + "loss": 1.4134, + "theoretical_loss": 3.696717803622115, + "tokens_seen": 946995200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005762407221664996, + "loss": 1.4542, + "theoretical_loss": 3.696693402023744, + "tokens_seen": 947060736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005762246740220662, + "loss": 1.4112, + "theoretical_loss": 3.696669002586653, + "tokens_seen": 947126272 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576208625877633, + "loss": 1.403, + "theoretical_loss": 3.696644605310502, + "tokens_seen": 947191808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005761925777331996, + "loss": 1.4535, + "theoretical_loss": 3.696620210194949, + "tokens_seen": 947257344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005761765295887664, + "loss": 1.3034, + "theoretical_loss": 3.696595817239654, + "tokens_seen": 947322880 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576160481444333, + "loss": 1.43, + "theoretical_loss": 3.6965714264442764, + "tokens_seen": 947388416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005761444332998998, + "loss": 1.3952, + "theoretical_loss": 3.6965470378084744, + "tokens_seen": 947453952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005761283851554664, + "loss": 1.3982, + "theoretical_loss": 3.696522651331909, + "tokens_seen": 947519488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005761123370110332, + "loss": 1.4256, + "theoretical_loss": 3.6964982670142383, + "tokens_seen": 947585024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005760962888665998, + "loss": 1.4405, + "theoretical_loss": 3.6964738848551226, + "tokens_seen": 947650560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005760802407221666, + "loss": 1.5135, + "theoretical_loss": 3.696449504854222, + "tokens_seen": 947716096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005760641925777332, + "loss": 1.3825, + "theoretical_loss": 3.696425127011195, + "tokens_seen": 947781632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005760481444333, + "loss": 1.4355, + "theoretical_loss": 3.6964007513257022, + "tokens_seen": 947847168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005760320962888666, + "loss": 1.4849, + "theoretical_loss": 3.696376377797404, + "tokens_seen": 947912704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005760160481444334, + "loss": 1.353, + "theoretical_loss": 3.696352006425959, + "tokens_seen": 947978240 + }, + { + "epoch": 0.29, + "learning_rate": 0.000576, + "loss": 1.4759, + "theoretical_loss": 3.696327637211028, + "tokens_seen": 948043776 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005759839518555668, + "loss": 1.3498, + "theoretical_loss": 3.6963032701522716, + "tokens_seen": 948109312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005759679037111334, + "loss": 1.3996, + "theoretical_loss": 3.6962789052493497, + "tokens_seen": 948174848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005759518555667002, + "loss": 1.4454, + "theoretical_loss": 3.696254542501922, + "tokens_seen": 948240384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005759358074222668, + "loss": 1.4197, + "theoretical_loss": 3.6962301819096495, + "tokens_seen": 948305920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005759197592778336, + "loss": 1.4658, + "theoretical_loss": 3.696205823472192, + "tokens_seen": 948371456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005759037111334002, + "loss": 1.3548, + "theoretical_loss": 3.696181467189211, + "tokens_seen": 948436992 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575887662988967, + "loss": 1.3631, + "theoretical_loss": 3.6961571130603663, + "tokens_seen": 948502528 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005758716148445336, + "loss": 1.3497, + "theoretical_loss": 3.6961327610853187, + "tokens_seen": 948568064 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005758555667001004, + "loss": 1.3761, + "theoretical_loss": 3.6961084112637295, + "tokens_seen": 948633600 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575839518555667, + "loss": 1.3947, + "theoretical_loss": 3.6960840635952583, + "tokens_seen": 948699136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005758234704112338, + "loss": 1.328, + "theoretical_loss": 3.696059718079567, + "tokens_seen": 948764672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005758074222668004, + "loss": 1.4697, + "theoretical_loss": 3.696035374716316, + "tokens_seen": 948830208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005757913741223672, + "loss": 1.426, + "theoretical_loss": 3.696011033505167, + "tokens_seen": 948895744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005757753259779338, + "loss": 1.4018, + "theoretical_loss": 3.6959866944457813, + "tokens_seen": 948961280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005757592778335006, + "loss": 1.3941, + "theoretical_loss": 3.6959623575378187, + "tokens_seen": 949026816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005757432296890672, + "loss": 1.3764, + "theoretical_loss": 3.6959380227809415, + "tokens_seen": 949092352 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575727181544634, + "loss": 1.4238, + "theoretical_loss": 3.6959136901748115, + "tokens_seen": 949157888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005757111334002006, + "loss": 1.4044, + "theoretical_loss": 3.695889359719089, + "tokens_seen": 949223424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005756950852557674, + "loss": 1.443, + "theoretical_loss": 3.695865031413436, + "tokens_seen": 949288960 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575679037111334, + "loss": 1.2825, + "theoretical_loss": 3.695840705257514, + "tokens_seen": 949354496 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005756629889669008, + "loss": 1.4454, + "theoretical_loss": 3.6958163812509857, + "tokens_seen": 949420032 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005756469408224674, + "loss": 1.381, + "theoretical_loss": 3.695792059393511, + "tokens_seen": 949485568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005756308926780342, + "loss": 1.4229, + "theoretical_loss": 3.695767739684753, + "tokens_seen": 949551104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005756148445336008, + "loss": 1.3433, + "theoretical_loss": 3.695743422124373, + "tokens_seen": 949616640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005755987963891676, + "loss": 1.3934, + "theoretical_loss": 3.6957191067120334, + "tokens_seen": 949682176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005755827482447342, + "loss": 1.4416, + "theoretical_loss": 3.6956947934473963, + "tokens_seen": 949747712 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575566700100301, + "loss": 1.3913, + "theoretical_loss": 3.695670482330123, + "tokens_seen": 949813248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005755506519558676, + "loss": 1.3274, + "theoretical_loss": 3.6956461733598767, + "tokens_seen": 949878784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005755346038114344, + "loss": 1.4088, + "theoretical_loss": 3.695621866536319, + "tokens_seen": 949944320 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575518555667001, + "loss": 1.4538, + "theoretical_loss": 3.6955975618591133, + "tokens_seen": 950009856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005755025075225678, + "loss": 1.3233, + "theoretical_loss": 3.6955732593279205, + "tokens_seen": 950075392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005754864593781344, + "loss": 1.3608, + "theoretical_loss": 3.6955489589424033, + "tokens_seen": 950140928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005754704112337011, + "loss": 1.3454, + "theoretical_loss": 3.6955246607022256, + "tokens_seen": 950206464 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 356596, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4497437477111816, + "objective/train/theoretical_loss": 3.695500364607049, + "objective/train/tokens_used": 970732000, + "theoretical_loss": 3.695500364607049, + "tokens_seen": 950272000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005754543630892678, + "loss": 1.4497, + "theoretical_loss": 3.695500364607049, + "tokens_seen": 950272000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005754383149448345, + "loss": 1.424, + "theoretical_loss": 3.695476070656537, + "tokens_seen": 950337536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005754222668004012, + "loss": 1.4205, + "theoretical_loss": 3.6954517788503507, + "tokens_seen": 950403072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005754062186559679, + "loss": 1.5036, + "theoretical_loss": 3.695427489188155, + "tokens_seen": 950468608 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005753901705115346, + "loss": 1.5075, + "theoretical_loss": 3.6954032016696123, + "tokens_seen": 950534144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005753741223671013, + "loss": 1.4232, + "theoretical_loss": 3.6953789162943846, + "tokens_seen": 950599680 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575358074222668, + "loss": 1.4669, + "theoretical_loss": 3.695354633062136, + "tokens_seen": 950665216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005753420260782347, + "loss": 1.4354, + "theoretical_loss": 3.6953303519725296, + "tokens_seen": 950730752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005753259779338014, + "loss": 1.4211, + "theoretical_loss": 3.695306073025229, + "tokens_seen": 950796288 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005753099297893681, + "loss": 1.3391, + "theoretical_loss": 3.6952817962198967, + "tokens_seen": 950861824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005752938816449348, + "loss": 1.4505, + "theoretical_loss": 3.6952575215561962, + "tokens_seen": 950927360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005752778335005015, + "loss": 1.369, + "theoretical_loss": 3.695233249033792, + "tokens_seen": 950992896 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005752617853560682, + "loss": 1.2678, + "theoretical_loss": 3.6952089786523468, + "tokens_seen": 951058432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005752457372116349, + "loss": 1.399, + "theoretical_loss": 3.6951847104115236, + "tokens_seen": 951123968 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005752296890672016, + "loss": 1.4776, + "theoretical_loss": 3.6951604443109876, + "tokens_seen": 951189504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005752136409227683, + "loss": 1.5069, + "theoretical_loss": 3.695136180350402, + "tokens_seen": 951255040 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575197592778335, + "loss": 1.4682, + "theoretical_loss": 3.69511191852943, + "tokens_seen": 951320576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005751815446339017, + "loss": 1.3562, + "theoretical_loss": 3.695087658847737, + "tokens_seen": 951386112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005751654964894684, + "loss": 1.4215, + "theoretical_loss": 3.695063401304986, + "tokens_seen": 951451648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005751494483450351, + "loss": 1.3523, + "theoretical_loss": 3.695039145900841, + "tokens_seen": 951517184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005751334002006018, + "loss": 1.3632, + "theoretical_loss": 3.6950148926349664, + "tokens_seen": 951582720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005751173520561685, + "loss": 1.452, + "theoretical_loss": 3.694990641507027, + "tokens_seen": 951648256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005751013039117353, + "loss": 1.4605, + "theoretical_loss": 3.694966392516686, + "tokens_seen": 951713792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005750852557673019, + "loss": 1.3557, + "theoretical_loss": 3.6949421456636085, + "tokens_seen": 951779328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005750692076228687, + "loss": 1.5047, + "theoretical_loss": 3.694917900947459, + "tokens_seen": 951844864 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005750531594784353, + "loss": 1.5339, + "theoretical_loss": 3.6948936583679024, + "tokens_seen": 951910400 + }, + { + "epoch": 0.29, + "learning_rate": 0.000575037111334002, + "loss": 1.4128, + "theoretical_loss": 3.694869417924603, + "tokens_seen": 951975936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005750210631895687, + "loss": 1.414, + "theoretical_loss": 3.694845179617225, + "tokens_seen": 952041472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005750050150451355, + "loss": 1.3975, + "theoretical_loss": 3.6948209434454338, + "tokens_seen": 952107008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005749889669007021, + "loss": 1.3916, + "theoretical_loss": 3.6947967094088936, + "tokens_seen": 952172544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005749729187562689, + "loss": 1.3748, + "theoretical_loss": 3.6947724775072706, + "tokens_seen": 952238080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005749568706118355, + "loss": 1.5422, + "theoretical_loss": 3.6947482477402285, + "tokens_seen": 952303616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005749408224674023, + "loss": 1.4405, + "theoretical_loss": 3.694724020107433, + "tokens_seen": 952369152 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005749247743229689, + "loss": 1.2321, + "theoretical_loss": 3.6946997946085487, + "tokens_seen": 952434688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005749087261785357, + "loss": 1.3226, + "theoretical_loss": 3.694675571243242, + "tokens_seen": 952500224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005748926780341023, + "loss": 1.4067, + "theoretical_loss": 3.6946513500111777, + "tokens_seen": 952565760 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005748766298896691, + "loss": 1.4967, + "theoretical_loss": 3.6946271309120204, + "tokens_seen": 952631296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005748605817452357, + "loss": 1.3457, + "theoretical_loss": 3.6946029139454364, + "tokens_seen": 952696832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005748445336008025, + "loss": 1.2925, + "theoretical_loss": 3.6945786991110907, + "tokens_seen": 952762368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005748284854563691, + "loss": 1.3404, + "theoretical_loss": 3.6945544864086495, + "tokens_seen": 952827904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005748124373119359, + "loss": 1.3618, + "theoretical_loss": 3.6945302758377783, + "tokens_seen": 952893440 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747963891675025, + "loss": 1.4247, + "theoretical_loss": 3.6945060673981427, + "tokens_seen": 952958976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747803410230693, + "loss": 1.4448, + "theoretical_loss": 3.6944818610894083, + "tokens_seen": 953024512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747642928786359, + "loss": 1.354, + "theoretical_loss": 3.6944576569112417, + "tokens_seen": 953090048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747482447342027, + "loss": 1.4647, + "theoretical_loss": 3.694433454863309, + "tokens_seen": 953155584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747321965897693, + "loss": 1.4646, + "theoretical_loss": 3.694409254945275, + "tokens_seen": 953221120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747161484453361, + "loss": 1.39, + "theoretical_loss": 3.694385057156807, + "tokens_seen": 953286656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005747001003009027, + "loss": 1.3463, + "theoretical_loss": 3.6943608614975707, + "tokens_seen": 953352192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005746840521564695, + "loss": 1.297, + "theoretical_loss": 3.6943366679672325, + "tokens_seen": 953417728 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005746680040120361, + "loss": 1.4381, + "theoretical_loss": 3.6943124765654587, + "tokens_seen": 953483264 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005746519558676029, + "loss": 1.4154, + "theoretical_loss": 3.694288287291916, + "tokens_seen": 953548800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005746359077231695, + "loss": 1.479, + "theoretical_loss": 3.6942641001462713, + "tokens_seen": 953614336 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005746198595787363, + "loss": 1.3096, + "theoretical_loss": 3.6942399151281897, + "tokens_seen": 953679872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005746038114343029, + "loss": 1.377, + "theoretical_loss": 3.6942157322373395, + "tokens_seen": 953745408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005745877632898697, + "loss": 1.4169, + "theoretical_loss": 3.694191551473387, + "tokens_seen": 953810944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005745717151454363, + "loss": 1.4063, + "theoretical_loss": 3.694167372835998, + "tokens_seen": 953876480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005745556670010031, + "loss": 1.3227, + "theoretical_loss": 3.6941431963248403, + "tokens_seen": 953942016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005745396188565697, + "loss": 1.4891, + "theoretical_loss": 3.6941190219395814, + "tokens_seen": 954007552 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005745235707121365, + "loss": 1.4249, + "theoretical_loss": 3.6940948496798875, + "tokens_seen": 954073088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005745075225677031, + "loss": 1.492, + "theoretical_loss": 3.694070679545426, + "tokens_seen": 954138624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005744914744232699, + "loss": 1.3326, + "theoretical_loss": 3.6940465115358636, + "tokens_seen": 954204160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005744754262788365, + "loss": 1.4654, + "theoretical_loss": 3.6940223456508683, + "tokens_seen": 954269696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005744593781344033, + "loss": 1.3852, + "theoretical_loss": 3.693998181890107, + "tokens_seen": 954335232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005744433299899699, + "loss": 1.3916, + "theoretical_loss": 3.693974020253248, + "tokens_seen": 954400768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005744272818455367, + "loss": 1.3962, + "theoretical_loss": 3.6939498607399566, + "tokens_seen": 954466304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005744112337011033, + "loss": 1.318, + "theoretical_loss": 3.693925703349903, + "tokens_seen": 954531840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005743951855566701, + "loss": 1.3594, + "theoretical_loss": 3.6939015480827533, + "tokens_seen": 954597376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005743791374122367, + "loss": 1.3577, + "theoretical_loss": 3.6938773949381756, + "tokens_seen": 954662912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005743630892678035, + "loss": 1.3766, + "theoretical_loss": 3.6938532439158376, + "tokens_seen": 954728448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005743470411233701, + "loss": 1.2482, + "theoretical_loss": 3.693829095015407, + "tokens_seen": 954793984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005743309929789369, + "loss": 1.3529, + "theoretical_loss": 3.6938049482365525, + "tokens_seen": 954859520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005743149448345035, + "loss": 1.4463, + "theoretical_loss": 3.6937808035789415, + "tokens_seen": 954925056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742988966900703, + "loss": 1.3393, + "theoretical_loss": 3.6937566610422423, + "tokens_seen": 954990592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742828485456369, + "loss": 1.414, + "theoretical_loss": 3.6937325206261225, + "tokens_seen": 955056128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742668004012037, + "loss": 1.4021, + "theoretical_loss": 3.693708382330251, + "tokens_seen": 955121664 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742507522567703, + "loss": 1.3464, + "theoretical_loss": 3.6936842461542962, + "tokens_seen": 955187200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742347041123371, + "loss": 1.348, + "theoretical_loss": 3.693660112097926, + "tokens_seen": 955252736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742186559679037, + "loss": 1.4148, + "theoretical_loss": 3.693635980160809, + "tokens_seen": 955318272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005742026078234705, + "loss": 1.4008, + "theoretical_loss": 3.693611850342614, + "tokens_seen": 955383808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005741865596790371, + "loss": 1.4251, + "theoretical_loss": 3.6935877226430094, + "tokens_seen": 955449344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005741705115346039, + "loss": 1.3945, + "theoretical_loss": 3.6935635970616643, + "tokens_seen": 955514880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005741544633901705, + "loss": 1.4562, + "theoretical_loss": 3.6935394735982463, + "tokens_seen": 955580416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005741384152457373, + "loss": 1.323, + "theoretical_loss": 3.6935153522524256, + "tokens_seen": 955645952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005741223671013039, + "loss": 1.3163, + "theoretical_loss": 3.6934912330238703, + "tokens_seen": 955711488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005741063189568707, + "loss": 1.4333, + "theoretical_loss": 3.6934671159122496, + "tokens_seen": 955777024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005740902708124373, + "loss": 1.4022, + "theoretical_loss": 3.6934430009172328, + "tokens_seen": 955842560 + }, + { + "epoch": 0.29, + "learning_rate": 0.000574074222668004, + "loss": 1.4303, + "theoretical_loss": 3.693418888038489, + "tokens_seen": 955908096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005740581745235707, + "loss": 1.3152, + "theoretical_loss": 3.693394777275687, + "tokens_seen": 955973632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005740421263791374, + "loss": 1.4448, + "theoretical_loss": 3.693370668628496, + "tokens_seen": 956039168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005740260782347041, + "loss": 1.3734, + "theoretical_loss": 3.693346562096586, + "tokens_seen": 956104704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005740100300902708, + "loss": 1.5086, + "theoretical_loss": 3.693322457679626, + "tokens_seen": 956170240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005739939819458375, + "loss": 1.4161, + "theoretical_loss": 3.693298355377286, + "tokens_seen": 956235776 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005739779338014042, + "loss": 1.5015, + "theoretical_loss": 3.693274255189235, + "tokens_seen": 956301312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005739618856569709, + "loss": 1.5039, + "theoretical_loss": 3.693250157115142, + "tokens_seen": 956366848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005739458375125376, + "loss": 1.4488, + "theoretical_loss": 3.6932260611546788, + "tokens_seen": 956432384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005739297893681043, + "loss": 1.3876, + "theoretical_loss": 3.693201967307514, + "tokens_seen": 956497920 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573913741223671, + "loss": 1.411, + "theoretical_loss": 3.6931778755733164, + "tokens_seen": 956563456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005738976930792377, + "loss": 1.3902, + "theoretical_loss": 3.6931537859517585, + "tokens_seen": 956628992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005738816449348044, + "loss": 1.4507, + "theoretical_loss": 3.6931296984425077, + "tokens_seen": 956694528 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005738655967903711, + "loss": 1.3728, + "theoretical_loss": 3.693105613045236, + "tokens_seen": 956760064 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 359021, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3924574851989746, + "objective/train/theoretical_loss": 3.693081529759612, + "objective/train/tokens_used": 977285600, + "theoretical_loss": 3.693081529759612, + "tokens_seen": 956825600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005738495486459378, + "loss": 1.3925, + "theoretical_loss": 3.693081529759612, + "tokens_seen": 956825600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005738335005015045, + "loss": 1.5064, + "theoretical_loss": 3.6930574485853076, + "tokens_seen": 956891136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005738174523570712, + "loss": 1.4148, + "theoretical_loss": 3.6930333695219915, + "tokens_seen": 956956672 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573801404212638, + "loss": 1.4275, + "theoretical_loss": 3.693009292569336, + "tokens_seen": 957022208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005737853560682046, + "loss": 1.4086, + "theoretical_loss": 3.69298521772701, + "tokens_seen": 957087744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005737693079237713, + "loss": 1.3733, + "theoretical_loss": 3.692961144994684, + "tokens_seen": 957153280 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573753259779338, + "loss": 1.4362, + "theoretical_loss": 3.6929370743720296, + "tokens_seen": 957218816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005737372116349047, + "loss": 1.4913, + "theoretical_loss": 3.692913005858717, + "tokens_seen": 957284352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005737211634904714, + "loss": 1.3495, + "theoretical_loss": 3.6928889394544173, + "tokens_seen": 957349888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005737051153460382, + "loss": 1.4362, + "theoretical_loss": 3.6928648751588007, + "tokens_seen": 957415424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005736890672016048, + "loss": 1.4254, + "theoretical_loss": 3.6928408129715384, + "tokens_seen": 957480960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005736730190571716, + "loss": 1.428, + "theoretical_loss": 3.692816752892302, + "tokens_seen": 957546496 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005736569709127382, + "loss": 1.3845, + "theoretical_loss": 3.6927926949207617, + "tokens_seen": 957612032 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573640922768305, + "loss": 1.4061, + "theoretical_loss": 3.692768639056589, + "tokens_seen": 957677568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005736248746238716, + "loss": 1.4475, + "theoretical_loss": 3.692744585299455, + "tokens_seen": 957743104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005736088264794384, + "loss": 1.4006, + "theoretical_loss": 3.692720533649031, + "tokens_seen": 957808640 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573592778335005, + "loss": 1.4782, + "theoretical_loss": 3.6926964841049887, + "tokens_seen": 957874176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005735767301905718, + "loss": 1.3145, + "theoretical_loss": 3.692672436666999, + "tokens_seen": 957939712 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005735606820461384, + "loss": 1.4012, + "theoretical_loss": 3.6926483913347337, + "tokens_seen": 958005248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005735446339017052, + "loss": 1.4018, + "theoretical_loss": 3.692624348107864, + "tokens_seen": 958070784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005735285857572718, + "loss": 1.4819, + "theoretical_loss": 3.692600306986063, + "tokens_seen": 958136320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005735125376128386, + "loss": 1.4357, + "theoretical_loss": 3.692576267969, + "tokens_seen": 958201856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005734964894684052, + "loss": 1.4887, + "theoretical_loss": 3.6925522310563488, + "tokens_seen": 958267392 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573480441323972, + "loss": 1.4494, + "theoretical_loss": 3.6925281962477805, + "tokens_seen": 958332928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005734643931795386, + "loss": 1.3876, + "theoretical_loss": 3.692504163542967, + "tokens_seen": 958398464 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005734483450351054, + "loss": 1.4394, + "theoretical_loss": 3.692480132941581, + "tokens_seen": 958464000 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573432296890672, + "loss": 1.363, + "theoretical_loss": 3.6924561044432935, + "tokens_seen": 958529536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005734162487462388, + "loss": 1.5355, + "theoretical_loss": 3.6924320780477773, + "tokens_seen": 958595072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005734002006018054, + "loss": 1.3661, + "theoretical_loss": 3.692408053754705, + "tokens_seen": 958660608 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005733841524573722, + "loss": 1.3842, + "theoretical_loss": 3.6923840315637477, + "tokens_seen": 958726144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005733681043129388, + "loss": 1.3823, + "theoretical_loss": 3.6923600114745794, + "tokens_seen": 958791680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005733520561685056, + "loss": 1.4863, + "theoretical_loss": 3.6923359934868714, + "tokens_seen": 958857216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005733360080240722, + "loss": 1.3879, + "theoretical_loss": 3.692311977600297, + "tokens_seen": 958922752 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573319959879639, + "loss": 1.3816, + "theoretical_loss": 3.6922879638145276, + "tokens_seen": 958988288 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005733039117352056, + "loss": 1.3412, + "theoretical_loss": 3.6922639521292373, + "tokens_seen": 959053824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005732878635907724, + "loss": 1.3226, + "theoretical_loss": 3.6922399425440977, + "tokens_seen": 959119360 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573271815446339, + "loss": 1.4325, + "theoretical_loss": 3.6922159350587824, + "tokens_seen": 959184896 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005732557673019058, + "loss": 1.4363, + "theoretical_loss": 3.692191929672964, + "tokens_seen": 959250432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005732397191574724, + "loss": 1.4239, + "theoretical_loss": 3.6921679263863156, + "tokens_seen": 959315968 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005732236710130392, + "loss": 1.2842, + "theoretical_loss": 3.6921439251985104, + "tokens_seen": 959381504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005732076228686058, + "loss": 1.4548, + "theoretical_loss": 3.6921199261092212, + "tokens_seen": 959447040 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005731915747241726, + "loss": 1.4161, + "theoretical_loss": 3.6920959291181212, + "tokens_seen": 959512576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005731755265797392, + "loss": 1.4386, + "theoretical_loss": 3.692071934224884, + "tokens_seen": 959578112 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573159478435306, + "loss": 1.5463, + "theoretical_loss": 3.6920479414291822, + "tokens_seen": 959643648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005731434302908726, + "loss": 1.5083, + "theoretical_loss": 3.6920239507306905, + "tokens_seen": 959709184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005731273821464394, + "loss": 1.4522, + "theoretical_loss": 3.6919999621290813, + "tokens_seen": 959774720 + }, + { + "epoch": 0.29, + "learning_rate": 0.000573111334002006, + "loss": 1.4186, + "theoretical_loss": 3.691975975624028, + "tokens_seen": 959840256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005730952858575728, + "loss": 1.4655, + "theoretical_loss": 3.6919519912152055, + "tokens_seen": 959905792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005730792377131394, + "loss": 1.4585, + "theoretical_loss": 3.691928008902286, + "tokens_seen": 959971328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005730631895687062, + "loss": 1.5037, + "theoretical_loss": 3.6919040286849443, + "tokens_seen": 960036864 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005730471414242728, + "loss": 1.4327, + "theoretical_loss": 3.6918800505628546, + "tokens_seen": 960102400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005730310932798396, + "loss": 1.3288, + "theoretical_loss": 3.691856074535689, + "tokens_seen": 960167936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005730150451354062, + "loss": 1.443, + "theoretical_loss": 3.691832100603124, + "tokens_seen": 960233472 + }, + { + "epoch": 0.29, + "learning_rate": 0.000572998996990973, + "loss": 1.4273, + "theoretical_loss": 3.691808128764831, + "tokens_seen": 960299008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005729829488465396, + "loss": 1.3503, + "theoretical_loss": 3.6917841590204867, + "tokens_seen": 960364544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005729669007021064, + "loss": 1.4442, + "theoretical_loss": 3.6917601913697635, + "tokens_seen": 960430080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005729508525576731, + "loss": 1.4088, + "theoretical_loss": 3.6917362258123365, + "tokens_seen": 960495616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005729348044132398, + "loss": 1.3101, + "theoretical_loss": 3.69171226234788, + "tokens_seen": 960561152 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005729187562688065, + "loss": 1.3961, + "theoretical_loss": 3.691688300976068, + "tokens_seen": 960626688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005729027081243732, + "loss": 1.2714, + "theoretical_loss": 3.6916643416965753, + "tokens_seen": 960692224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005728866599799399, + "loss": 1.3963, + "theoretical_loss": 3.6916403845090766, + "tokens_seen": 960757760 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005728706118355066, + "loss": 1.4644, + "theoretical_loss": 3.691616429413247, + "tokens_seen": 960823296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005728545636910733, + "loss": 1.4371, + "theoretical_loss": 3.6915924764087604, + "tokens_seen": 960888832 + }, + { + "epoch": 0.29, + "learning_rate": 0.00057283851554664, + "loss": 1.4418, + "theoretical_loss": 3.6915685254952915, + "tokens_seen": 960954368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005728224674022067, + "loss": 1.3165, + "theoretical_loss": 3.6915445766725163, + "tokens_seen": 961019904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005728064192577734, + "loss": 1.3796, + "theoretical_loss": 3.691520629940108, + "tokens_seen": 961085440 + }, + { + "epoch": 0.29, + "learning_rate": 0.00057279037111334, + "loss": 1.4641, + "theoretical_loss": 3.6914966852977438, + "tokens_seen": 961150976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005727743229689068, + "loss": 1.4221, + "theoretical_loss": 3.691472742745097, + "tokens_seen": 961216512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005727582748244734, + "loss": 1.3525, + "theoretical_loss": 3.6914488022818435, + "tokens_seen": 961282048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005727422266800402, + "loss": 1.3778, + "theoretical_loss": 3.691424863907659, + "tokens_seen": 961347584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005727261785356068, + "loss": 1.3786, + "theoretical_loss": 3.691400927622217, + "tokens_seen": 961413120 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005727101303911736, + "loss": 1.3411, + "theoretical_loss": 3.6913769934251954, + "tokens_seen": 961478656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005726940822467402, + "loss": 1.3479, + "theoretical_loss": 3.691353061316268, + "tokens_seen": 961544192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005726780341023069, + "loss": 1.4366, + "theoretical_loss": 3.6913291312951104, + "tokens_seen": 961609728 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005726619859578736, + "loss": 1.4881, + "theoretical_loss": 3.6913052033613987, + "tokens_seen": 961675264 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005726459378134403, + "loss": 1.3908, + "theoretical_loss": 3.6912812775148085, + "tokens_seen": 961740800 + }, + { + "epoch": 0.29, + "learning_rate": 0.000572629889669007, + "loss": 1.515, + "theoretical_loss": 3.691257353755015, + "tokens_seen": 961806336 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005726138415245737, + "loss": 1.388, + "theoretical_loss": 3.691233432081695, + "tokens_seen": 961871872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725977933801404, + "loss": 1.3814, + "theoretical_loss": 3.691209512494524, + "tokens_seen": 961937408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725817452357071, + "loss": 1.3917, + "theoretical_loss": 3.6911855949931773, + "tokens_seen": 962002944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725656970912738, + "loss": 1.358, + "theoretical_loss": 3.6911616795773314, + "tokens_seen": 962068480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725496489468405, + "loss": 1.4446, + "theoretical_loss": 3.691137766246663, + "tokens_seen": 962134016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725336008024072, + "loss": 1.3756, + "theoretical_loss": 3.6911138550008475, + "tokens_seen": 962199552 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725175526579739, + "loss": 1.4396, + "theoretical_loss": 3.691089945839561, + "tokens_seen": 962265088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005725015045135406, + "loss": 1.4132, + "theoretical_loss": 3.6910660387624805, + "tokens_seen": 962330624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005724854563691073, + "loss": 1.3769, + "theoretical_loss": 3.6910421337692823, + "tokens_seen": 962396160 + }, + { + "epoch": 0.29, + "learning_rate": 0.000572469408224674, + "loss": 1.4298, + "theoretical_loss": 3.6910182308596426, + "tokens_seen": 962461696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005724533600802407, + "loss": 1.3909, + "theoretical_loss": 3.6909943300332384, + "tokens_seen": 962527232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005724373119358074, + "loss": 1.4042, + "theoretical_loss": 3.6909704312897453, + "tokens_seen": 962592768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005724212637913741, + "loss": 1.4908, + "theoretical_loss": 3.690946534628841, + "tokens_seen": 962658304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005724052156469408, + "loss": 1.3347, + "theoretical_loss": 3.6909226400502018, + "tokens_seen": 962723840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005723891675025075, + "loss": 1.4248, + "theoretical_loss": 3.6908987475535047, + "tokens_seen": 962789376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005723731193580742, + "loss": 1.388, + "theoretical_loss": 3.690874857138426, + "tokens_seen": 962854912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005723570712136409, + "loss": 1.3995, + "theoretical_loss": 3.690850968804644, + "tokens_seen": 962920448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005723410230692077, + "loss": 1.4017, + "theoretical_loss": 3.6908270825518343, + "tokens_seen": 962985984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005723249749247743, + "loss": 1.4565, + "theoretical_loss": 3.690803198379675, + "tokens_seen": 963051520 + }, + { + "epoch": 0.29, + "learning_rate": 0.000572308926780341, + "loss": 1.3582, + "theoretical_loss": 3.690779316287843, + "tokens_seen": 963117056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005722928786359077, + "loss": 1.4813, + "theoretical_loss": 3.6907554362760155, + "tokens_seen": 963182592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005722768304914745, + "loss": 1.3928, + "theoretical_loss": 3.6907315583438693, + "tokens_seen": 963248128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005722607823470411, + "loss": 1.4402, + "theoretical_loss": 3.690707682491083, + "tokens_seen": 963313664 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 361323, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.37824285030365, + "objective/train/theoretical_loss": 3.690683808717333, + "objective/train/tokens_used": 983839200, + "theoretical_loss": 3.690683808717333, + "tokens_seen": 963379200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005722447342026079, + "loss": 1.3782, + "theoretical_loss": 3.690683808717333, + "tokens_seen": 963379200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005722286860581745, + "loss": 1.4223, + "theoretical_loss": 3.6906599370222977, + "tokens_seen": 963444736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005722126379137413, + "loss": 1.4715, + "theoretical_loss": 3.690636067405654, + "tokens_seen": 963510272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721965897693079, + "loss": 1.4365, + "theoretical_loss": 3.69061219986708, + "tokens_seen": 963575808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721805416248747, + "loss": 1.3985, + "theoretical_loss": 3.6905883344062533, + "tokens_seen": 963641344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721644934804413, + "loss": 1.4185, + "theoretical_loss": 3.690564471022852, + "tokens_seen": 963706880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721484453360081, + "loss": 1.4335, + "theoretical_loss": 3.690540609716554, + "tokens_seen": 963772416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721323971915747, + "loss": 1.4417, + "theoretical_loss": 3.6905167504870366, + "tokens_seen": 963837952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721163490471415, + "loss": 1.3722, + "theoretical_loss": 3.6904928933339787, + "tokens_seen": 963903488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005721003009027081, + "loss": 1.3947, + "theoretical_loss": 3.690469038257058, + "tokens_seen": 963969024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005720842527582749, + "loss": 1.4181, + "theoretical_loss": 3.6904451852559528, + "tokens_seen": 964034560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005720682046138415, + "loss": 1.3267, + "theoretical_loss": 3.6904213343303414, + "tokens_seen": 964100096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005720521564694083, + "loss": 1.4668, + "theoretical_loss": 3.690397485479902, + "tokens_seen": 964165632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005720361083249749, + "loss": 1.3615, + "theoretical_loss": 3.690373638704313, + "tokens_seen": 964231168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005720200601805417, + "loss": 1.3424, + "theoretical_loss": 3.6903497940032532, + "tokens_seen": 964296704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005720040120361083, + "loss": 1.3619, + "theoretical_loss": 3.690325951376401, + "tokens_seen": 964362240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005719879638916751, + "loss": 1.376, + "theoretical_loss": 3.690302110823435, + "tokens_seen": 964427776 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005719719157472417, + "loss": 1.3977, + "theoretical_loss": 3.6902782723440337, + "tokens_seen": 964493312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005719558676028085, + "loss": 1.2589, + "theoretical_loss": 3.6902544359378755, + "tokens_seen": 964558848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005719398194583751, + "loss": 1.4027, + "theoretical_loss": 3.6902306016046404, + "tokens_seen": 964624384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005719237713139419, + "loss": 1.4344, + "theoretical_loss": 3.690206769344007, + "tokens_seen": 964689920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005719077231695085, + "loss": 1.4551, + "theoretical_loss": 3.6901829391556538, + "tokens_seen": 964755456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005718916750250753, + "loss": 1.3201, + "theoretical_loss": 3.69015911103926, + "tokens_seen": 964820992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005718756268806419, + "loss": 1.4856, + "theoretical_loss": 3.690135284994504, + "tokens_seen": 964886528 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005718595787362087, + "loss": 1.4117, + "theoretical_loss": 3.6901114610210666, + "tokens_seen": 964952064 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005718435305917753, + "loss": 1.4036, + "theoretical_loss": 3.6900876391186266, + "tokens_seen": 965017600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005718274824473421, + "loss": 1.3068, + "theoretical_loss": 3.6900638192868627, + "tokens_seen": 965083136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005718114343029087, + "loss": 1.3713, + "theoretical_loss": 3.690040001525454, + "tokens_seen": 965148672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005717953861584755, + "loss": 1.3862, + "theoretical_loss": 3.6900161858340814, + "tokens_seen": 965214208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005717793380140421, + "loss": 1.4055, + "theoretical_loss": 3.689992372212423, + "tokens_seen": 965279744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005717632898696089, + "loss": 1.4302, + "theoretical_loss": 3.6899685606601595, + "tokens_seen": 965345280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005717472417251756, + "loss": 1.4282, + "theoretical_loss": 3.68994475117697, + "tokens_seen": 965410816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005717311935807423, + "loss": 1.4018, + "theoretical_loss": 3.6899209437625347, + "tokens_seen": 965476352 + }, + { + "epoch": 0.29, + "learning_rate": 0.000571715145436309, + "loss": 1.3488, + "theoretical_loss": 3.6898971384165327, + "tokens_seen": 965541888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716990972918757, + "loss": 1.3328, + "theoretical_loss": 3.6898733351386444, + "tokens_seen": 965607424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716830491474424, + "loss": 1.3832, + "theoretical_loss": 3.68984953392855, + "tokens_seen": 965672960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716670010030091, + "loss": 1.3304, + "theoretical_loss": 3.68982573478593, + "tokens_seen": 965738496 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716509528585758, + "loss": 1.5406, + "theoretical_loss": 3.689801937710463, + "tokens_seen": 965804032 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716349047141425, + "loss": 1.3605, + "theoretical_loss": 3.6897781427018304, + "tokens_seen": 965869568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716188565697092, + "loss": 1.4437, + "theoretical_loss": 3.6897543497597125, + "tokens_seen": 965935104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005716028084252759, + "loss": 1.4088, + "theoretical_loss": 3.689730558883789, + "tokens_seen": 966000640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005715867602808426, + "loss": 1.3218, + "theoretical_loss": 3.68970677007374, + "tokens_seen": 966066176 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005715707121364093, + "loss": 1.4019, + "theoretical_loss": 3.6896829833292477, + "tokens_seen": 966131712 + }, + { + "epoch": 0.29, + "learning_rate": 0.000571554663991976, + "loss": 1.4271, + "theoretical_loss": 3.6896591986499905, + "tokens_seen": 966197248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005715386158475427, + "loss": 1.3848, + "theoretical_loss": 3.689635416035651, + "tokens_seen": 966262784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005715225677031094, + "loss": 1.4145, + "theoretical_loss": 3.6896116354859085, + "tokens_seen": 966328320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005715065195586761, + "loss": 1.4304, + "theoretical_loss": 3.6895878570004443, + "tokens_seen": 966393856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005714904714142428, + "loss": 1.444, + "theoretical_loss": 3.689564080578939, + "tokens_seen": 966459392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005714744232698095, + "loss": 1.3846, + "theoretical_loss": 3.689540306221074, + "tokens_seen": 966524928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005714583751253762, + "loss": 1.5117, + "theoretical_loss": 3.6895165339265295, + "tokens_seen": 966590464 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005714423269809429, + "loss": 1.3436, + "theoretical_loss": 3.6894927636949877, + "tokens_seen": 966656000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005714262788365096, + "loss": 1.3222, + "theoretical_loss": 3.689468995526128, + "tokens_seen": 966721536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005714102306920763, + "loss": 1.2983, + "theoretical_loss": 3.6894452294196336, + "tokens_seen": 966787072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005713941825476429, + "loss": 1.4934, + "theoretical_loss": 3.6894214653751845, + "tokens_seen": 966852608 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005713781344032097, + "loss": 1.3755, + "theoretical_loss": 3.689397703392462, + "tokens_seen": 966918144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005713620862587763, + "loss": 1.4286, + "theoretical_loss": 3.689373943471148, + "tokens_seen": 966983680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005713460381143431, + "loss": 1.3277, + "theoretical_loss": 3.689350185610924, + "tokens_seen": 967049216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005713299899699097, + "loss": 1.3429, + "theoretical_loss": 3.6893264298114716, + "tokens_seen": 967114752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005713139418254765, + "loss": 1.3983, + "theoretical_loss": 3.689302676072472, + "tokens_seen": 967180288 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712978936810431, + "loss": 1.4586, + "theoretical_loss": 3.689278924393607, + "tokens_seen": 967245824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712818455366099, + "loss": 1.4969, + "theoretical_loss": 3.689255174774558, + "tokens_seen": 967311360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712657973921765, + "loss": 1.4893, + "theoretical_loss": 3.6892314272150077, + "tokens_seen": 967376896 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712497492477432, + "loss": 1.4354, + "theoretical_loss": 3.6892076817146373, + "tokens_seen": 967442432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712337011033099, + "loss": 1.4091, + "theoretical_loss": 3.6891839382731297, + "tokens_seen": 967507968 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712176529588766, + "loss": 1.3668, + "theoretical_loss": 3.6891601968901657, + "tokens_seen": 967573504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005712016048144433, + "loss": 1.4466, + "theoretical_loss": 3.689136457565428, + "tokens_seen": 967639040 + }, + { + "epoch": 0.29, + "learning_rate": 0.00057118555667001, + "loss": 1.3796, + "theoretical_loss": 3.689112720298599, + "tokens_seen": 967704576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005711695085255767, + "loss": 1.3715, + "theoretical_loss": 3.6890889850893602, + "tokens_seen": 967770112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005711534603811434, + "loss": 1.4133, + "theoretical_loss": 3.689065251937395, + "tokens_seen": 967835648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005711374122367101, + "loss": 1.4173, + "theoretical_loss": 3.6890415208423857, + "tokens_seen": 967901184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005711213640922768, + "loss": 1.3736, + "theoretical_loss": 3.6890177918040132, + "tokens_seen": 967966720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005711053159478435, + "loss": 1.4179, + "theoretical_loss": 3.688994064821962, + "tokens_seen": 968032256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005710892678034102, + "loss": 1.3747, + "theoretical_loss": 3.6889703398959135, + "tokens_seen": 968097792 + }, + { + "epoch": 0.29, + "learning_rate": 0.000571073219658977, + "loss": 1.3845, + "theoretical_loss": 3.6889466170255503, + "tokens_seen": 968163328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005710571715145436, + "loss": 1.363, + "theoretical_loss": 3.6889228962105562, + "tokens_seen": 968228864 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005710411233701103, + "loss": 1.4055, + "theoretical_loss": 3.6888991774506135, + "tokens_seen": 968294400 + }, + { + "epoch": 0.29, + "learning_rate": 0.000571025075225677, + "loss": 1.513, + "theoretical_loss": 3.688875460745405, + "tokens_seen": 968359936 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005710090270812437, + "loss": 1.298, + "theoretical_loss": 3.6888517460946133, + "tokens_seen": 968425472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005709929789368104, + "loss": 1.4666, + "theoretical_loss": 3.6888280334979218, + "tokens_seen": 968491008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005709769307923772, + "loss": 1.3805, + "theoretical_loss": 3.688804322955014, + "tokens_seen": 968556544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005709608826479438, + "loss": 1.3588, + "theoretical_loss": 3.688780614465572, + "tokens_seen": 968622080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005709448345035106, + "loss": 1.2772, + "theoretical_loss": 3.68875690802928, + "tokens_seen": 968687616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005709287863590772, + "loss": 1.4617, + "theoretical_loss": 3.688733203645821, + "tokens_seen": 968753152 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570912738214644, + "loss": 1.5691, + "theoretical_loss": 3.6887095013148787, + "tokens_seen": 968818688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005708966900702106, + "loss": 1.352, + "theoretical_loss": 3.6886858010361356, + "tokens_seen": 968884224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005708806419257774, + "loss": 1.4569, + "theoretical_loss": 3.6886621028092765, + "tokens_seen": 968949760 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570864593781344, + "loss": 1.3523, + "theoretical_loss": 3.688638406633984, + "tokens_seen": 969015296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005708485456369108, + "loss": 1.4548, + "theoretical_loss": 3.688614712509942, + "tokens_seen": 969080832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005708324974924774, + "loss": 1.4734, + "theoretical_loss": 3.688591020436835, + "tokens_seen": 969146368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005708164493480442, + "loss": 1.4159, + "theoretical_loss": 3.6885673304143456, + "tokens_seen": 969211904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005708004012036108, + "loss": 1.4553, + "theoretical_loss": 3.6885436424421583, + "tokens_seen": 969277440 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005707843530591776, + "loss": 1.3526, + "theoretical_loss": 3.688519956519957, + "tokens_seen": 969342976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005707683049147442, + "loss": 1.3753, + "theoretical_loss": 3.6884962726474253, + "tokens_seen": 969408512 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570752256770311, + "loss": 1.4074, + "theoretical_loss": 3.688472590824248, + "tokens_seen": 969474048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005707362086258776, + "loss": 1.4446, + "theoretical_loss": 3.6884489110501093, + "tokens_seen": 969539584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005707201604814444, + "loss": 1.4833, + "theoretical_loss": 3.6884252333246925, + "tokens_seen": 969605120 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570704112337011, + "loss": 1.3504, + "theoretical_loss": 3.6884015576476825, + "tokens_seen": 969670656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005706880641925778, + "loss": 1.4549, + "theoretical_loss": 3.688377884018764, + "tokens_seen": 969736192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005706720160481444, + "loss": 1.3698, + "theoretical_loss": 3.6883542124376207, + "tokens_seen": 969801728 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005706559679037112, + "loss": 1.3241, + "theoretical_loss": 3.6883305429039375, + "tokens_seen": 969867264 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 363666, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4186903238296509, + "objective/train/theoretical_loss": 3.688306875417399, + "objective/train/tokens_used": 990392800, + "theoretical_loss": 3.688306875417399, + "tokens_seen": 969932800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005706399197592778, + "loss": 1.4187, + "theoretical_loss": 3.688306875417399, + "tokens_seen": 969932800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005706238716148446, + "loss": 1.3453, + "theoretical_loss": 3.688283209977689, + "tokens_seen": 969998336 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005706078234704112, + "loss": 1.3888, + "theoretical_loss": 3.6882595465844936, + "tokens_seen": 970063872 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570591775325978, + "loss": 1.4118, + "theoretical_loss": 3.6882358852374972, + "tokens_seen": 970129408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005705757271815446, + "loss": 1.5882, + "theoretical_loss": 3.688212225936384, + "tokens_seen": 970194944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005705596790371114, + "loss": 1.4551, + "theoretical_loss": 3.6881885686808396, + "tokens_seen": 970260480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005705436308926781, + "loss": 1.3923, + "theoretical_loss": 3.688164913470548, + "tokens_seen": 970326016 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005705275827482448, + "loss": 1.424, + "theoretical_loss": 3.6881412603051955, + "tokens_seen": 970391552 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005705115346038115, + "loss": 1.44, + "theoretical_loss": 3.6881176091844665, + "tokens_seen": 970457088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005704954864593782, + "loss": 1.3827, + "theoretical_loss": 3.6880939601080467, + "tokens_seen": 970522624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005704794383149449, + "loss": 1.4155, + "theoretical_loss": 3.688070313075621, + "tokens_seen": 970588160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005704633901705116, + "loss": 1.4159, + "theoretical_loss": 3.6880466680868746, + "tokens_seen": 970653696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005704473420260783, + "loss": 1.4814, + "theoretical_loss": 3.6880230251414936, + "tokens_seen": 970719232 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570431293881645, + "loss": 1.441, + "theoretical_loss": 3.6879993842391627, + "tokens_seen": 970784768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005704152457372117, + "loss": 1.3468, + "theoretical_loss": 3.6879757453795676, + "tokens_seen": 970850304 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703991975927784, + "loss": 1.4258, + "theoretical_loss": 3.6879521085623943, + "tokens_seen": 970915840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703831494483451, + "loss": 1.4635, + "theoretical_loss": 3.687928473787328, + "tokens_seen": 970981376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703671013039118, + "loss": 1.4496, + "theoretical_loss": 3.6879048410540554, + "tokens_seen": 971046912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703510531594785, + "loss": 1.4703, + "theoretical_loss": 3.687881210362261, + "tokens_seen": 971112448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703350050150452, + "loss": 1.3585, + "theoretical_loss": 3.687857581711631, + "tokens_seen": 971177984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703189568706119, + "loss": 1.3485, + "theoretical_loss": 3.687833955101852, + "tokens_seen": 971243520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005703029087261786, + "loss": 1.4463, + "theoretical_loss": 3.68781033053261, + "tokens_seen": 971309056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005702868605817453, + "loss": 1.3382, + "theoretical_loss": 3.687786708003591, + "tokens_seen": 971374592 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570270812437312, + "loss": 1.4974, + "theoretical_loss": 3.6877630875144805, + "tokens_seen": 971440128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005702547642928787, + "loss": 1.4264, + "theoretical_loss": 3.687739469064966, + "tokens_seen": 971505664 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005702387161484454, + "loss": 1.3301, + "theoretical_loss": 3.6877158526547325, + "tokens_seen": 971571200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005702226680040121, + "loss": 1.4727, + "theoretical_loss": 3.6876922382834674, + "tokens_seen": 971636736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005702066198595788, + "loss": 1.3984, + "theoretical_loss": 3.687668625950856, + "tokens_seen": 971702272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005701905717151455, + "loss": 1.3256, + "theoretical_loss": 3.687645015656586, + "tokens_seen": 971767808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005701745235707122, + "loss": 1.4443, + "theoretical_loss": 3.6876214074003437, + "tokens_seen": 971833344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005701584754262789, + "loss": 1.3701, + "theoretical_loss": 3.6875978011818153, + "tokens_seen": 971898880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005701424272818456, + "loss": 1.3369, + "theoretical_loss": 3.6875741970006883, + "tokens_seen": 971964416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005701263791374123, + "loss": 1.3865, + "theoretical_loss": 3.6875505948566487, + "tokens_seen": 972029952 + }, + { + "epoch": 0.29, + "learning_rate": 0.000570110330992979, + "loss": 1.433, + "theoretical_loss": 3.6875269947493834, + "tokens_seen": 972095488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005700942828485457, + "loss": 1.3944, + "theoretical_loss": 3.68750339667858, + "tokens_seen": 972161024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005700782347041124, + "loss": 1.3137, + "theoretical_loss": 3.687479800643925, + "tokens_seen": 972226560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005700621865596791, + "loss": 1.4108, + "theoretical_loss": 3.687456206645106, + "tokens_seen": 972292096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005700461384152458, + "loss": 1.3267, + "theoretical_loss": 3.6874326146818097, + "tokens_seen": 972357632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005700300902708125, + "loss": 1.3732, + "theoretical_loss": 3.687409024753723, + "tokens_seen": 972423168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005700140421263792, + "loss": 1.3185, + "theoretical_loss": 3.6873854368605334, + "tokens_seen": 972488704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005699979939819458, + "loss": 1.4503, + "theoretical_loss": 3.687361851001929, + "tokens_seen": 972554240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005699819458375126, + "loss": 1.4365, + "theoretical_loss": 3.687338267177596, + "tokens_seen": 972619776 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005699658976930792, + "loss": 1.4649, + "theoretical_loss": 3.687314685387223, + "tokens_seen": 972685312 + }, + { + "epoch": 0.29, + "learning_rate": 0.000569949849548646, + "loss": 1.5141, + "theoretical_loss": 3.6872911056304973, + "tokens_seen": 972750848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005699338014042126, + "loss": 1.4192, + "theoretical_loss": 3.687267527907106, + "tokens_seen": 972816384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005699177532597794, + "loss": 1.3984, + "theoretical_loss": 3.687243952216737, + "tokens_seen": 972881920 + }, + { + "epoch": 0.29, + "learning_rate": 0.000569901705115346, + "loss": 1.4687, + "theoretical_loss": 3.6872203785590782, + "tokens_seen": 972947456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005698856569709128, + "loss": 1.3729, + "theoretical_loss": 3.6871968069338177, + "tokens_seen": 973012992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005698696088264794, + "loss": 1.2914, + "theoretical_loss": 3.6871732373406436, + "tokens_seen": 973078528 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005698535606820461, + "loss": 1.4988, + "theoretical_loss": 3.6871496697792425, + "tokens_seen": 973144064 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005698375125376128, + "loss": 1.4126, + "theoretical_loss": 3.6871261042493035, + "tokens_seen": 973209600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005698214643931795, + "loss": 1.5242, + "theoretical_loss": 3.687102540750515, + "tokens_seen": 973275136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005698054162487462, + "loss": 1.4767, + "theoretical_loss": 3.687078979282564, + "tokens_seen": 973340672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005697893681043129, + "loss": 1.3817, + "theoretical_loss": 3.6870554198451404, + "tokens_seen": 973406208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0005697733199598796, + "loss": 1.4312, + "theoretical_loss": 3.687031862437931, + "tokens_seen": 973471744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005697572718154463, + "loss": 1.4589, + "theoretical_loss": 3.687008307060625, + "tokens_seen": 973537280 + }, + { + "epoch": 0.3, + "learning_rate": 0.000569741223671013, + "loss": 1.4157, + "theoretical_loss": 3.6869847537129106, + "tokens_seen": 973602816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005697251755265797, + "loss": 1.3198, + "theoretical_loss": 3.6869612023944764, + "tokens_seen": 973668352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005697091273821464, + "loss": 1.3223, + "theoretical_loss": 3.6869376531050113, + "tokens_seen": 973733888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005696930792377131, + "loss": 1.3062, + "theoretical_loss": 3.686914105844203, + "tokens_seen": 973799424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005696770310932798, + "loss": 1.3312, + "theoretical_loss": 3.686890560611741, + "tokens_seen": 973864960 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005696609829488465, + "loss": 1.519, + "theoretical_loss": 3.686867017407314, + "tokens_seen": 973930496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005696449348044132, + "loss": 1.3855, + "theoretical_loss": 3.686843476230611, + "tokens_seen": 973996032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005696288866599799, + "loss": 1.5286, + "theoretical_loss": 3.686819937081321, + "tokens_seen": 974061568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005696128385155467, + "loss": 1.3617, + "theoretical_loss": 3.6867963999591327, + "tokens_seen": 974127104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005695967903711133, + "loss": 1.3921, + "theoretical_loss": 3.686772864863735, + "tokens_seen": 974192640 + }, + { + "epoch": 0.3, + "learning_rate": 0.00056958074222668, + "loss": 1.4824, + "theoretical_loss": 3.6867493317948172, + "tokens_seen": 974258176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005695646940822467, + "loss": 1.473, + "theoretical_loss": 3.686725800752069, + "tokens_seen": 974323712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005695486459378135, + "loss": 1.3966, + "theoretical_loss": 3.6867022717351787, + "tokens_seen": 974389248 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005695325977933801, + "loss": 1.3533, + "theoretical_loss": 3.686678744743837, + "tokens_seen": 974454784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005695165496489469, + "loss": 1.3108, + "theoretical_loss": 3.686655219777732, + "tokens_seen": 974520320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005695005015045135, + "loss": 1.3741, + "theoretical_loss": 3.686631696836554, + "tokens_seen": 974585856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005694844533600803, + "loss": 1.5128, + "theoretical_loss": 3.686608175919992, + "tokens_seen": 974651392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005694684052156469, + "loss": 1.2811, + "theoretical_loss": 3.6865846570277365, + "tokens_seen": 974716928 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005694523570712137, + "loss": 1.3272, + "theoretical_loss": 3.6865611401594762, + "tokens_seen": 974782464 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005694363089267803, + "loss": 1.351, + "theoretical_loss": 3.6865376253149016, + "tokens_seen": 974848000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005694202607823471, + "loss": 1.3605, + "theoretical_loss": 3.6865141124937013, + "tokens_seen": 974913536 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005694042126379137, + "loss": 1.4189, + "theoretical_loss": 3.686490601695567, + "tokens_seen": 974979072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005693881644934805, + "loss": 1.3011, + "theoretical_loss": 3.6864670929201875, + "tokens_seen": 975044608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005693721163490471, + "loss": 1.4103, + "theoretical_loss": 3.686443586167253, + "tokens_seen": 975110144 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005693560682046139, + "loss": 1.4282, + "theoretical_loss": 3.686420081436454, + "tokens_seen": 975175680 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005693400200601806, + "loss": 1.3302, + "theoretical_loss": 3.6863965787274804, + "tokens_seen": 975241216 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005693239719157473, + "loss": 1.4039, + "theoretical_loss": 3.686373078040022, + "tokens_seen": 975306752 + }, + { + "epoch": 0.3, + "learning_rate": 0.000569307923771314, + "loss": 1.4326, + "theoretical_loss": 3.6863495793737697, + "tokens_seen": 975372288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005692918756268807, + "loss": 1.3622, + "theoretical_loss": 3.6863260827284137, + "tokens_seen": 975437824 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005692758274824474, + "loss": 1.5491, + "theoretical_loss": 3.6863025881036444, + "tokens_seen": 975503360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005692597793380141, + "loss": 1.3649, + "theoretical_loss": 3.6862790954991524, + "tokens_seen": 975568896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005692437311935808, + "loss": 1.4103, + "theoretical_loss": 3.686255604914628, + "tokens_seen": 975634432 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005692276830491475, + "loss": 1.3988, + "theoretical_loss": 3.6862321163497622, + "tokens_seen": 975699968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005692116349047142, + "loss": 1.3427, + "theoretical_loss": 3.686208629804246, + "tokens_seen": 975765504 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005691955867602809, + "loss": 1.3823, + "theoretical_loss": 3.686185145277769, + "tokens_seen": 975831040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005691795386158476, + "loss": 1.3962, + "theoretical_loss": 3.686161662770023, + "tokens_seen": 975896576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005691634904714143, + "loss": 1.323, + "theoretical_loss": 3.686138182280699, + "tokens_seen": 975962112 + }, + { + "epoch": 0.3, + "learning_rate": 0.000569147442326981, + "loss": 1.2776, + "theoretical_loss": 3.686114703809488, + "tokens_seen": 976027648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005691313941825477, + "loss": 1.4301, + "theoretical_loss": 3.68609122735608, + "tokens_seen": 976093184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005691153460381144, + "loss": 1.5376, + "theoretical_loss": 3.6860677529201675, + "tokens_seen": 976158720 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690992978936811, + "loss": 1.3533, + "theoretical_loss": 3.686044280501441, + "tokens_seen": 976224256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690832497492478, + "loss": 1.4948, + "theoretical_loss": 3.686020810099592, + "tokens_seen": 976289792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690672016048145, + "loss": 1.4524, + "theoretical_loss": 3.6859973417143115, + "tokens_seen": 976355328 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690511534603812, + "loss": 1.4617, + "theoretical_loss": 3.685973875345291, + "tokens_seen": 976420864 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 366434, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4689242839813232, + "objective/train/theoretical_loss": 3.6859504109922225, + "objective/train/tokens_used": 996946400, + "theoretical_loss": 3.6859504109922225, + "tokens_seen": 976486400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690351053159479, + "loss": 1.4689, + "theoretical_loss": 3.6859504109922225, + "tokens_seen": 976486400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690190571715146, + "loss": 1.3831, + "theoretical_loss": 3.6859269486547968, + "tokens_seen": 976551936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005690030090270813, + "loss": 1.3691, + "theoretical_loss": 3.6859034883327055, + "tokens_seen": 976617472 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568986960882648, + "loss": 1.438, + "theoretical_loss": 3.6858800300256407, + "tokens_seen": 976683008 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005689709127382147, + "loss": 1.3607, + "theoretical_loss": 3.685856573733294, + "tokens_seen": 976748544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005689548645937814, + "loss": 1.3703, + "theoretical_loss": 3.6858331194553573, + "tokens_seen": 976814080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005689388164493481, + "loss": 1.3663, + "theoretical_loss": 3.6858096671915224, + "tokens_seen": 976879616 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005689227683049148, + "loss": 1.3576, + "theoretical_loss": 3.685786216941481, + "tokens_seen": 976945152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005689067201604815, + "loss": 1.4888, + "theoretical_loss": 3.6857627687049255, + "tokens_seen": 977010688 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005688906720160482, + "loss": 1.4965, + "theoretical_loss": 3.6857393224815476, + "tokens_seen": 977076224 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005688746238716149, + "loss": 1.3641, + "theoretical_loss": 3.68571587827104, + "tokens_seen": 977141760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005688585757271816, + "loss": 1.3992, + "theoretical_loss": 3.6856924360730936, + "tokens_seen": 977207296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005688425275827483, + "loss": 1.3367, + "theoretical_loss": 3.6856689958874025, + "tokens_seen": 977272832 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568826479438315, + "loss": 1.488, + "theoretical_loss": 3.6856455577136584, + "tokens_seen": 977338368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005688104312938817, + "loss": 1.4263, + "theoretical_loss": 3.6856221215515528, + "tokens_seen": 977403904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005687943831494484, + "loss": 1.3662, + "theoretical_loss": 3.685598687400779, + "tokens_seen": 977469440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005687783350050151, + "loss": 1.368, + "theoretical_loss": 3.6855752552610292, + "tokens_seen": 977534976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005687622868605818, + "loss": 1.3966, + "theoretical_loss": 3.6855518251319968, + "tokens_seen": 977600512 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005687462387161485, + "loss": 1.4399, + "theoretical_loss": 3.6855283970133734, + "tokens_seen": 977666048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005687301905717152, + "loss": 1.3549, + "theoretical_loss": 3.6855049709048524, + "tokens_seen": 977731584 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005687141424272819, + "loss": 1.448, + "theoretical_loss": 3.685481546806127, + "tokens_seen": 977797120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005686980942828486, + "loss": 1.3973, + "theoretical_loss": 3.685458124716889, + "tokens_seen": 977862656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005686820461384153, + "loss": 1.3912, + "theoretical_loss": 3.6854347046368314, + "tokens_seen": 977928192 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568665997993982, + "loss": 1.3634, + "theoretical_loss": 3.6854112865656488, + "tokens_seen": 977993728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005686499498495487, + "loss": 1.446, + "theoretical_loss": 3.6853878705030327, + "tokens_seen": 978059264 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005686339017051154, + "loss": 1.4515, + "theoretical_loss": 3.685364456448677, + "tokens_seen": 978124800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005686178535606821, + "loss": 1.3596, + "theoretical_loss": 3.685341044402274, + "tokens_seen": 978190336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005686018054162487, + "loss": 1.3376, + "theoretical_loss": 3.6853176343635186, + "tokens_seen": 978255872 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005685857572718155, + "loss": 1.4272, + "theoretical_loss": 3.6852942263321022, + "tokens_seen": 978321408 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005685697091273821, + "loss": 1.3644, + "theoretical_loss": 3.6852708203077205, + "tokens_seen": 978386944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005685536609829489, + "loss": 1.3531, + "theoretical_loss": 3.685247416290065, + "tokens_seen": 978452480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005685376128385155, + "loss": 1.4246, + "theoretical_loss": 3.6852240142788304, + "tokens_seen": 978518016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005685215646940823, + "loss": 1.4445, + "theoretical_loss": 3.6852006142737093, + "tokens_seen": 978583552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005685055165496489, + "loss": 1.3904, + "theoretical_loss": 3.685177216274397, + "tokens_seen": 978649088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005684894684052157, + "loss": 1.3756, + "theoretical_loss": 3.6851538202805854, + "tokens_seen": 978714624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005684734202607823, + "loss": 1.2768, + "theoretical_loss": 3.6851304262919697, + "tokens_seen": 978780160 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568457372116349, + "loss": 1.465, + "theoretical_loss": 3.6851070343082433, + "tokens_seen": 978845696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005684413239719157, + "loss": 1.3819, + "theoretical_loss": 3.6850836443291004, + "tokens_seen": 978911232 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005684252758274824, + "loss": 1.2772, + "theoretical_loss": 3.6850602563542347, + "tokens_seen": 978976768 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005684092276830491, + "loss": 1.4044, + "theoretical_loss": 3.6850368703833403, + "tokens_seen": 979042304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005683931795386158, + "loss": 1.4297, + "theoretical_loss": 3.6850134864161115, + "tokens_seen": 979107840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005683771313941825, + "loss": 1.3973, + "theoretical_loss": 3.684990104452243, + "tokens_seen": 979173376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005683610832497492, + "loss": 1.3082, + "theoretical_loss": 3.684966724491428, + "tokens_seen": 979238912 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568345035105316, + "loss": 1.4373, + "theoretical_loss": 3.6849433465333616, + "tokens_seen": 979304448 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005683289869608826, + "loss": 1.3997, + "theoretical_loss": 3.6849199705777385, + "tokens_seen": 979369984 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005683129388164493, + "loss": 1.3155, + "theoretical_loss": 3.6848965966242524, + "tokens_seen": 979435520 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568296890672016, + "loss": 1.3185, + "theoretical_loss": 3.684873224672599, + "tokens_seen": 979501056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005682808425275827, + "loss": 1.3883, + "theoretical_loss": 3.6848498547224717, + "tokens_seen": 979566592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005682647943831494, + "loss": 1.3754, + "theoretical_loss": 3.6848264867735656, + "tokens_seen": 979632128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005682487462387162, + "loss": 1.3956, + "theoretical_loss": 3.684803120825576, + "tokens_seen": 979697664 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005682326980942828, + "loss": 1.4364, + "theoretical_loss": 3.6847797568781973, + "tokens_seen": 979763200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005682166499498496, + "loss": 1.3204, + "theoretical_loss": 3.6847563949311244, + "tokens_seen": 979828736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005682006018054162, + "loss": 1.413, + "theoretical_loss": 3.684733034984052, + "tokens_seen": 979894272 + }, + { + "epoch": 0.3, + "learning_rate": 0.000568184553660983, + "loss": 1.4974, + "theoretical_loss": 3.684709677036676, + "tokens_seen": 979959808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005681685055165496, + "loss": 1.4135, + "theoretical_loss": 3.6846863210886904, + "tokens_seen": 980025344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005681524573721164, + "loss": 1.3198, + "theoretical_loss": 3.6846629671397912, + "tokens_seen": 980090880 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005681364092276831, + "loss": 1.4907, + "theoretical_loss": 3.684639615189673, + "tokens_seen": 980156416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005681203610832498, + "loss": 1.433, + "theoretical_loss": 3.6846162652380325, + "tokens_seen": 980221952 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005681043129388165, + "loss": 1.3846, + "theoretical_loss": 3.6845929172845633, + "tokens_seen": 980287488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005680882647943832, + "loss": 1.5346, + "theoretical_loss": 3.6845695713289617, + "tokens_seen": 980353024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005680722166499499, + "loss": 1.4124, + "theoretical_loss": 3.684546227370923, + "tokens_seen": 980418560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005680561685055166, + "loss": 1.3862, + "theoretical_loss": 3.684522885410143, + "tokens_seen": 980484096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005680401203610833, + "loss": 1.4209, + "theoretical_loss": 3.684499545446317, + "tokens_seen": 980549632 + }, + { + "epoch": 0.3, + "learning_rate": 0.00056802407221665, + "loss": 1.3631, + "theoretical_loss": 3.684476207479141, + "tokens_seen": 980615168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005680080240722167, + "loss": 1.5128, + "theoretical_loss": 3.6844528715083107, + "tokens_seen": 980680704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005679919759277834, + "loss": 1.3044, + "theoretical_loss": 3.6844295375335223, + "tokens_seen": 980746240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005679759277833501, + "loss": 1.3577, + "theoretical_loss": 3.684406205554471, + "tokens_seen": 980811776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005679598796389168, + "loss": 1.4395, + "theoretical_loss": 3.6843828755708525, + "tokens_seen": 980877312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005679438314944835, + "loss": 1.3823, + "theoretical_loss": 3.684359547582364, + "tokens_seen": 980942848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005679277833500502, + "loss": 1.3867, + "theoretical_loss": 3.684336221588701, + "tokens_seen": 981008384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005679117352056169, + "loss": 1.3254, + "theoretical_loss": 3.6843128975895603, + "tokens_seen": 981073920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005678956870611836, + "loss": 1.45, + "theoretical_loss": 3.6842895755846365, + "tokens_seen": 981139456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005678796389167503, + "loss": 1.4239, + "theoretical_loss": 3.684266255573627, + "tokens_seen": 981204992 + }, + { + "epoch": 0.3, + "learning_rate": 0.000567863590772317, + "loss": 1.3867, + "theoretical_loss": 3.684242937556229, + "tokens_seen": 981270528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005678475426278837, + "loss": 1.3588, + "theoretical_loss": 3.6842196215321374, + "tokens_seen": 981336064 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005678314944834504, + "loss": 1.3274, + "theoretical_loss": 3.6841963075010495, + "tokens_seen": 981401600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005678154463390171, + "loss": 1.4794, + "theoretical_loss": 3.684172995462662, + "tokens_seen": 981467136 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005677993981945838, + "loss": 1.4074, + "theoretical_loss": 3.6841496854166706, + "tokens_seen": 981532672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005677833500501505, + "loss": 1.4479, + "theoretical_loss": 3.6841263773627735, + "tokens_seen": 981598208 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005677673019057172, + "loss": 1.3507, + "theoretical_loss": 3.684103071300666, + "tokens_seen": 981663744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005677512537612839, + "loss": 1.319, + "theoretical_loss": 3.6840797672300463, + "tokens_seen": 981729280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005677352056168506, + "loss": 1.393, + "theoretical_loss": 3.6840564651506096, + "tokens_seen": 981794816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005677191574724173, + "loss": 1.3384, + "theoretical_loss": 3.684033165062055, + "tokens_seen": 981860352 + }, + { + "epoch": 0.3, + "learning_rate": 0.000567703109327984, + "loss": 1.3219, + "theoretical_loss": 3.6840098669640775, + "tokens_seen": 981925888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005676870611835507, + "loss": 1.3944, + "theoretical_loss": 3.6839865708563755, + "tokens_seen": 981991424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005676710130391174, + "loss": 1.2895, + "theoretical_loss": 3.6839632767386457, + "tokens_seen": 982056960 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005676549648946841, + "loss": 1.4411, + "theoretical_loss": 3.6839399846105847, + "tokens_seen": 982122496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005676389167502508, + "loss": 1.4159, + "theoretical_loss": 3.6839166944718915, + "tokens_seen": 982188032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005676228686058175, + "loss": 1.3841, + "theoretical_loss": 3.683893406322262, + "tokens_seen": 982253568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005676068204613842, + "loss": 1.4488, + "theoretical_loss": 3.6838701201613944, + "tokens_seen": 982319104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005675907723169509, + "loss": 1.3848, + "theoretical_loss": 3.6838468359889855, + "tokens_seen": 982384640 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005675747241725176, + "loss": 1.4253, + "theoretical_loss": 3.683823553804733, + "tokens_seen": 982450176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005675586760280843, + "loss": 1.5543, + "theoretical_loss": 3.6838002736083353, + "tokens_seen": 982515712 + }, + { + "epoch": 0.3, + "learning_rate": 0.000567542627883651, + "loss": 1.4146, + "theoretical_loss": 3.6837769953994894, + "tokens_seen": 982581248 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005675265797392177, + "loss": 1.4161, + "theoretical_loss": 3.6837537191778935, + "tokens_seen": 982646784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005675105315947844, + "loss": 1.3216, + "theoretical_loss": 3.6837304449432446, + "tokens_seen": 982712320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005674944834503511, + "loss": 1.37, + "theoretical_loss": 3.683707172695242, + "tokens_seen": 982777856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005674784353059178, + "loss": 1.3395, + "theoretical_loss": 3.6836839024335823, + "tokens_seen": 982843392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005674623871614845, + "loss": 1.4485, + "theoretical_loss": 3.683660634157964, + "tokens_seen": 982908928 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005674463390170512, + "loss": 1.3134, + "theoretical_loss": 3.6836373678680854, + "tokens_seen": 982974464 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 368773, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4319138526916504, + "objective/train/theoretical_loss": 3.6836141035636447, + "objective/train/tokens_used": 1003500000, + "theoretical_loss": 3.6836141035636447, + "tokens_seen": 983040000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005674302908726179, + "loss": 1.4319, + "theoretical_loss": 3.6836141035636447, + "tokens_seen": 983040000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005674142427281846, + "loss": 1.3287, + "theoretical_loss": 3.6835908412443397, + "tokens_seen": 983105536 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005673981945837513, + "loss": 1.3194, + "theoretical_loss": 3.683567580909869, + "tokens_seen": 983171072 + }, + { + "epoch": 0.3, + "learning_rate": 0.000567382146439318, + "loss": 1.333, + "theoretical_loss": 3.683544322559931, + "tokens_seen": 983236608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005673660982948847, + "loss": 1.4202, + "theoretical_loss": 3.683521066194224, + "tokens_seen": 983302144 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005673500501504514, + "loss": 1.4506, + "theoretical_loss": 3.6834978118124466, + "tokens_seen": 983367680 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005673340020060181, + "loss": 1.4448, + "theoretical_loss": 3.683474559414297, + "tokens_seen": 983433216 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005673179538615848, + "loss": 1.4445, + "theoretical_loss": 3.6834513089994743, + "tokens_seen": 983498752 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005673019057171515, + "loss": 1.3687, + "theoretical_loss": 3.683428060567677, + "tokens_seen": 983564288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005672858575727182, + "loss": 1.3049, + "theoretical_loss": 3.6834048141186035, + "tokens_seen": 983629824 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005672698094282849, + "loss": 1.3188, + "theoretical_loss": 3.683381569651953, + "tokens_seen": 983695360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005672537612838516, + "loss": 1.4008, + "theoretical_loss": 3.683358327167425, + "tokens_seen": 983760896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005672377131394183, + "loss": 1.3657, + "theoretical_loss": 3.683335086664717, + "tokens_seen": 983826432 + }, + { + "epoch": 0.3, + "learning_rate": 0.000567221664994985, + "loss": 1.4723, + "theoretical_loss": 3.6833118481435294, + "tokens_seen": 983891968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005672056168505516, + "loss": 1.4164, + "theoretical_loss": 3.6832886116035604, + "tokens_seen": 983957504 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005671895687061184, + "loss": 1.3849, + "theoretical_loss": 3.6832653770445094, + "tokens_seen": 984023040 + }, + { + "epoch": 0.3, + "learning_rate": 0.000567173520561685, + "loss": 1.4216, + "theoretical_loss": 3.6832421444660755, + "tokens_seen": 984088576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005671574724172518, + "loss": 1.4801, + "theoretical_loss": 3.6832189138679583, + "tokens_seen": 984154112 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005671414242728184, + "loss": 1.5069, + "theoretical_loss": 3.6831956852498573, + "tokens_seen": 984219648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005671253761283852, + "loss": 1.2926, + "theoretical_loss": 3.6831724586114714, + "tokens_seen": 984285184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005671093279839518, + "loss": 1.3056, + "theoretical_loss": 3.6831492339525, + "tokens_seen": 984350720 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005670932798395186, + "loss": 1.374, + "theoretical_loss": 3.683126011272643, + "tokens_seen": 984416256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005670772316950852, + "loss": 1.38, + "theoretical_loss": 3.6831027905716, + "tokens_seen": 984481792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005670611835506519, + "loss": 1.4815, + "theoretical_loss": 3.6830795718490705, + "tokens_seen": 984547328 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005670451354062186, + "loss": 1.4786, + "theoretical_loss": 3.683056355104754, + "tokens_seen": 984612864 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005670290872617853, + "loss": 1.4243, + "theoretical_loss": 3.683033140338351, + "tokens_seen": 984678400 + }, + { + "epoch": 0.3, + "learning_rate": 0.000567013039117352, + "loss": 1.3716, + "theoretical_loss": 3.6830099275495614, + "tokens_seen": 984743936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005669969909729187, + "loss": 1.3837, + "theoretical_loss": 3.6829867167380845, + "tokens_seen": 984809472 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005669809428284854, + "loss": 1.3144, + "theoretical_loss": 3.6829635079036205, + "tokens_seen": 984875008 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005669648946840522, + "loss": 1.3584, + "theoretical_loss": 3.682940301045869, + "tokens_seen": 984940544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005669488465396188, + "loss": 1.3517, + "theoretical_loss": 3.6829170961645312, + "tokens_seen": 985006080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005669327983951856, + "loss": 1.3695, + "theoretical_loss": 3.6828938932593065, + "tokens_seen": 985071616 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005669167502507522, + "loss": 1.3824, + "theoretical_loss": 3.6828706923298955, + "tokens_seen": 985137152 + }, + { + "epoch": 0.3, + "learning_rate": 0.000566900702106319, + "loss": 1.3438, + "theoretical_loss": 3.6828474933759985, + "tokens_seen": 985202688 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005668846539618856, + "loss": 1.3485, + "theoretical_loss": 3.6828242963973157, + "tokens_seen": 985268224 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005668686058174524, + "loss": 1.5045, + "theoretical_loss": 3.6828011013935473, + "tokens_seen": 985333760 + }, + { + "epoch": 0.3, + "learning_rate": 0.000566852557673019, + "loss": 1.5064, + "theoretical_loss": 3.682777908364395, + "tokens_seen": 985399296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005668365095285858, + "loss": 1.3931, + "theoretical_loss": 3.682754717309558, + "tokens_seen": 985464832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005668204613841525, + "loss": 1.3264, + "theoretical_loss": 3.6827315282287376, + "tokens_seen": 985530368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005668044132397192, + "loss": 1.3188, + "theoretical_loss": 3.682708341121635, + "tokens_seen": 985595904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005667883650952859, + "loss": 1.4915, + "theoretical_loss": 3.68268515598795, + "tokens_seen": 985661440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005667723169508526, + "loss": 1.4104, + "theoretical_loss": 3.6826619728273835, + "tokens_seen": 985726976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005667562688064193, + "loss": 1.3974, + "theoretical_loss": 3.6826387916396373, + "tokens_seen": 985792512 + }, + { + "epoch": 0.3, + "learning_rate": 0.000566740220661986, + "loss": 1.3717, + "theoretical_loss": 3.6826156124244123, + "tokens_seen": 985858048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005667241725175527, + "loss": 1.3291, + "theoretical_loss": 3.6825924351814088, + "tokens_seen": 985923584 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005667081243731194, + "loss": 1.4469, + "theoretical_loss": 3.682569259910328, + "tokens_seen": 985989120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005666920762286861, + "loss": 1.4357, + "theoretical_loss": 3.6825460866108717, + "tokens_seen": 986054656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005666760280842528, + "loss": 1.382, + "theoretical_loss": 3.6825229152827403, + "tokens_seen": 986120192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005666599799398195, + "loss": 1.4224, + "theoretical_loss": 3.682499745925636, + "tokens_seen": 986185728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005666439317953862, + "loss": 1.3853, + "theoretical_loss": 3.6824765785392595, + "tokens_seen": 986251264 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005666278836509529, + "loss": 1.4249, + "theoretical_loss": 3.682453413123313, + "tokens_seen": 986316800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005666118355065196, + "loss": 1.4045, + "theoretical_loss": 3.682430249677497, + "tokens_seen": 986382336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005665957873620863, + "loss": 1.3792, + "theoretical_loss": 3.682407088201514, + "tokens_seen": 986447872 + }, + { + "epoch": 0.3, + "learning_rate": 0.000566579739217653, + "loss": 1.419, + "theoretical_loss": 3.6823839286950646, + "tokens_seen": 986513408 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005665636910732197, + "loss": 1.4543, + "theoretical_loss": 3.6823607711578514, + "tokens_seen": 986578944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005665476429287864, + "loss": 1.4273, + "theoretical_loss": 3.682337615589576, + "tokens_seen": 986644480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005665315947843531, + "loss": 1.3971, + "theoretical_loss": 3.68231446198994, + "tokens_seen": 986710016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005665155466399198, + "loss": 1.4037, + "theoretical_loss": 3.682291310358645, + "tokens_seen": 986775552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005664994984954865, + "loss": 1.392, + "theoretical_loss": 3.682268160695394, + "tokens_seen": 986841088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005664834503510532, + "loss": 1.3844, + "theoretical_loss": 3.6822450129998883, + "tokens_seen": 986906624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005664674022066199, + "loss": 1.3261, + "theoretical_loss": 3.6822218672718297, + "tokens_seen": 986972160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005664513540621866, + "loss": 1.4472, + "theoretical_loss": 3.6821987235109206, + "tokens_seen": 987037696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005664353059177533, + "loss": 1.4583, + "theoretical_loss": 3.6821755817168635, + "tokens_seen": 987103232 + }, + { + "epoch": 0.3, + "learning_rate": 0.00056641925777332, + "loss": 1.3625, + "theoretical_loss": 3.6821524418893605, + "tokens_seen": 987168768 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005664032096288867, + "loss": 1.3721, + "theoretical_loss": 3.682129304028114, + "tokens_seen": 987234304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005663871614844534, + "loss": 1.2728, + "theoretical_loss": 3.682106168132826, + "tokens_seen": 987299840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005663711133400201, + "loss": 1.2935, + "theoretical_loss": 3.6820830342032, + "tokens_seen": 987365376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005663550651955868, + "loss": 1.4422, + "theoretical_loss": 3.6820599022389375, + "tokens_seen": 987430912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005663390170511535, + "loss": 1.3921, + "theoretical_loss": 3.6820367722397416, + "tokens_seen": 987496448 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005663229689067202, + "loss": 1.4471, + "theoretical_loss": 3.6820136442053144, + "tokens_seen": 987561984 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005663069207622869, + "loss": 1.4636, + "theoretical_loss": 3.6819905181353594, + "tokens_seen": 987627520 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005662908726178536, + "loss": 1.4044, + "theoretical_loss": 3.6819673940295794, + "tokens_seen": 987693056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005662748244734203, + "loss": 1.338, + "theoretical_loss": 3.6819442718876765, + "tokens_seen": 987758592 + }, + { + "epoch": 0.3, + "learning_rate": 0.000566258776328987, + "loss": 1.4567, + "theoretical_loss": 3.6819211517093544, + "tokens_seen": 987824128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005662427281845537, + "loss": 1.3777, + "theoretical_loss": 3.6818980334943157, + "tokens_seen": 987889664 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005662266800401204, + "loss": 1.3896, + "theoretical_loss": 3.681874917242263, + "tokens_seen": 987955200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005662106318956871, + "loss": 1.4392, + "theoretical_loss": 3.6818518029529006, + "tokens_seen": 988020736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005661945837512538, + "loss": 1.369, + "theoretical_loss": 3.681828690625931, + "tokens_seen": 988086272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005661785356068205, + "loss": 1.3991, + "theoretical_loss": 3.681805580261057, + "tokens_seen": 988151808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005661624874623872, + "loss": 1.3938, + "theoretical_loss": 3.681782471857983, + "tokens_seen": 988217344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005661464393179539, + "loss": 1.3921, + "theoretical_loss": 3.6817593654164114, + "tokens_seen": 988282880 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005661303911735206, + "loss": 1.4356, + "theoretical_loss": 3.681736260936046, + "tokens_seen": 988348416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005661143430290873, + "loss": 1.3917, + "theoretical_loss": 3.6817131584165907, + "tokens_seen": 988413952 + }, + { + "epoch": 0.3, + "learning_rate": 0.000566098294884654, + "loss": 1.3511, + "theoretical_loss": 3.6816900578577485, + "tokens_seen": 988479488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005660822467402207, + "loss": 1.4025, + "theoretical_loss": 3.681666959259223, + "tokens_seen": 988545024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005660661985957874, + "loss": 1.3948, + "theoretical_loss": 3.681643862620719, + "tokens_seen": 988610560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005660501504513541, + "loss": 1.4219, + "theoretical_loss": 3.681620767941939, + "tokens_seen": 988676096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005660341023069208, + "loss": 1.3477, + "theoretical_loss": 3.6815976752225867, + "tokens_seen": 988741632 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005660180541624875, + "loss": 1.3849, + "theoretical_loss": 3.681574584462367, + "tokens_seen": 988807168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005660020060180542, + "loss": 1.3556, + "theoretical_loss": 3.6815514956609836, + "tokens_seen": 988872704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005659859578736209, + "loss": 1.3723, + "theoretical_loss": 3.6815284088181404, + "tokens_seen": 988938240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005659699097291876, + "loss": 1.3773, + "theoretical_loss": 3.681505323933541, + "tokens_seen": 989003776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005659538615847543, + "loss": 1.3727, + "theoretical_loss": 3.68148224100689, + "tokens_seen": 989069312 + }, + { + "epoch": 0.3, + "learning_rate": 0.000565937813440321, + "loss": 1.3869, + "theoretical_loss": 3.681459160037892, + "tokens_seen": 989134848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005659217652958877, + "loss": 1.3141, + "theoretical_loss": 3.6814360810262508, + "tokens_seen": 989200384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005659057171514544, + "loss": 1.4208, + "theoretical_loss": 3.681413003971671, + "tokens_seen": 989265920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005658896690070211, + "loss": 1.4796, + "theoretical_loss": 3.6813899288738563, + "tokens_seen": 989331456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005658736208625878, + "loss": 1.3821, + "theoretical_loss": 3.681366855732512, + "tokens_seen": 989396992 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005658575727181545, + "loss": 1.3328, + "theoretical_loss": 3.6813437845473427, + "tokens_seen": 989462528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005658415245737212, + "loss": 1.3366, + "theoretical_loss": 3.6813207153180523, + "tokens_seen": 989528064 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 371080, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.372469186782837, + "objective/train/theoretical_loss": 3.681297648044346, + "objective/train/tokens_used": 1010053600, + "theoretical_loss": 3.681297648044346, + "tokens_seen": 989593600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005658254764292879, + "loss": 1.3725, + "theoretical_loss": 3.681297648044346, + "tokens_seen": 989593600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005658094282848546, + "loss": 1.3686, + "theoretical_loss": 3.681274582725928, + "tokens_seen": 989659136 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005657933801404213, + "loss": 1.4577, + "theoretical_loss": 3.681251519362504, + "tokens_seen": 989724672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005657773319959879, + "loss": 1.3614, + "theoretical_loss": 3.681228457953778, + "tokens_seen": 989790208 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005657612838515547, + "loss": 1.4238, + "theoretical_loss": 3.681205398499456, + "tokens_seen": 989855744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005657452357071213, + "loss": 1.3577, + "theoretical_loss": 3.681182340999241, + "tokens_seen": 989921280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005657291875626881, + "loss": 1.3575, + "theoretical_loss": 3.68115928545284, + "tokens_seen": 989986816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005657131394182547, + "loss": 1.3995, + "theoretical_loss": 3.6811362318599574, + "tokens_seen": 990052352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005656970912738215, + "loss": 1.3728, + "theoretical_loss": 3.681113180220298, + "tokens_seen": 990117888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005656810431293881, + "loss": 1.2922, + "theoretical_loss": 3.681090130533568, + "tokens_seen": 990183424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005656649949849549, + "loss": 1.3688, + "theoretical_loss": 3.6810670827994723, + "tokens_seen": 990248960 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005656489468405215, + "loss": 1.4188, + "theoretical_loss": 3.6810440370177155, + "tokens_seen": 990314496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005656328986960883, + "loss": 1.3713, + "theoretical_loss": 3.681020993188004, + "tokens_seen": 990380032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005656168505516549, + "loss": 1.3881, + "theoretical_loss": 3.680997951310043, + "tokens_seen": 990445568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005656008024072217, + "loss": 1.3402, + "theoretical_loss": 3.6809749113835375, + "tokens_seen": 990511104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005655847542627883, + "loss": 1.3906, + "theoretical_loss": 3.680951873408194, + "tokens_seen": 990576640 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005655687061183551, + "loss": 1.4646, + "theoretical_loss": 3.680928837383718, + "tokens_seen": 990642176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005655526579739217, + "loss": 1.3077, + "theoretical_loss": 3.680905803309815, + "tokens_seen": 990707712 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005655366098294885, + "loss": 1.4227, + "theoretical_loss": 3.680882771186191, + "tokens_seen": 990773248 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005655205616850551, + "loss": 1.4269, + "theoretical_loss": 3.6808597410125516, + "tokens_seen": 990838784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005655045135406219, + "loss": 1.376, + "theoretical_loss": 3.680836712788603, + "tokens_seen": 990904320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005654884653961886, + "loss": 1.4804, + "theoretical_loss": 3.6808136865140515, + "tokens_seen": 990969856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005654724172517553, + "loss": 1.3051, + "theoretical_loss": 3.6807906621886026, + "tokens_seen": 991035392 + }, + { + "epoch": 0.3, + "learning_rate": 0.000565456369107322, + "loss": 1.4527, + "theoretical_loss": 3.6807676398119624, + "tokens_seen": 991100928 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005654403209628887, + "loss": 1.4199, + "theoretical_loss": 3.6807446193838373, + "tokens_seen": 991166464 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005654242728184554, + "loss": 1.3762, + "theoretical_loss": 3.6807216009039343, + "tokens_seen": 991232000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005654082246740221, + "loss": 1.3821, + "theoretical_loss": 3.680698584371959, + "tokens_seen": 991297536 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005653921765295888, + "loss": 1.332, + "theoretical_loss": 3.680675569787617, + "tokens_seen": 991363072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005653761283851555, + "loss": 1.3677, + "theoretical_loss": 3.6806525571506166, + "tokens_seen": 991428608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005653600802407222, + "loss": 1.3649, + "theoretical_loss": 3.680629546460663, + "tokens_seen": 991494144 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005653440320962889, + "loss": 1.3438, + "theoretical_loss": 3.680606537717463, + "tokens_seen": 991559680 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005653279839518556, + "loss": 1.3487, + "theoretical_loss": 3.6805835309207238, + "tokens_seen": 991625216 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005653119358074223, + "loss": 1.4473, + "theoretical_loss": 3.680560526070151, + "tokens_seen": 991690752 + }, + { + "epoch": 0.3, + "learning_rate": 0.000565295887662989, + "loss": 1.4143, + "theoretical_loss": 3.6805375231654525, + "tokens_seen": 991756288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005652798395185557, + "loss": 1.4415, + "theoretical_loss": 3.6805145222063347, + "tokens_seen": 991821824 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005652637913741224, + "loss": 1.435, + "theoretical_loss": 3.6804915231925044, + "tokens_seen": 991887360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005652477432296891, + "loss": 1.4669, + "theoretical_loss": 3.680468526123669, + "tokens_seen": 991952896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005652316950852558, + "loss": 1.3896, + "theoretical_loss": 3.6804455309995348, + "tokens_seen": 992018432 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005652156469408225, + "loss": 1.3912, + "theoretical_loss": 3.6804225378198097, + "tokens_seen": 992083968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005651995987963892, + "loss": 1.3819, + "theoretical_loss": 3.6803995465842, + "tokens_seen": 992149504 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005651835506519559, + "loss": 1.3204, + "theoretical_loss": 3.680376557292414, + "tokens_seen": 992215040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005651675025075226, + "loss": 1.3585, + "theoretical_loss": 3.680353569944158, + "tokens_seen": 992280576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005651514543630893, + "loss": 1.3781, + "theoretical_loss": 3.6803305845391394, + "tokens_seen": 992346112 + }, + { + "epoch": 0.3, + "learning_rate": 0.000565135406218656, + "loss": 1.346, + "theoretical_loss": 3.680307601077066, + "tokens_seen": 992411648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005651193580742227, + "loss": 1.4988, + "theoretical_loss": 3.6802846195576455, + "tokens_seen": 992477184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005651033099297894, + "loss": 1.4182, + "theoretical_loss": 3.6802616399805848, + "tokens_seen": 992542720 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005650872617853561, + "loss": 1.3852, + "theoretical_loss": 3.680238662345592, + "tokens_seen": 992608256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005650712136409228, + "loss": 1.3885, + "theoretical_loss": 3.6802156866523745, + "tokens_seen": 992673792 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005650551654964895, + "loss": 1.4994, + "theoretical_loss": 3.6801927129006407, + "tokens_seen": 992739328 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005650391173520562, + "loss": 1.3769, + "theoretical_loss": 3.680169741090097, + "tokens_seen": 992804864 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005650230692076229, + "loss": 1.3439, + "theoretical_loss": 3.6801467712204525, + "tokens_seen": 992870400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005650070210631896, + "loss": 1.4463, + "theoretical_loss": 3.6801238032914148, + "tokens_seen": 992935936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005649909729187563, + "loss": 1.3962, + "theoretical_loss": 3.6801008373026916, + "tokens_seen": 993001472 + }, + { + "epoch": 0.3, + "learning_rate": 0.000564974924774323, + "loss": 1.3651, + "theoretical_loss": 3.6800778732539907, + "tokens_seen": 993067008 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005649588766298897, + "loss": 1.4253, + "theoretical_loss": 3.680054911145021, + "tokens_seen": 993132544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005649428284854564, + "loss": 1.4694, + "theoretical_loss": 3.68003195097549, + "tokens_seen": 993198080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005649267803410231, + "loss": 1.4568, + "theoretical_loss": 3.6800089927451065, + "tokens_seen": 993263616 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005649107321965898, + "loss": 1.4672, + "theoretical_loss": 3.6799860364535784, + "tokens_seen": 993329152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005648946840521565, + "loss": 1.3214, + "theoretical_loss": 3.679963082100614, + "tokens_seen": 993394688 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005648786359077232, + "loss": 1.4669, + "theoretical_loss": 3.6799401296859227, + "tokens_seen": 993460224 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005648625877632899, + "loss": 1.4356, + "theoretical_loss": 3.6799171792092116, + "tokens_seen": 993525760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005648465396188566, + "loss": 1.4013, + "theoretical_loss": 3.6798942306701896, + "tokens_seen": 993591296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005648304914744233, + "loss": 1.4812, + "theoretical_loss": 3.6798712840685655, + "tokens_seen": 993656832 + }, + { + "epoch": 0.3, + "learning_rate": 0.00056481444332999, + "loss": 1.3206, + "theoretical_loss": 3.679848339404048, + "tokens_seen": 993722368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005647983951855567, + "loss": 1.2846, + "theoretical_loss": 3.679825396676346, + "tokens_seen": 993787904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005647823470411234, + "loss": 1.3486, + "theoretical_loss": 3.6798024558851683, + "tokens_seen": 993853440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005647662988966901, + "loss": 1.3364, + "theoretical_loss": 3.679779517030224, + "tokens_seen": 993918976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005647502507522568, + "loss": 1.3113, + "theoretical_loss": 3.6797565801112206, + "tokens_seen": 993984512 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005647342026078235, + "loss": 1.3923, + "theoretical_loss": 3.679733645127869, + "tokens_seen": 994050048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005647181544633902, + "loss": 1.3506, + "theoretical_loss": 3.679710712079877, + "tokens_seen": 994115584 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005647021063189569, + "loss": 1.4256, + "theoretical_loss": 3.679687780966954, + "tokens_seen": 994181120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005646860581745236, + "loss": 1.3518, + "theoretical_loss": 3.679664851788809, + "tokens_seen": 994246656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005646700100300903, + "loss": 1.3419, + "theoretical_loss": 3.6796419245451526, + "tokens_seen": 994312192 + }, + { + "epoch": 0.3, + "learning_rate": 0.000564653961885657, + "loss": 1.4137, + "theoretical_loss": 3.679618999235692, + "tokens_seen": 994377728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005646379137412237, + "loss": 1.456, + "theoretical_loss": 3.679596075860138, + "tokens_seen": 994443264 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005646218655967904, + "loss": 1.3503, + "theoretical_loss": 3.6795731544181995, + "tokens_seen": 994508800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005646058174523571, + "loss": 1.4013, + "theoretical_loss": 3.679550234909586, + "tokens_seen": 994574336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005645897693079238, + "loss": 1.386, + "theoretical_loss": 3.6795273173340073, + "tokens_seen": 994639872 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005645737211634905, + "loss": 1.3557, + "theoretical_loss": 3.6795044016911724, + "tokens_seen": 994705408 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005645576730190573, + "loss": 1.333, + "theoretical_loss": 3.679481487980792, + "tokens_seen": 994770944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005645416248746239, + "loss": 1.3175, + "theoretical_loss": 3.6794585762025753, + "tokens_seen": 994836480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005645255767301906, + "loss": 1.3708, + "theoretical_loss": 3.679435666356232, + "tokens_seen": 994902016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005645095285857573, + "loss": 1.3777, + "theoretical_loss": 3.679412758441472, + "tokens_seen": 994967552 + }, + { + "epoch": 0.3, + "learning_rate": 0.000564493480441324, + "loss": 1.3276, + "theoretical_loss": 3.679389852458005, + "tokens_seen": 995033088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005644774322968907, + "loss": 1.4098, + "theoretical_loss": 3.6793669484055416, + "tokens_seen": 995098624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005644613841524574, + "loss": 1.3736, + "theoretical_loss": 3.6793440462837923, + "tokens_seen": 995164160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005644453360080241, + "loss": 1.3594, + "theoretical_loss": 3.6793211460924655, + "tokens_seen": 995229696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005644292878635908, + "loss": 1.4243, + "theoretical_loss": 3.679298247831272, + "tokens_seen": 995295232 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005644132397191575, + "loss": 1.3515, + "theoretical_loss": 3.679275351499924, + "tokens_seen": 995360768 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005643971915747242, + "loss": 1.4047, + "theoretical_loss": 3.679252457098129, + "tokens_seen": 995426304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005643811434302908, + "loss": 1.3509, + "theoretical_loss": 3.6792295646255986, + "tokens_seen": 995491840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005643650952858576, + "loss": 1.4047, + "theoretical_loss": 3.6792066740820433, + "tokens_seen": 995557376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005643490471414242, + "loss": 1.3948, + "theoretical_loss": 3.679183785467173, + "tokens_seen": 995622912 + }, + { + "epoch": 0.3, + "learning_rate": 0.000564332998996991, + "loss": 1.3983, + "theoretical_loss": 3.6791608987807, + "tokens_seen": 995688448 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005643169508525576, + "loss": 1.4185, + "theoretical_loss": 3.6791380140223326, + "tokens_seen": 995753984 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005643009027081244, + "loss": 1.4169, + "theoretical_loss": 3.679115131191783, + "tokens_seen": 995819520 + }, + { + "epoch": 0.3, + "learning_rate": 0.000564284854563691, + "loss": 1.3641, + "theoretical_loss": 3.6790922502887615, + "tokens_seen": 995885056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005642688064192578, + "loss": 1.3015, + "theoretical_loss": 3.6790693713129787, + "tokens_seen": 995950592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005642527582748244, + "loss": 1.4849, + "theoretical_loss": 3.679046494264146, + "tokens_seen": 996016128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005642367101303912, + "loss": 1.4747, + "theoretical_loss": 3.679023619141974, + "tokens_seen": 996081664 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 373501, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4506340026855469, + "objective/train/theoretical_loss": 3.6790007459461735, + "objective/train/tokens_used": 1016607200, + "theoretical_loss": 3.6790007459461735, + "tokens_seen": 996147200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005642206619859578, + "loss": 1.4506, + "theoretical_loss": 3.6790007459461735, + "tokens_seen": 996147200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005642046138415246, + "loss": 1.3966, + "theoretical_loss": 3.678977874676456, + "tokens_seen": 996212736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005641885656970912, + "loss": 1.3633, + "theoretical_loss": 3.678955005332532, + "tokens_seen": 996278272 + }, + { + "epoch": 0.3, + "learning_rate": 0.000564172517552658, + "loss": 1.5342, + "theoretical_loss": 3.6789321379141136, + "tokens_seen": 996343808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005641564694082246, + "loss": 1.4504, + "theoretical_loss": 3.6789092724209107, + "tokens_seen": 996409344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005641404212637914, + "loss": 1.4973, + "theoretical_loss": 3.6788864088526356, + "tokens_seen": 996474880 + }, + { + "epoch": 0.3, + "learning_rate": 0.000564124373119358, + "loss": 1.3595, + "theoretical_loss": 3.678863547209, + "tokens_seen": 996540416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005641083249749248, + "loss": 1.4087, + "theoretical_loss": 3.6788406874897146, + "tokens_seen": 996605952 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005640922768304915, + "loss": 1.4281, + "theoretical_loss": 3.6788178296944913, + "tokens_seen": 996671488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005640762286860582, + "loss": 1.4515, + "theoretical_loss": 3.6787949738230408, + "tokens_seen": 996737024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005640601805416249, + "loss": 1.3784, + "theoretical_loss": 3.678772119875076, + "tokens_seen": 996802560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005640441323971916, + "loss": 1.4693, + "theoretical_loss": 3.678749267850308, + "tokens_seen": 996868096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005640280842527583, + "loss": 1.3411, + "theoretical_loss": 3.678726417748448, + "tokens_seen": 996933632 + }, + { + "epoch": 0.3, + "learning_rate": 0.000564012036108325, + "loss": 1.4052, + "theoretical_loss": 3.6787035695692083, + "tokens_seen": 996999168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005639959879638917, + "loss": 1.3768, + "theoretical_loss": 3.678680723312301, + "tokens_seen": 997064704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005639799398194584, + "loss": 1.2458, + "theoretical_loss": 3.678657878977438, + "tokens_seen": 997130240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005639638916750251, + "loss": 1.3246, + "theoretical_loss": 3.6786350365643306, + "tokens_seen": 997195776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005639478435305918, + "loss": 1.4523, + "theoretical_loss": 3.6786121960726916, + "tokens_seen": 997261312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005639317953861585, + "loss": 1.3918, + "theoretical_loss": 3.678589357502233, + "tokens_seen": 997326848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005639157472417252, + "loss": 1.4208, + "theoretical_loss": 3.6785665208526668, + "tokens_seen": 997392384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005638996990972919, + "loss": 1.3605, + "theoretical_loss": 3.678543686123705, + "tokens_seen": 997457920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005638836509528586, + "loss": 1.3656, + "theoretical_loss": 3.67852085331506, + "tokens_seen": 997523456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005638676028084253, + "loss": 1.2997, + "theoretical_loss": 3.6784980224264445, + "tokens_seen": 997588992 + }, + { + "epoch": 0.3, + "learning_rate": 0.000563851554663992, + "loss": 1.3147, + "theoretical_loss": 3.6784751934575706, + "tokens_seen": 997654528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005638355065195587, + "loss": 1.332, + "theoretical_loss": 3.6784523664081514, + "tokens_seen": 997720064 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005638194583751254, + "loss": 1.3768, + "theoretical_loss": 3.6784295412778985, + "tokens_seen": 997785600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005638034102306921, + "loss": 1.3263, + "theoretical_loss": 3.678406718066525, + "tokens_seen": 997851136 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005637873620862588, + "loss": 1.3165, + "theoretical_loss": 3.6783838967737443, + "tokens_seen": 997916672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005637713139418255, + "loss": 1.4145, + "theoretical_loss": 3.6783610773992677, + "tokens_seen": 997982208 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005637552657973922, + "loss": 1.3199, + "theoretical_loss": 3.6783382599428087, + "tokens_seen": 998047744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005637392176529589, + "loss": 1.3697, + "theoretical_loss": 3.67831544440408, + "tokens_seen": 998113280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005637231695085256, + "loss": 1.4719, + "theoretical_loss": 3.6782926307827952, + "tokens_seen": 998178816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005637071213640923, + "loss": 1.3809, + "theoretical_loss": 3.6782698190786665, + "tokens_seen": 998244352 + }, + { + "epoch": 0.3, + "learning_rate": 0.000563691073219659, + "loss": 1.4112, + "theoretical_loss": 3.6782470092914075, + "tokens_seen": 998309888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005636750250752257, + "loss": 1.4016, + "theoretical_loss": 3.67822420142073, + "tokens_seen": 998375424 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005636589769307924, + "loss": 1.3359, + "theoretical_loss": 3.678201395466349, + "tokens_seen": 998440960 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005636429287863591, + "loss": 1.5046, + "theoretical_loss": 3.6781785914279768, + "tokens_seen": 998506496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005636268806419258, + "loss": 1.4377, + "theoretical_loss": 3.6781557893053267, + "tokens_seen": 998572032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005636108324974925, + "loss": 1.4868, + "theoretical_loss": 3.6781329890981125, + "tokens_seen": 998637568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005635947843530592, + "loss": 1.4191, + "theoretical_loss": 3.6781101908060467, + "tokens_seen": 998703104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005635787362086259, + "loss": 1.4992, + "theoretical_loss": 3.678087394428843, + "tokens_seen": 998768640 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005635626880641926, + "loss": 1.2911, + "theoretical_loss": 3.6780645999662163, + "tokens_seen": 998834176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005635466399197593, + "loss": 1.4104, + "theoretical_loss": 3.6780418074178782, + "tokens_seen": 998899712 + }, + { + "epoch": 0.3, + "learning_rate": 0.000563530591775326, + "loss": 1.3701, + "theoretical_loss": 3.678019016783544, + "tokens_seen": 998965248 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005635145436308927, + "loss": 1.2773, + "theoretical_loss": 3.677996228062926, + "tokens_seen": 999030784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005634984954864594, + "loss": 1.4505, + "theoretical_loss": 3.677973441255739, + "tokens_seen": 999096320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005634824473420261, + "loss": 1.3537, + "theoretical_loss": 3.6779506563616966, + "tokens_seen": 999161856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005634663991975928, + "loss": 1.5353, + "theoretical_loss": 3.677927873380513, + "tokens_seen": 999227392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005634503510531595, + "loss": 1.4486, + "theoretical_loss": 3.6779050923119017, + "tokens_seen": 999292928 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005634343029087262, + "loss": 1.4098, + "theoretical_loss": 3.6778823131555765, + "tokens_seen": 999358464 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005634182547642929, + "loss": 1.3692, + "theoretical_loss": 3.677859535911252, + "tokens_seen": 999424000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005634022066198596, + "loss": 1.367, + "theoretical_loss": 3.677836760578643, + "tokens_seen": 999489536 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005633861584754263, + "loss": 1.4535, + "theoretical_loss": 3.677813987157462, + "tokens_seen": 999555072 + }, + { + "epoch": 0.3, + "learning_rate": 0.000563370110330993, + "loss": 1.4322, + "theoretical_loss": 3.6777912156474244, + "tokens_seen": 999620608 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005633540621865598, + "loss": 1.4273, + "theoretical_loss": 3.677768446048244, + "tokens_seen": 999686144 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005633380140421264, + "loss": 1.4485, + "theoretical_loss": 3.677745678359636, + "tokens_seen": 999751680 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005633219658976932, + "loss": 1.3294, + "theoretical_loss": 3.6777229125813142, + "tokens_seen": 999817216 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005633059177532598, + "loss": 1.4734, + "theoretical_loss": 3.6777001487129937, + "tokens_seen": 999882752 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005632898696088266, + "loss": 1.3607, + "theoretical_loss": 3.677677386754388, + "tokens_seen": 999948288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005632738214643932, + "loss": 1.453, + "theoretical_loss": 3.677654626705213, + "tokens_seen": 1000013824 + }, + { + "epoch": 0.3, + "learning_rate": 0.00056325777331996, + "loss": 1.4469, + "theoretical_loss": 3.6776318685651823, + "tokens_seen": 1000079360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005632417251755266, + "loss": 1.4138, + "theoretical_loss": 3.6776091123340118, + "tokens_seen": 1000144896 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005632256770310934, + "loss": 1.3747, + "theoretical_loss": 3.6775863580114154, + "tokens_seen": 1000210432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00056320962888666, + "loss": 1.4433, + "theoretical_loss": 3.6775636055971086, + "tokens_seen": 1000275968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005631935807422268, + "loss": 1.3991, + "theoretical_loss": 3.6775408550908057, + "tokens_seen": 1000341504 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005631775325977934, + "loss": 1.4185, + "theoretical_loss": 3.677518106492222, + "tokens_seen": 1000407040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005631614844533602, + "loss": 1.4748, + "theoretical_loss": 3.6774953598010733, + "tokens_seen": 1000472576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005631454363089268, + "loss": 1.4424, + "theoretical_loss": 3.6774726150170736, + "tokens_seen": 1000538112 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005631293881644935, + "loss": 1.3685, + "theoretical_loss": 3.6774498721399387, + "tokens_seen": 1000603648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005631133400200602, + "loss": 1.4993, + "theoretical_loss": 3.6774271311693836, + "tokens_seen": 1000669184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005630972918756269, + "loss": 1.3912, + "theoretical_loss": 3.6774043921051236, + "tokens_seen": 1000734720 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005630812437311936, + "loss": 1.3582, + "theoretical_loss": 3.677381654946875, + "tokens_seen": 1000800256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005630651955867603, + "loss": 1.4274, + "theoretical_loss": 3.6773589196943517, + "tokens_seen": 1000865792 + }, + { + "epoch": 0.3, + "learning_rate": 0.000563049147442327, + "loss": 1.4607, + "theoretical_loss": 3.6773361863472704, + "tokens_seen": 1000931328 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005630330992978937, + "loss": 1.3408, + "theoretical_loss": 3.677313454905346, + "tokens_seen": 1000996864 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005630170511534604, + "loss": 1.3845, + "theoretical_loss": 3.6772907253682945, + "tokens_seen": 1001062400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005630010030090271, + "loss": 1.4952, + "theoretical_loss": 3.6772679977358314, + "tokens_seen": 1001127936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005629849548645937, + "loss": 1.4509, + "theoretical_loss": 3.6772452720076725, + "tokens_seen": 1001193472 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005629689067201605, + "loss": 1.3715, + "theoretical_loss": 3.6772225481835337, + "tokens_seen": 1001259008 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005629528585757271, + "loss": 1.5053, + "theoretical_loss": 3.6771998262631307, + "tokens_seen": 1001324544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005629368104312939, + "loss": 1.3971, + "theoretical_loss": 3.6771771062461793, + "tokens_seen": 1001390080 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005629207622868605, + "loss": 1.3687, + "theoretical_loss": 3.6771543881323963, + "tokens_seen": 1001455616 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005629047141424273, + "loss": 1.4189, + "theoretical_loss": 3.6771316719214964, + "tokens_seen": 1001521152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005628886659979939, + "loss": 1.4022, + "theoretical_loss": 3.677108957613197, + "tokens_seen": 1001586688 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005628726178535607, + "loss": 1.4191, + "theoretical_loss": 3.6770862452072133, + "tokens_seen": 1001652224 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005628565697091273, + "loss": 1.3599, + "theoretical_loss": 3.6770635347032616, + "tokens_seen": 1001717760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005628405215646941, + "loss": 1.4055, + "theoretical_loss": 3.6770408261010594, + "tokens_seen": 1001783296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005628244734202607, + "loss": 1.4113, + "theoretical_loss": 3.677018119400322, + "tokens_seen": 1001848832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005628084252758275, + "loss": 1.4412, + "theoretical_loss": 3.6769954146007655, + "tokens_seen": 1001914368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005627923771313941, + "loss": 1.3943, + "theoretical_loss": 3.676972711702107, + "tokens_seen": 1001979904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005627763289869609, + "loss": 1.4027, + "theoretical_loss": 3.676950010704063, + "tokens_seen": 1002045440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005627602808425276, + "loss": 1.409, + "theoretical_loss": 3.6769273116063506, + "tokens_seen": 1002110976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005627442326980943, + "loss": 1.489, + "theoretical_loss": 3.676904614408685, + "tokens_seen": 1002176512 + }, + { + "epoch": 0.3, + "learning_rate": 0.000562728184553661, + "loss": 1.3667, + "theoretical_loss": 3.6768819191107847, + "tokens_seen": 1002242048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005627121364092277, + "loss": 1.4104, + "theoretical_loss": 3.6768592257123647, + "tokens_seen": 1002307584 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005626960882647944, + "loss": 1.4246, + "theoretical_loss": 3.6768365342131437, + "tokens_seen": 1002373120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005626800401203611, + "loss": 1.5198, + "theoretical_loss": 3.6768138446128367, + "tokens_seen": 1002438656 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005626639919759278, + "loss": 1.4203, + "theoretical_loss": 3.676791156911162, + "tokens_seen": 1002504192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005626479438314945, + "loss": 1.422, + "theoretical_loss": 3.6767684711078363, + "tokens_seen": 1002569728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005626318956870612, + "loss": 1.3475, + "theoretical_loss": 3.6767457872025764, + "tokens_seen": 1002635264 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 375948, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3362542390823364, + "objective/train/theoretical_loss": 3.6767231051950997, + "objective/train/tokens_used": 1023160800, + "theoretical_loss": 3.6767231051950997, + "tokens_seen": 1002700800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005626158475426279, + "loss": 1.3363, + "theoretical_loss": 3.6767231051950997, + "tokens_seen": 1002700800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005625997993981946, + "loss": 1.5263, + "theoretical_loss": 3.676700425085123, + "tokens_seen": 1002766336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005625837512537613, + "loss": 1.391, + "theoretical_loss": 3.6766777468723646, + "tokens_seen": 1002831872 + }, + { + "epoch": 0.3, + "learning_rate": 0.000562567703109328, + "loss": 1.3853, + "theoretical_loss": 3.6766550705565404, + "tokens_seen": 1002897408 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005625516549648947, + "loss": 1.4765, + "theoretical_loss": 3.676632396137369, + "tokens_seen": 1002962944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005625356068204614, + "loss": 1.3205, + "theoretical_loss": 3.676609723614567, + "tokens_seen": 1003028480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005625195586760281, + "loss": 1.3774, + "theoretical_loss": 3.6765870529878524, + "tokens_seen": 1003094016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005625035105315948, + "loss": 1.4815, + "theoretical_loss": 3.6765643842569427, + "tokens_seen": 1003159552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005624874623871615, + "loss": 1.3943, + "theoretical_loss": 3.6765417174215553, + "tokens_seen": 1003225088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005624714142427282, + "loss": 1.425, + "theoretical_loss": 3.6765190524814084, + "tokens_seen": 1003290624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005624553660982949, + "loss": 1.3986, + "theoretical_loss": 3.6764963894362195, + "tokens_seen": 1003356160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005624393179538616, + "loss": 1.3958, + "theoretical_loss": 3.676473728285706, + "tokens_seen": 1003421696 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005624232698094283, + "loss": 1.3748, + "theoretical_loss": 3.676451069029586, + "tokens_seen": 1003487232 + }, + { + "epoch": 0.3, + "learning_rate": 0.000562407221664995, + "loss": 1.3119, + "theoretical_loss": 3.676428411667578, + "tokens_seen": 1003552768 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005623911735205617, + "loss": 1.3313, + "theoretical_loss": 3.676405756199399, + "tokens_seen": 1003618304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005623751253761284, + "loss": 1.4849, + "theoretical_loss": 3.6763831026247678, + "tokens_seen": 1003683840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005623590772316951, + "loss": 1.2802, + "theoretical_loss": 3.6763604509434025, + "tokens_seen": 1003749376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005623430290872618, + "loss": 1.4235, + "theoretical_loss": 3.6763378011550207, + "tokens_seen": 1003814912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005623269809428285, + "loss": 1.4284, + "theoretical_loss": 3.676315153259341, + "tokens_seen": 1003880448 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005623109327983952, + "loss": 1.3183, + "theoretical_loss": 3.6762925072560817, + "tokens_seen": 1003945984 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005622948846539619, + "loss": 1.5055, + "theoretical_loss": 3.6762698631449613, + "tokens_seen": 1004011520 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005622788365095286, + "loss": 1.5214, + "theoretical_loss": 3.676247220925698, + "tokens_seen": 1004077056 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005622627883650953, + "loss": 1.347, + "theoretical_loss": 3.67622458059801, + "tokens_seen": 1004142592 + }, + { + "epoch": 0.3, + "learning_rate": 0.000562246740220662, + "loss": 1.4541, + "theoretical_loss": 3.6762019421616166, + "tokens_seen": 1004208128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005622306920762287, + "loss": 1.2898, + "theoretical_loss": 3.6761793056162357, + "tokens_seen": 1004273664 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005622146439317954, + "loss": 1.4277, + "theoretical_loss": 3.676156670961586, + "tokens_seen": 1004339200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005621985957873621, + "loss": 1.4467, + "theoretical_loss": 3.676134038197387, + "tokens_seen": 1004404736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005621825476429288, + "loss": 1.3886, + "theoretical_loss": 3.6761114073233565, + "tokens_seen": 1004470272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005621664994984955, + "loss": 1.3792, + "theoretical_loss": 3.6760887783392135, + "tokens_seen": 1004535808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005621504513540623, + "loss": 1.3955, + "theoretical_loss": 3.6760661512446777, + "tokens_seen": 1004601344 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005621344032096289, + "loss": 1.3037, + "theoretical_loss": 3.676043526039467, + "tokens_seen": 1004666880 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005621183550651957, + "loss": 1.3091, + "theoretical_loss": 3.6760209027233013, + "tokens_seen": 1004732416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005621023069207623, + "loss": 1.367, + "theoretical_loss": 3.6759982812958993, + "tokens_seen": 1004797952 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005620862587763291, + "loss": 1.3544, + "theoretical_loss": 3.67597566175698, + "tokens_seen": 1004863488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005620702106318957, + "loss": 1.4817, + "theoretical_loss": 3.6759530441062624, + "tokens_seen": 1004929024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005620541624874625, + "loss": 1.5089, + "theoretical_loss": 3.675930428343466, + "tokens_seen": 1004994560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005620381143430291, + "loss": 1.318, + "theoretical_loss": 3.6759078144683106, + "tokens_seen": 1005060096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005620220661985959, + "loss": 1.4168, + "theoretical_loss": 3.675885202480515, + "tokens_seen": 1005125632 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005620060180541625, + "loss": 1.5052, + "theoretical_loss": 3.675862592379799, + "tokens_seen": 1005191168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005619899699097293, + "loss": 1.3678, + "theoretical_loss": 3.6758399841658815, + "tokens_seen": 1005256704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005619739217652959, + "loss": 1.3488, + "theoretical_loss": 3.6758173778384826, + "tokens_seen": 1005322240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005619578736208627, + "loss": 1.2484, + "theoretical_loss": 3.675794773397322, + "tokens_seen": 1005387776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005619418254764293, + "loss": 1.4298, + "theoretical_loss": 3.675772170842119, + "tokens_seen": 1005453312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005619257773319961, + "loss": 1.3874, + "theoretical_loss": 3.6757495701725933, + "tokens_seen": 1005518848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005619097291875627, + "loss": 1.3831, + "theoretical_loss": 3.675726971388465, + "tokens_seen": 1005584384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005618936810431295, + "loss": 1.3625, + "theoretical_loss": 3.6757043744894538, + "tokens_seen": 1005649920 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005618776328986961, + "loss": 1.4073, + "theoretical_loss": 3.67568177947528, + "tokens_seen": 1005715456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005618615847542629, + "loss": 1.3309, + "theoretical_loss": 3.675659186345663, + "tokens_seen": 1005780992 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005618455366098295, + "loss": 1.4094, + "theoretical_loss": 3.6756365951003227, + "tokens_seen": 1005846528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005618294884653963, + "loss": 1.4346, + "theoretical_loss": 3.6756140057389795, + "tokens_seen": 1005912064 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005618134403209629, + "loss": 1.4179, + "theoretical_loss": 3.6755914182613543, + "tokens_seen": 1005977600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005617973921765297, + "loss": 1.4824, + "theoretical_loss": 3.675568832667166, + "tokens_seen": 1006043136 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005617813440320963, + "loss": 1.4319, + "theoretical_loss": 3.6755462489561355, + "tokens_seen": 1006108672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005617652958876631, + "loss": 1.3903, + "theoretical_loss": 3.6755236671279836, + "tokens_seen": 1006174208 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005617492477432297, + "loss": 1.4288, + "theoretical_loss": 3.67550108718243, + "tokens_seen": 1006239744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005617331995987964, + "loss": 1.3976, + "theoretical_loss": 3.6754785091191957, + "tokens_seen": 1006305280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005617171514543631, + "loss": 1.3595, + "theoretical_loss": 3.675455932938, + "tokens_seen": 1006370816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0005617011033099298, + "loss": 1.3876, + "theoretical_loss": 3.675433358638565, + "tokens_seen": 1006436352 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005616850551654965, + "loss": 1.4174, + "theoretical_loss": 3.6754107862206107, + "tokens_seen": 1006501888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005616690070210632, + "loss": 1.2999, + "theoretical_loss": 3.6753882156838578, + "tokens_seen": 1006567424 + }, + { + "epoch": 0.31, + "learning_rate": 0.00056165295887663, + "loss": 1.3531, + "theoretical_loss": 3.675365647028027, + "tokens_seen": 1006632960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005616369107321966, + "loss": 1.3493, + "theoretical_loss": 3.675343080252839, + "tokens_seen": 1006698496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005616208625877633, + "loss": 1.5069, + "theoretical_loss": 3.675320515358015, + "tokens_seen": 1006764032 + }, + { + "epoch": 0.31, + "learning_rate": 0.00056160481444333, + "loss": 1.4113, + "theoretical_loss": 3.675297952343276, + "tokens_seen": 1006829568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005615887662988966, + "loss": 1.3343, + "theoretical_loss": 3.6752753912083422, + "tokens_seen": 1006895104 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005615727181544634, + "loss": 1.3967, + "theoretical_loss": 3.6752528319529354, + "tokens_seen": 1006960640 + }, + { + "epoch": 0.31, + "learning_rate": 0.00056155667001003, + "loss": 1.3392, + "theoretical_loss": 3.675230274576777, + "tokens_seen": 1007026176 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005615406218655968, + "loss": 1.3982, + "theoretical_loss": 3.6752077190795873, + "tokens_seen": 1007091712 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005615245737211634, + "loss": 1.3191, + "theoretical_loss": 3.6751851654610874, + "tokens_seen": 1007157248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005615085255767302, + "loss": 1.4422, + "theoretical_loss": 3.6751626137209996, + "tokens_seen": 1007222784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005614924774322968, + "loss": 1.4126, + "theoretical_loss": 3.675140063859045, + "tokens_seen": 1007288320 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005614764292878636, + "loss": 1.3864, + "theoretical_loss": 3.6751175158749447, + "tokens_seen": 1007353856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005614603811434302, + "loss": 1.4031, + "theoretical_loss": 3.67509496976842, + "tokens_seen": 1007419392 + }, + { + "epoch": 0.31, + "learning_rate": 0.000561444332998997, + "loss": 1.4281, + "theoretical_loss": 3.6750724255391924, + "tokens_seen": 1007484928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005614282848545636, + "loss": 1.3077, + "theoretical_loss": 3.6750498831869844, + "tokens_seen": 1007550464 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005614122367101304, + "loss": 1.3355, + "theoretical_loss": 3.6750273427115165, + "tokens_seen": 1007616000 + }, + { + "epoch": 0.31, + "learning_rate": 0.000561396188565697, + "loss": 1.4451, + "theoretical_loss": 3.675004804112511, + "tokens_seen": 1007681536 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005613801404212638, + "loss": 1.3754, + "theoretical_loss": 3.67498226738969, + "tokens_seen": 1007747072 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005613640922768305, + "loss": 1.3114, + "theoretical_loss": 3.674959732542774, + "tokens_seen": 1007812608 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005613480441323972, + "loss": 1.5092, + "theoretical_loss": 3.6749371995714863, + "tokens_seen": 1007878144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005613319959879639, + "loss": 1.3265, + "theoretical_loss": 3.674914668475548, + "tokens_seen": 1007943680 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005613159478435306, + "loss": 1.4437, + "theoretical_loss": 3.674892139254682, + "tokens_seen": 1008009216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005612998996990973, + "loss": 1.3306, + "theoretical_loss": 3.6748696119086093, + "tokens_seen": 1008074752 + }, + { + "epoch": 0.31, + "learning_rate": 0.000561283851554664, + "loss": 1.3021, + "theoretical_loss": 3.674847086437052, + "tokens_seen": 1008140288 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005612678034102307, + "loss": 1.3292, + "theoretical_loss": 3.6748245628397336, + "tokens_seen": 1008205824 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005612517552657974, + "loss": 1.3223, + "theoretical_loss": 3.6748020411163753, + "tokens_seen": 1008271360 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005612357071213641, + "loss": 1.3689, + "theoretical_loss": 3.674779521266699, + "tokens_seen": 1008336896 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005612196589769308, + "loss": 1.4245, + "theoretical_loss": 3.674757003290429, + "tokens_seen": 1008402432 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005612036108324975, + "loss": 1.4178, + "theoretical_loss": 3.674734487187285, + "tokens_seen": 1008467968 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005611875626880642, + "loss": 1.4782, + "theoretical_loss": 3.6747119729569917, + "tokens_seen": 1008533504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005611715145436309, + "loss": 1.3095, + "theoretical_loss": 3.6746894605992706, + "tokens_seen": 1008599040 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005611554663991976, + "loss": 1.3221, + "theoretical_loss": 3.674666950113844, + "tokens_seen": 1008664576 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005611394182547643, + "loss": 1.4579, + "theoretical_loss": 3.674644441500435, + "tokens_seen": 1008730112 + }, + { + "epoch": 0.31, + "learning_rate": 0.000561123370110331, + "loss": 1.5137, + "theoretical_loss": 3.674621934758767, + "tokens_seen": 1008795648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005611073219658977, + "loss": 1.4436, + "theoretical_loss": 3.674599429888562, + "tokens_seen": 1008861184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005610912738214644, + "loss": 1.3546, + "theoretical_loss": 3.6745769268895425, + "tokens_seen": 1008926720 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005610752256770311, + "loss": 1.4, + "theoretical_loss": 3.674554425761432, + "tokens_seen": 1008992256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005610591775325978, + "loss": 1.4892, + "theoretical_loss": 3.6745319265039535, + "tokens_seen": 1009057792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005610431293881645, + "loss": 1.5369, + "theoretical_loss": 3.6745094291168297, + "tokens_seen": 1009123328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005610270812437312, + "loss": 1.3749, + "theoretical_loss": 3.6744869335997836, + "tokens_seen": 1009188864 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 378457, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5162826776504517, + "objective/train/theoretical_loss": 3.6744644399525384, + "objective/train/tokens_used": 1029714400, + "theoretical_loss": 3.6744644399525384, + "tokens_seen": 1009254400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005610110330992979, + "loss": 1.5163, + "theoretical_loss": 3.6744644399525384, + "tokens_seen": 1009254400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005609949849548646, + "loss": 1.4072, + "theoretical_loss": 3.6744419481748176, + "tokens_seen": 1009319936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005609789368104313, + "loss": 1.4045, + "theoretical_loss": 3.674419458266344, + "tokens_seen": 1009385472 + }, + { + "epoch": 0.31, + "learning_rate": 0.000560962888665998, + "loss": 1.3664, + "theoretical_loss": 3.674396970226841, + "tokens_seen": 1009451008 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005609468405215648, + "loss": 1.4465, + "theoretical_loss": 3.674374484056032, + "tokens_seen": 1009516544 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005609307923771314, + "loss": 1.4346, + "theoretical_loss": 3.6743519997536414, + "tokens_seen": 1009582080 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005609147442326982, + "loss": 1.366, + "theoretical_loss": 3.674329517319391, + "tokens_seen": 1009647616 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005608986960882648, + "loss": 1.3533, + "theoretical_loss": 3.674307036753005, + "tokens_seen": 1009713152 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005608826479438316, + "loss": 1.4079, + "theoretical_loss": 3.6742845580542074, + "tokens_seen": 1009778688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005608665997993982, + "loss": 1.4014, + "theoretical_loss": 3.6742620812227216, + "tokens_seen": 1009844224 + }, + { + "epoch": 0.31, + "learning_rate": 0.000560850551654965, + "loss": 1.3765, + "theoretical_loss": 3.6742396062582703, + "tokens_seen": 1009909760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005608345035105316, + "loss": 1.454, + "theoretical_loss": 3.674217133160579, + "tokens_seen": 1009975296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005608184553660984, + "loss": 1.4336, + "theoretical_loss": 3.6741946619293713, + "tokens_seen": 1010040832 + }, + { + "epoch": 0.31, + "learning_rate": 0.000560802407221665, + "loss": 1.3691, + "theoretical_loss": 3.6741721925643698, + "tokens_seen": 1010106368 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005607863590772318, + "loss": 1.4289, + "theoretical_loss": 3.6741497250652997, + "tokens_seen": 1010171904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005607703109327984, + "loss": 1.3984, + "theoretical_loss": 3.6741272594318843, + "tokens_seen": 1010237440 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005607542627883652, + "loss": 1.3273, + "theoretical_loss": 3.6741047956638475, + "tokens_seen": 1010302976 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005607382146439318, + "loss": 1.3674, + "theoretical_loss": 3.6740823337609143, + "tokens_seen": 1010368512 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005607221664994986, + "loss": 1.416, + "theoretical_loss": 3.6740598737228085, + "tokens_seen": 1010434048 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005607061183550652, + "loss": 1.2699, + "theoretical_loss": 3.674037415549254, + "tokens_seen": 1010499584 + }, + { + "epoch": 0.31, + "learning_rate": 0.000560690070210632, + "loss": 1.4648, + "theoretical_loss": 3.674014959239975, + "tokens_seen": 1010565120 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005606740220661986, + "loss": 1.3774, + "theoretical_loss": 3.6739925047946964, + "tokens_seen": 1010630656 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005606579739217654, + "loss": 1.4467, + "theoretical_loss": 3.6739700522131424, + "tokens_seen": 1010696192 + }, + { + "epoch": 0.31, + "learning_rate": 0.000560641925777332, + "loss": 1.3941, + "theoretical_loss": 3.673947601495038, + "tokens_seen": 1010761728 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005606258776328988, + "loss": 1.3423, + "theoretical_loss": 3.6739251526401064, + "tokens_seen": 1010827264 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005606098294884654, + "loss": 1.3751, + "theoretical_loss": 3.6739027056480733, + "tokens_seen": 1010892800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005605937813440322, + "loss": 1.3735, + "theoretical_loss": 3.673880260518663, + "tokens_seen": 1010958336 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005605777331995988, + "loss": 1.3761, + "theoretical_loss": 3.6738578172516005, + "tokens_seen": 1011023872 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005605616850551656, + "loss": 1.3606, + "theoretical_loss": 3.6738353758466102, + "tokens_seen": 1011089408 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005605456369107322, + "loss": 1.3755, + "theoretical_loss": 3.6738129363034178, + "tokens_seen": 1011154944 + }, + { + "epoch": 0.31, + "learning_rate": 0.000560529588766299, + "loss": 1.4081, + "theoretical_loss": 3.6737904986217464, + "tokens_seen": 1011220480 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005605135406218656, + "loss": 1.382, + "theoretical_loss": 3.6737680628013223, + "tokens_seen": 1011286016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005604974924774324, + "loss": 1.333, + "theoretical_loss": 3.6737456288418704, + "tokens_seen": 1011351552 + }, + { + "epoch": 0.31, + "learning_rate": 0.000560481444332999, + "loss": 1.4375, + "theoretical_loss": 3.673723196743116, + "tokens_seen": 1011417088 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005604653961885658, + "loss": 1.3359, + "theoretical_loss": 3.673700766504783, + "tokens_seen": 1011482624 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005604493480441324, + "loss": 1.3851, + "theoretical_loss": 3.6736783381265976, + "tokens_seen": 1011548160 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005604332998996992, + "loss": 1.4613, + "theoretical_loss": 3.673655911608285, + "tokens_seen": 1011613696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005604172517552658, + "loss": 1.445, + "theoretical_loss": 3.67363348694957, + "tokens_seen": 1011679232 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005604012036108326, + "loss": 1.4023, + "theoretical_loss": 3.6736110641501787, + "tokens_seen": 1011744768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005603851554663992, + "loss": 1.3275, + "theoretical_loss": 3.673588643209836, + "tokens_seen": 1011810304 + }, + { + "epoch": 0.31, + "learning_rate": 0.000560369107321966, + "loss": 1.3787, + "theoretical_loss": 3.673566224128267, + "tokens_seen": 1011875840 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005603530591775326, + "loss": 1.3527, + "theoretical_loss": 3.673543806905198, + "tokens_seen": 1011941376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005603370110330994, + "loss": 1.3998, + "theoretical_loss": 3.6735213915403544, + "tokens_seen": 1012006912 + }, + { + "epoch": 0.31, + "learning_rate": 0.000560320962888666, + "loss": 1.4078, + "theoretical_loss": 3.6734989780334617, + "tokens_seen": 1012072448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005603049147442327, + "loss": 1.3459, + "theoretical_loss": 3.6734765663842452, + "tokens_seen": 1012137984 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005602888665997994, + "loss": 1.4077, + "theoretical_loss": 3.6734541565924315, + "tokens_seen": 1012203520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005602728184553661, + "loss": 1.4246, + "theoretical_loss": 3.673431748657746, + "tokens_seen": 1012269056 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005602567703109328, + "loss": 1.4378, + "theoretical_loss": 3.673409342579914, + "tokens_seen": 1012334592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005602407221664995, + "loss": 1.3539, + "theoretical_loss": 3.6733869383586626, + "tokens_seen": 1012400128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005602246740220663, + "loss": 1.4492, + "theoretical_loss": 3.6733645359937173, + "tokens_seen": 1012465664 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005602086258776329, + "loss": 1.4988, + "theoretical_loss": 3.6733421354848037, + "tokens_seen": 1012531200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005601925777331995, + "loss": 1.3661, + "theoretical_loss": 3.6733197368316484, + "tokens_seen": 1012596736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005601765295887663, + "loss": 1.4, + "theoretical_loss": 3.6732973400339772, + "tokens_seen": 1012662272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005601604814443329, + "loss": 1.3635, + "theoretical_loss": 3.6732749450915168, + "tokens_seen": 1012727808 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005601444332998997, + "loss": 1.1896, + "theoretical_loss": 3.6732525520039934, + "tokens_seen": 1012793344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005601283851554663, + "loss": 1.4149, + "theoretical_loss": 3.6732301607711326, + "tokens_seen": 1012858880 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005601123370110331, + "loss": 1.434, + "theoretical_loss": 3.6732077713926623, + "tokens_seen": 1012924416 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005600962888665997, + "loss": 1.4343, + "theoretical_loss": 3.673185383868307, + "tokens_seen": 1012989952 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005600802407221665, + "loss": 1.3782, + "theoretical_loss": 3.673162998197795, + "tokens_seen": 1013055488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005600641925777331, + "loss": 1.3477, + "theoretical_loss": 3.673140614380852, + "tokens_seen": 1013121024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005600481444332999, + "loss": 1.3614, + "theoretical_loss": 3.673118232417205, + "tokens_seen": 1013186560 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005600320962888665, + "loss": 1.4275, + "theoretical_loss": 3.6730958523065804, + "tokens_seen": 1013252096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005600160481444333, + "loss": 1.3368, + "theoretical_loss": 3.6730734740487048, + "tokens_seen": 1013317632 + }, + { + "epoch": 0.31, + "learning_rate": 0.00056, + "loss": 1.416, + "theoretical_loss": 3.6730510976433055, + "tokens_seen": 1013383168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005599839518555667, + "loss": 1.4075, + "theoretical_loss": 3.673028723090109, + "tokens_seen": 1013448704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005599679037111334, + "loss": 1.3583, + "theoretical_loss": 3.6730063503888424, + "tokens_seen": 1013514240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005599518555667001, + "loss": 1.4114, + "theoretical_loss": 3.6729839795392323, + "tokens_seen": 1013579776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005599358074222668, + "loss": 1.3741, + "theoretical_loss": 3.672961610541006, + "tokens_seen": 1013645312 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005599197592778335, + "loss": 1.3941, + "theoretical_loss": 3.672939243393891, + "tokens_seen": 1013710848 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005599037111334002, + "loss": 1.3624, + "theoretical_loss": 3.672916878097614, + "tokens_seen": 1013776384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005598876629889669, + "loss": 1.3788, + "theoretical_loss": 3.6728945146519023, + "tokens_seen": 1013841920 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005598716148445336, + "loss": 1.4471, + "theoretical_loss": 3.6728721530564834, + "tokens_seen": 1013907456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005598555667001003, + "loss": 1.2961, + "theoretical_loss": 3.672849793311084, + "tokens_seen": 1013972992 + }, + { + "epoch": 0.31, + "learning_rate": 0.000559839518555667, + "loss": 1.3476, + "theoretical_loss": 3.6728274354154324, + "tokens_seen": 1014038528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005598234704112337, + "loss": 1.3474, + "theoretical_loss": 3.672805079369255, + "tokens_seen": 1014104064 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005598074222668004, + "loss": 1.4313, + "theoretical_loss": 3.67278272517228, + "tokens_seen": 1014169600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005597913741223671, + "loss": 1.4657, + "theoretical_loss": 3.6727603728242353, + "tokens_seen": 1014235136 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005597753259779338, + "loss": 1.3953, + "theoretical_loss": 3.6727380223248476, + "tokens_seen": 1014300672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005597592778335005, + "loss": 1.3612, + "theoretical_loss": 3.672715673673845, + "tokens_seen": 1014366208 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005597432296890673, + "loss": 1.3266, + "theoretical_loss": 3.672693326870955, + "tokens_seen": 1014431744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005597271815446339, + "loss": 1.3922, + "theoretical_loss": 3.672670981915906, + "tokens_seen": 1014497280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005597111334002007, + "loss": 1.4209, + "theoretical_loss": 3.6726486388084254, + "tokens_seen": 1014562816 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005596950852557673, + "loss": 1.3053, + "theoretical_loss": 3.6726262975482413, + "tokens_seen": 1014628352 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005596790371113341, + "loss": 1.341, + "theoretical_loss": 3.6726039581350816, + "tokens_seen": 1014693888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005596629889669007, + "loss": 1.3981, + "theoretical_loss": 3.672581620568674, + "tokens_seen": 1014759424 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005596469408224675, + "loss": 1.3497, + "theoretical_loss": 3.672559284848747, + "tokens_seen": 1014824960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005596308926780341, + "loss": 1.3978, + "theoretical_loss": 3.6725369509750285, + "tokens_seen": 1014890496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005596148445336009, + "loss": 1.3948, + "theoretical_loss": 3.672514618947247, + "tokens_seen": 1014956032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005595987963891675, + "loss": 1.4267, + "theoretical_loss": 3.67249228876513, + "tokens_seen": 1015021568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005595827482447343, + "loss": 1.4623, + "theoretical_loss": 3.672469960428407, + "tokens_seen": 1015087104 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005595667001003009, + "loss": 1.3583, + "theoretical_loss": 3.6724476339368053, + "tokens_seen": 1015152640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005595506519558677, + "loss": 1.3258, + "theoretical_loss": 3.672425309290054, + "tokens_seen": 1015218176 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005595346038114343, + "loss": 1.4278, + "theoretical_loss": 3.6724029864878807, + "tokens_seen": 1015283712 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005595185556670011, + "loss": 1.4191, + "theoretical_loss": 3.6723806655300155, + "tokens_seen": 1015349248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005595025075225677, + "loss": 1.397, + "theoretical_loss": 3.6723583464161855, + "tokens_seen": 1015414784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005594864593781345, + "loss": 1.3768, + "theoretical_loss": 3.67233602914612, + "tokens_seen": 1015480320 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005594704112337011, + "loss": 1.3023, + "theoretical_loss": 3.6723137137195474, + "tokens_seen": 1015545856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005594543630892679, + "loss": 1.3364, + "theoretical_loss": 3.672291400136197, + "tokens_seen": 1015611392 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005594383149448345, + "loss": 1.3436, + "theoretical_loss": 3.6722690883957974, + "tokens_seen": 1015676928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005594222668004013, + "loss": 1.3806, + "theoretical_loss": 3.672246778498077, + "tokens_seen": 1015742464 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 380743, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.400584101676941, + "objective/train/theoretical_loss": 3.6722244704427656, + "objective/train/tokens_used": 1036268000, + "theoretical_loss": 3.6722244704427656, + "tokens_seen": 1015808000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005594062186559679, + "loss": 1.4006, + "theoretical_loss": 3.6722244704427656, + "tokens_seen": 1015808000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005593901705115347, + "loss": 1.3123, + "theoretical_loss": 3.6722021642295912, + "tokens_seen": 1015873536 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005593741223671013, + "loss": 1.3707, + "theoretical_loss": 3.6721798598582835, + "tokens_seen": 1015939072 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005593580742226681, + "loss": 1.4066, + "theoretical_loss": 3.6721575573285716, + "tokens_seen": 1016004608 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005593420260782347, + "loss": 1.2934, + "theoretical_loss": 3.6721352566401846, + "tokens_seen": 1016070144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005593259779338015, + "loss": 1.3837, + "theoretical_loss": 3.6721129577928524, + "tokens_seen": 1016135680 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005593099297893681, + "loss": 1.3574, + "theoretical_loss": 3.672090660786303, + "tokens_seen": 1016201216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005592938816449349, + "loss": 1.3664, + "theoretical_loss": 3.672068365620266, + "tokens_seen": 1016266752 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005592778335005015, + "loss": 1.2867, + "theoretical_loss": 3.6720460722944717, + "tokens_seen": 1016332288 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005592617853560683, + "loss": 1.3553, + "theoretical_loss": 3.672023780808648, + "tokens_seen": 1016397824 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005592457372116349, + "loss": 1.4558, + "theoretical_loss": 3.6720014911625265, + "tokens_seen": 1016463360 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005592296890672017, + "loss": 1.2957, + "theoretical_loss": 3.671979203355835, + "tokens_seen": 1016528896 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005592136409227683, + "loss": 1.4107, + "theoretical_loss": 3.671956917388304, + "tokens_seen": 1016594432 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005591975927783351, + "loss": 1.4133, + "theoretical_loss": 3.6719346332596636, + "tokens_seen": 1016659968 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005591815446339017, + "loss": 1.3891, + "theoretical_loss": 3.6719123509696425, + "tokens_seen": 1016725504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005591654964894685, + "loss": 1.4232, + "theoretical_loss": 3.6718900705179705, + "tokens_seen": 1016791040 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005591494483450351, + "loss": 1.426, + "theoretical_loss": 3.6718677919043783, + "tokens_seen": 1016856576 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005591334002006019, + "loss": 1.3141, + "theoretical_loss": 3.671845515128595, + "tokens_seen": 1016922112 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005591173520561685, + "loss": 1.3565, + "theoretical_loss": 3.671823240190352, + "tokens_seen": 1016987648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005591013039117353, + "loss": 1.3907, + "theoretical_loss": 3.6718009670893776, + "tokens_seen": 1017053184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005590852557673019, + "loss": 1.3316, + "theoretical_loss": 3.6717786958254024, + "tokens_seen": 1017118720 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005590692076228687, + "loss": 1.3481, + "theoretical_loss": 3.671756426398157, + "tokens_seen": 1017184256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005590531594784353, + "loss": 1.4242, + "theoretical_loss": 3.6717341588073715, + "tokens_seen": 1017249792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005590371113340021, + "loss": 1.4889, + "theoretical_loss": 3.671711893052776, + "tokens_seen": 1017315328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005590210631895687, + "loss": 1.5038, + "theoretical_loss": 3.6716896291341, + "tokens_seen": 1017380864 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005590050150451355, + "loss": 1.3557, + "theoretical_loss": 3.671667367051076, + "tokens_seen": 1017446400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005589889669007021, + "loss": 1.3395, + "theoretical_loss": 3.671645106803432, + "tokens_seen": 1017511936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005589729187562689, + "loss": 1.2987, + "theoretical_loss": 3.6716228483909, + "tokens_seen": 1017577472 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005589568706118355, + "loss": 1.3176, + "theoretical_loss": 3.6716005918132097, + "tokens_seen": 1017643008 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005589408224674023, + "loss": 1.4029, + "theoretical_loss": 3.671578337070093, + "tokens_seen": 1017708544 + }, + { + "epoch": 0.31, + "learning_rate": 0.000558924774322969, + "loss": 1.4117, + "theoretical_loss": 3.6715560841612795, + "tokens_seen": 1017774080 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005589087261785356, + "loss": 1.3439, + "theoretical_loss": 3.6715338330864995, + "tokens_seen": 1017839616 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005588926780341023, + "loss": 1.4268, + "theoretical_loss": 3.6715115838454846, + "tokens_seen": 1017905152 + }, + { + "epoch": 0.31, + "learning_rate": 0.000558876629889669, + "loss": 1.495, + "theoretical_loss": 3.6714893364379657, + "tokens_seen": 1017970688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005588605817452358, + "loss": 1.3385, + "theoretical_loss": 3.671467090863673, + "tokens_seen": 1018036224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005588445336008024, + "loss": 1.4355, + "theoretical_loss": 3.6714448471223378, + "tokens_seen": 1018101760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005588284854563692, + "loss": 1.367, + "theoretical_loss": 3.6714226052136913, + "tokens_seen": 1018167296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005588124373119358, + "loss": 1.3833, + "theoretical_loss": 3.6714003651374645, + "tokens_seen": 1018232832 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005587963891675026, + "loss": 1.4164, + "theoretical_loss": 3.671378126893388, + "tokens_seen": 1018298368 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005587803410230692, + "loss": 1.3873, + "theoretical_loss": 3.671355890481194, + "tokens_seen": 1018363904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005587642928786358, + "loss": 1.349, + "theoretical_loss": 3.6713336559006127, + "tokens_seen": 1018429440 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005587482447342026, + "loss": 1.3507, + "theoretical_loss": 3.6713114231513755, + "tokens_seen": 1018494976 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005587321965897692, + "loss": 1.4024, + "theoretical_loss": 3.671289192233214, + "tokens_seen": 1018560512 + }, + { + "epoch": 0.31, + "learning_rate": 0.000558716148445336, + "loss": 1.3879, + "theoretical_loss": 3.6712669631458597, + "tokens_seen": 1018626048 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005587001003009026, + "loss": 1.448, + "theoretical_loss": 3.671244735889044, + "tokens_seen": 1018691584 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005586840521564694, + "loss": 1.3774, + "theoretical_loss": 3.6712225104624983, + "tokens_seen": 1018757120 + }, + { + "epoch": 0.31, + "learning_rate": 0.000558668004012036, + "loss": 1.3626, + "theoretical_loss": 3.6712002868659543, + "tokens_seen": 1018822656 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005586519558676028, + "loss": 1.4039, + "theoretical_loss": 3.6711780650991432, + "tokens_seen": 1018888192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005586359077231695, + "loss": 1.371, + "theoretical_loss": 3.671155845161797, + "tokens_seen": 1018953728 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005586198595787362, + "loss": 1.3258, + "theoretical_loss": 3.671133627053648, + "tokens_seen": 1019019264 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005586038114343029, + "loss": 1.4259, + "theoretical_loss": 3.6711114107744267, + "tokens_seen": 1019084800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005585877632898696, + "loss": 1.3132, + "theoretical_loss": 3.671089196323866, + "tokens_seen": 1019150336 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005585717151454363, + "loss": 1.4089, + "theoretical_loss": 3.6710669837016976, + "tokens_seen": 1019215872 + }, + { + "epoch": 0.31, + "learning_rate": 0.000558555667001003, + "loss": 1.3544, + "theoretical_loss": 3.671044772907653, + "tokens_seen": 1019281408 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005585396188565698, + "loss": 1.3298, + "theoretical_loss": 3.671022563941465, + "tokens_seen": 1019346944 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005585235707121364, + "loss": 1.3257, + "theoretical_loss": 3.671000356802865, + "tokens_seen": 1019412480 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005585075225677032, + "loss": 1.3791, + "theoretical_loss": 3.6709781514915854, + "tokens_seen": 1019478016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005584914744232698, + "loss": 1.3409, + "theoretical_loss": 3.670955948007358, + "tokens_seen": 1019543552 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005584754262788366, + "loss": 1.2614, + "theoretical_loss": 3.6709337463499154, + "tokens_seen": 1019609088 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005584593781344032, + "loss": 1.3539, + "theoretical_loss": 3.6709115465189903, + "tokens_seen": 1019674624 + }, + { + "epoch": 0.31, + "learning_rate": 0.00055844332998997, + "loss": 1.4327, + "theoretical_loss": 3.670889348514314, + "tokens_seen": 1019740160 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005584272818455366, + "loss": 1.4088, + "theoretical_loss": 3.67086715233562, + "tokens_seen": 1019805696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005584112337011034, + "loss": 1.3739, + "theoretical_loss": 3.6708449579826405, + "tokens_seen": 1019871232 + }, + { + "epoch": 0.31, + "learning_rate": 0.00055839518555667, + "loss": 1.4393, + "theoretical_loss": 3.6708227654551076, + "tokens_seen": 1019936768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005583791374122368, + "loss": 1.6472, + "theoretical_loss": 3.6708005747527537, + "tokens_seen": 1020002304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005583630892678034, + "loss": 1.449, + "theoretical_loss": 3.6707783858753125, + "tokens_seen": 1020067840 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005583470411233702, + "loss": 1.3603, + "theoretical_loss": 3.6707561988225157, + "tokens_seen": 1020133376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005583309929789368, + "loss": 1.3441, + "theoretical_loss": 3.670734013594096, + "tokens_seen": 1020198912 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005583149448345036, + "loss": 1.371, + "theoretical_loss": 3.670711830189787, + "tokens_seen": 1020264448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005582988966900702, + "loss": 1.3644, + "theoretical_loss": 3.6706896486093212, + "tokens_seen": 1020329984 + }, + { + "epoch": 0.31, + "learning_rate": 0.000558282848545637, + "loss": 1.3678, + "theoretical_loss": 3.6706674688524314, + "tokens_seen": 1020395520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005582668004012036, + "loss": 1.3148, + "theoretical_loss": 3.670645290918851, + "tokens_seen": 1020461056 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005582507522567704, + "loss": 1.3429, + "theoretical_loss": 3.670623114808312, + "tokens_seen": 1020526592 + }, + { + "epoch": 0.31, + "learning_rate": 0.000558234704112337, + "loss": 1.3971, + "theoretical_loss": 3.670600940520549, + "tokens_seen": 1020592128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005582186559679038, + "loss": 1.3612, + "theoretical_loss": 3.670578768055294, + "tokens_seen": 1020657664 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005582026078234704, + "loss": 1.4171, + "theoretical_loss": 3.67055659741228, + "tokens_seen": 1020723200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005581865596790372, + "loss": 1.4864, + "theoretical_loss": 3.670534428591241, + "tokens_seen": 1020788736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005581705115346038, + "loss": 1.3754, + "theoretical_loss": 3.6705122615919104, + "tokens_seen": 1020854272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005581544633901706, + "loss": 1.4194, + "theoretical_loss": 3.6704900964140212, + "tokens_seen": 1020919808 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005581384152457372, + "loss": 1.4032, + "theoretical_loss": 3.6704679330573065, + "tokens_seen": 1020985344 + }, + { + "epoch": 0.31, + "learning_rate": 0.000558122367101304, + "loss": 1.3501, + "theoretical_loss": 3.6704457715215004, + "tokens_seen": 1021050880 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005581063189568706, + "loss": 1.3636, + "theoretical_loss": 3.6704236118063363, + "tokens_seen": 1021116416 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005580902708124374, + "loss": 1.3674, + "theoretical_loss": 3.6704014539115475, + "tokens_seen": 1021181952 + }, + { + "epoch": 0.31, + "learning_rate": 0.000558074222668004, + "loss": 1.2343, + "theoretical_loss": 3.670379297836868, + "tokens_seen": 1021247488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005580581745235708, + "loss": 1.235, + "theoretical_loss": 3.6703571435820312, + "tokens_seen": 1021313024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005580421263791374, + "loss": 1.3622, + "theoretical_loss": 3.6703349911467713, + "tokens_seen": 1021378560 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005580260782347042, + "loss": 1.383, + "theoretical_loss": 3.670312840530821, + "tokens_seen": 1021444096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005580100300902708, + "loss": 1.2905, + "theoretical_loss": 3.670290691733916, + "tokens_seen": 1021509632 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005579939819458376, + "loss": 1.4657, + "theoretical_loss": 3.6702685447557886, + "tokens_seen": 1021575168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005579779338014042, + "loss": 1.4797, + "theoretical_loss": 3.6702463995961736, + "tokens_seen": 1021640704 + }, + { + "epoch": 0.31, + "learning_rate": 0.000557961885656971, + "loss": 1.3372, + "theoretical_loss": 3.6702242562548046, + "tokens_seen": 1021706240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005579458375125376, + "loss": 1.455, + "theoretical_loss": 3.670202114731416, + "tokens_seen": 1021771776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005579297893681044, + "loss": 1.3344, + "theoretical_loss": 3.670179975025742, + "tokens_seen": 1021837312 + }, + { + "epoch": 0.31, + "learning_rate": 0.000557913741223671, + "loss": 1.4904, + "theoretical_loss": 3.670157837137517, + "tokens_seen": 1021902848 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005578976930792378, + "loss": 1.4544, + "theoretical_loss": 3.6701357010664744, + "tokens_seen": 1021968384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005578816449348044, + "loss": 1.3569, + "theoretical_loss": 3.6701135668123497, + "tokens_seen": 1022033920 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005578655967903712, + "loss": 1.4213, + "theoretical_loss": 3.670091434374876, + "tokens_seen": 1022099456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005578495486459378, + "loss": 1.3625, + "theoretical_loss": 3.670069303753789, + "tokens_seen": 1022164992 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005578335005015046, + "loss": 1.4273, + "theoretical_loss": 3.670047174948822, + "tokens_seen": 1022230528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005578174523570712, + "loss": 1.4347, + "theoretical_loss": 3.6700250479597107, + "tokens_seen": 1022296064 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 382990, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.420756459236145, + "objective/train/theoretical_loss": 3.6700029227861886, + "objective/train/tokens_used": 1042821600, + "theoretical_loss": 3.6700029227861886, + "tokens_seen": 1022361600 + }, + { + "epoch": 0.31, + "learning_rate": 0.000557801404212638, + "loss": 1.4208, + "theoretical_loss": 3.6700029227861886, + "tokens_seen": 1022361600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005577853560682046, + "loss": 1.3326, + "theoretical_loss": 3.669980799427991, + "tokens_seen": 1022427136 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005577693079237714, + "loss": 1.3818, + "theoretical_loss": 3.6699586778848525, + "tokens_seen": 1022492672 + }, + { + "epoch": 0.31, + "learning_rate": 0.000557753259779338, + "loss": 1.4688, + "theoretical_loss": 3.6699365581565084, + "tokens_seen": 1022558208 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005577372116349048, + "loss": 1.4319, + "theoretical_loss": 3.6699144402426924, + "tokens_seen": 1022623744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005577211634904714, + "loss": 1.4253, + "theoretical_loss": 3.66989232414314, + "tokens_seen": 1022689280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005577051153460382, + "loss": 1.3616, + "theoretical_loss": 3.6698702098575864, + "tokens_seen": 1022754816 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005576890672016048, + "loss": 1.4092, + "theoretical_loss": 3.669848097385766, + "tokens_seen": 1022820352 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005576730190571716, + "loss": 1.3928, + "theoretical_loss": 3.669825986727414, + "tokens_seen": 1022885888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005576569709127382, + "loss": 1.3943, + "theoretical_loss": 3.669803877882266, + "tokens_seen": 1022951424 + }, + { + "epoch": 0.31, + "learning_rate": 0.000557640922768305, + "loss": 1.3625, + "theoretical_loss": 3.6697817708500566, + "tokens_seen": 1023016960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005576248746238716, + "loss": 1.3535, + "theoretical_loss": 3.6697596656305214, + "tokens_seen": 1023082496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005576088264794384, + "loss": 1.374, + "theoretical_loss": 3.6697375622233954, + "tokens_seen": 1023148032 + }, + { + "epoch": 0.31, + "learning_rate": 0.000557592778335005, + "loss": 1.3358, + "theoretical_loss": 3.6697154606284137, + "tokens_seen": 1023213568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005575767301905718, + "loss": 1.416, + "theoretical_loss": 3.669693360845312, + "tokens_seen": 1023279104 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005575606820461384, + "loss": 1.3814, + "theoretical_loss": 3.669671262873826, + "tokens_seen": 1023344640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005575446339017052, + "loss": 1.3209, + "theoretical_loss": 3.6696491667136906, + "tokens_seen": 1023410176 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005575285857572718, + "loss": 1.443, + "theoretical_loss": 3.6696270723646416, + "tokens_seen": 1023475712 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005575125376128385, + "loss": 1.3374, + "theoretical_loss": 3.669604979826415, + "tokens_seen": 1023541248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005574964894684052, + "loss": 1.3597, + "theoretical_loss": 3.669582889098746, + "tokens_seen": 1023606784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005574804413239719, + "loss": 1.2991, + "theoretical_loss": 3.6695608001813707, + "tokens_seen": 1023672320 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005574643931795387, + "loss": 1.3067, + "theoretical_loss": 3.669538713074024, + "tokens_seen": 1023737856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005574483450351053, + "loss": 1.3892, + "theoretical_loss": 3.669516627776442, + "tokens_seen": 1023803392 + }, + { + "epoch": 0.31, + "learning_rate": 0.000557432296890672, + "loss": 1.3378, + "theoretical_loss": 3.669494544288362, + "tokens_seen": 1023868928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005574162487462387, + "loss": 1.3545, + "theoretical_loss": 3.6694724626095185, + "tokens_seen": 1023934464 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005574002006018055, + "loss": 1.3907, + "theoretical_loss": 3.6694503827396474, + "tokens_seen": 1024000000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005573841524573721, + "loss": 1.5303, + "theoretical_loss": 3.669428304678485, + "tokens_seen": 1024065536 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005573681043129387, + "loss": 1.3674, + "theoretical_loss": 3.669406228425768, + "tokens_seen": 1024131072 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005573520561685055, + "loss": 1.4427, + "theoretical_loss": 3.669384153981232, + "tokens_seen": 1024196608 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005573360080240723, + "loss": 1.4436, + "theoretical_loss": 3.6693620813446133, + "tokens_seen": 1024262144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005573199598796389, + "loss": 1.3192, + "theoretical_loss": 3.669340010515648, + "tokens_seen": 1024327680 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005573039117352057, + "loss": 1.2815, + "theoretical_loss": 3.6693179414940733, + "tokens_seen": 1024393216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005572878635907723, + "loss": 1.4161, + "theoretical_loss": 3.6692958742796242, + "tokens_seen": 1024458752 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005572718154463391, + "loss": 1.3749, + "theoretical_loss": 3.6692738088720382, + "tokens_seen": 1024524288 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005572557673019057, + "loss": 1.3807, + "theoretical_loss": 3.669251745271051, + "tokens_seen": 1024589824 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005572397191574725, + "loss": 1.4612, + "theoretical_loss": 3.6692296834764, + "tokens_seen": 1024655360 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005572236710130391, + "loss": 1.376, + "theoretical_loss": 3.6692076234878215, + "tokens_seen": 1024720896 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005572076228686059, + "loss": 1.2931, + "theoretical_loss": 3.6691855653050514, + "tokens_seen": 1024786432 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005571915747241725, + "loss": 1.3709, + "theoretical_loss": 3.669163508927827, + "tokens_seen": 1024851968 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005571755265797393, + "loss": 1.2835, + "theoretical_loss": 3.6691414543558856, + "tokens_seen": 1024917504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005571594784353059, + "loss": 1.3691, + "theoretical_loss": 3.669119401588963, + "tokens_seen": 1024983040 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005571434302908727, + "loss": 1.4132, + "theoretical_loss": 3.669097350626797, + "tokens_seen": 1025048576 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005571273821464393, + "loss": 1.3689, + "theoretical_loss": 3.669075301469124, + "tokens_seen": 1025114112 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005571113340020061, + "loss": 1.4564, + "theoretical_loss": 3.6690532541156813, + "tokens_seen": 1025179648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005570952858575727, + "loss": 1.329, + "theoretical_loss": 3.6690312085662056, + "tokens_seen": 1025245184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005570792377131395, + "loss": 1.3372, + "theoretical_loss": 3.6690091648204337, + "tokens_seen": 1025310720 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005570631895687061, + "loss": 1.3525, + "theoretical_loss": 3.6689871228781037, + "tokens_seen": 1025376256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005570471414242729, + "loss": 1.4429, + "theoretical_loss": 3.6689650827389517, + "tokens_seen": 1025441792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005570310932798395, + "loss": 1.3291, + "theoretical_loss": 3.6689430444027162, + "tokens_seen": 1025507328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005570150451354063, + "loss": 1.3203, + "theoretical_loss": 3.668921007869134, + "tokens_seen": 1025572864 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005569989969909729, + "loss": 1.3817, + "theoretical_loss": 3.6688989731379413, + "tokens_seen": 1025638400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005569829488465397, + "loss": 1.4763, + "theoretical_loss": 3.6688769402088774, + "tokens_seen": 1025703936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005569669007021063, + "loss": 1.2664, + "theoretical_loss": 3.668854909081679, + "tokens_seen": 1025769472 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005569508525576731, + "loss": 1.3179, + "theoretical_loss": 3.668832879756083, + "tokens_seen": 1025835008 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005569348044132397, + "loss": 1.272, + "theoretical_loss": 3.668810852231828, + "tokens_seen": 1025900544 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005569187562688065, + "loss": 1.3871, + "theoretical_loss": 3.668788826508651, + "tokens_seen": 1025966080 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005569027081243731, + "loss": 1.3113, + "theoretical_loss": 3.6687668025862905, + "tokens_seen": 1026031616 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005568866599799399, + "loss": 1.3159, + "theoretical_loss": 3.668744780464483, + "tokens_seen": 1026097152 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005568706118355065, + "loss": 1.349, + "theoretical_loss": 3.668722760142967, + "tokens_seen": 1026162688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005568545636910733, + "loss": 1.3393, + "theoretical_loss": 3.668700741621481, + "tokens_seen": 1026228224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005568385155466399, + "loss": 1.362, + "theoretical_loss": 3.6686787248997614, + "tokens_seen": 1026293760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005568224674022067, + "loss": 1.3102, + "theoretical_loss": 3.6686567099775473, + "tokens_seen": 1026359296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005568064192577733, + "loss": 1.4575, + "theoretical_loss": 3.6686346968545767, + "tokens_seen": 1026424832 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005567903711133401, + "loss": 1.4328, + "theoretical_loss": 3.6686126855305874, + "tokens_seen": 1026490368 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005567743229689067, + "loss": 1.3827, + "theoretical_loss": 3.6685906760053175, + "tokens_seen": 1026555904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005567582748244735, + "loss": 1.3766, + "theoretical_loss": 3.6685686682785055, + "tokens_seen": 1026621440 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005567422266800401, + "loss": 1.3979, + "theoretical_loss": 3.6685466623498892, + "tokens_seen": 1026686976 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005567261785356069, + "loss": 1.3488, + "theoretical_loss": 3.6685246582192077, + "tokens_seen": 1026752512 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005567101303911735, + "loss": 1.5075, + "theoretical_loss": 3.668502655886198, + "tokens_seen": 1026818048 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005566940822467403, + "loss": 1.3772, + "theoretical_loss": 3.6684806553505998, + "tokens_seen": 1026883584 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005566780341023069, + "loss": 1.2772, + "theoretical_loss": 3.6684586566121506, + "tokens_seen": 1026949120 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005566619859578737, + "loss": 1.3675, + "theoretical_loss": 3.66843665967059, + "tokens_seen": 1027014656 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005566459378134403, + "loss": 1.4619, + "theoretical_loss": 3.668414664525656, + "tokens_seen": 1027080192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005566298896690071, + "loss": 1.3555, + "theoretical_loss": 3.668392671177087, + "tokens_seen": 1027145728 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005566138415245737, + "loss": 1.314, + "theoretical_loss": 3.668370679624622, + "tokens_seen": 1027211264 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005565977933801405, + "loss": 1.2966, + "theoretical_loss": 3.6683486898679996, + "tokens_seen": 1027276800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005565817452357071, + "loss": 1.4109, + "theoretical_loss": 3.6683267019069588, + "tokens_seen": 1027342336 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005565656970912739, + "loss": 1.3353, + "theoretical_loss": 3.668304715741238, + "tokens_seen": 1027407872 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005565496489468405, + "loss": 1.3071, + "theoretical_loss": 3.6682827313705766, + "tokens_seen": 1027473408 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005565336008024073, + "loss": 1.3451, + "theoretical_loss": 3.6682607487947134, + "tokens_seen": 1027538944 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005565175526579739, + "loss": 1.4268, + "theoretical_loss": 3.668238768013387, + "tokens_seen": 1027604480 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005565015045135407, + "loss": 1.5068, + "theoretical_loss": 3.6682167890263377, + "tokens_seen": 1027670016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005564854563691073, + "loss": 1.456, + "theoretical_loss": 3.668194811833303, + "tokens_seen": 1027735552 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005564694082246741, + "loss": 1.3245, + "theoretical_loss": 3.6681728364340236, + "tokens_seen": 1027801088 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005564533600802407, + "loss": 1.341, + "theoretical_loss": 3.668150862828237, + "tokens_seen": 1027866624 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005564373119358075, + "loss": 1.3989, + "theoretical_loss": 3.668128891015684, + "tokens_seen": 1027932160 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005564212637913741, + "loss": 1.2441, + "theoretical_loss": 3.6681069209961037, + "tokens_seen": 1027997696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005564052156469409, + "loss": 1.3318, + "theoretical_loss": 3.668084952769235, + "tokens_seen": 1028063232 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005563891675025075, + "loss": 1.3082, + "theoretical_loss": 3.6680629863348178, + "tokens_seen": 1028128768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005563731193580743, + "loss": 1.3283, + "theoretical_loss": 3.6680410216925914, + "tokens_seen": 1028194304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005563570712136409, + "loss": 1.46, + "theoretical_loss": 3.6680190588422947, + "tokens_seen": 1028259840 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005563410230692077, + "loss": 1.3639, + "theoretical_loss": 3.6679970977836684, + "tokens_seen": 1028325376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005563249749247743, + "loss": 1.4377, + "theoretical_loss": 3.667975138516452, + "tokens_seen": 1028390912 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005563089267803411, + "loss": 1.4057, + "theoretical_loss": 3.6679531810403843, + "tokens_seen": 1028456448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005562928786359077, + "loss": 1.4047, + "theoretical_loss": 3.6679312253552063, + "tokens_seen": 1028521984 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005562768304914745, + "loss": 1.3563, + "theoretical_loss": 3.6679092714606574, + "tokens_seen": 1028587520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005562607823470411, + "loss": 1.3068, + "theoretical_loss": 3.6678873193564767, + "tokens_seen": 1028653056 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005562447342026079, + "loss": 1.3506, + "theoretical_loss": 3.6678653690424055, + "tokens_seen": 1028718592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005562286860581745, + "loss": 1.4095, + "theoretical_loss": 3.6678434205181825, + "tokens_seen": 1028784128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005562126379137413, + "loss": 1.4601, + "theoretical_loss": 3.6678214737835484, + "tokens_seen": 1028849664 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 385302, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3480840921401978, + "objective/train/theoretical_loss": 3.667799528838244, + "objective/train/tokens_used": 1049375200, + "theoretical_loss": 3.667799528838244, + "tokens_seen": 1028915200 + }, + { + "epoch": 0.31, + "learning_rate": 0.000556196589769308, + "loss": 1.3481, + "theoretical_loss": 3.667799528838244, + "tokens_seen": 1028915200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005561805416248747, + "loss": 1.36, + "theoretical_loss": 3.667777585682008, + "tokens_seen": 1028980736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005561644934804413, + "loss": 1.2668, + "theoretical_loss": 3.667755644314581, + "tokens_seen": 1029046272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005561484453360081, + "loss": 1.4205, + "theoretical_loss": 3.667733704735704, + "tokens_seen": 1029111808 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005561323971915747, + "loss": 1.3045, + "theoretical_loss": 3.667711766945117, + "tokens_seen": 1029177344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005561163490471414, + "loss": 1.2833, + "theoretical_loss": 3.6676898309425603, + "tokens_seen": 1029242880 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005561003009027082, + "loss": 1.3239, + "theoretical_loss": 3.6676678967277745, + "tokens_seen": 1029308416 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005560842527582748, + "loss": 1.4592, + "theoretical_loss": 3.6676459643004993, + "tokens_seen": 1029373952 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005560682046138416, + "loss": 1.3881, + "theoretical_loss": 3.667624033660476, + "tokens_seen": 1029439488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005560521564694082, + "loss": 1.4198, + "theoretical_loss": 3.6676021048074454, + "tokens_seen": 1029505024 + }, + { + "epoch": 0.31, + "learning_rate": 0.000556036108324975, + "loss": 1.3283, + "theoretical_loss": 3.6675801777411476, + "tokens_seen": 1029570560 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005560200601805416, + "loss": 1.2505, + "theoretical_loss": 3.6675582524613235, + "tokens_seen": 1029636096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005560040120361084, + "loss": 1.364, + "theoretical_loss": 3.667536328967714, + "tokens_seen": 1029701632 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555987963891675, + "loss": 1.2612, + "theoretical_loss": 3.6675144072600596, + "tokens_seen": 1029767168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005559719157472418, + "loss": 1.366, + "theoretical_loss": 3.667492487338101, + "tokens_seen": 1029832704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005559558676028084, + "loss": 1.3702, + "theoretical_loss": 3.66747056920158, + "tokens_seen": 1029898240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005559398194583752, + "loss": 1.2692, + "theoretical_loss": 3.667448652850237, + "tokens_seen": 1029963776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005559237713139418, + "loss": 1.4032, + "theoretical_loss": 3.6674267382838126, + "tokens_seen": 1030029312 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005559077231695086, + "loss": 1.2823, + "theoretical_loss": 3.6674048255020484, + "tokens_seen": 1030094848 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005558916750250752, + "loss": 1.1999, + "theoretical_loss": 3.6673829145046852, + "tokens_seen": 1030160384 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555875626880642, + "loss": 1.3042, + "theoretical_loss": 3.667361005291465, + "tokens_seen": 1030225920 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005558595787362086, + "loss": 1.3645, + "theoretical_loss": 3.6673390978621283, + "tokens_seen": 1030291456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005558435305917754, + "loss": 1.3961, + "theoretical_loss": 3.6673171922164163, + "tokens_seen": 1030356992 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555827482447342, + "loss": 1.4593, + "theoretical_loss": 3.6672952883540706, + "tokens_seen": 1030422528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005558114343029088, + "loss": 1.5048, + "theoretical_loss": 3.6672733862748332, + "tokens_seen": 1030488064 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005557953861584754, + "loss": 1.3878, + "theoretical_loss": 3.6672514859784444, + "tokens_seen": 1030553600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005557793380140422, + "loss": 1.3801, + "theoretical_loss": 3.6672295874646466, + "tokens_seen": 1030619136 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005557632898696088, + "loss": 1.4183, + "theoretical_loss": 3.6672076907331808, + "tokens_seen": 1030684672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005557472417251756, + "loss": 1.314, + "theoretical_loss": 3.6671857957837886, + "tokens_seen": 1030750208 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005557311935807422, + "loss": 1.2916, + "theoretical_loss": 3.6671639026162124, + "tokens_seen": 1030815744 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555715145436309, + "loss": 1.4576, + "theoretical_loss": 3.6671420112301933, + "tokens_seen": 1030881280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005556990972918756, + "loss": 1.4299, + "theoretical_loss": 3.6671201216254734, + "tokens_seen": 1030946816 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005556830491474424, + "loss": 1.3711, + "theoretical_loss": 3.667098233801794, + "tokens_seen": 1031012352 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555667001003009, + "loss": 1.275, + "theoretical_loss": 3.667076347758898, + "tokens_seen": 1031077888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005556509528585758, + "loss": 1.3179, + "theoretical_loss": 3.6670544634965263, + "tokens_seen": 1031143424 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005556349047141424, + "loss": 1.3506, + "theoretical_loss": 3.6670325810144213, + "tokens_seen": 1031208960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005556188565697092, + "loss": 1.3837, + "theoretical_loss": 3.6670107003123253, + "tokens_seen": 1031274496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005556028084252758, + "loss": 1.3648, + "theoretical_loss": 3.66698882138998, + "tokens_seen": 1031340032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005555867602808426, + "loss": 1.3261, + "theoretical_loss": 3.666966944247128, + "tokens_seen": 1031405568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005555707121364092, + "loss": 1.412, + "theoretical_loss": 3.666945068883511, + "tokens_seen": 1031471104 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555554663991976, + "loss": 1.4082, + "theoretical_loss": 3.6669231952988715, + "tokens_seen": 1031536640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005555386158475426, + "loss": 1.3915, + "theoretical_loss": 3.6669013234929517, + "tokens_seen": 1031602176 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005555225677031094, + "loss": 1.3941, + "theoretical_loss": 3.6668794534654943, + "tokens_seen": 1031667712 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555506519558676, + "loss": 1.3069, + "theoretical_loss": 3.6668575852162415, + "tokens_seen": 1031733248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005554904714142428, + "loss": 1.2215, + "theoretical_loss": 3.6668357187449354, + "tokens_seen": 1031798784 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005554744232698094, + "loss": 1.3611, + "theoretical_loss": 3.6668138540513198, + "tokens_seen": 1031864320 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005554583751253762, + "loss": 1.298, + "theoretical_loss": 3.6667919911351357, + "tokens_seen": 1031929856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005554423269809428, + "loss": 1.309, + "theoretical_loss": 3.6667701299961264, + "tokens_seen": 1031995392 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005554262788365096, + "loss": 1.3553, + "theoretical_loss": 3.6667482706340353, + "tokens_seen": 1032060928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005554102306920762, + "loss": 1.3999, + "theoretical_loss": 3.666726413048604, + "tokens_seen": 1032126464 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555394182547643, + "loss": 1.3929, + "theoretical_loss": 3.6667045572395756, + "tokens_seen": 1032192000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005553781344032096, + "loss": 1.3727, + "theoretical_loss": 3.6666827032066935, + "tokens_seen": 1032257536 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005553620862587764, + "loss": 1.4619, + "theoretical_loss": 3.6666608509497003, + "tokens_seen": 1032323072 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555346038114343, + "loss": 1.2926, + "theoretical_loss": 3.666639000468339, + "tokens_seen": 1032388608 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005553299899699098, + "loss": 1.3542, + "theoretical_loss": 3.6666171517623525, + "tokens_seen": 1032454144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005553139418254764, + "loss": 1.4046, + "theoretical_loss": 3.6665953048314837, + "tokens_seen": 1032519680 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005552978936810432, + "loss": 1.4319, + "theoretical_loss": 3.666573459675476, + "tokens_seen": 1032585216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005552818455366098, + "loss": 1.3742, + "theoretical_loss": 3.666551616294073, + "tokens_seen": 1032650752 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005552657973921766, + "loss": 1.3644, + "theoretical_loss": 3.6665297746870165, + "tokens_seen": 1032716288 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005552497492477432, + "loss": 1.3529, + "theoretical_loss": 3.6665079348540512, + "tokens_seen": 1032781824 + }, + { + "epoch": 0.31, + "learning_rate": 0.00055523370110331, + "loss": 1.364, + "theoretical_loss": 3.6664860967949204, + "tokens_seen": 1032847360 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005552176529588766, + "loss": 1.3583, + "theoretical_loss": 3.6664642605093665, + "tokens_seen": 1032912896 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005552016048144434, + "loss": 1.295, + "theoretical_loss": 3.6664424259971335, + "tokens_seen": 1032978432 + }, + { + "epoch": 0.31, + "learning_rate": 0.00055518555667001, + "loss": 1.3522, + "theoretical_loss": 3.666420593257965, + "tokens_seen": 1033043968 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005551695085255768, + "loss": 1.3841, + "theoretical_loss": 3.6663987622916046, + "tokens_seen": 1033109504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005551534603811434, + "loss": 1.3572, + "theoretical_loss": 3.6663769330977956, + "tokens_seen": 1033175040 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005551374122367102, + "loss": 1.1841, + "theoretical_loss": 3.6663551056762818, + "tokens_seen": 1033240576 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005551213640922768, + "loss": 1.4043, + "theoretical_loss": 3.666333280026807, + "tokens_seen": 1033306112 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005551053159478436, + "loss": 1.2789, + "theoretical_loss": 3.6663114561491144, + "tokens_seen": 1033371648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005550892678034102, + "loss": 1.4211, + "theoretical_loss": 3.6662896340429487, + "tokens_seen": 1033437184 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555073219658977, + "loss": 1.3789, + "theoretical_loss": 3.6662678137080533, + "tokens_seen": 1033502720 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005550571715145436, + "loss": 1.4442, + "theoretical_loss": 3.6662459951441724, + "tokens_seen": 1033568256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005550411233701104, + "loss": 1.4096, + "theoretical_loss": 3.6662241783510496, + "tokens_seen": 1033633792 + }, + { + "epoch": 0.31, + "learning_rate": 0.000555025075225677, + "loss": 1.3349, + "theoretical_loss": 3.6662023633284293, + "tokens_seen": 1033699328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005550090270812438, + "loss": 1.4083, + "theoretical_loss": 3.6661805500760547, + "tokens_seen": 1033764864 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005549929789368104, + "loss": 1.3662, + "theoretical_loss": 3.6661587385936714, + "tokens_seen": 1033830400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005549769307923772, + "loss": 1.3669, + "theoretical_loss": 3.6661369288810226, + "tokens_seen": 1033895936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005549608826479438, + "loss": 1.3294, + "theoretical_loss": 3.666115120937852, + "tokens_seen": 1033961472 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005549448345035106, + "loss": 1.4227, + "theoretical_loss": 3.6660933147639057, + "tokens_seen": 1034027008 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005549287863590774, + "loss": 1.377, + "theoretical_loss": 3.6660715103589263, + "tokens_seen": 1034092544 + }, + { + "epoch": 0.31, + "learning_rate": 0.000554912738214644, + "loss": 1.4855, + "theoretical_loss": 3.6660497077226593, + "tokens_seen": 1034158080 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005548966900702108, + "loss": 1.365, + "theoretical_loss": 3.6660279068548487, + "tokens_seen": 1034223616 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005548806419257774, + "loss": 1.3943, + "theoretical_loss": 3.666006107755239, + "tokens_seen": 1034289152 + }, + { + "epoch": 0.31, + "learning_rate": 0.000554864593781344, + "loss": 1.2954, + "theoretical_loss": 3.665984310423574, + "tokens_seen": 1034354688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005548485456369108, + "loss": 1.4227, + "theoretical_loss": 3.6659625148596002, + "tokens_seen": 1034420224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005548324974924774, + "loss": 1.2827, + "theoretical_loss": 3.665940721063061, + "tokens_seen": 1034485760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005548164493480442, + "loss": 1.312, + "theoretical_loss": 3.6659189290337006, + "tokens_seen": 1034551296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005548004012036108, + "loss": 1.3295, + "theoretical_loss": 3.665897138771265, + "tokens_seen": 1034616832 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005547843530591776, + "loss": 1.3047, + "theoretical_loss": 3.6658753502754986, + "tokens_seen": 1034682368 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005547683049147442, + "loss": 1.3276, + "theoretical_loss": 3.665853563546146, + "tokens_seen": 1034747904 + }, + { + "epoch": 0.31, + "learning_rate": 0.000554752256770311, + "loss": 1.3142, + "theoretical_loss": 3.6658317785829526, + "tokens_seen": 1034813440 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005547362086258777, + "loss": 1.404, + "theoretical_loss": 3.665809995385663, + "tokens_seen": 1034878976 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005547201604814443, + "loss": 1.4464, + "theoretical_loss": 3.665788213954022, + "tokens_seen": 1034944512 + }, + { + "epoch": 0.31, + "learning_rate": 0.000554704112337011, + "loss": 1.3824, + "theoretical_loss": 3.6657664342877756, + "tokens_seen": 1035010048 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005546880641925777, + "loss": 1.381, + "theoretical_loss": 3.665744656386668, + "tokens_seen": 1035075584 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005546720160481445, + "loss": 1.366, + "theoretical_loss": 3.6657228802504447, + "tokens_seen": 1035141120 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005546559679037111, + "loss": 1.3943, + "theoretical_loss": 3.6657011058788513, + "tokens_seen": 1035206656 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005546399197592779, + "loss": 1.3477, + "theoretical_loss": 3.665679333271633, + "tokens_seen": 1035272192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005546238716148445, + "loss": 1.3615, + "theoretical_loss": 3.6656575624285344, + "tokens_seen": 1035337728 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005546078234704113, + "loss": 1.3703, + "theoretical_loss": 3.665635793349302, + "tokens_seen": 1035403264 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 387689, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3519833087921143, + "objective/train/theoretical_loss": 3.6656140260336807, + "objective/train/tokens_used": 1055928800, + "theoretical_loss": 3.6656140260336807, + "tokens_seen": 1035468800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005545917753259779, + "loss": 1.352, + "theoretical_loss": 3.6656140260336807, + "tokens_seen": 1035468800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005545757271815447, + "loss": 1.2344, + "theoretical_loss": 3.665592260481416, + "tokens_seen": 1035534336 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005545596790371113, + "loss": 1.33, + "theoretical_loss": 3.6655704966922533, + "tokens_seen": 1035599872 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005545436308926781, + "loss": 1.3912, + "theoretical_loss": 3.665548734665939, + "tokens_seen": 1035665408 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005545275827482447, + "loss": 1.4187, + "theoretical_loss": 3.6655269744022174, + "tokens_seen": 1035730944 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005545115346038115, + "loss": 1.4349, + "theoretical_loss": 3.665505215900836, + "tokens_seen": 1035796480 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005544954864593781, + "loss": 1.2991, + "theoretical_loss": 3.665483459161539, + "tokens_seen": 1035862016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005544794383149449, + "loss": 1.3239, + "theoretical_loss": 3.6654617041840734, + "tokens_seen": 1035927552 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005544633901705115, + "loss": 1.3676, + "theoretical_loss": 3.6654399509681843, + "tokens_seen": 1035993088 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005544473420260783, + "loss": 1.3563, + "theoretical_loss": 3.665418199513618, + "tokens_seen": 1036058624 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005544312938816449, + "loss": 1.3276, + "theoretical_loss": 3.6653964498201197, + "tokens_seen": 1036124160 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005544152457372117, + "loss": 1.368, + "theoretical_loss": 3.665374701887437, + "tokens_seen": 1036189696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005543991975927783, + "loss": 1.2814, + "theoretical_loss": 3.6653529557153144, + "tokens_seen": 1036255232 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005543831494483451, + "loss": 1.3295, + "theoretical_loss": 3.6653312113034993, + "tokens_seen": 1036320768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005543671013039117, + "loss": 1.4365, + "theoretical_loss": 3.665309468651737, + "tokens_seen": 1036386304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005543510531594785, + "loss": 1.3802, + "theoretical_loss": 3.6652877277597744, + "tokens_seen": 1036451840 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005543350050150451, + "loss": 1.2382, + "theoretical_loss": 3.6652659886273575, + "tokens_seen": 1036517376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005543189568706119, + "loss": 1.3423, + "theoretical_loss": 3.6652442512542325, + "tokens_seen": 1036582912 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005543029087261785, + "loss": 1.3368, + "theoretical_loss": 3.6652225156401457, + "tokens_seen": 1036648448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005542868605817453, + "loss": 1.3618, + "theoretical_loss": 3.6652007817848444, + "tokens_seen": 1036713984 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005542708124373119, + "loss": 1.3762, + "theoretical_loss": 3.6651790496880747, + "tokens_seen": 1036779520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005542547642928787, + "loss": 1.3533, + "theoretical_loss": 3.6651573193495826, + "tokens_seen": 1036845056 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005542387161484453, + "loss": 1.2913, + "theoretical_loss": 3.665135590769115, + "tokens_seen": 1036910592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005542226680040121, + "loss": 1.4072, + "theoretical_loss": 3.665113863946419, + "tokens_seen": 1036976128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005542066198595787, + "loss": 1.4247, + "theoretical_loss": 3.6650921388812416, + "tokens_seen": 1037041664 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005541905717151455, + "loss": 1.4063, + "theoretical_loss": 3.6650704155733287, + "tokens_seen": 1037107200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005541745235707121, + "loss": 1.3022, + "theoretical_loss": 3.6650486940224267, + "tokens_seen": 1037172736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005541584754262789, + "loss": 1.3037, + "theoretical_loss": 3.6650269742282844, + "tokens_seen": 1037238272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005541424272818455, + "loss": 1.3422, + "theoretical_loss": 3.6650052561906468, + "tokens_seen": 1037303808 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005541263791374123, + "loss": 1.3429, + "theoretical_loss": 3.664983539909262, + "tokens_seen": 1037369344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005541103309929789, + "loss": 1.3874, + "theoretical_loss": 3.6649618253838767, + "tokens_seen": 1037434880 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005540942828485457, + "loss": 1.3205, + "theoretical_loss": 3.664940112614238, + "tokens_seen": 1037500416 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005540782347041123, + "loss": 1.4747, + "theoretical_loss": 3.664918401600093, + "tokens_seen": 1037565952 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005540621865596791, + "loss": 1.3287, + "theoretical_loss": 3.664896692341189, + "tokens_seen": 1037631488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005540461384152457, + "loss": 1.3859, + "theoretical_loss": 3.6648749848372733, + "tokens_seen": 1037697024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005540300902708125, + "loss": 1.3247, + "theoretical_loss": 3.664853279088093, + "tokens_seen": 1037762560 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005540140421263791, + "loss": 1.3327, + "theoretical_loss": 3.6648315750933955, + "tokens_seen": 1037828096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005539979939819459, + "loss": 1.3282, + "theoretical_loss": 3.664809872852928, + "tokens_seen": 1037893632 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005539819458375125, + "loss": 1.3358, + "theoretical_loss": 3.6647881723664386, + "tokens_seen": 1037959168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005539658976930793, + "loss": 1.2916, + "theoretical_loss": 3.6647664736336747, + "tokens_seen": 1038024704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005539498495486459, + "loss": 1.3769, + "theoretical_loss": 3.6647447766543833, + "tokens_seen": 1038090240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005539338014042127, + "loss": 1.367, + "theoretical_loss": 3.664723081428312, + "tokens_seen": 1038155776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005539177532597793, + "loss": 1.5205, + "theoretical_loss": 3.6647013879552093, + "tokens_seen": 1038221312 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005539017051153461, + "loss": 1.4299, + "theoretical_loss": 3.664679696234822, + "tokens_seen": 1038286848 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005538856569709127, + "loss": 1.3177, + "theoretical_loss": 3.6646580062668983, + "tokens_seen": 1038352384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005538696088264795, + "loss": 1.343, + "theoretical_loss": 3.664636318051186, + "tokens_seen": 1038417920 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005538535606820461, + "loss": 1.2487, + "theoretical_loss": 3.6646146315874333, + "tokens_seen": 1038483456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005538375125376129, + "loss": 1.4344, + "theoretical_loss": 3.6645929468753877, + "tokens_seen": 1038548992 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005538214643931795, + "loss": 1.3606, + "theoretical_loss": 3.664571263914797, + "tokens_seen": 1038614528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005538054162487463, + "loss": 1.4101, + "theoretical_loss": 3.66454958270541, + "tokens_seen": 1038680064 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005537893681043129, + "loss": 1.397, + "theoretical_loss": 3.6645279032469746, + "tokens_seen": 1038745600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005537733199598797, + "loss": 1.3619, + "theoretical_loss": 3.664506225539238, + "tokens_seen": 1038811136 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005537572718154463, + "loss": 1.4008, + "theoretical_loss": 3.664484549581949, + "tokens_seen": 1038876672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005537412236710131, + "loss": 1.3729, + "theoretical_loss": 3.6644628753748565, + "tokens_seen": 1038942208 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005537251755265798, + "loss": 1.3718, + "theoretical_loss": 3.6644412029177076, + "tokens_seen": 1039007744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005537091273821465, + "loss": 1.2331, + "theoretical_loss": 3.6644195322102515, + "tokens_seen": 1039073280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005536930792377132, + "loss": 1.3894, + "theoretical_loss": 3.664397863252236, + "tokens_seen": 1039138816 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005536770310932799, + "loss": 1.4726, + "theoretical_loss": 3.6643761960434107, + "tokens_seen": 1039204352 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005536609829488466, + "loss": 1.3226, + "theoretical_loss": 3.6643545305835223, + "tokens_seen": 1039269888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005536449348044133, + "loss": 1.3899, + "theoretical_loss": 3.664332866872321, + "tokens_seen": 1039335424 + }, + { + "epoch": 0.31, + "learning_rate": 0.00055362888665998, + "loss": 1.3434, + "theoretical_loss": 3.6643112049095543, + "tokens_seen": 1039400960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0005536128385155467, + "loss": 1.3587, + "theoretical_loss": 3.6642895446949715, + "tokens_seen": 1039466496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005535967903711134, + "loss": 1.3191, + "theoretical_loss": 3.664267886228321, + "tokens_seen": 1039532032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005535807422266801, + "loss": 1.359, + "theoretical_loss": 3.6642462295093523, + "tokens_seen": 1039597568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005535646940822469, + "loss": 1.3286, + "theoretical_loss": 3.664224574537813, + "tokens_seen": 1039663104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005535486459378135, + "loss": 1.3576, + "theoretical_loss": 3.6642029213134526, + "tokens_seen": 1039728640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005535325977933803, + "loss": 1.432, + "theoretical_loss": 3.66418126983602, + "tokens_seen": 1039794176 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005535165496489469, + "loss": 1.3392, + "theoretical_loss": 3.6641596201052637, + "tokens_seen": 1039859712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005535005015045137, + "loss": 1.3556, + "theoretical_loss": 3.6641379721209337, + "tokens_seen": 1039925248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005534844533600803, + "loss": 1.3791, + "theoretical_loss": 3.6641163258827785, + "tokens_seen": 1039990784 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005534684052156471, + "loss": 1.4463, + "theoretical_loss": 3.664094681390547, + "tokens_seen": 1040056320 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005534523570712137, + "loss": 1.4307, + "theoretical_loss": 3.6640730386439895, + "tokens_seen": 1040121856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005534363089267803, + "loss": 1.5391, + "theoretical_loss": 3.6640513976428535, + "tokens_seen": 1040187392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005534202607823471, + "loss": 1.3742, + "theoretical_loss": 3.6640297583868895, + "tokens_seen": 1040252928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005534042126379137, + "loss": 1.2645, + "theoretical_loss": 3.664008120875846, + "tokens_seen": 1040318464 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005533881644934805, + "loss": 1.4027, + "theoretical_loss": 3.663986485109474, + "tokens_seen": 1040384000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005533721163490472, + "loss": 1.3465, + "theoretical_loss": 3.6639648510875205, + "tokens_seen": 1040449536 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005533560682046139, + "loss": 1.2937, + "theoretical_loss": 3.663943218809737, + "tokens_seen": 1040515072 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005533400200601806, + "loss": 1.3472, + "theoretical_loss": 3.6639215882758727, + "tokens_seen": 1040580608 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005533239719157473, + "loss": 1.3193, + "theoretical_loss": 3.663899959485676, + "tokens_seen": 1040646144 + }, + { + "epoch": 0.32, + "learning_rate": 0.000553307923771314, + "loss": 1.3074, + "theoretical_loss": 3.6638783324388977, + "tokens_seen": 1040711680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005532918756268806, + "loss": 1.3699, + "theoretical_loss": 3.663856707135287, + "tokens_seen": 1040777216 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005532758274824474, + "loss": 1.423, + "theoretical_loss": 3.6638350835745945, + "tokens_seen": 1040842752 + }, + { + "epoch": 0.32, + "learning_rate": 0.000553259779338014, + "loss": 1.3429, + "theoretical_loss": 3.663813461756569, + "tokens_seen": 1040908288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005532437311935808, + "loss": 1.3962, + "theoretical_loss": 3.6637918416809603, + "tokens_seen": 1040973824 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005532276830491474, + "loss": 1.3297, + "theoretical_loss": 3.663770223347519, + "tokens_seen": 1041039360 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005532116349047142, + "loss": 1.2241, + "theoretical_loss": 3.663748606755995, + "tokens_seen": 1041104896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005531955867602808, + "loss": 1.4459, + "theoretical_loss": 3.6637269919061373, + "tokens_seen": 1041170432 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005531795386158476, + "loss": 1.4275, + "theoretical_loss": 3.6637053787976974, + "tokens_seen": 1041235968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005531634904714142, + "loss": 1.33, + "theoretical_loss": 3.663683767430424, + "tokens_seen": 1041301504 + }, + { + "epoch": 0.32, + "learning_rate": 0.000553147442326981, + "loss": 1.371, + "theoretical_loss": 3.6636621578040685, + "tokens_seen": 1041367040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005531313941825476, + "loss": 1.3752, + "theoretical_loss": 3.66364054991838, + "tokens_seen": 1041432576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005531153460381144, + "loss": 1.4024, + "theoretical_loss": 3.66361894377311, + "tokens_seen": 1041498112 + }, + { + "epoch": 0.32, + "learning_rate": 0.000553099297893681, + "loss": 1.406, + "theoretical_loss": 3.6635973393680077, + "tokens_seen": 1041563648 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005530832497492478, + "loss": 1.3164, + "theoretical_loss": 3.6635757367028248, + "tokens_seen": 1041629184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005530672016048144, + "loss": 1.4598, + "theoretical_loss": 3.66355413577731, + "tokens_seen": 1041694720 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005530511534603812, + "loss": 1.4003, + "theoretical_loss": 3.6635325365912146, + "tokens_seen": 1041760256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005530351053159478, + "loss": 1.4144, + "theoretical_loss": 3.663510939144289, + "tokens_seen": 1041825792 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005530190571715146, + "loss": 1.4998, + "theoretical_loss": 3.6634893434362845, + "tokens_seen": 1041891328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005530030090270812, + "loss": 1.4211, + "theoretical_loss": 3.6634677494669505, + "tokens_seen": 1041956864 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 390282, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.412980079650879, + "objective/train/theoretical_loss": 3.663446157236039, + "objective/train/tokens_used": 1062482400, + "theoretical_loss": 3.663446157236039, + "tokens_seen": 1042022400 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552986960882648, + "loss": 1.413, + "theoretical_loss": 3.663446157236039, + "tokens_seen": 1042022400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005529709127382146, + "loss": 1.3768, + "theoretical_loss": 3.6634245667432994, + "tokens_seen": 1042087936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005529548645937814, + "loss": 1.3865, + "theoretical_loss": 3.6634029779884836, + "tokens_seen": 1042153472 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552938816449348, + "loss": 1.2739, + "theoretical_loss": 3.6633813909713417, + "tokens_seen": 1042219008 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005529227683049148, + "loss": 1.3874, + "theoretical_loss": 3.6633598056916243, + "tokens_seen": 1042284544 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005529067201604814, + "loss": 1.297, + "theoretical_loss": 3.6633382221490836, + "tokens_seen": 1042350080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005528906720160482, + "loss": 1.319, + "theoretical_loss": 3.66331664034347, + "tokens_seen": 1042415616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005528746238716148, + "loss": 1.3017, + "theoretical_loss": 3.6632950602745336, + "tokens_seen": 1042481152 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005528585757271816, + "loss": 1.4234, + "theoretical_loss": 3.663273481942026, + "tokens_seen": 1042546688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005528425275827482, + "loss": 1.4234, + "theoretical_loss": 3.6632519053456996, + "tokens_seen": 1042612224 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552826479438315, + "loss": 1.4056, + "theoretical_loss": 3.6632303304853044, + "tokens_seen": 1042677760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005528104312938816, + "loss": 1.3758, + "theoretical_loss": 3.663208757360591, + "tokens_seen": 1042743296 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005527943831494484, + "loss": 1.3192, + "theoretical_loss": 3.6631871859713123, + "tokens_seen": 1042808832 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552778335005015, + "loss": 1.4996, + "theoretical_loss": 3.663165616317219, + "tokens_seen": 1042874368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005527622868605818, + "loss": 1.4285, + "theoretical_loss": 3.6631440483980615, + "tokens_seen": 1042939904 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005527462387161484, + "loss": 1.3108, + "theoretical_loss": 3.663122482213593, + "tokens_seen": 1043005440 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005527301905717152, + "loss": 1.3651, + "theoretical_loss": 3.6631009177635634, + "tokens_seen": 1043070976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005527141424272818, + "loss": 1.4558, + "theoretical_loss": 3.663079355047725, + "tokens_seen": 1043136512 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005526980942828486, + "loss": 1.3893, + "theoretical_loss": 3.6630577940658298, + "tokens_seen": 1043202048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005526820461384152, + "loss": 1.346, + "theoretical_loss": 3.6630362348176284, + "tokens_seen": 1043267584 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552665997993982, + "loss": 1.3461, + "theoretical_loss": 3.663014677302874, + "tokens_seen": 1043333120 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005526499498495486, + "loss": 1.4654, + "theoretical_loss": 3.6629931215213167, + "tokens_seen": 1043398656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005526339017051154, + "loss": 1.3814, + "theoretical_loss": 3.6629715674727095, + "tokens_seen": 1043464192 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552617853560682, + "loss": 1.5062, + "theoretical_loss": 3.6629500151568033, + "tokens_seen": 1043529728 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005526018054162488, + "loss": 1.3398, + "theoretical_loss": 3.6629284645733504, + "tokens_seen": 1043595264 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005525857572718154, + "loss": 1.4166, + "theoretical_loss": 3.662906915722103, + "tokens_seen": 1043660800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005525697091273822, + "loss": 1.4087, + "theoretical_loss": 3.6628853686028133, + "tokens_seen": 1043726336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005525536609829488, + "loss": 1.4223, + "theoretical_loss": 3.662863823215233, + "tokens_seen": 1043791872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005525376128385156, + "loss": 1.4063, + "theoretical_loss": 3.662842279559114, + "tokens_seen": 1043857408 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005525215646940823, + "loss": 1.3292, + "theoretical_loss": 3.6628207376342092, + "tokens_seen": 1043922944 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552505516549649, + "loss": 1.4256, + "theoretical_loss": 3.66279919744027, + "tokens_seen": 1043988480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005524894684052157, + "loss": 1.373, + "theoretical_loss": 3.6627776589770487, + "tokens_seen": 1044054016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005524734202607824, + "loss": 1.3696, + "theoretical_loss": 3.662756122244298, + "tokens_seen": 1044119552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005524573721163491, + "loss": 1.2793, + "theoretical_loss": 3.6627345872417703, + "tokens_seen": 1044185088 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005524413239719158, + "loss": 1.5225, + "theoretical_loss": 3.662713053969218, + "tokens_seen": 1044250624 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005524252758274825, + "loss": 1.3696, + "theoretical_loss": 3.6626915224263934, + "tokens_seen": 1044316160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005524092276830492, + "loss": 1.2573, + "theoretical_loss": 3.662669992613049, + "tokens_seen": 1044381696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005523931795386159, + "loss": 1.4214, + "theoretical_loss": 3.6626484645289374, + "tokens_seen": 1044447232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005523771313941826, + "loss": 1.4251, + "theoretical_loss": 3.662626938173811, + "tokens_seen": 1044512768 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005523610832497493, + "loss": 1.3732, + "theoretical_loss": 3.662605413547423, + "tokens_seen": 1044578304 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552345035105316, + "loss": 1.3852, + "theoretical_loss": 3.6625838906495263, + "tokens_seen": 1044643840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005523289869608827, + "loss": 1.3648, + "theoretical_loss": 3.6625623694798723, + "tokens_seen": 1044709376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005523129388164494, + "loss": 1.3041, + "theoretical_loss": 3.6625408500382153, + "tokens_seen": 1044774912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005522968906720161, + "loss": 1.3885, + "theoretical_loss": 3.6625193323243073, + "tokens_seen": 1044840448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005522808425275828, + "loss": 1.2271, + "theoretical_loss": 3.6624978163379014, + "tokens_seen": 1044905984 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005522647943831495, + "loss": 1.4392, + "theoretical_loss": 3.662476302078751, + "tokens_seen": 1044971520 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005522487462387162, + "loss": 1.3102, + "theoretical_loss": 3.6624547895466093, + "tokens_seen": 1045037056 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552232698094283, + "loss": 1.3846, + "theoretical_loss": 3.6624332787412284, + "tokens_seen": 1045102592 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005522166499498496, + "loss": 1.3733, + "theoretical_loss": 3.662411769662362, + "tokens_seen": 1045168128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005522006018054164, + "loss": 1.3886, + "theoretical_loss": 3.6623902623097635, + "tokens_seen": 1045233664 + }, + { + "epoch": 0.32, + "learning_rate": 0.000552184553660983, + "loss": 1.3989, + "theoretical_loss": 3.6623687566831857, + "tokens_seen": 1045299200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005521685055165498, + "loss": 1.3373, + "theoretical_loss": 3.662347252782382, + "tokens_seen": 1045364736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005521524573721164, + "loss": 1.2977, + "theoretical_loss": 3.6623257506071054, + "tokens_seen": 1045430272 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005521364092276832, + "loss": 1.4041, + "theoretical_loss": 3.66230425015711, + "tokens_seen": 1045495808 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005521203610832498, + "loss": 1.3485, + "theoretical_loss": 3.662282751432149, + "tokens_seen": 1045561344 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005521043129388166, + "loss": 1.3755, + "theoretical_loss": 3.662261254431976, + "tokens_seen": 1045626880 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005520882647943832, + "loss": 1.4355, + "theoretical_loss": 3.6622397591563436, + "tokens_seen": 1045692416 + }, + { + "epoch": 0.32, + "learning_rate": 0.00055207221664995, + "loss": 1.4134, + "theoretical_loss": 3.6622182656050066, + "tokens_seen": 1045757952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005520561685055166, + "loss": 1.3084, + "theoretical_loss": 3.662196773777718, + "tokens_seen": 1045823488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005520401203610832, + "loss": 1.4362, + "theoretical_loss": 3.6621752836742316, + "tokens_seen": 1045889024 + }, + { + "epoch": 0.32, + "learning_rate": 0.00055202407221665, + "loss": 1.3655, + "theoretical_loss": 3.662153795294301, + "tokens_seen": 1045954560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005520080240722167, + "loss": 1.3615, + "theoretical_loss": 3.6621323086376805, + "tokens_seen": 1046020096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005519919759277834, + "loss": 1.3501, + "theoretical_loss": 3.6621108237041233, + "tokens_seen": 1046085632 + }, + { + "epoch": 0.32, + "learning_rate": 0.00055197592778335, + "loss": 1.53, + "theoretical_loss": 3.6620893404933836, + "tokens_seen": 1046151168 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005519598796389168, + "loss": 1.4042, + "theoretical_loss": 3.6620678590052154, + "tokens_seen": 1046216704 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005519438314944835, + "loss": 1.3872, + "theoretical_loss": 3.6620463792393725, + "tokens_seen": 1046282240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005519277833500502, + "loss": 1.3521, + "theoretical_loss": 3.6620249011956094, + "tokens_seen": 1046347776 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005519117352056169, + "loss": 1.316, + "theoretical_loss": 3.662003424873679, + "tokens_seen": 1046413312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005518956870611835, + "loss": 1.3666, + "theoretical_loss": 3.6619819502733373, + "tokens_seen": 1046478848 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005518796389167503, + "loss": 1.315, + "theoretical_loss": 3.6619604773943366, + "tokens_seen": 1046544384 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005518635907723169, + "loss": 1.3662, + "theoretical_loss": 3.661939006236432, + "tokens_seen": 1046609920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005518475426278837, + "loss": 1.3645, + "theoretical_loss": 3.661917536799378, + "tokens_seen": 1046675456 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005518314944834503, + "loss": 1.4975, + "theoretical_loss": 3.661896069082929, + "tokens_seen": 1046740992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005518154463390171, + "loss": 1.3145, + "theoretical_loss": 3.661874603086839, + "tokens_seen": 1046806528 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005517993981945837, + "loss": 1.3209, + "theoretical_loss": 3.661853138810863, + "tokens_seen": 1046872064 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005517833500501505, + "loss": 1.2741, + "theoretical_loss": 3.6618316762547543, + "tokens_seen": 1046937600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005517673019057171, + "loss": 1.4018, + "theoretical_loss": 3.6618102154182686, + "tokens_seen": 1047003136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005517512537612839, + "loss": 1.4, + "theoretical_loss": 3.66178875630116, + "tokens_seen": 1047068672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005517352056168505, + "loss": 1.3474, + "theoretical_loss": 3.6617672989031833, + "tokens_seen": 1047134208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005517191574724173, + "loss": 1.2951, + "theoretical_loss": 3.661745843224093, + "tokens_seen": 1047199744 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005517031093279839, + "loss": 1.3381, + "theoretical_loss": 3.6617243892636444, + "tokens_seen": 1047265280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005516870611835507, + "loss": 1.366, + "theoretical_loss": 3.6617029370215914, + "tokens_seen": 1047330816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005516710130391173, + "loss": 1.4059, + "theoretical_loss": 3.66168148649769, + "tokens_seen": 1047396352 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005516549648946841, + "loss": 1.4024, + "theoretical_loss": 3.6616600376916937, + "tokens_seen": 1047461888 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005516389167502507, + "loss": 1.4048, + "theoretical_loss": 3.6616385906033586, + "tokens_seen": 1047527424 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005516228686058175, + "loss": 1.3395, + "theoretical_loss": 3.661617145232439, + "tokens_seen": 1047592960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005516068204613841, + "loss": 1.4944, + "theoretical_loss": 3.6615957015786904, + "tokens_seen": 1047658496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005515907723169509, + "loss": 1.3786, + "theoretical_loss": 3.661574259641868, + "tokens_seen": 1047724032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005515747241725175, + "loss": 1.4025, + "theoretical_loss": 3.6615528194217264, + "tokens_seen": 1047789568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005515586760280843, + "loss": 1.3597, + "theoretical_loss": 3.661531380918021, + "tokens_seen": 1047855104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005515426278836509, + "loss": 1.3347, + "theoretical_loss": 3.6615099441305077, + "tokens_seen": 1047920640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005515265797392177, + "loss": 1.4141, + "theoretical_loss": 3.6614885090589406, + "tokens_seen": 1047986176 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005515105315947843, + "loss": 1.2832, + "theoretical_loss": 3.661467075703076, + "tokens_seen": 1048051712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005514944834503511, + "loss": 1.3026, + "theoretical_loss": 3.6614456440626686, + "tokens_seen": 1048117248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005514784353059177, + "loss": 1.4487, + "theoretical_loss": 3.6614242141374747, + "tokens_seen": 1048182784 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005514623871614845, + "loss": 1.3547, + "theoretical_loss": 3.6614027859272493, + "tokens_seen": 1048248320 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005514463390170511, + "loss": 1.4202, + "theoretical_loss": 3.6613813594317475, + "tokens_seen": 1048313856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005514302908726179, + "loss": 1.4794, + "theoretical_loss": 3.6613599346507257, + "tokens_seen": 1048379392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005514142427281845, + "loss": 1.3837, + "theoretical_loss": 3.6613385115839394, + "tokens_seen": 1048444928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005513981945837513, + "loss": 1.325, + "theoretical_loss": 3.661317090231144, + "tokens_seen": 1048510464 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 392596, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.443063735961914, + "objective/train/theoretical_loss": 3.661295670592095, + "objective/train/tokens_used": 1069036000, + "theoretical_loss": 3.661295670592095, + "tokens_seen": 1048576000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005513821464393179, + "loss": 1.4431, + "theoretical_loss": 3.661295670592095, + "tokens_seen": 1048576000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005513660982948847, + "loss": 1.3731, + "theoretical_loss": 3.661274252666549, + "tokens_seen": 1048641536 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005513500501504514, + "loss": 1.3286, + "theoretical_loss": 3.6612528364542616, + "tokens_seen": 1048707072 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005513340020060181, + "loss": 1.4432, + "theoretical_loss": 3.661231421954988, + "tokens_seen": 1048772608 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005513179538615848, + "loss": 1.4146, + "theoretical_loss": 3.661210009168485, + "tokens_seen": 1048838144 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005513019057171515, + "loss": 1.3041, + "theoretical_loss": 3.6611885980945083, + "tokens_seen": 1048903680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005512858575727182, + "loss": 1.3555, + "theoretical_loss": 3.661167188732814, + "tokens_seen": 1048969216 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005512698094282849, + "loss": 1.3591, + "theoretical_loss": 3.661145781083158, + "tokens_seen": 1049034752 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005512537612838516, + "loss": 1.3203, + "theoretical_loss": 3.6611243751452966, + "tokens_seen": 1049100288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005512377131394183, + "loss": 1.3778, + "theoretical_loss": 3.661102970918986, + "tokens_seen": 1049165824 + }, + { + "epoch": 0.32, + "learning_rate": 0.000551221664994985, + "loss": 1.4316, + "theoretical_loss": 3.661081568403983, + "tokens_seen": 1049231360 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005512056168505517, + "loss": 1.3795, + "theoretical_loss": 3.6610601676000423, + "tokens_seen": 1049296896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005511895687061184, + "loss": 1.43, + "theoretical_loss": 3.6610387685069217, + "tokens_seen": 1049362432 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005511735205616851, + "loss": 1.3974, + "theoretical_loss": 3.6610173711243776, + "tokens_seen": 1049427968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005511574724172518, + "loss": 1.4144, + "theoretical_loss": 3.660995975452166, + "tokens_seen": 1049493504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005511414242728185, + "loss": 1.3453, + "theoretical_loss": 3.660974581490043, + "tokens_seen": 1049559040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005511253761283852, + "loss": 1.3688, + "theoretical_loss": 3.660953189237766, + "tokens_seen": 1049624576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005511093279839519, + "loss": 1.2454, + "theoretical_loss": 3.660931798695091, + "tokens_seen": 1049690112 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005510932798395186, + "loss": 1.4181, + "theoretical_loss": 3.6609104098617746, + "tokens_seen": 1049755648 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005510772316950853, + "loss": 1.4, + "theoretical_loss": 3.6608890227375737, + "tokens_seen": 1049821184 + }, + { + "epoch": 0.32, + "learning_rate": 0.000551061183550652, + "loss": 1.3549, + "theoretical_loss": 3.6608676373222457, + "tokens_seen": 1049886720 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005510451354062187, + "loss": 1.443, + "theoretical_loss": 3.6608462536155466, + "tokens_seen": 1049952256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005510290872617854, + "loss": 1.3792, + "theoretical_loss": 3.6608248716172334, + "tokens_seen": 1050017792 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005510130391173521, + "loss": 1.3019, + "theoretical_loss": 3.660803491327063, + "tokens_seen": 1050083328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005509969909729188, + "loss": 1.3596, + "theoretical_loss": 3.6607821127447924, + "tokens_seen": 1050148864 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005509809428284855, + "loss": 1.3251, + "theoretical_loss": 3.6607607358701784, + "tokens_seen": 1050214400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005509648946840522, + "loss": 1.3615, + "theoretical_loss": 3.660739360702978, + "tokens_seen": 1050279936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005509488465396189, + "loss": 1.378, + "theoretical_loss": 3.660717987242949, + "tokens_seen": 1050345472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005509327983951856, + "loss": 1.2841, + "theoretical_loss": 3.660696615489848, + "tokens_seen": 1050411008 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005509167502507523, + "loss": 1.3183, + "theoretical_loss": 3.6606752454434326, + "tokens_seen": 1050476544 + }, + { + "epoch": 0.32, + "learning_rate": 0.000550900702106319, + "loss": 1.3607, + "theoretical_loss": 3.660653877103459, + "tokens_seen": 1050542080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005508846539618857, + "loss": 1.3904, + "theoretical_loss": 3.660632510469686, + "tokens_seen": 1050607616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005508686058174524, + "loss": 1.4378, + "theoretical_loss": 3.66061114554187, + "tokens_seen": 1050673152 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005508525576730191, + "loss": 1.3736, + "theoretical_loss": 3.6605897823197684, + "tokens_seen": 1050738688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005508365095285859, + "loss": 1.3254, + "theoretical_loss": 3.6605684208031386, + "tokens_seen": 1050804224 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005508204613841525, + "loss": 1.3892, + "theoretical_loss": 3.660547060991739, + "tokens_seen": 1050869760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005508044132397193, + "loss": 1.3636, + "theoretical_loss": 3.6605257028853257, + "tokens_seen": 1050935296 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005507883650952859, + "loss": 1.3789, + "theoretical_loss": 3.6605043464836573, + "tokens_seen": 1051000832 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005507723169508527, + "loss": 1.3805, + "theoretical_loss": 3.660482991786491, + "tokens_seen": 1051066368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005507562688064193, + "loss": 1.3757, + "theoretical_loss": 3.660461638793585, + "tokens_seen": 1051131904 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005507402206619861, + "loss": 1.3436, + "theoretical_loss": 3.660440287504697, + "tokens_seen": 1051197440 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005507241725175527, + "loss": 1.3548, + "theoretical_loss": 3.6604189379195837, + "tokens_seen": 1051262976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005507081243731195, + "loss": 1.371, + "theoretical_loss": 3.6603975900380044, + "tokens_seen": 1051328512 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005506920762286861, + "loss": 1.3208, + "theoretical_loss": 3.660376243859716, + "tokens_seen": 1051394048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005506760280842529, + "loss": 1.3309, + "theoretical_loss": 3.660354899384477, + "tokens_seen": 1051459584 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005506599799398195, + "loss": 1.3443, + "theoretical_loss": 3.6603335566120454, + "tokens_seen": 1051525120 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005506439317953861, + "loss": 1.3902, + "theoretical_loss": 3.660312215542179, + "tokens_seen": 1051590656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005506278836509529, + "loss": 1.3521, + "theoretical_loss": 3.6602908761746358, + "tokens_seen": 1051656192 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005506118355065196, + "loss": 1.4538, + "theoretical_loss": 3.6602695385091737, + "tokens_seen": 1051721728 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005505957873620863, + "loss": 1.3487, + "theoretical_loss": 3.660248202545551, + "tokens_seen": 1051787264 + }, + { + "epoch": 0.32, + "learning_rate": 0.000550579739217653, + "loss": 1.4289, + "theoretical_loss": 3.660226868283527, + "tokens_seen": 1051852800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005505636910732197, + "loss": 1.3451, + "theoretical_loss": 3.660205535722859, + "tokens_seen": 1051918336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005505476429287864, + "loss": 1.3476, + "theoretical_loss": 3.660184204863305, + "tokens_seen": 1051983872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005505315947843531, + "loss": 1.2839, + "theoretical_loss": 3.6601628757046245, + "tokens_seen": 1052049408 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005505155466399198, + "loss": 1.4139, + "theoretical_loss": 3.660141548246575, + "tokens_seen": 1052114944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005504994984954864, + "loss": 1.3599, + "theoretical_loss": 3.6601202224889153, + "tokens_seen": 1052180480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005504834503510532, + "loss": 1.328, + "theoretical_loss": 3.6600988984314045, + "tokens_seen": 1052246016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005504674022066198, + "loss": 1.3973, + "theoretical_loss": 3.6600775760738, + "tokens_seen": 1052311552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005504513540621866, + "loss": 1.4649, + "theoretical_loss": 3.660056255415861, + "tokens_seen": 1052377088 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005504353059177532, + "loss": 1.3864, + "theoretical_loss": 3.660034936457347, + "tokens_seen": 1052442624 + }, + { + "epoch": 0.32, + "learning_rate": 0.00055041925777332, + "loss": 1.4108, + "theoretical_loss": 3.660013619198015, + "tokens_seen": 1052508160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005504032096288866, + "loss": 1.2919, + "theoretical_loss": 3.659992303637625, + "tokens_seen": 1052573696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005503871614844534, + "loss": 1.2909, + "theoretical_loss": 3.6599709897759363, + "tokens_seen": 1052639232 + }, + { + "epoch": 0.32, + "learning_rate": 0.00055037111334002, + "loss": 1.4121, + "theoretical_loss": 3.6599496776127065, + "tokens_seen": 1052704768 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005503550651955868, + "loss": 1.3013, + "theoretical_loss": 3.659928367147695, + "tokens_seen": 1052770304 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005503390170511534, + "loss": 1.461, + "theoretical_loss": 3.6599070583806617, + "tokens_seen": 1052835840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005503229689067202, + "loss": 1.3272, + "theoretical_loss": 3.6598857513113634, + "tokens_seen": 1052901376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005503069207622868, + "loss": 1.404, + "theoretical_loss": 3.6598644459395615, + "tokens_seen": 1052966912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005502908726178536, + "loss": 1.4547, + "theoretical_loss": 3.6598431422650144, + "tokens_seen": 1053032448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005502748244734202, + "loss": 1.3523, + "theoretical_loss": 3.6598218402874805, + "tokens_seen": 1053097984 + }, + { + "epoch": 0.32, + "learning_rate": 0.000550258776328987, + "loss": 1.4776, + "theoretical_loss": 3.65980054000672, + "tokens_seen": 1053163520 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005502427281845536, + "loss": 1.3137, + "theoretical_loss": 3.6597792414224912, + "tokens_seen": 1053229056 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005502266800401204, + "loss": 1.2979, + "theoretical_loss": 3.6597579445345545, + "tokens_seen": 1053294592 + }, + { + "epoch": 0.32, + "learning_rate": 0.000550210631895687, + "loss": 1.3407, + "theoretical_loss": 3.6597366493426686, + "tokens_seen": 1053360128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005501945837512538, + "loss": 1.3552, + "theoretical_loss": 3.6597153558465934, + "tokens_seen": 1053425664 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005501785356068204, + "loss": 1.4248, + "theoretical_loss": 3.6596940640460875, + "tokens_seen": 1053491200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005501624874623872, + "loss": 1.3538, + "theoretical_loss": 3.6596727739409114, + "tokens_seen": 1053556736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005501464393179539, + "loss": 1.2939, + "theoretical_loss": 3.6596514855308238, + "tokens_seen": 1053622272 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005501303911735206, + "loss": 1.324, + "theoretical_loss": 3.6596301988155853, + "tokens_seen": 1053687808 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005501143430290873, + "loss": 1.4518, + "theoretical_loss": 3.659608913794955, + "tokens_seen": 1053753344 + }, + { + "epoch": 0.32, + "learning_rate": 0.000550098294884654, + "loss": 1.4075, + "theoretical_loss": 3.659587630468692, + "tokens_seen": 1053818880 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005500822467402207, + "loss": 1.3934, + "theoretical_loss": 3.6595663488365573, + "tokens_seen": 1053884416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005500661985957874, + "loss": 1.4478, + "theoretical_loss": 3.6595450688983098, + "tokens_seen": 1053949952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005500501504513541, + "loss": 1.4386, + "theoretical_loss": 3.6595237906537097, + "tokens_seen": 1054015488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005500341023069208, + "loss": 1.3915, + "theoretical_loss": 3.659502514102517, + "tokens_seen": 1054081024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005500180541624875, + "loss": 1.435, + "theoretical_loss": 3.6594812392444913, + "tokens_seen": 1054146560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005500020060180542, + "loss": 1.3717, + "theoretical_loss": 3.6594599660793934, + "tokens_seen": 1054212096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005499859578736209, + "loss": 1.429, + "theoretical_loss": 3.6594386946069823, + "tokens_seen": 1054277632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005499699097291876, + "loss": 1.3627, + "theoretical_loss": 3.659417424827019, + "tokens_seen": 1054343168 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005499538615847543, + "loss": 1.3638, + "theoretical_loss": 3.6593961567392626, + "tokens_seen": 1054408704 + }, + { + "epoch": 0.32, + "learning_rate": 0.000549937813440321, + "loss": 1.3651, + "theoretical_loss": 3.6593748903434746, + "tokens_seen": 1054474240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005499217652958877, + "loss": 1.3802, + "theoretical_loss": 3.6593536256394144, + "tokens_seen": 1054539776 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005499057171514544, + "loss": 1.4536, + "theoretical_loss": 3.6593323626268424, + "tokens_seen": 1054605312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005498896690070211, + "loss": 1.3398, + "theoretical_loss": 3.6593111013055193, + "tokens_seen": 1054670848 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005498736208625878, + "loss": 1.2914, + "theoretical_loss": 3.6592898416752053, + "tokens_seen": 1054736384 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005498575727181545, + "loss": 1.444, + "theoretical_loss": 3.6592685837356607, + "tokens_seen": 1054801920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005498415245737212, + "loss": 1.4221, + "theoretical_loss": 3.6592473274866464, + "tokens_seen": 1054867456 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005498254764292879, + "loss": 1.3847, + "theoretical_loss": 3.6592260729279222, + "tokens_seen": 1054932992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005498094282848546, + "loss": 1.4191, + "theoretical_loss": 3.6592048200592497, + "tokens_seen": 1054998528 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005497933801404213, + "loss": 1.346, + "theoretical_loss": 3.6591835688803886, + "tokens_seen": 1055064064 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 395013, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.398438572883606, + "objective/train/theoretical_loss": 3.6591623193911, + "objective/train/tokens_used": 1075589600, + "theoretical_loss": 3.6591623193911, + "tokens_seen": 1055129600 + }, + { + "epoch": 0.32, + "learning_rate": 0.000549777331995988, + "loss": 1.3984, + "theoretical_loss": 3.6591623193911, + "tokens_seen": 1055129600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005497612838515547, + "loss": 1.4395, + "theoretical_loss": 3.6591410715911445, + "tokens_seen": 1055195136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005497452357071214, + "loss": 1.471, + "theoretical_loss": 3.6591198254802837, + "tokens_seen": 1055260672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005497291875626881, + "loss": 1.4041, + "theoretical_loss": 3.6590985810582772, + "tokens_seen": 1055326208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005497131394182548, + "loss": 1.2939, + "theoretical_loss": 3.6590773383248862, + "tokens_seen": 1055391744 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005496970912738215, + "loss": 1.237, + "theoretical_loss": 3.6590560972798727, + "tokens_seen": 1055457280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005496810431293882, + "loss": 1.3442, + "theoretical_loss": 3.6590348579229963, + "tokens_seen": 1055522816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005496649949849549, + "loss": 1.3399, + "theoretical_loss": 3.6590136202540187, + "tokens_seen": 1055588352 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005496489468405216, + "loss": 1.4203, + "theoretical_loss": 3.6589923842727012, + "tokens_seen": 1055653888 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005496328986960883, + "loss": 1.5158, + "theoretical_loss": 3.6589711499788042, + "tokens_seen": 1055719424 + }, + { + "epoch": 0.32, + "learning_rate": 0.000549616850551655, + "loss": 1.4483, + "theoretical_loss": 3.658949917372089, + "tokens_seen": 1055784960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005496008024072217, + "loss": 1.4349, + "theoretical_loss": 3.6589286864523176, + "tokens_seen": 1055850496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005495847542627884, + "loss": 1.3944, + "theoretical_loss": 3.658907457219251, + "tokens_seen": 1055916032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005495687061183551, + "loss": 1.4325, + "theoretical_loss": 3.65888622967265, + "tokens_seen": 1055981568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005495526579739218, + "loss": 1.3198, + "theoretical_loss": 3.6588650038122763, + "tokens_seen": 1056047104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005495366098294885, + "loss": 1.419, + "theoretical_loss": 3.6588437796378916, + "tokens_seen": 1056112640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005495205616850552, + "loss": 1.2915, + "theoretical_loss": 3.658822557149257, + "tokens_seen": 1056178176 + }, + { + "epoch": 0.32, + "learning_rate": 0.000549504513540622, + "loss": 1.3824, + "theoretical_loss": 3.658801336346134, + "tokens_seen": 1056243712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005494884653961886, + "loss": 1.3576, + "theoretical_loss": 3.658780117228284, + "tokens_seen": 1056309248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005494724172517554, + "loss": 1.4114, + "theoretical_loss": 3.65875889979547, + "tokens_seen": 1056374784 + }, + { + "epoch": 0.32, + "learning_rate": 0.000549456369107322, + "loss": 1.327, + "theoretical_loss": 3.658737684047451, + "tokens_seen": 1056440320 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005494403209628888, + "loss": 1.448, + "theoretical_loss": 3.6587164699839914, + "tokens_seen": 1056505856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005494242728184554, + "loss": 1.4512, + "theoretical_loss": 3.6586952576048515, + "tokens_seen": 1056571392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005494082246740222, + "loss": 1.3881, + "theoretical_loss": 3.658674046909793, + "tokens_seen": 1056636928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005493921765295888, + "loss": 1.3484, + "theoretical_loss": 3.658652837898579, + "tokens_seen": 1056702464 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005493761283851556, + "loss": 1.3104, + "theoretical_loss": 3.6586316305709703, + "tokens_seen": 1056768000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005493600802407222, + "loss": 1.2856, + "theoretical_loss": 3.658610424926729, + "tokens_seen": 1056833536 + }, + { + "epoch": 0.32, + "learning_rate": 0.000549344032096289, + "loss": 1.3436, + "theoretical_loss": 3.6585892209656175, + "tokens_seen": 1056899072 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005493279839518556, + "loss": 1.4484, + "theoretical_loss": 3.6585680186873972, + "tokens_seen": 1056964608 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005493119358074224, + "loss": 1.4098, + "theoretical_loss": 3.658546818091831, + "tokens_seen": 1057030144 + }, + { + "epoch": 0.32, + "learning_rate": 0.000549295887662989, + "loss": 1.2676, + "theoretical_loss": 3.658525619178681, + "tokens_seen": 1057095680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005492798395185558, + "loss": 1.3376, + "theoretical_loss": 3.658504421947708, + "tokens_seen": 1057161216 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005492637913741224, + "loss": 1.3985, + "theoretical_loss": 3.658483226398676, + "tokens_seen": 1057226752 + }, + { + "epoch": 0.32, + "learning_rate": 0.000549247743229689, + "loss": 1.3692, + "theoretical_loss": 3.6584620325313466, + "tokens_seen": 1057292288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005492316950852558, + "loss": 1.3899, + "theoretical_loss": 3.6584408403454822, + "tokens_seen": 1057357824 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005492156469408225, + "loss": 1.3954, + "theoretical_loss": 3.6584196498408454, + "tokens_seen": 1057423360 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005491995987963892, + "loss": 1.3592, + "theoretical_loss": 3.658398461017198, + "tokens_seen": 1057488896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005491835506519559, + "loss": 1.3766, + "theoretical_loss": 3.6583772738743026, + "tokens_seen": 1057554432 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005491675025075226, + "loss": 1.4456, + "theoretical_loss": 3.6583560884119226, + "tokens_seen": 1057619968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005491514543630893, + "loss": 1.3895, + "theoretical_loss": 3.6583349046298195, + "tokens_seen": 1057685504 + }, + { + "epoch": 0.32, + "learning_rate": 0.000549135406218656, + "loss": 1.3679, + "theoretical_loss": 3.6583137225277564, + "tokens_seen": 1057751040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005491193580742227, + "loss": 1.3982, + "theoretical_loss": 3.6582925421054964, + "tokens_seen": 1057816576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005491033099297893, + "loss": 1.5015, + "theoretical_loss": 3.6582713633628017, + "tokens_seen": 1057882112 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005490872617853561, + "loss": 1.3944, + "theoretical_loss": 3.658250186299435, + "tokens_seen": 1057947648 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005490712136409227, + "loss": 1.3812, + "theoretical_loss": 3.65822901091516, + "tokens_seen": 1058013184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005490551654964895, + "loss": 1.4124, + "theoretical_loss": 3.658207837209738, + "tokens_seen": 1058078720 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005490391173520561, + "loss": 1.3674, + "theoretical_loss": 3.6581866651829333, + "tokens_seen": 1058144256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005490230692076229, + "loss": 1.4111, + "theoretical_loss": 3.658165494834508, + "tokens_seen": 1058209792 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005490070210631895, + "loss": 1.3969, + "theoretical_loss": 3.6581443261642264, + "tokens_seen": 1058275328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005489909729187563, + "loss": 1.312, + "theoretical_loss": 3.6581231591718497, + "tokens_seen": 1058340864 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005489749247743229, + "loss": 1.4286, + "theoretical_loss": 3.6581019938571426, + "tokens_seen": 1058406400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005489588766298897, + "loss": 1.3935, + "theoretical_loss": 3.658080830219867, + "tokens_seen": 1058471936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005489428284854564, + "loss": 1.416, + "theoretical_loss": 3.6580596682597877, + "tokens_seen": 1058537472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005489267803410231, + "loss": 1.4446, + "theoretical_loss": 3.6580385079766664, + "tokens_seen": 1058603008 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005489107321965898, + "loss": 1.365, + "theoretical_loss": 3.658017349370267, + "tokens_seen": 1058668544 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005488946840521565, + "loss": 1.4849, + "theoretical_loss": 3.6579961924403532, + "tokens_seen": 1058734080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005488786359077232, + "loss": 1.3087, + "theoretical_loss": 3.657975037186688, + "tokens_seen": 1058799616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005488625877632899, + "loss": 1.3639, + "theoretical_loss": 3.657953883609035, + "tokens_seen": 1058865152 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005488465396188566, + "loss": 1.3927, + "theoretical_loss": 3.6579327317071573, + "tokens_seen": 1058930688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005488304914744233, + "loss": 1.3452, + "theoretical_loss": 3.6579115814808185, + "tokens_seen": 1058996224 + }, + { + "epoch": 0.32, + "learning_rate": 0.00054881444332999, + "loss": 1.3123, + "theoretical_loss": 3.6578904329297837, + "tokens_seen": 1059061760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005487983951855567, + "loss": 1.4673, + "theoretical_loss": 3.6578692860538142, + "tokens_seen": 1059127296 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005487823470411234, + "loss": 1.2746, + "theoretical_loss": 3.6578481408526753, + "tokens_seen": 1059192832 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005487662988966901, + "loss": 1.3347, + "theoretical_loss": 3.65782699732613, + "tokens_seen": 1059258368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005487502507522568, + "loss": 1.3506, + "theoretical_loss": 3.657805855473942, + "tokens_seen": 1059323904 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005487342026078235, + "loss": 1.3494, + "theoretical_loss": 3.6577847152958762, + "tokens_seen": 1059389440 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005487181544633902, + "loss": 1.4904, + "theoretical_loss": 3.6577635767916954, + "tokens_seen": 1059454976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005487021063189569, + "loss": 1.3789, + "theoretical_loss": 3.657742439961164, + "tokens_seen": 1059520512 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005486860581745236, + "loss": 1.3121, + "theoretical_loss": 3.657721304804046, + "tokens_seen": 1059586048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005486700100300903, + "loss": 1.3974, + "theoretical_loss": 3.657700171320105, + "tokens_seen": 1059651584 + }, + { + "epoch": 0.32, + "learning_rate": 0.000548653961885657, + "loss": 1.4214, + "theoretical_loss": 3.6576790395091052, + "tokens_seen": 1059717120 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005486379137412237, + "loss": 1.2228, + "theoretical_loss": 3.6576579093708115, + "tokens_seen": 1059782656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005486218655967904, + "loss": 1.3737, + "theoretical_loss": 3.657636780904987, + "tokens_seen": 1059848192 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005486058174523571, + "loss": 1.4061, + "theoretical_loss": 3.6576156541113964, + "tokens_seen": 1059913728 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005485897693079238, + "loss": 1.3903, + "theoretical_loss": 3.657594528989804, + "tokens_seen": 1059979264 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005485737211634905, + "loss": 1.3011, + "theoretical_loss": 3.6575734055399742, + "tokens_seen": 1060044800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005485576730190572, + "loss": 1.369, + "theoretical_loss": 3.657552283761671, + "tokens_seen": 1060110336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005485416248746239, + "loss": 1.3592, + "theoretical_loss": 3.6575311636546584, + "tokens_seen": 1060175872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005485255767301906, + "loss": 1.4422, + "theoretical_loss": 3.6575100452187024, + "tokens_seen": 1060241408 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005485095285857573, + "loss": 1.3626, + "theoretical_loss": 3.6574889284535663, + "tokens_seen": 1060306944 + }, + { + "epoch": 0.32, + "learning_rate": 0.000548493480441324, + "loss": 1.4884, + "theoretical_loss": 3.657467813359015, + "tokens_seen": 1060372480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005484774322968907, + "loss": 1.4545, + "theoretical_loss": 3.6574466999348125, + "tokens_seen": 1060438016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005484613841524574, + "loss": 1.3181, + "theoretical_loss": 3.6574255881807245, + "tokens_seen": 1060503552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005484453360080241, + "loss": 1.3814, + "theoretical_loss": 3.657404478096515, + "tokens_seen": 1060569088 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005484292878635908, + "loss": 1.4467, + "theoretical_loss": 3.657383369681949, + "tokens_seen": 1060634624 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005484132397191575, + "loss": 1.3215, + "theoretical_loss": 3.657362262936791, + "tokens_seen": 1060700160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005483971915747242, + "loss": 1.3492, + "theoretical_loss": 3.6573411578608064, + "tokens_seen": 1060765696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005483811434302909, + "loss": 1.4381, + "theoretical_loss": 3.657320054453759, + "tokens_seen": 1060831232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005483650952858576, + "loss": 1.3041, + "theoretical_loss": 3.6572989527154154, + "tokens_seen": 1060896768 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005483490471414243, + "loss": 1.4072, + "theoretical_loss": 3.6572778526455387, + "tokens_seen": 1060962304 + }, + { + "epoch": 0.32, + "learning_rate": 0.000548332998996991, + "loss": 1.3705, + "theoretical_loss": 3.6572567542438956, + "tokens_seen": 1061027840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005483169508525577, + "loss": 1.4662, + "theoretical_loss": 3.6572356575102503, + "tokens_seen": 1061093376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005483009027081244, + "loss": 1.2855, + "theoretical_loss": 3.6572145624443677, + "tokens_seen": 1061158912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005482848545636911, + "loss": 1.2885, + "theoretical_loss": 3.6571934690460135, + "tokens_seen": 1061224448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005482688064192578, + "loss": 1.4102, + "theoretical_loss": 3.6571723773149527, + "tokens_seen": 1061289984 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005482527582748245, + "loss": 1.3526, + "theoretical_loss": 3.657151287250951, + "tokens_seen": 1061355520 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005482367101303912, + "loss": 1.2792, + "theoretical_loss": 3.657130198853773, + "tokens_seen": 1061421056 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005482206619859579, + "loss": 1.4263, + "theoretical_loss": 3.6571091121231847, + "tokens_seen": 1061486592 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005482046138415246, + "loss": 1.4084, + "theoretical_loss": 3.6570880270589505, + "tokens_seen": 1061552128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005481885656970913, + "loss": 1.4443, + "theoretical_loss": 3.657066943660837, + "tokens_seen": 1061617664 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 397308, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4307212829589844, + "objective/train/theoretical_loss": 3.6570458619286095, + "objective/train/tokens_used": 1082143200, + "theoretical_loss": 3.6570458619286095, + "tokens_seen": 1061683200 + }, + { + "epoch": 0.32, + "learning_rate": 0.000548172517552658, + "loss": 1.4307, + "theoretical_loss": 3.6570458619286095, + "tokens_seen": 1061683200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005481564694082247, + "loss": 1.296, + "theoretical_loss": 3.6570247818620327, + "tokens_seen": 1061748736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005481404212637914, + "loss": 1.3659, + "theoretical_loss": 3.657003703460873, + "tokens_seen": 1061814272 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005481243731193581, + "loss": 1.3826, + "theoretical_loss": 3.656982626724896, + "tokens_seen": 1061879808 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005481083249749249, + "loss": 1.3084, + "theoretical_loss": 3.6569615516538674, + "tokens_seen": 1061945344 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005480922768304915, + "loss": 1.4276, + "theoretical_loss": 3.656940478247553, + "tokens_seen": 1062010880 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005480762286860583, + "loss": 1.3931, + "theoretical_loss": 3.6569194065057182, + "tokens_seen": 1062076416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005480601805416249, + "loss": 1.368, + "theoretical_loss": 3.656898336428129, + "tokens_seen": 1062141952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005480441323971917, + "loss": 1.2993, + "theoretical_loss": 3.6568772680145507, + "tokens_seen": 1062207488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005480280842527583, + "loss": 1.3231, + "theoretical_loss": 3.65685620126475, + "tokens_seen": 1062273024 + }, + { + "epoch": 0.32, + "learning_rate": 0.000548012036108325, + "loss": 1.3349, + "theoretical_loss": 3.6568351361784934, + "tokens_seen": 1062338560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005479959879638917, + "loss": 1.4164, + "theoretical_loss": 3.656814072755546, + "tokens_seen": 1062404096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005479799398194585, + "loss": 1.3847, + "theoretical_loss": 3.656793010995674, + "tokens_seen": 1062469632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005479638916750251, + "loss": 1.3567, + "theoretical_loss": 3.6567719508986434, + "tokens_seen": 1062535168 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005479478435305919, + "loss": 1.3475, + "theoretical_loss": 3.6567508924642205, + "tokens_seen": 1062600704 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005479317953861585, + "loss": 1.4161, + "theoretical_loss": 3.656729835692172, + "tokens_seen": 1062666240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005479157472417253, + "loss": 1.2711, + "theoretical_loss": 3.656708780582264, + "tokens_seen": 1062731776 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005478996990972919, + "loss": 1.3984, + "theoretical_loss": 3.656687727134262, + "tokens_seen": 1062797312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005478836509528587, + "loss": 1.4524, + "theoretical_loss": 3.656666675347933, + "tokens_seen": 1062862848 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005478676028084253, + "loss": 1.4905, + "theoretical_loss": 3.6566456252230437, + "tokens_seen": 1062928384 + }, + { + "epoch": 0.32, + "learning_rate": 0.000547851554663992, + "loss": 1.323, + "theoretical_loss": 3.65662457675936, + "tokens_seen": 1062993920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005478355065195587, + "loss": 1.3992, + "theoretical_loss": 3.6566035299566484, + "tokens_seen": 1063059456 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005478194583751254, + "loss": 1.4238, + "theoretical_loss": 3.6565824848146757, + "tokens_seen": 1063124992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005478034102306921, + "loss": 1.3454, + "theoretical_loss": 3.656561441333209, + "tokens_seen": 1063190528 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005477873620862588, + "loss": 1.4347, + "theoretical_loss": 3.6565403995120134, + "tokens_seen": 1063256064 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005477713139418255, + "loss": 1.3441, + "theoretical_loss": 3.656519359350857, + "tokens_seen": 1063321600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005477552657973922, + "loss": 1.3185, + "theoretical_loss": 3.656498320849506, + "tokens_seen": 1063387136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005477392176529589, + "loss": 1.3763, + "theoretical_loss": 3.6564772840077278, + "tokens_seen": 1063452672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005477231695085256, + "loss": 1.4079, + "theoretical_loss": 3.6564562488252883, + "tokens_seen": 1063518208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005477071213640923, + "loss": 1.3521, + "theoretical_loss": 3.6564352153019546, + "tokens_seen": 1063583744 + }, + { + "epoch": 0.32, + "learning_rate": 0.000547691073219659, + "loss": 1.3672, + "theoretical_loss": 3.656414183437494, + "tokens_seen": 1063649280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005476750250752257, + "loss": 1.3022, + "theoretical_loss": 3.656393153231673, + "tokens_seen": 1063714816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005476589769307924, + "loss": 1.3235, + "theoretical_loss": 3.6563721246842595, + "tokens_seen": 1063780352 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005476429287863591, + "loss": 1.4655, + "theoretical_loss": 3.656351097795019, + "tokens_seen": 1063845888 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005476268806419258, + "loss": 1.3781, + "theoretical_loss": 3.6563300725637204, + "tokens_seen": 1063911424 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005476108324974925, + "loss": 1.485, + "theoretical_loss": 3.6563090489901295, + "tokens_seen": 1063976960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005475947843530592, + "loss": 1.4348, + "theoretical_loss": 3.656288027074014, + "tokens_seen": 1064042496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005475787362086259, + "loss": 1.3955, + "theoretical_loss": 3.656267006815142, + "tokens_seen": 1064108032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005475626880641926, + "loss": 1.3517, + "theoretical_loss": 3.6562459882132794, + "tokens_seen": 1064173568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005475466399197593, + "loss": 1.3812, + "theoretical_loss": 3.656224971268194, + "tokens_seen": 1064239104 + }, + { + "epoch": 0.32, + "learning_rate": 0.000547530591775326, + "loss": 1.3938, + "theoretical_loss": 3.6562039559796533, + "tokens_seen": 1064304640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005475145436308927, + "loss": 1.3572, + "theoretical_loss": 3.6561829423474252, + "tokens_seen": 1064370176 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005474984954864594, + "loss": 1.2861, + "theoretical_loss": 3.656161930371276, + "tokens_seen": 1064435712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005474824473420261, + "loss": 1.4191, + "theoretical_loss": 3.6561409200509747, + "tokens_seen": 1064501248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005474663991975928, + "loss": 1.5006, + "theoretical_loss": 3.6561199113862877, + "tokens_seen": 1064566784 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005474503510531595, + "loss": 1.3706, + "theoretical_loss": 3.656098904376983, + "tokens_seen": 1064632320 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005474343029087262, + "loss": 1.4692, + "theoretical_loss": 3.6560778990228284, + "tokens_seen": 1064697856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005474182547642929, + "loss": 1.3258, + "theoretical_loss": 3.6560568953235917, + "tokens_seen": 1064763392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005474022066198596, + "loss": 1.3506, + "theoretical_loss": 3.6560358932790407, + "tokens_seen": 1064828928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005473861584754263, + "loss": 1.3824, + "theoretical_loss": 3.656014892888943, + "tokens_seen": 1064894464 + }, + { + "epoch": 0.32, + "learning_rate": 0.000547370110330993, + "loss": 1.3644, + "theoretical_loss": 3.6559938941530667, + "tokens_seen": 1064960000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005473540621865597, + "loss": 1.3312, + "theoretical_loss": 3.655972897071179, + "tokens_seen": 1065025536 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005473380140421264, + "loss": 1.2806, + "theoretical_loss": 3.655951901643049, + "tokens_seen": 1065091072 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005473219658976931, + "loss": 1.3921, + "theoretical_loss": 3.655930907868443, + "tokens_seen": 1065156608 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005473059177532598, + "loss": 1.4845, + "theoretical_loss": 3.6559099157471313, + "tokens_seen": 1065222144 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005472898696088265, + "loss": 1.3503, + "theoretical_loss": 3.65588892527888, + "tokens_seen": 1065287680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005472738214643932, + "loss": 1.3773, + "theoretical_loss": 3.655867936463459, + "tokens_seen": 1065353216 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005472577733199599, + "loss": 1.4118, + "theoretical_loss": 3.655846949300635, + "tokens_seen": 1065418752 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005472417251755266, + "loss": 1.3773, + "theoretical_loss": 3.6558259637901767, + "tokens_seen": 1065484288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005472256770310933, + "loss": 1.376, + "theoretical_loss": 3.655804979931853, + "tokens_seen": 1065549824 + }, + { + "epoch": 0.32, + "learning_rate": 0.00054720962888666, + "loss": 1.3099, + "theoretical_loss": 3.655783997725431, + "tokens_seen": 1065615360 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005471935807422267, + "loss": 1.3242, + "theoretical_loss": 3.6557630171706803, + "tokens_seen": 1065680896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005471775325977934, + "loss": 1.3539, + "theoretical_loss": 3.655742038267369, + "tokens_seen": 1065746432 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005471614844533601, + "loss": 1.312, + "theoretical_loss": 3.655721061015265, + "tokens_seen": 1065811968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005471454363089268, + "loss": 1.3577, + "theoretical_loss": 3.6557000854141375, + "tokens_seen": 1065877504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005471293881644935, + "loss": 1.3414, + "theoretical_loss": 3.6556791114637543, + "tokens_seen": 1065943040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005471133400200602, + "loss": 1.3665, + "theoretical_loss": 3.6556581391638847, + "tokens_seen": 1066008576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005470972918756269, + "loss": 1.3604, + "theoretical_loss": 3.655637168514297, + "tokens_seen": 1066074112 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005470812437311936, + "loss": 1.3908, + "theoretical_loss": 3.6556161995147605, + "tokens_seen": 1066139648 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005470651955867603, + "loss": 1.3842, + "theoretical_loss": 3.6555952321650436, + "tokens_seen": 1066205184 + }, + { + "epoch": 0.32, + "learning_rate": 0.000547049147442327, + "loss": 1.435, + "theoretical_loss": 3.6555742664649147, + "tokens_seen": 1066270720 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005470330992978937, + "loss": 1.3842, + "theoretical_loss": 3.6555533024141424, + "tokens_seen": 1066336256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005470170511534604, + "loss": 1.3495, + "theoretical_loss": 3.655532340012497, + "tokens_seen": 1066401792 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005470010030090271, + "loss": 1.2778, + "theoretical_loss": 3.655511379259746, + "tokens_seen": 1066467328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005469849548645938, + "loss": 1.3419, + "theoretical_loss": 3.655490420155659, + "tokens_seen": 1066532864 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005469689067201605, + "loss": 1.3309, + "theoretical_loss": 3.655469462700005, + "tokens_seen": 1066598400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005469528585757272, + "loss": 1.4276, + "theoretical_loss": 3.6554485068925535, + "tokens_seen": 1066663936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005469368104312939, + "loss": 1.3592, + "theoretical_loss": 3.655427552733072, + "tokens_seen": 1066729472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005469207622868606, + "loss": 1.3617, + "theoretical_loss": 3.6554066002213323, + "tokens_seen": 1066795008 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005469047141424273, + "loss": 1.4296, + "theoretical_loss": 3.6553856493571013, + "tokens_seen": 1066860544 + }, + { + "epoch": 0.32, + "learning_rate": 0.000546888665997994, + "loss": 1.4092, + "theoretical_loss": 3.6553647001401495, + "tokens_seen": 1066926080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005468726178535607, + "loss": 1.3036, + "theoretical_loss": 3.6553437525702455, + "tokens_seen": 1066991616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005468565697091274, + "loss": 1.3405, + "theoretical_loss": 3.6553228066471593, + "tokens_seen": 1067057152 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005468405215646941, + "loss": 1.3933, + "theoretical_loss": 3.65530186237066, + "tokens_seen": 1067122688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005468244734202608, + "loss": 1.3844, + "theoretical_loss": 3.655280919740517, + "tokens_seen": 1067188224 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005468084252758275, + "loss": 1.3106, + "theoretical_loss": 3.6552599787564994, + "tokens_seen": 1067253760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005467923771313942, + "loss": 1.3006, + "theoretical_loss": 3.655239039418378, + "tokens_seen": 1067319296 + }, + { + "epoch": 0.32, + "learning_rate": 0.000546776328986961, + "loss": 1.3282, + "theoretical_loss": 3.6552181017259207, + "tokens_seen": 1067384832 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005467602808425276, + "loss": 1.3925, + "theoretical_loss": 3.6551971656788984, + "tokens_seen": 1067450368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005467442326980943, + "loss": 1.4282, + "theoretical_loss": 3.65517623127708, + "tokens_seen": 1067515904 + }, + { + "epoch": 0.32, + "learning_rate": 0.000546728184553661, + "loss": 1.3955, + "theoretical_loss": 3.6551552985202362, + "tokens_seen": 1067581440 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005467121364092278, + "loss": 1.4319, + "theoretical_loss": 3.6551343674081362, + "tokens_seen": 1067646976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005466960882647944, + "loss": 1.4451, + "theoretical_loss": 3.655113437940549, + "tokens_seen": 1067712512 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005466800401203612, + "loss": 1.3614, + "theoretical_loss": 3.655092510117246, + "tokens_seen": 1067778048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005466639919759278, + "loss": 1.4937, + "theoretical_loss": 3.655071583937996, + "tokens_seen": 1067843584 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005466479438314946, + "loss": 1.337, + "theoretical_loss": 3.65505065940257, + "tokens_seen": 1067909120 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005466318956870612, + "loss": 1.2696, + "theoretical_loss": 3.6550297365107367, + "tokens_seen": 1067974656 + }, + { + "epoch": 0.32, + "learning_rate": 0.000546615847542628, + "loss": 1.396, + "theoretical_loss": 3.655008815262267, + "tokens_seen": 1068040192 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005465997993981946, + "loss": 1.3781, + "theoretical_loss": 3.65498789565693, + "tokens_seen": 1068105728 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005465837512537614, + "loss": 1.3417, + "theoretical_loss": 3.6549669776944977, + "tokens_seen": 1068171264 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 399599, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.334648847579956, + "objective/train/theoretical_loss": 3.654946061374739, + "objective/train/tokens_used": 1088696800, + "theoretical_loss": 3.654946061374739, + "tokens_seen": 1068236800 + }, + { + "epoch": 0.32, + "learning_rate": 0.000546567703109328, + "loss": 1.3346, + "theoretical_loss": 3.654946061374739, + "tokens_seen": 1068236800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005465516549648948, + "loss": 1.3695, + "theoretical_loss": 3.6549251466974244, + "tokens_seen": 1068302336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005465356068204614, + "loss": 1.3645, + "theoretical_loss": 3.6549042336623234, + "tokens_seen": 1068367872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005465195586760282, + "loss": 1.4289, + "theoretical_loss": 3.6548833222692076, + "tokens_seen": 1068433408 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005465035105315948, + "loss": 1.3578, + "theoretical_loss": 3.654862412517847, + "tokens_seen": 1068498944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005464874623871616, + "loss": 1.3703, + "theoretical_loss": 3.654841504408012, + "tokens_seen": 1068564480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005464714142427282, + "loss": 1.4185, + "theoretical_loss": 3.6548205979394726, + "tokens_seen": 1068630016 + }, + { + "epoch": 0.32, + "learning_rate": 0.000546455366098295, + "loss": 1.3323, + "theoretical_loss": 3.6547996931119995, + "tokens_seen": 1068695552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005464393179538616, + "loss": 1.3699, + "theoretical_loss": 3.6547787899253636, + "tokens_seen": 1068761088 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005464232698094283, + "loss": 1.3012, + "theoretical_loss": 3.6547578883793355, + "tokens_seen": 1068826624 + }, + { + "epoch": 0.32, + "learning_rate": 0.000546407221664995, + "loss": 1.3751, + "theoretical_loss": 3.6547369884736858, + "tokens_seen": 1068892160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005463911735205617, + "loss": 1.4, + "theoretical_loss": 3.6547160902081846, + "tokens_seen": 1068957696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005463751253761284, + "loss": 1.3819, + "theoretical_loss": 3.6546951935826035, + "tokens_seen": 1069023232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005463590772316951, + "loss": 1.3905, + "theoretical_loss": 3.6546742985967127, + "tokens_seen": 1069088768 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005463430290872618, + "loss": 1.3749, + "theoretical_loss": 3.654653405250283, + "tokens_seen": 1069154304 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005463269809428285, + "loss": 1.3379, + "theoretical_loss": 3.6546325135430857, + "tokens_seen": 1069219840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005463109327983952, + "loss": 1.3996, + "theoretical_loss": 3.6546116234748913, + "tokens_seen": 1069285376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005462948846539619, + "loss": 1.4133, + "theoretical_loss": 3.6545907350454714, + "tokens_seen": 1069350912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005462788365095286, + "loss": 1.3277, + "theoretical_loss": 3.6545698482545963, + "tokens_seen": 1069416448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005462627883650953, + "loss": 1.3713, + "theoretical_loss": 3.6545489631020374, + "tokens_seen": 1069481984 + }, + { + "epoch": 0.32, + "learning_rate": 0.000546246740220662, + "loss": 1.405, + "theoretical_loss": 3.654528079587566, + "tokens_seen": 1069547520 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005462306920762287, + "loss": 1.3629, + "theoretical_loss": 3.654507197710953, + "tokens_seen": 1069613056 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005462146439317954, + "loss": 1.2748, + "theoretical_loss": 3.65448631747197, + "tokens_seen": 1069678592 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005461985957873621, + "loss": 1.3775, + "theoretical_loss": 3.6544654388703877, + "tokens_seen": 1069744128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005461825476429288, + "loss": 1.3575, + "theoretical_loss": 3.6544445619059776, + "tokens_seen": 1069809664 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005461664994984955, + "loss": 1.3985, + "theoretical_loss": 3.654423686578511, + "tokens_seen": 1069875200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005461504513540622, + "loss": 1.2718, + "theoretical_loss": 3.65440281288776, + "tokens_seen": 1069940736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005461344032096289, + "loss": 1.4507, + "theoretical_loss": 3.6543819408334945, + "tokens_seen": 1070006272 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005461183550651956, + "loss": 1.3595, + "theoretical_loss": 3.6543610704154874, + "tokens_seen": 1070071808 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005461023069207623, + "loss": 1.4402, + "theoretical_loss": 3.6543402016335094, + "tokens_seen": 1070137344 + }, + { + "epoch": 0.32, + "learning_rate": 0.000546086258776329, + "loss": 1.3584, + "theoretical_loss": 3.6543193344873326, + "tokens_seen": 1070202880 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005460702106318957, + "loss": 1.3965, + "theoretical_loss": 3.6542984689767284, + "tokens_seen": 1070268416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005460541624874624, + "loss": 1.3463, + "theoretical_loss": 3.654277605101468, + "tokens_seen": 1070333952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005460381143430291, + "loss": 1.3819, + "theoretical_loss": 3.6542567428613237, + "tokens_seen": 1070399488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005460220661985958, + "loss": 1.3951, + "theoretical_loss": 3.654235882256067, + "tokens_seen": 1070465024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005460060180541625, + "loss": 1.3184, + "theoretical_loss": 3.6542150232854707, + "tokens_seen": 1070530560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005459899699097292, + "loss": 1.4859, + "theoretical_loss": 3.6541941659493054, + "tokens_seen": 1070596096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005459739217652959, + "loss": 1.3421, + "theoretical_loss": 3.6541733102473426, + "tokens_seen": 1070661632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005459578736208626, + "loss": 1.4076, + "theoretical_loss": 3.654152456179356, + "tokens_seen": 1070727168 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005459418254764293, + "loss": 1.3702, + "theoretical_loss": 3.6541316037451157, + "tokens_seen": 1070792704 + }, + { + "epoch": 0.32, + "learning_rate": 0.000545925777331996, + "loss": 1.3358, + "theoretical_loss": 3.654110752944395, + "tokens_seen": 1070858240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005459097291875627, + "loss": 1.3096, + "theoretical_loss": 3.6540899037769656, + "tokens_seen": 1070923776 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005458936810431294, + "loss": 1.3225, + "theoretical_loss": 3.6540690562425997, + "tokens_seen": 1070989312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005458776328986961, + "loss": 1.3336, + "theoretical_loss": 3.6540482103410694, + "tokens_seen": 1071054848 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005458615847542628, + "loss": 1.3104, + "theoretical_loss": 3.654027366072147, + "tokens_seen": 1071120384 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005458455366098295, + "loss": 1.3798, + "theoretical_loss": 3.654006523435604, + "tokens_seen": 1071185920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005458294884653962, + "loss": 1.3786, + "theoretical_loss": 3.6539856824312142, + "tokens_seen": 1071251456 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005458134403209629, + "loss": 1.371, + "theoretical_loss": 3.6539648430587484, + "tokens_seen": 1071316992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005457973921765296, + "loss": 1.3662, + "theoretical_loss": 3.65394400531798, + "tokens_seen": 1071382528 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005457813440320963, + "loss": 1.3526, + "theoretical_loss": 3.653923169208681, + "tokens_seen": 1071448064 + }, + { + "epoch": 0.32, + "learning_rate": 0.000545765295887663, + "loss": 1.4202, + "theoretical_loss": 3.6539023347306245, + "tokens_seen": 1071513600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005457492477432297, + "loss": 1.372, + "theoretical_loss": 3.653881501883582, + "tokens_seen": 1071579136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005457331995987964, + "loss": 1.3589, + "theoretical_loss": 3.653860670667327, + "tokens_seen": 1071644672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005457171514543631, + "loss": 1.3631, + "theoretical_loss": 3.6538398410816315, + "tokens_seen": 1071710208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005457011033099298, + "loss": 1.4519, + "theoretical_loss": 3.6538190131262684, + "tokens_seen": 1071775744 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005456850551654965, + "loss": 1.3526, + "theoretical_loss": 3.653798186801011, + "tokens_seen": 1071841280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005456690070210632, + "loss": 1.352, + "theoretical_loss": 3.6537773621056306, + "tokens_seen": 1071906816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005456529588766299, + "loss": 1.5267, + "theoretical_loss": 3.653756539039902, + "tokens_seen": 1071972352 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005456369107321966, + "loss": 1.4831, + "theoretical_loss": 3.6537357176035963, + "tokens_seen": 1072037888 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005456208625877633, + "loss": 1.4293, + "theoretical_loss": 3.653714897796487, + "tokens_seen": 1072103424 + }, + { + "epoch": 0.32, + "learning_rate": 0.00054560481444333, + "loss": 1.3538, + "theoretical_loss": 3.6536940796183472, + "tokens_seen": 1072168960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005455887662988967, + "loss": 1.4603, + "theoretical_loss": 3.65367326306895, + "tokens_seen": 1072234496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005455727181544634, + "loss": 1.4814, + "theoretical_loss": 3.653652448148068, + "tokens_seen": 1072300032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005455566700100301, + "loss": 1.4676, + "theoretical_loss": 3.6536316348554747, + "tokens_seen": 1072365568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005455406218655968, + "loss": 1.4352, + "theoretical_loss": 3.653610823190943, + "tokens_seen": 1072431104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0005455245737211635, + "loss": 1.5169, + "theoretical_loss": 3.653590013154246, + "tokens_seen": 1072496640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005455085255767302, + "loss": 1.5616, + "theoretical_loss": 3.653569204745157, + "tokens_seen": 1072562176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005454924774322969, + "loss": 1.4826, + "theoretical_loss": 3.6535483979634495, + "tokens_seen": 1072627712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005454764292878636, + "loss": 1.3051, + "theoretical_loss": 3.6535275928088966, + "tokens_seen": 1072693248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005454603811434303, + "loss": 1.4593, + "theoretical_loss": 3.6535067892812716, + "tokens_seen": 1072758784 + }, + { + "epoch": 0.33, + "learning_rate": 0.000545444332998997, + "loss": 1.3796, + "theoretical_loss": 3.6534859873803476, + "tokens_seen": 1072824320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005454282848545637, + "loss": 1.3467, + "theoretical_loss": 3.6534651871058985, + "tokens_seen": 1072889856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005454122367101304, + "loss": 1.3696, + "theoretical_loss": 3.653444388457698, + "tokens_seen": 1072955392 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005453961885656971, + "loss": 1.4731, + "theoretical_loss": 3.6534235914355193, + "tokens_seen": 1073020928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005453801404212638, + "loss": 1.3647, + "theoretical_loss": 3.6534027960391358, + "tokens_seen": 1073086464 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005453640922768305, + "loss": 1.3402, + "theoretical_loss": 3.653382002268321, + "tokens_seen": 1073152000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005453480441323973, + "loss": 1.3085, + "theoretical_loss": 3.653361210122849, + "tokens_seen": 1073217536 + }, + { + "epoch": 0.33, + "learning_rate": 0.000545331995987964, + "loss": 1.4434, + "theoretical_loss": 3.653340419602494, + "tokens_seen": 1073283072 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005453159478435307, + "loss": 1.3414, + "theoretical_loss": 3.6533196307070286, + "tokens_seen": 1073348608 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005452998996990974, + "loss": 1.4704, + "theoretical_loss": 3.6532988434362275, + "tokens_seen": 1073414144 + }, + { + "epoch": 0.33, + "learning_rate": 0.000545283851554664, + "loss": 1.378, + "theoretical_loss": 3.653278057789864, + "tokens_seen": 1073479680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005452678034102308, + "loss": 1.4704, + "theoretical_loss": 3.6532572737677125, + "tokens_seen": 1073545216 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005452517552657975, + "loss": 1.3606, + "theoretical_loss": 3.6532364913695465, + "tokens_seen": 1073610752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005452357071213642, + "loss": 1.4306, + "theoretical_loss": 3.6532157105951395, + "tokens_seen": 1073676288 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005452196589769309, + "loss": 1.3845, + "theoretical_loss": 3.6531949314442667, + "tokens_seen": 1073741824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005452036108324976, + "loss": 1.3775, + "theoretical_loss": 3.6531741539167024, + "tokens_seen": 1073807360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005451875626880643, + "loss": 1.561, + "theoretical_loss": 3.653153378012219, + "tokens_seen": 1073872896 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005451715145436309, + "loss": 1.386, + "theoretical_loss": 3.653132603730592, + "tokens_seen": 1073938432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005451554663991977, + "loss": 1.3364, + "theoretical_loss": 3.6531118310715947, + "tokens_seen": 1074003968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005451394182547643, + "loss": 1.3847, + "theoretical_loss": 3.6530910600350026, + "tokens_seen": 1074069504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005451233701103311, + "loss": 1.3434, + "theoretical_loss": 3.6530702906205894, + "tokens_seen": 1074135040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005451073219658977, + "loss": 1.4038, + "theoretical_loss": 3.6530495228281286, + "tokens_seen": 1074200576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005450912738214645, + "loss": 1.3149, + "theoretical_loss": 3.6530287566573962, + "tokens_seen": 1074266112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005450752256770311, + "loss": 1.418, + "theoretical_loss": 3.653007992108165, + "tokens_seen": 1074331648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005450591775325979, + "loss": 1.4021, + "theoretical_loss": 3.6529872291802112, + "tokens_seen": 1074397184 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005450431293881645, + "loss": 1.4032, + "theoretical_loss": 3.6529664678733074, + "tokens_seen": 1074462720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005450270812437312, + "loss": 1.4225, + "theoretical_loss": 3.65294570818723, + "tokens_seen": 1074528256 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005450110330992979, + "loss": 1.3712, + "theoretical_loss": 3.6529249501217524, + "tokens_seen": 1074593792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005449949849548646, + "loss": 1.3924, + "theoretical_loss": 3.65290419367665, + "tokens_seen": 1074659328 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005449789368104313, + "loss": 1.3922, + "theoretical_loss": 3.652883438851697, + "tokens_seen": 1074724864 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 402021, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3588179349899292, + "objective/train/theoretical_loss": 3.652862685646668, + "objective/train/tokens_used": 1095250400, + "theoretical_loss": 3.652862685646668, + "tokens_seen": 1074790400 + }, + { + "epoch": 0.33, + "learning_rate": 0.000544962888665998, + "loss": 1.3588, + "theoretical_loss": 3.652862685646668, + "tokens_seen": 1074790400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005449468405215647, + "loss": 1.4699, + "theoretical_loss": 3.652841934061339, + "tokens_seen": 1074855936 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005449307923771314, + "loss": 1.377, + "theoretical_loss": 3.652821184095483, + "tokens_seen": 1074921472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005449147442326981, + "loss": 1.5025, + "theoretical_loss": 3.652800435748876, + "tokens_seen": 1074987008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005448986960882648, + "loss": 1.4182, + "theoretical_loss": 3.6527796890212936, + "tokens_seen": 1075052544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005448826479438315, + "loss": 1.3967, + "theoretical_loss": 3.6527589439125094, + "tokens_seen": 1075118080 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005448665997993982, + "loss": 1.3803, + "theoretical_loss": 3.6527382004222995, + "tokens_seen": 1075183616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005448505516549649, + "loss": 1.388, + "theoretical_loss": 3.6527174585504376, + "tokens_seen": 1075249152 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005448345035105316, + "loss": 1.4274, + "theoretical_loss": 3.6526967182967, + "tokens_seen": 1075314688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005448184553660983, + "loss": 1.4128, + "theoretical_loss": 3.652675979660862, + "tokens_seen": 1075380224 + }, + { + "epoch": 0.33, + "learning_rate": 0.000544802407221665, + "loss": 1.4955, + "theoretical_loss": 3.652655242642698, + "tokens_seen": 1075445760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005447863590772317, + "loss": 1.2949, + "theoretical_loss": 3.652634507241984, + "tokens_seen": 1075511296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005447703109327984, + "loss": 1.4064, + "theoretical_loss": 3.6526137734584943, + "tokens_seen": 1075576832 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005447542627883651, + "loss": 1.3011, + "theoretical_loss": 3.6525930412920053, + "tokens_seen": 1075642368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005447382146439318, + "loss": 1.4101, + "theoretical_loss": 3.6525723107422916, + "tokens_seen": 1075707904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005447221664994985, + "loss": 1.4069, + "theoretical_loss": 3.6525515818091296, + "tokens_seen": 1075773440 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005447061183550652, + "loss": 1.3974, + "theoretical_loss": 3.6525308544922934, + "tokens_seen": 1075838976 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005446900702106319, + "loss": 1.3963, + "theoretical_loss": 3.6525101287915596, + "tokens_seen": 1075904512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005446740220661986, + "loss": 1.4033, + "theoretical_loss": 3.6524894047067034, + "tokens_seen": 1075970048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005446579739217653, + "loss": 1.3634, + "theoretical_loss": 3.6524686822375, + "tokens_seen": 1076035584 + }, + { + "epoch": 0.33, + "learning_rate": 0.000544641925777332, + "loss": 1.4037, + "theoretical_loss": 3.652447961383726, + "tokens_seen": 1076101120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005446258776328987, + "loss": 1.4832, + "theoretical_loss": 3.6524272421451567, + "tokens_seen": 1076166656 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005446098294884654, + "loss": 1.3271, + "theoretical_loss": 3.6524065245215676, + "tokens_seen": 1076232192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005445937813440321, + "loss": 1.3117, + "theoretical_loss": 3.652385808512734, + "tokens_seen": 1076297728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005445777331995988, + "loss": 1.4034, + "theoretical_loss": 3.6523650941184327, + "tokens_seen": 1076363264 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005445616850551655, + "loss": 1.3238, + "theoretical_loss": 3.6523443813384393, + "tokens_seen": 1076428800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005445456369107322, + "loss": 1.4025, + "theoretical_loss": 3.65232367017253, + "tokens_seen": 1076494336 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005445295887662989, + "loss": 1.3424, + "theoretical_loss": 3.6523029606204798, + "tokens_seen": 1076559872 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005445135406218656, + "loss": 1.4559, + "theoretical_loss": 3.6522822526820655, + "tokens_seen": 1076625408 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005444974924774323, + "loss": 1.3881, + "theoretical_loss": 3.652261546357063, + "tokens_seen": 1076690944 + }, + { + "epoch": 0.33, + "learning_rate": 0.000544481444332999, + "loss": 1.4161, + "theoretical_loss": 3.6522408416452485, + "tokens_seen": 1076756480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005444653961885657, + "loss": 1.3657, + "theoretical_loss": 3.6522201385463977, + "tokens_seen": 1076822016 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005444493480441324, + "loss": 1.3964, + "theoretical_loss": 3.6521994370602875, + "tokens_seen": 1076887552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005444332998996991, + "loss": 1.4178, + "theoretical_loss": 3.6521787371866936, + "tokens_seen": 1076953088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005444172517552658, + "loss": 1.4092, + "theoretical_loss": 3.6521580389253927, + "tokens_seen": 1077018624 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005444012036108325, + "loss": 1.3338, + "theoretical_loss": 3.65213734227616, + "tokens_seen": 1077084160 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005443851554663992, + "loss": 1.3467, + "theoretical_loss": 3.652116647238773, + "tokens_seen": 1077149696 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005443691073219659, + "loss": 1.4234, + "theoretical_loss": 3.6520959538130087, + "tokens_seen": 1077215232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005443530591775326, + "loss": 1.4533, + "theoretical_loss": 3.652075261998642, + "tokens_seen": 1077280768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005443370110330993, + "loss": 1.2117, + "theoretical_loss": 3.65205457179545, + "tokens_seen": 1077346304 + }, + { + "epoch": 0.33, + "learning_rate": 0.000544320962888666, + "loss": 1.3985, + "theoretical_loss": 3.6520338832032095, + "tokens_seen": 1077411840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005443049147442327, + "loss": 1.3459, + "theoretical_loss": 3.652013196221697, + "tokens_seen": 1077477376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005442888665997994, + "loss": 1.5381, + "theoretical_loss": 3.6519925108506888, + "tokens_seen": 1077542912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005442728184553661, + "loss": 1.4383, + "theoretical_loss": 3.6519718270899624, + "tokens_seen": 1077608448 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005442567703109328, + "loss": 1.3348, + "theoretical_loss": 3.651951144939294, + "tokens_seen": 1077673984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005442407221664995, + "loss": 1.3765, + "theoretical_loss": 3.6519304643984594, + "tokens_seen": 1077739520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005442246740220662, + "loss": 1.3654, + "theoretical_loss": 3.651909785467237, + "tokens_seen": 1077805056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005442086258776329, + "loss": 1.4338, + "theoretical_loss": 3.651889108145403, + "tokens_seen": 1077870592 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005441925777331996, + "loss": 1.3713, + "theoretical_loss": 3.6518684324327344, + "tokens_seen": 1077936128 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005441765295887663, + "loss": 1.3499, + "theoretical_loss": 3.6518477583290077, + "tokens_seen": 1078001664 + }, + { + "epoch": 0.33, + "learning_rate": 0.000544160481444333, + "loss": 1.3782, + "theoretical_loss": 3.6518270858340007, + "tokens_seen": 1078067200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005441444332998997, + "loss": 1.3686, + "theoretical_loss": 3.6518064149474903, + "tokens_seen": 1078132736 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005441283851554665, + "loss": 1.4103, + "theoretical_loss": 3.651785745669253, + "tokens_seen": 1078198272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005441123370110331, + "loss": 1.3184, + "theoretical_loss": 3.6517650779990656, + "tokens_seen": 1078263808 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005440962888665999, + "loss": 1.4108, + "theoretical_loss": 3.6517444119367064, + "tokens_seen": 1078329344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005440802407221665, + "loss": 1.4086, + "theoretical_loss": 3.651723747481952, + "tokens_seen": 1078394880 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005440641925777333, + "loss": 1.4142, + "theoretical_loss": 3.6517030846345797, + "tokens_seen": 1078460416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005440481444333, + "loss": 1.4256, + "theoretical_loss": 3.6516824233943668, + "tokens_seen": 1078525952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005440320962888667, + "loss": 1.3238, + "theoretical_loss": 3.651661763761091, + "tokens_seen": 1078591488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005440160481444333, + "loss": 1.3763, + "theoretical_loss": 3.6516411057345293, + "tokens_seen": 1078657024 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005440000000000001, + "loss": 1.3797, + "theoretical_loss": 3.651620449314459, + "tokens_seen": 1078722560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005439839518555668, + "loss": 1.3115, + "theoretical_loss": 3.651599794500658, + "tokens_seen": 1078788096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005439679037111335, + "loss": 1.4024, + "theoretical_loss": 3.6515791412929035, + "tokens_seen": 1078853632 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005439518555667002, + "loss": 1.3563, + "theoretical_loss": 3.6515584896909736, + "tokens_seen": 1078919168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005439358074222669, + "loss": 1.3623, + "theoretical_loss": 3.651537839694645, + "tokens_seen": 1078984704 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005439197592778336, + "loss": 1.3696, + "theoretical_loss": 3.6515171913036966, + "tokens_seen": 1079050240 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005439037111334003, + "loss": 1.3131, + "theoretical_loss": 3.651496544517905, + "tokens_seen": 1079115776 + }, + { + "epoch": 0.33, + "learning_rate": 0.000543887662988967, + "loss": 1.4193, + "theoretical_loss": 3.651475899337048, + "tokens_seen": 1079181312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005438716148445337, + "loss": 1.4446, + "theoretical_loss": 3.651455255760904, + "tokens_seen": 1079246848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005438555667001004, + "loss": 1.421, + "theoretical_loss": 3.6514346137892506, + "tokens_seen": 1079312384 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005438395185556671, + "loss": 1.3067, + "theoretical_loss": 3.6514139734218656, + "tokens_seen": 1079377920 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005438234704112338, + "loss": 1.4382, + "theoretical_loss": 3.6513933346585272, + "tokens_seen": 1079443456 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005438074222668005, + "loss": 1.4676, + "theoretical_loss": 3.6513726974990126, + "tokens_seen": 1079508992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005437913741223672, + "loss": 1.432, + "theoretical_loss": 3.6513520619431006, + "tokens_seen": 1079574528 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005437753259779338, + "loss": 1.4805, + "theoretical_loss": 3.651331427990569, + "tokens_seen": 1079640064 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005437592778335006, + "loss": 1.3475, + "theoretical_loss": 3.651310795641196, + "tokens_seen": 1079705600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005437432296890672, + "loss": 1.2964, + "theoretical_loss": 3.65129016489476, + "tokens_seen": 1079771136 + }, + { + "epoch": 0.33, + "learning_rate": 0.000543727181544634, + "loss": 1.3683, + "theoretical_loss": 3.6512695357510383, + "tokens_seen": 1079836672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005437111334002006, + "loss": 1.4371, + "theoretical_loss": 3.6512489082098103, + "tokens_seen": 1079902208 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005436950852557674, + "loss": 1.4728, + "theoretical_loss": 3.6512282822708526, + "tokens_seen": 1079967744 + }, + { + "epoch": 0.33, + "learning_rate": 0.000543679037111334, + "loss": 1.3333, + "theoretical_loss": 3.6512076579339454, + "tokens_seen": 1080033280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005436629889669008, + "loss": 1.3921, + "theoretical_loss": 3.6511870351988662, + "tokens_seen": 1080098816 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005436469408224674, + "loss": 1.3909, + "theoretical_loss": 3.6511664140653934, + "tokens_seen": 1080164352 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005436308926780341, + "loss": 1.3622, + "theoretical_loss": 3.6511457945333055, + "tokens_seen": 1080229888 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005436148445336008, + "loss": 1.3686, + "theoretical_loss": 3.6511251766023807, + "tokens_seen": 1080295424 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005435987963891675, + "loss": 1.3947, + "theoretical_loss": 3.651104560272398, + "tokens_seen": 1080360960 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005435827482447342, + "loss": 1.3768, + "theoretical_loss": 3.6510839455431356, + "tokens_seen": 1080426496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005435667001003009, + "loss": 1.4575, + "theoretical_loss": 3.6510633324143726, + "tokens_seen": 1080492032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005435506519558676, + "loss": 1.2961, + "theoretical_loss": 3.6510427208858873, + "tokens_seen": 1080557568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005435346038114343, + "loss": 1.4467, + "theoretical_loss": 3.651022110957459, + "tokens_seen": 1080623104 + }, + { + "epoch": 0.33, + "learning_rate": 0.000543518555667001, + "loss": 1.3794, + "theoretical_loss": 3.6510015026288656, + "tokens_seen": 1080688640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005435025075225677, + "loss": 1.4126, + "theoretical_loss": 3.6509808958998864, + "tokens_seen": 1080754176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005434864593781344, + "loss": 1.3969, + "theoretical_loss": 3.6509602907703003, + "tokens_seen": 1080819712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005434704112337011, + "loss": 1.4059, + "theoretical_loss": 3.650939687239886, + "tokens_seen": 1080885248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005434543630892678, + "loss": 1.4428, + "theoretical_loss": 3.6509190853084226, + "tokens_seen": 1080950784 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005434383149448345, + "loss": 1.4149, + "theoretical_loss": 3.6508984849756887, + "tokens_seen": 1081016320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005434222668004012, + "loss": 1.3309, + "theoretical_loss": 3.6508778862414637, + "tokens_seen": 1081081856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005434062186559679, + "loss": 1.2839, + "theoretical_loss": 3.650857289105527, + "tokens_seen": 1081147392 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005433901705115346, + "loss": 1.4361, + "theoretical_loss": 3.6508366935676566, + "tokens_seen": 1081212928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005433741223671013, + "loss": 1.3708, + "theoretical_loss": 3.650816099627633, + "tokens_seen": 1081278464 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 404745, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.318852186203003, + "objective/train/theoretical_loss": 3.6507955072852343, + "objective/train/tokens_used": 1101804000, + "theoretical_loss": 3.6507955072852343, + "tokens_seen": 1081344000 + }, + { + "epoch": 0.33, + "learning_rate": 0.000543358074222668, + "loss": 1.3189, + "theoretical_loss": 3.6507955072852343, + "tokens_seen": 1081344000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005433420260782347, + "loss": 1.3589, + "theoretical_loss": 3.6507749165402403, + "tokens_seen": 1081409536 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005433259779338014, + "loss": 1.401, + "theoretical_loss": 3.6507543273924306, + "tokens_seen": 1081475072 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005433099297893681, + "loss": 1.4173, + "theoretical_loss": 3.6507337398415833, + "tokens_seen": 1081540608 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005432938816449348, + "loss": 1.3859, + "theoretical_loss": 3.6507131538874793, + "tokens_seen": 1081606144 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005432778335005015, + "loss": 1.4024, + "theoretical_loss": 3.6506925695298973, + "tokens_seen": 1081671680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005432617853560682, + "loss": 1.3133, + "theoretical_loss": 3.650671986768616, + "tokens_seen": 1081737216 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005432457372116349, + "loss": 1.2727, + "theoretical_loss": 3.6506514056034165, + "tokens_seen": 1081802752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005432296890672016, + "loss": 1.2284, + "theoretical_loss": 3.6506308260340776, + "tokens_seen": 1081868288 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005432136409227683, + "loss": 1.4568, + "theoretical_loss": 3.6506102480603784, + "tokens_seen": 1081933824 + }, + { + "epoch": 0.33, + "learning_rate": 0.000543197592778335, + "loss": 1.4361, + "theoretical_loss": 3.6505896716820994, + "tokens_seen": 1081999360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005431815446339017, + "loss": 1.4151, + "theoretical_loss": 3.6505690968990194, + "tokens_seen": 1082064896 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005431654964894684, + "loss": 1.3534, + "theoretical_loss": 3.650548523710919, + "tokens_seen": 1082130432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005431494483450351, + "loss": 1.3615, + "theoretical_loss": 3.6505279521175775, + "tokens_seen": 1082195968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005431334002006018, + "loss": 1.3847, + "theoretical_loss": 3.650507382118775, + "tokens_seen": 1082261504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005431173520561685, + "loss": 1.4196, + "theoretical_loss": 3.6504868137142905, + "tokens_seen": 1082327040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005431013039117352, + "loss": 1.4277, + "theoretical_loss": 3.650466246903905, + "tokens_seen": 1082392576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005430852557673019, + "loss": 1.3406, + "theoretical_loss": 3.6504456816873976, + "tokens_seen": 1082458112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005430692076228686, + "loss": 1.3335, + "theoretical_loss": 3.6504251180645495, + "tokens_seen": 1082523648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005430531594784353, + "loss": 1.3284, + "theoretical_loss": 3.6504045560351392, + "tokens_seen": 1082589184 + }, + { + "epoch": 0.33, + "learning_rate": 0.000543037111334002, + "loss": 1.2572, + "theoretical_loss": 3.650383995598948, + "tokens_seen": 1082654720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005430210631895687, + "loss": 1.3542, + "theoretical_loss": 3.6503634367557547, + "tokens_seen": 1082720256 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005430050150451354, + "loss": 1.3022, + "theoretical_loss": 3.6503428795053408, + "tokens_seen": 1082785792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005429889669007021, + "loss": 1.2551, + "theoretical_loss": 3.6503223238474862, + "tokens_seen": 1082851328 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005429729187562688, + "loss": 1.3825, + "theoretical_loss": 3.650301769781971, + "tokens_seen": 1082916864 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005429568706118355, + "loss": 1.3428, + "theoretical_loss": 3.6502812173085752, + "tokens_seen": 1082982400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005429408224674022, + "loss": 1.3765, + "theoretical_loss": 3.650260666427079, + "tokens_seen": 1083047936 + }, + { + "epoch": 0.33, + "learning_rate": 0.000542924774322969, + "loss": 1.3222, + "theoretical_loss": 3.6502401171372636, + "tokens_seen": 1083113472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005429087261785356, + "loss": 1.4358, + "theoretical_loss": 3.650219569438909, + "tokens_seen": 1083179008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005428926780341024, + "loss": 1.3918, + "theoretical_loss": 3.650199023331796, + "tokens_seen": 1083244544 + }, + { + "epoch": 0.33, + "learning_rate": 0.000542876629889669, + "loss": 1.3043, + "theoretical_loss": 3.650178478815704, + "tokens_seen": 1083310080 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005428605817452358, + "loss": 1.3587, + "theoretical_loss": 3.6501579358904146, + "tokens_seen": 1083375616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005428445336008024, + "loss": 1.3882, + "theoretical_loss": 3.650137394555708, + "tokens_seen": 1083441152 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005428284854563692, + "loss": 1.4371, + "theoretical_loss": 3.6501168548113654, + "tokens_seen": 1083506688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005428124373119358, + "loss": 1.4005, + "theoretical_loss": 3.6500963166571667, + "tokens_seen": 1083572224 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005427963891675026, + "loss": 1.4067, + "theoretical_loss": 3.650075780092893, + "tokens_seen": 1083637760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005427803410230692, + "loss": 1.3625, + "theoretical_loss": 3.6500552451183252, + "tokens_seen": 1083703296 + }, + { + "epoch": 0.33, + "learning_rate": 0.000542764292878636, + "loss": 1.3071, + "theoretical_loss": 3.650034711733244, + "tokens_seen": 1083768832 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005427482447342026, + "loss": 1.4, + "theoretical_loss": 3.6500141799374304, + "tokens_seen": 1083834368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005427321965897694, + "loss": 1.2801, + "theoretical_loss": 3.649993649730665, + "tokens_seen": 1083899904 + }, + { + "epoch": 0.33, + "learning_rate": 0.000542716148445336, + "loss": 1.323, + "theoretical_loss": 3.649973121112729, + "tokens_seen": 1083965440 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005427001003009028, + "loss": 1.3488, + "theoretical_loss": 3.649952594083403, + "tokens_seen": 1084030976 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005426840521564694, + "loss": 1.3352, + "theoretical_loss": 3.6499320686424683, + "tokens_seen": 1084096512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005426680040120362, + "loss": 1.3502, + "theoretical_loss": 3.649911544789706, + "tokens_seen": 1084162048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005426519558676028, + "loss": 1.2405, + "theoretical_loss": 3.6498910225248977, + "tokens_seen": 1084227584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005426359077231696, + "loss": 1.3922, + "theoretical_loss": 3.649870501847824, + "tokens_seen": 1084293120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005426198595787363, + "loss": 1.4293, + "theoretical_loss": 3.6498499827582664, + "tokens_seen": 1084358656 + }, + { + "epoch": 0.33, + "learning_rate": 0.000542603811434303, + "loss": 1.3304, + "theoretical_loss": 3.6498294652560053, + "tokens_seen": 1084424192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005425877632898697, + "loss": 1.3167, + "theoretical_loss": 3.649808949340823, + "tokens_seen": 1084489728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005425717151454364, + "loss": 1.3392, + "theoretical_loss": 3.6497884350125007, + "tokens_seen": 1084555264 + }, + { + "epoch": 0.33, + "learning_rate": 0.000542555667001003, + "loss": 1.3796, + "theoretical_loss": 3.649767922270819, + "tokens_seen": 1084620800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005425396188565698, + "loss": 1.3958, + "theoretical_loss": 3.64974741111556, + "tokens_seen": 1084686336 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005425235707121365, + "loss": 1.3033, + "theoretical_loss": 3.6497269015465053, + "tokens_seen": 1084751872 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005425075225677032, + "loss": 1.3542, + "theoretical_loss": 3.649706393563436, + "tokens_seen": 1084817408 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005424914744232699, + "loss": 1.3747, + "theoretical_loss": 3.649685887166134, + "tokens_seen": 1084882944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005424754262788366, + "loss": 1.3569, + "theoretical_loss": 3.6496653823543816, + "tokens_seen": 1084948480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005424593781344033, + "loss": 1.3206, + "theoretical_loss": 3.6496448791279583, + "tokens_seen": 1085014016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00054244332998997, + "loss": 1.3204, + "theoretical_loss": 3.6496243774866475, + "tokens_seen": 1085079552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005424272818455367, + "loss": 1.4912, + "theoretical_loss": 3.6496038774302306, + "tokens_seen": 1085145088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005424112337011034, + "loss": 1.3795, + "theoretical_loss": 3.6495833789584893, + "tokens_seen": 1085210624 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005423951855566701, + "loss": 1.3546, + "theoretical_loss": 3.649562882071205, + "tokens_seen": 1085276160 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005423791374122367, + "loss": 1.3671, + "theoretical_loss": 3.64954238676816, + "tokens_seen": 1085341696 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005423630892678035, + "loss": 1.4275, + "theoretical_loss": 3.649521893049137, + "tokens_seen": 1085407232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005423470411233701, + "loss": 1.3427, + "theoretical_loss": 3.649501400913916, + "tokens_seen": 1085472768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005423309929789369, + "loss": 1.4166, + "theoretical_loss": 3.6494809103622803, + "tokens_seen": 1085538304 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005423149448345035, + "loss": 1.3847, + "theoretical_loss": 3.6494604213940125, + "tokens_seen": 1085603840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005422988966900703, + "loss": 1.4334, + "theoretical_loss": 3.649439934008893, + "tokens_seen": 1085669376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005422828485456369, + "loss": 1.4084, + "theoretical_loss": 3.649419448206705, + "tokens_seen": 1085734912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005422668004012037, + "loss": 1.3756, + "theoretical_loss": 3.6493989639872306, + "tokens_seen": 1085800448 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005422507522567703, + "loss": 1.3274, + "theoretical_loss": 3.6493784813502517, + "tokens_seen": 1085865984 + }, + { + "epoch": 0.33, + "learning_rate": 0.000542234704112337, + "loss": 1.482, + "theoretical_loss": 3.649358000295551, + "tokens_seen": 1085931520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005422186559679037, + "loss": 1.5377, + "theoretical_loss": 3.64933752082291, + "tokens_seen": 1085997056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005422026078234704, + "loss": 1.4125, + "theoretical_loss": 3.649317042932112, + "tokens_seen": 1086062592 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005421865596790371, + "loss": 1.3099, + "theoretical_loss": 3.649296566622939, + "tokens_seen": 1086128128 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005421705115346038, + "loss": 1.4311, + "theoretical_loss": 3.6492760918951728, + "tokens_seen": 1086193664 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005421544633901705, + "loss": 1.3869, + "theoretical_loss": 3.6492556187485965, + "tokens_seen": 1086259200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005421384152457372, + "loss": 1.4075, + "theoretical_loss": 3.649235147182993, + "tokens_seen": 1086324736 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005421223671013039, + "loss": 1.3538, + "theoretical_loss": 3.649214677198144, + "tokens_seen": 1086390272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005421063189568706, + "loss": 1.4482, + "theoretical_loss": 3.6491942087938325, + "tokens_seen": 1086455808 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005420902708124373, + "loss": 1.4327, + "theoretical_loss": 3.649173741969841, + "tokens_seen": 1086521344 + }, + { + "epoch": 0.33, + "learning_rate": 0.000542074222668004, + "loss": 1.4836, + "theoretical_loss": 3.6491532767259525, + "tokens_seen": 1086586880 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005420581745235707, + "loss": 1.2979, + "theoretical_loss": 3.649132813061949, + "tokens_seen": 1086652416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005420421263791374, + "loss": 1.3666, + "theoretical_loss": 3.6491123509776138, + "tokens_seen": 1086717952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005420260782347041, + "loss": 1.4762, + "theoretical_loss": 3.64909189047273, + "tokens_seen": 1086783488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005420100300902708, + "loss": 1.4025, + "theoretical_loss": 3.64907143154708, + "tokens_seen": 1086849024 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005419939819458375, + "loss": 1.3338, + "theoretical_loss": 3.6490509742004464, + "tokens_seen": 1086914560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005419779338014042, + "loss": 1.3082, + "theoretical_loss": 3.649030518432613, + "tokens_seen": 1086980096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005419618856569709, + "loss": 1.3748, + "theoretical_loss": 3.6490100642433623, + "tokens_seen": 1087045632 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005419458375125376, + "loss": 1.3925, + "theoretical_loss": 3.648989611632477, + "tokens_seen": 1087111168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005419297893681043, + "loss": 1.3894, + "theoretical_loss": 3.6489691605997403, + "tokens_seen": 1087176704 + }, + { + "epoch": 0.33, + "learning_rate": 0.000541913741223671, + "loss": 1.3221, + "theoretical_loss": 3.6489487111449357, + "tokens_seen": 1087242240 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005418976930792377, + "loss": 1.4917, + "theoretical_loss": 3.6489282632678464, + "tokens_seen": 1087307776 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005418816449348044, + "loss": 1.3341, + "theoretical_loss": 3.648907816968255, + "tokens_seen": 1087373312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005418655967903711, + "loss": 1.2762, + "theoretical_loss": 3.6488873722459454, + "tokens_seen": 1087438848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005418495486459378, + "loss": 1.3271, + "theoretical_loss": 3.6488669291007003, + "tokens_seen": 1087504384 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005418335005015045, + "loss": 1.3252, + "theoretical_loss": 3.6488464875323032, + "tokens_seen": 1087569920 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005418174523570712, + "loss": 1.3196, + "theoretical_loss": 3.6488260475405374, + "tokens_seen": 1087635456 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005418014042126379, + "loss": 1.3911, + "theoretical_loss": 3.648805609125187, + "tokens_seen": 1087700992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005417853560682046, + "loss": 1.3992, + "theoretical_loss": 3.6487851722860345, + "tokens_seen": 1087766528 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005417693079237713, + "loss": 1.4354, + "theoretical_loss": 3.648764737022864, + "tokens_seen": 1087832064 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 407194, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3681126832962036, + "objective/train/theoretical_loss": 3.6487443033354583, + "objective/train/tokens_used": 1108357600, + "theoretical_loss": 3.6487443033354583, + "tokens_seen": 1087897600 + }, + { + "epoch": 0.33, + "learning_rate": 0.000541753259779338, + "loss": 1.3681, + "theoretical_loss": 3.6487443033354583, + "tokens_seen": 1087897600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005417372116349047, + "loss": 1.3484, + "theoretical_loss": 3.648723871223602, + "tokens_seen": 1087963136 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005417211634904715, + "loss": 1.3408, + "theoretical_loss": 3.6487034406870777, + "tokens_seen": 1088028672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005417051153460381, + "loss": 1.4675, + "theoretical_loss": 3.6486830117256703, + "tokens_seen": 1088094208 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005416890672016049, + "loss": 1.5255, + "theoretical_loss": 3.648662584339162, + "tokens_seen": 1088159744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005416730190571715, + "loss": 1.4105, + "theoretical_loss": 3.6486421585273376, + "tokens_seen": 1088225280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005416569709127383, + "loss": 1.4469, + "theoretical_loss": 3.648621734289981, + "tokens_seen": 1088290816 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005416409227683049, + "loss": 1.383, + "theoretical_loss": 3.648601311626875, + "tokens_seen": 1088356352 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005416248746238717, + "loss": 1.4894, + "theoretical_loss": 3.6485808905378048, + "tokens_seen": 1088421888 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005416088264794383, + "loss": 1.3961, + "theoretical_loss": 3.648560471022553, + "tokens_seen": 1088487424 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005415927783350051, + "loss": 1.3457, + "theoretical_loss": 3.6485400530809047, + "tokens_seen": 1088552960 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005415767301905717, + "loss": 1.3825, + "theoretical_loss": 3.6485196367126433, + "tokens_seen": 1088618496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005415606820461385, + "loss": 1.3153, + "theoretical_loss": 3.648499221917553, + "tokens_seen": 1088684032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005415446339017051, + "loss": 1.318, + "theoretical_loss": 3.6484788086954176, + "tokens_seen": 1088749568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005415285857572719, + "loss": 1.4147, + "theoretical_loss": 3.6484583970460216, + "tokens_seen": 1088815104 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005415125376128385, + "loss": 1.3644, + "theoretical_loss": 3.6484379869691494, + "tokens_seen": 1088880640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005414964894684053, + "loss": 1.4715, + "theoretical_loss": 3.6484175784645845, + "tokens_seen": 1088946176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005414804413239719, + "loss": 1.4317, + "theoretical_loss": 3.6483971715321117, + "tokens_seen": 1089011712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005414643931795387, + "loss": 1.4797, + "theoretical_loss": 3.6483767661715154, + "tokens_seen": 1089077248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005414483450351053, + "loss": 1.339, + "theoretical_loss": 3.648356362382579, + "tokens_seen": 1089142784 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005414322968906721, + "loss": 1.3625, + "theoretical_loss": 3.648335960165088, + "tokens_seen": 1089208320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005414162487462387, + "loss": 1.3242, + "theoretical_loss": 3.6483155595188266, + "tokens_seen": 1089273856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005414002006018055, + "loss": 1.5713, + "theoretical_loss": 3.648295160443579, + "tokens_seen": 1089339392 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005413841524573721, + "loss": 1.3874, + "theoretical_loss": 3.6482747629391294, + "tokens_seen": 1089404928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005413681043129389, + "loss": 1.4084, + "theoretical_loss": 3.648254367005263, + "tokens_seen": 1089470464 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005413520561685055, + "loss": 1.3252, + "theoretical_loss": 3.6482339726417643, + "tokens_seen": 1089536000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005413360080240723, + "loss": 1.3143, + "theoretical_loss": 3.6482135798484174, + "tokens_seen": 1089601536 + }, + { + "epoch": 0.33, + "learning_rate": 0.000541319959879639, + "loss": 1.4371, + "theoretical_loss": 3.648193188625008, + "tokens_seen": 1089667072 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005413039117352057, + "loss": 1.3293, + "theoretical_loss": 3.64817279897132, + "tokens_seen": 1089732608 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005412878635907723, + "loss": 1.4073, + "theoretical_loss": 3.648152410887138, + "tokens_seen": 1089798144 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005412718154463391, + "loss": 1.339, + "theoretical_loss": 3.6481320243722477, + "tokens_seen": 1089863680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005412557673019058, + "loss": 1.3133, + "theoretical_loss": 3.6481116394264337, + "tokens_seen": 1089929216 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005412397191574725, + "loss": 1.3363, + "theoretical_loss": 3.64809125604948, + "tokens_seen": 1089994752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005412236710130392, + "loss": 1.4011, + "theoretical_loss": 3.6480708742411725, + "tokens_seen": 1090060288 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005412076228686059, + "loss": 1.3699, + "theoretical_loss": 3.6480504940012954, + "tokens_seen": 1090125824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005411915747241726, + "loss": 1.448, + "theoretical_loss": 3.648030115329635, + "tokens_seen": 1090191360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005411755265797393, + "loss": 1.2556, + "theoretical_loss": 3.648009738225975, + "tokens_seen": 1090256896 + }, + { + "epoch": 0.33, + "learning_rate": 0.000541159478435306, + "loss": 1.3539, + "theoretical_loss": 3.647989362690101, + "tokens_seen": 1090322432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005411434302908727, + "loss": 1.2833, + "theoretical_loss": 3.6479689887217988, + "tokens_seen": 1090387968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005411273821464394, + "loss": 1.3434, + "theoretical_loss": 3.6479486163208525, + "tokens_seen": 1090453504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005411113340020061, + "loss": 1.3431, + "theoretical_loss": 3.6479282454870487, + "tokens_seen": 1090519040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005410952858575728, + "loss": 1.3964, + "theoretical_loss": 3.647907876220171, + "tokens_seen": 1090584576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005410792377131395, + "loss": 1.3345, + "theoretical_loss": 3.6478875085200055, + "tokens_seen": 1090650112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005410631895687062, + "loss": 1.3365, + "theoretical_loss": 3.647867142386338, + "tokens_seen": 1090715648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005410471414242729, + "loss": 1.4346, + "theoretical_loss": 3.6478467778189536, + "tokens_seen": 1090781184 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005410310932798396, + "loss": 1.3844, + "theoretical_loss": 3.647826414817638, + "tokens_seen": 1090846720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005410150451354063, + "loss": 1.3398, + "theoretical_loss": 3.6478060533821757, + "tokens_seen": 1090912256 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540998996990973, + "loss": 1.3669, + "theoretical_loss": 3.647785693512353, + "tokens_seen": 1090977792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005409829488465397, + "loss": 1.4559, + "theoretical_loss": 3.6477653352079558, + "tokens_seen": 1091043328 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005409669007021064, + "loss": 1.3199, + "theoretical_loss": 3.647744978468769, + "tokens_seen": 1091108864 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540950852557673, + "loss": 1.4128, + "theoretical_loss": 3.647724623294579, + "tokens_seen": 1091174400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005409348044132398, + "loss": 1.3347, + "theoretical_loss": 3.6477042696851707, + "tokens_seen": 1091239936 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005409187562688064, + "loss": 1.3944, + "theoretical_loss": 3.64768391764033, + "tokens_seen": 1091305472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005409027081243732, + "loss": 1.3716, + "theoretical_loss": 3.647663567159843, + "tokens_seen": 1091371008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005408866599799398, + "loss": 1.3537, + "theoretical_loss": 3.6476432182434957, + "tokens_seen": 1091436544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005408706118355066, + "loss": 1.3481, + "theoretical_loss": 3.647622870891074, + "tokens_seen": 1091502080 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005408545636910732, + "loss": 1.3953, + "theoretical_loss": 3.647602525102363, + "tokens_seen": 1091567616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005408385155466399, + "loss": 1.4431, + "theoretical_loss": 3.6475821808771496, + "tokens_seen": 1091633152 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005408224674022066, + "loss": 1.2626, + "theoretical_loss": 3.6475618382152186, + "tokens_seen": 1091698688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005408064192577733, + "loss": 1.3165, + "theoretical_loss": 3.6475414971163573, + "tokens_seen": 1091764224 + }, + { + "epoch": 0.33, + "learning_rate": 0.00054079037111334, + "loss": 1.3546, + "theoretical_loss": 3.6475211575803517, + "tokens_seen": 1091829760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005407743229689067, + "loss": 1.4754, + "theoretical_loss": 3.6475008196069867, + "tokens_seen": 1091895296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005407582748244734, + "loss": 1.29, + "theoretical_loss": 3.6474804831960497, + "tokens_seen": 1091960832 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005407422266800401, + "loss": 1.4665, + "theoretical_loss": 3.6474601483473266, + "tokens_seen": 1092026368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005407261785356068, + "loss": 1.3996, + "theoretical_loss": 3.6474398150606033, + "tokens_seen": 1092091904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005407101303911735, + "loss": 1.347, + "theoretical_loss": 3.647419483335667, + "tokens_seen": 1092157440 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005406940822467402, + "loss": 1.3558, + "theoretical_loss": 3.6473991531723025, + "tokens_seen": 1092222976 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005406780341023069, + "loss": 1.3384, + "theoretical_loss": 3.647378824570297, + "tokens_seen": 1092288512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005406619859578736, + "loss": 1.3949, + "theoretical_loss": 3.6473584975294377, + "tokens_seen": 1092354048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005406459378134403, + "loss": 1.4105, + "theoretical_loss": 3.64733817204951, + "tokens_seen": 1092419584 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540629889669007, + "loss": 1.3707, + "theoretical_loss": 3.6473178481303004, + "tokens_seen": 1092485120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005406138415245737, + "loss": 1.3506, + "theoretical_loss": 3.647297525771596, + "tokens_seen": 1092550656 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005405977933801404, + "loss": 1.3765, + "theoretical_loss": 3.647277204973183, + "tokens_seen": 1092616192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005405817452357071, + "loss": 1.3583, + "theoretical_loss": 3.6472568857348486, + "tokens_seen": 1092681728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005405656970912738, + "loss": 1.418, + "theoretical_loss": 3.647236568056379, + "tokens_seen": 1092747264 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005405496489468405, + "loss": 1.4195, + "theoretical_loss": 3.647216251937561, + "tokens_seen": 1092812800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005405336008024072, + "loss": 1.4313, + "theoretical_loss": 3.647195937378181, + "tokens_seen": 1092878336 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540517552657974, + "loss": 1.3617, + "theoretical_loss": 3.6471756243780264, + "tokens_seen": 1092943872 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005405015045135406, + "loss": 1.2559, + "theoretical_loss": 3.647155312936883, + "tokens_seen": 1093009408 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005404854563691074, + "loss": 1.4009, + "theoretical_loss": 3.6471350030545393, + "tokens_seen": 1093074944 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540469408224674, + "loss": 1.3957, + "theoretical_loss": 3.6471146947307815, + "tokens_seen": 1093140480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005404533600802408, + "loss": 1.3307, + "theoretical_loss": 3.647094387965396, + "tokens_seen": 1093206016 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005404373119358074, + "loss": 1.2773, + "theoretical_loss": 3.64707408275817, + "tokens_seen": 1093271552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005404212637913742, + "loss": 1.3962, + "theoretical_loss": 3.6470537791088913, + "tokens_seen": 1093337088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005404052156469408, + "loss": 1.4317, + "theoretical_loss": 3.647033477017346, + "tokens_seen": 1093402624 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005403891675025076, + "loss": 1.3712, + "theoretical_loss": 3.6470131764833225, + "tokens_seen": 1093468160 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005403731193580742, + "loss": 1.38, + "theoretical_loss": 3.646992877506606, + "tokens_seen": 1093533696 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540357071213641, + "loss": 1.2798, + "theoretical_loss": 3.646972580086986, + "tokens_seen": 1093599232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005403410230692076, + "loss": 1.37, + "theoretical_loss": 3.646952284224248, + "tokens_seen": 1093664768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005403249749247744, + "loss": 1.396, + "theoretical_loss": 3.64693198991818, + "tokens_seen": 1093730304 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540308926780341, + "loss": 1.3587, + "theoretical_loss": 3.6469116971685693, + "tokens_seen": 1093795840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005402928786359078, + "loss": 1.4311, + "theoretical_loss": 3.6468914059752033, + "tokens_seen": 1093861376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005402768304914744, + "loss": 1.4013, + "theoretical_loss": 3.6468711163378695, + "tokens_seen": 1093926912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005402607823470412, + "loss": 1.322, + "theoretical_loss": 3.646850828256355, + "tokens_seen": 1093992448 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005402447342026078, + "loss": 1.4214, + "theoretical_loss": 3.646830541730448, + "tokens_seen": 1094057984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005402286860581746, + "loss": 1.4086, + "theoretical_loss": 3.6468102567599354, + "tokens_seen": 1094123520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005402126379137412, + "loss": 1.4671, + "theoretical_loss": 3.646789973344605, + "tokens_seen": 1094189056 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540196589769308, + "loss": 1.384, + "theoretical_loss": 3.6467696914842445, + "tokens_seen": 1094254592 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005401805416248746, + "loss": 1.3917, + "theoretical_loss": 3.646749411178641, + "tokens_seen": 1094320128 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005401644934804414, + "loss": 1.3877, + "theoretical_loss": 3.6467291324275832, + "tokens_seen": 1094385664 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 409330, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3387359380722046, + "objective/train/theoretical_loss": 3.646708855230859, + "objective/train/tokens_used": 1114911200, + "theoretical_loss": 3.646708855230859, + "tokens_seen": 1094451200 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540148445336008, + "loss": 1.3387, + "theoretical_loss": 3.646708855230859, + "tokens_seen": 1094451200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005401323971915748, + "loss": 1.5156, + "theoretical_loss": 3.646688579588255, + "tokens_seen": 1094516736 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005401163490471414, + "loss": 1.4782, + "theoretical_loss": 3.646668305499559, + "tokens_seen": 1094582272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005401003009027082, + "loss": 1.2537, + "theoretical_loss": 3.6466480329645603, + "tokens_seen": 1094647808 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005400842527582748, + "loss": 1.3721, + "theoretical_loss": 3.646627761983046, + "tokens_seen": 1094713344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005400682046138416, + "loss": 1.3356, + "theoretical_loss": 3.6466074925548035, + "tokens_seen": 1094778880 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005400521564694082, + "loss": 1.3683, + "theoretical_loss": 3.646587224679622, + "tokens_seen": 1094844416 + }, + { + "epoch": 0.33, + "learning_rate": 0.000540036108324975, + "loss": 1.4247, + "theoretical_loss": 3.646566958357289, + "tokens_seen": 1094909952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005400200601805416, + "loss": 1.4413, + "theoretical_loss": 3.6465466935875925, + "tokens_seen": 1094975488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005400040120361084, + "loss": 1.2654, + "theoretical_loss": 3.6465264303703204, + "tokens_seen": 1095041024 + }, + { + "epoch": 0.33, + "learning_rate": 0.000539987963891675, + "loss": 1.3926, + "theoretical_loss": 3.6465061687052613, + "tokens_seen": 1095106560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005399719157472418, + "loss": 1.3646, + "theoretical_loss": 3.6464859085922035, + "tokens_seen": 1095172096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005399558676028084, + "loss": 1.3352, + "theoretical_loss": 3.646465650030935, + "tokens_seen": 1095237632 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005399398194583752, + "loss": 1.3336, + "theoretical_loss": 3.6464453930212435, + "tokens_seen": 1095303168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005399237713139418, + "loss": 1.3183, + "theoretical_loss": 3.6464251375629186, + "tokens_seen": 1095368704 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005399077231695086, + "loss": 1.3996, + "theoretical_loss": 3.646404883655748, + "tokens_seen": 1095434240 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005398916750250752, + "loss": 1.3827, + "theoretical_loss": 3.6463846312995196, + "tokens_seen": 1095499776 + }, + { + "epoch": 0.33, + "learning_rate": 0.000539875626880642, + "loss": 1.3667, + "theoretical_loss": 3.646364380494023, + "tokens_seen": 1095565312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005398595787362087, + "loss": 1.417, + "theoretical_loss": 3.6463441312390463, + "tokens_seen": 1095630848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005398435305917754, + "loss": 1.2936, + "theoretical_loss": 3.646323883534378, + "tokens_seen": 1095696384 + }, + { + "epoch": 0.33, + "learning_rate": 0.000539827482447342, + "loss": 1.425, + "theoretical_loss": 3.6463036373798063, + "tokens_seen": 1095761920 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005398114343029088, + "loss": 1.4285, + "theoretical_loss": 3.64628339277512, + "tokens_seen": 1095827456 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005397953861584755, + "loss": 1.425, + "theoretical_loss": 3.6462631497201077, + "tokens_seen": 1095892992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005397793380140422, + "loss": 1.3598, + "theoretical_loss": 3.6462429082145587, + "tokens_seen": 1095958528 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005397632898696089, + "loss": 1.3492, + "theoretical_loss": 3.646222668258261, + "tokens_seen": 1096024064 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005397472417251756, + "loss": 1.3213, + "theoretical_loss": 3.6462024298510043, + "tokens_seen": 1096089600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005397311935807423, + "loss": 1.4016, + "theoretical_loss": 3.6461821929925766, + "tokens_seen": 1096155136 + }, + { + "epoch": 0.33, + "learning_rate": 0.000539715145436309, + "loss": 1.3549, + "theoretical_loss": 3.646161957682767, + "tokens_seen": 1096220672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005396990972918757, + "loss": 1.3723, + "theoretical_loss": 3.6461417239213647, + "tokens_seen": 1096286208 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005396830491474424, + "loss": 1.3786, + "theoretical_loss": 3.6461214917081586, + "tokens_seen": 1096351744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005396670010030091, + "loss": 1.3676, + "theoretical_loss": 3.6461012610429373, + "tokens_seen": 1096417280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005396509528585758, + "loss": 1.4733, + "theoretical_loss": 3.64608103192549, + "tokens_seen": 1096482816 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005396349047141425, + "loss": 1.3166, + "theoretical_loss": 3.6460608043556064, + "tokens_seen": 1096548352 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005396188565697092, + "loss": 1.369, + "theoretical_loss": 3.6460405783330745, + "tokens_seen": 1096613888 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005396028084252759, + "loss": 1.388, + "theoretical_loss": 3.6460203538576845, + "tokens_seen": 1096679424 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005395867602808426, + "loss": 1.4207, + "theoretical_loss": 3.6460001309292247, + "tokens_seen": 1096744960 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005395707121364093, + "loss": 1.4, + "theoretical_loss": 3.6459799095474854, + "tokens_seen": 1096810496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005395546639919759, + "loss": 1.3701, + "theoretical_loss": 3.6459596897122553, + "tokens_seen": 1096876032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005395386158475427, + "loss": 1.3452, + "theoretical_loss": 3.6459394714233238, + "tokens_seen": 1096941568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005395225677031093, + "loss": 1.3612, + "theoretical_loss": 3.64591925468048, + "tokens_seen": 1097007104 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005395065195586761, + "loss": 1.4674, + "theoretical_loss": 3.645899039483514, + "tokens_seen": 1097072640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005394904714142427, + "loss": 1.3506, + "theoretical_loss": 3.6458788258322143, + "tokens_seen": 1097138176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005394744232698095, + "loss": 1.3615, + "theoretical_loss": 3.6458586137263715, + "tokens_seen": 1097203712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005394583751253761, + "loss": 1.4509, + "theoretical_loss": 3.645838403165774, + "tokens_seen": 1097269248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005394423269809429, + "loss": 1.3781, + "theoretical_loss": 3.6458181941502126, + "tokens_seen": 1097334784 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005394262788365095, + "loss": 1.314, + "theoretical_loss": 3.645797986679476, + "tokens_seen": 1097400320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005394102306920762, + "loss": 1.3713, + "theoretical_loss": 3.6457777807533542, + "tokens_seen": 1097465856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005393941825476429, + "loss": 1.3506, + "theoretical_loss": 3.6457575763716363, + "tokens_seen": 1097531392 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005393781344032096, + "loss": 1.4435, + "theoretical_loss": 3.645737373534113, + "tokens_seen": 1097596928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005393620862587763, + "loss": 1.3172, + "theoretical_loss": 3.645717172240574, + "tokens_seen": 1097662464 + }, + { + "epoch": 0.33, + "learning_rate": 0.000539346038114343, + "loss": 1.3306, + "theoretical_loss": 3.645696972490809, + "tokens_seen": 1097728000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005393299899699097, + "loss": 1.5103, + "theoretical_loss": 3.645676774284607, + "tokens_seen": 1097793536 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005393139418254765, + "loss": 1.4454, + "theoretical_loss": 3.645656577621758, + "tokens_seen": 1097859072 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005392978936810431, + "loss": 1.304, + "theoretical_loss": 3.6456363825020537, + "tokens_seen": 1097924608 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005392818455366099, + "loss": 1.395, + "theoretical_loss": 3.645616188925283, + "tokens_seen": 1097990144 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005392657973921765, + "loss": 1.3966, + "theoretical_loss": 3.6455959968912355, + "tokens_seen": 1098055680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005392497492477433, + "loss": 1.4597, + "theoretical_loss": 3.6455758063997017, + "tokens_seen": 1098121216 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005392337011033099, + "loss": 1.3478, + "theoretical_loss": 3.645555617450472, + "tokens_seen": 1098186752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005392176529588767, + "loss": 1.3461, + "theoretical_loss": 3.645535430043336, + "tokens_seen": 1098252288 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005392016048144433, + "loss": 1.4454, + "theoretical_loss": 3.6455152441780836, + "tokens_seen": 1098317824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005391855566700101, + "loss": 1.4669, + "theoretical_loss": 3.645495059854506, + "tokens_seen": 1098383360 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005391695085255767, + "loss": 1.3113, + "theoretical_loss": 3.6454748770723935, + "tokens_seen": 1098448896 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005391534603811435, + "loss": 1.4074, + "theoretical_loss": 3.6454546958315355, + "tokens_seen": 1098514432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005391374122367101, + "loss": 1.3578, + "theoretical_loss": 3.645434516131723, + "tokens_seen": 1098579968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005391213640922769, + "loss": 1.4569, + "theoretical_loss": 3.6454143379727464, + "tokens_seen": 1098645504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005391053159478435, + "loss": 1.3859, + "theoretical_loss": 3.645394161354395, + "tokens_seen": 1098711040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005390892678034103, + "loss": 1.3577, + "theoretical_loss": 3.6453739862764616, + "tokens_seen": 1098776576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005390732196589769, + "loss": 1.3329, + "theoretical_loss": 3.6453538127387346, + "tokens_seen": 1098842112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005390571715145437, + "loss": 1.4138, + "theoretical_loss": 3.645333640741006, + "tokens_seen": 1098907648 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005390411233701103, + "loss": 1.4331, + "theoretical_loss": 3.645313470283065, + "tokens_seen": 1098973184 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005390250752256771, + "loss": 1.4569, + "theoretical_loss": 3.6452933013647035, + "tokens_seen": 1099038720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005390090270812437, + "loss": 1.3487, + "theoretical_loss": 3.645273133985712, + "tokens_seen": 1099104256 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005389929789368105, + "loss": 1.358, + "theoretical_loss": 3.6452529681458805, + "tokens_seen": 1099169792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005389769307923771, + "loss": 1.3783, + "theoretical_loss": 3.6452328038450004, + "tokens_seen": 1099235328 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005389608826479439, + "loss": 1.3071, + "theoretical_loss": 3.6452126410828622, + "tokens_seen": 1099300864 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005389448345035105, + "loss": 1.3583, + "theoretical_loss": 3.645192479859257, + "tokens_seen": 1099366400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005389287863590773, + "loss": 1.3729, + "theoretical_loss": 3.645172320173976, + "tokens_seen": 1099431936 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005389127382146439, + "loss": 1.4293, + "theoretical_loss": 3.6451521620268093, + "tokens_seen": 1099497472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005388966900702107, + "loss": 1.4183, + "theoretical_loss": 3.645132005417548, + "tokens_seen": 1099563008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005388806419257773, + "loss": 1.3936, + "theoretical_loss": 3.645111850345984, + "tokens_seen": 1099628544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005388645937813441, + "loss": 1.3438, + "theoretical_loss": 3.6450916968119076, + "tokens_seen": 1099694080 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005388485456369107, + "loss": 1.3646, + "theoretical_loss": 3.64507154481511, + "tokens_seen": 1099759616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005388324974924775, + "loss": 1.3067, + "theoretical_loss": 3.645051394355382, + "tokens_seen": 1099825152 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005388164493480441, + "loss": 1.3267, + "theoretical_loss": 3.6450312454325164, + "tokens_seen": 1099890688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005388004012036109, + "loss": 1.4983, + "theoretical_loss": 3.645011098046302, + "tokens_seen": 1099956224 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005387843530591775, + "loss": 1.3445, + "theoretical_loss": 3.6449909521965322, + "tokens_seen": 1100021760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005387683049147443, + "loss": 1.3122, + "theoretical_loss": 3.6449708078829968, + "tokens_seen": 1100087296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005387522567703109, + "loss": 1.3933, + "theoretical_loss": 3.644950665105488, + "tokens_seen": 1100152832 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005387362086258777, + "loss": 1.3724, + "theoretical_loss": 3.6449305238637972, + "tokens_seen": 1100218368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005387201604814443, + "loss": 1.2864, + "theoretical_loss": 3.644910384157715, + "tokens_seen": 1100283904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005387041123370111, + "loss": 1.4321, + "theoretical_loss": 3.6448902459870336, + "tokens_seen": 1100349440 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005386880641925777, + "loss": 1.2689, + "theoretical_loss": 3.6448701093515448, + "tokens_seen": 1100414976 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005386720160481445, + "loss": 1.3682, + "theoretical_loss": 3.644849974251039, + "tokens_seen": 1100480512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005386559679037111, + "loss": 1.325, + "theoretical_loss": 3.6448298406853095, + "tokens_seen": 1100546048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005386399197592779, + "loss": 1.3461, + "theoretical_loss": 3.6448097086541456, + "tokens_seen": 1100611584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005386238716148445, + "loss": 1.2407, + "theoretical_loss": 3.644789578157341, + "tokens_seen": 1100677120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005386078234704113, + "loss": 1.4844, + "theoretical_loss": 3.6447694491946865, + "tokens_seen": 1100742656 + }, + { + "epoch": 0.33, + "learning_rate": 0.000538591775325978, + "loss": 1.3025, + "theoretical_loss": 3.6447493217659743, + "tokens_seen": 1100808192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005385757271815447, + "loss": 1.3856, + "theoretical_loss": 3.6447291958709958, + "tokens_seen": 1100873728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005385596790371113, + "loss": 1.3548, + "theoretical_loss": 3.644709071509543, + "tokens_seen": 1100939264 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 411765, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3470317125320435, + "objective/train/theoretical_loss": 3.6446889486814076, + "objective/train/tokens_used": 1121464800, + "theoretical_loss": 3.6446889486814076, + "tokens_seen": 1101004800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005385436308926781, + "loss": 1.347, + "theoretical_loss": 3.6446889486814076, + "tokens_seen": 1101004800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005385275827482447, + "loss": 1.3735, + "theoretical_loss": 3.6446688273863814, + "tokens_seen": 1101070336 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005385115346038115, + "loss": 1.3762, + "theoretical_loss": 3.644648707624257, + "tokens_seen": 1101135872 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005384954864593782, + "loss": 1.3676, + "theoretical_loss": 3.6446285893948263, + "tokens_seen": 1101201408 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005384794383149449, + "loss": 1.5252, + "theoretical_loss": 3.6446084726978807, + "tokens_seen": 1101266944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005384633901705116, + "loss": 1.4193, + "theoretical_loss": 3.6445883575332125, + "tokens_seen": 1101332480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005384473420260783, + "loss": 1.4446, + "theoretical_loss": 3.644568243900614, + "tokens_seen": 1101398016 + }, + { + "epoch": 0.33, + "learning_rate": 0.000538431293881645, + "loss": 1.3352, + "theoretical_loss": 3.644548131799877, + "tokens_seen": 1101463552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005384152457372117, + "loss": 1.3663, + "theoretical_loss": 3.6445280212307942, + "tokens_seen": 1101529088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005383991975927784, + "loss": 1.4453, + "theoretical_loss": 3.644507912193158, + "tokens_seen": 1101594624 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005383831494483451, + "loss": 1.3156, + "theoretical_loss": 3.64448780468676, + "tokens_seen": 1101660160 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005383671013039118, + "loss": 1.3995, + "theoretical_loss": 3.644467698711393, + "tokens_seen": 1101725696 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005383510531594785, + "loss": 1.3761, + "theoretical_loss": 3.644447594266849, + "tokens_seen": 1101791232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005383350050150452, + "loss": 1.4403, + "theoretical_loss": 3.644427491352921, + "tokens_seen": 1101856768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005383189568706119, + "loss": 1.391, + "theoretical_loss": 3.6444073899694005, + "tokens_seen": 1101922304 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005383029087261786, + "loss": 1.3893, + "theoretical_loss": 3.6443872901160805, + "tokens_seen": 1101987840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005382868605817453, + "loss": 1.2763, + "theoretical_loss": 3.644367191792754, + "tokens_seen": 1102053376 + }, + { + "epoch": 0.33, + "learning_rate": 0.000538270812437312, + "loss": 1.3198, + "theoretical_loss": 3.6443470949992127, + "tokens_seen": 1102118912 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005382547642928787, + "loss": 1.379, + "theoretical_loss": 3.64432699973525, + "tokens_seen": 1102184448 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005382387161484454, + "loss": 1.4883, + "theoretical_loss": 3.644306906000658, + "tokens_seen": 1102249984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005382226680040121, + "loss": 1.3552, + "theoretical_loss": 3.6442868137952296, + "tokens_seen": 1102315520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005382066198595788, + "loss": 1.3833, + "theoretical_loss": 3.6442667231187573, + "tokens_seen": 1102381056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005381905717151455, + "loss": 1.4474, + "theoretical_loss": 3.644246633971034, + "tokens_seen": 1102446592 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005381745235707122, + "loss": 1.3744, + "theoretical_loss": 3.6442265463518533, + "tokens_seen": 1102512128 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005381584754262788, + "loss": 1.4811, + "theoretical_loss": 3.6442064602610063, + "tokens_seen": 1102577664 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005381424272818456, + "loss": 1.3628, + "theoretical_loss": 3.644186375698287, + "tokens_seen": 1102643200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005381263791374122, + "loss": 1.3927, + "theoretical_loss": 3.6441662926634883, + "tokens_seen": 1102708736 + }, + { + "epoch": 0.33, + "learning_rate": 0.000538110330992979, + "loss": 1.4397, + "theoretical_loss": 3.6441462111564036, + "tokens_seen": 1102774272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005380942828485456, + "loss": 1.3514, + "theoretical_loss": 3.644126131176825, + "tokens_seen": 1102839808 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005380782347041124, + "loss": 1.4202, + "theoretical_loss": 3.6441060527245455, + "tokens_seen": 1102905344 + }, + { + "epoch": 0.33, + "learning_rate": 0.000538062186559679, + "loss": 1.4256, + "theoretical_loss": 3.6440859757993596, + "tokens_seen": 1102970880 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005380461384152458, + "loss": 1.3383, + "theoretical_loss": 3.644065900401059, + "tokens_seen": 1103036416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005380300902708124, + "loss": 1.3561, + "theoretical_loss": 3.644045826529437, + "tokens_seen": 1103101952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005380140421263792, + "loss": 1.3927, + "theoretical_loss": 3.644025754184287, + "tokens_seen": 1103167488 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005379979939819458, + "loss": 1.3451, + "theoretical_loss": 3.644005683365403, + "tokens_seen": 1103233024 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005379819458375126, + "loss": 1.3382, + "theoretical_loss": 3.643985614072577, + "tokens_seen": 1103298560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005379658976930792, + "loss": 1.2993, + "theoretical_loss": 3.643965546305603, + "tokens_seen": 1103364096 + }, + { + "epoch": 0.33, + "learning_rate": 0.000537949849548646, + "loss": 1.3657, + "theoretical_loss": 3.643945480064275, + "tokens_seen": 1103429632 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005379338014042126, + "loss": 1.3514, + "theoretical_loss": 3.6439254153483853, + "tokens_seen": 1103495168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005379177532597794, + "loss": 1.4135, + "theoretical_loss": 3.643905352157728, + "tokens_seen": 1103560704 + }, + { + "epoch": 0.33, + "learning_rate": 0.000537901705115346, + "loss": 1.3947, + "theoretical_loss": 3.6438852904920966, + "tokens_seen": 1103626240 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005378856569709128, + "loss": 1.4167, + "theoretical_loss": 3.6438652303512846, + "tokens_seen": 1103691776 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005378696088264794, + "loss": 1.3825, + "theoretical_loss": 3.6438451717350846, + "tokens_seen": 1103757312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005378535606820462, + "loss": 1.3689, + "theoretical_loss": 3.643825114643292, + "tokens_seen": 1103822848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005378375125376128, + "loss": 1.3957, + "theoretical_loss": 3.6438050590756985, + "tokens_seen": 1103888384 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005378214643931796, + "loss": 1.4131, + "theoretical_loss": 3.6437850050320995, + "tokens_seen": 1103953920 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005378054162487462, + "loss": 1.3589, + "theoretical_loss": 3.6437649525122877, + "tokens_seen": 1104019456 + }, + { + "epoch": 0.33, + "learning_rate": 0.000537789368104313, + "loss": 1.3403, + "theoretical_loss": 3.6437449015160572, + "tokens_seen": 1104084992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005377733199598796, + "loss": 1.4051, + "theoretical_loss": 3.643724852043202, + "tokens_seen": 1104150528 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005377572718154464, + "loss": 1.3066, + "theoretical_loss": 3.6437048040935154, + "tokens_seen": 1104216064 + }, + { + "epoch": 0.33, + "learning_rate": 0.000537741223671013, + "loss": 1.2375, + "theoretical_loss": 3.643684757666792, + "tokens_seen": 1104281600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005377251755265798, + "loss": 1.5065, + "theoretical_loss": 3.643664712762825, + "tokens_seen": 1104347136 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005377091273821464, + "loss": 1.3948, + "theoretical_loss": 3.643644669381409, + "tokens_seen": 1104412672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005376930792377132, + "loss": 1.4606, + "theoretical_loss": 3.6436246275223376, + "tokens_seen": 1104478208 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005376770310932798, + "loss": 1.3272, + "theoretical_loss": 3.643604587185405, + "tokens_seen": 1104543744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005376609829488466, + "loss": 1.4536, + "theoretical_loss": 3.6435845483704057, + "tokens_seen": 1104609280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005376449348044132, + "loss": 1.4225, + "theoretical_loss": 3.643564511077133, + "tokens_seen": 1104674816 + }, + { + "epoch": 0.33, + "learning_rate": 0.00053762888665998, + "loss": 1.2698, + "theoretical_loss": 3.643544475305382, + "tokens_seen": 1104740352 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005376128385155466, + "loss": 1.408, + "theoretical_loss": 3.6435244410549457, + "tokens_seen": 1104805888 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005375967903711134, + "loss": 1.3966, + "theoretical_loss": 3.6435044083256196, + "tokens_seen": 1104871424 + }, + { + "epoch": 0.33, + "learning_rate": 0.00053758074222668, + "loss": 1.3301, + "theoretical_loss": 3.6434843771171974, + "tokens_seen": 1104936960 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005375646940822468, + "loss": 1.317, + "theoretical_loss": 3.6434643474294734, + "tokens_seen": 1105002496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005375486459378134, + "loss": 1.3583, + "theoretical_loss": 3.643444319262242, + "tokens_seen": 1105068032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005375325977933802, + "loss": 1.3214, + "theoretical_loss": 3.643424292615298, + "tokens_seen": 1105133568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005375165496489468, + "loss": 1.3186, + "theoretical_loss": 3.6434042674884357, + "tokens_seen": 1105199104 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005375005015045136, + "loss": 1.3874, + "theoretical_loss": 3.6433842438814494, + "tokens_seen": 1105264640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005374844533600802, + "loss": 1.2671, + "theoretical_loss": 3.6433642217941333, + "tokens_seen": 1105330176 + }, + { + "epoch": 0.33, + "learning_rate": 0.000537468405215647, + "loss": 1.4313, + "theoretical_loss": 3.6433442012262827, + "tokens_seen": 1105395712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0005374523570712136, + "loss": 1.3311, + "theoretical_loss": 3.6433241821776914, + "tokens_seen": 1105461248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005374363089267804, + "loss": 1.4394, + "theoretical_loss": 3.643304164648155, + "tokens_seen": 1105526784 + }, + { + "epoch": 0.34, + "learning_rate": 0.000537420260782347, + "loss": 1.3518, + "theoretical_loss": 3.643284148637467, + "tokens_seen": 1105592320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005374042126379138, + "loss": 1.3953, + "theoretical_loss": 3.643264134145424, + "tokens_seen": 1105657856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005373881644934804, + "loss": 1.3837, + "theoretical_loss": 3.643244121171819, + "tokens_seen": 1105723392 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005373721163490472, + "loss": 1.3472, + "theoretical_loss": 3.6432241097164475, + "tokens_seen": 1105788928 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005373560682046138, + "loss": 1.2751, + "theoretical_loss": 3.643204099779104, + "tokens_seen": 1105854464 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005373400200601806, + "loss": 1.3246, + "theoretical_loss": 3.643184091359584, + "tokens_seen": 1105920000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005373239719157472, + "loss": 1.416, + "theoretical_loss": 3.6431640844576823, + "tokens_seen": 1105985536 + }, + { + "epoch": 0.34, + "learning_rate": 0.000537307923771314, + "loss": 1.3421, + "theoretical_loss": 3.6431440790731933, + "tokens_seen": 1106051072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005372918756268806, + "loss": 1.3932, + "theoretical_loss": 3.6431240752059124, + "tokens_seen": 1106116608 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005372758274824474, + "loss": 1.2558, + "theoretical_loss": 3.643104072855635, + "tokens_seen": 1106182144 + }, + { + "epoch": 0.34, + "learning_rate": 0.000537259779338014, + "loss": 1.3881, + "theoretical_loss": 3.6430840720221553, + "tokens_seen": 1106247680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005372437311935808, + "loss": 1.3323, + "theoretical_loss": 3.6430640727052697, + "tokens_seen": 1106313216 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005372276830491474, + "loss": 1.325, + "theoretical_loss": 3.643044074904772, + "tokens_seen": 1106378752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005372116349047142, + "loss": 1.3879, + "theoretical_loss": 3.6430240786204586, + "tokens_seen": 1106444288 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005371955867602808, + "loss": 1.3359, + "theoretical_loss": 3.643004083852124, + "tokens_seen": 1106509824 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005371795386158476, + "loss": 1.3409, + "theoretical_loss": 3.6429840905995636, + "tokens_seen": 1106575360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005371634904714142, + "loss": 1.3448, + "theoretical_loss": 3.6429640988625724, + "tokens_seen": 1106640896 + }, + { + "epoch": 0.34, + "learning_rate": 0.000537147442326981, + "loss": 1.5059, + "theoretical_loss": 3.642944108640947, + "tokens_seen": 1106706432 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005371313941825477, + "loss": 1.3422, + "theoretical_loss": 3.6429241199344817, + "tokens_seen": 1106771968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005371153460381144, + "loss": 1.2904, + "theoretical_loss": 3.6429041327429723, + "tokens_seen": 1106837504 + }, + { + "epoch": 0.34, + "learning_rate": 0.000537099297893681, + "loss": 1.288, + "theoretical_loss": 3.642884147066214, + "tokens_seen": 1106903040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005370832497492478, + "loss": 1.3871, + "theoretical_loss": 3.642864162904003, + "tokens_seen": 1106968576 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005370672016048145, + "loss": 1.2763, + "theoretical_loss": 3.6428441802561347, + "tokens_seen": 1107034112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005370511534603812, + "loss": 1.341, + "theoretical_loss": 3.642824199122404, + "tokens_seen": 1107099648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005370351053159479, + "loss": 1.3455, + "theoretical_loss": 3.642804219502607, + "tokens_seen": 1107165184 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005370190571715146, + "loss": 1.4359, + "theoretical_loss": 3.6427842413965394, + "tokens_seen": 1107230720 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005370030090270813, + "loss": 1.3423, + "theoretical_loss": 3.642764264803997, + "tokens_seen": 1107296256 + }, + { + "epoch": 0.34, + "learning_rate": 0.000536986960882648, + "loss": 1.4774, + "theoretical_loss": 3.6427442897247753, + "tokens_seen": 1107361792 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005369709127382147, + "loss": 1.3874, + "theoretical_loss": 3.6427243161586706, + "tokens_seen": 1107427328 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005369548645937814, + "loss": 1.4958, + "theoretical_loss": 3.6427043441054785, + "tokens_seen": 1107492864 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 414098, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3582265377044678, + "objective/train/theoretical_loss": 3.642684373564995, + "objective/train/tokens_used": 1128018400, + "theoretical_loss": 3.642684373564995, + "tokens_seen": 1107558400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005369388164493481, + "loss": 1.3582, + "theoretical_loss": 3.642684373564995, + "tokens_seen": 1107558400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005369227683049148, + "loss": 1.4301, + "theoretical_loss": 3.6426644045370153, + "tokens_seen": 1107623936 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005369067201604815, + "loss": 1.3194, + "theoretical_loss": 3.6426444370213362, + "tokens_seen": 1107689472 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005368906720160482, + "loss": 1.4041, + "theoretical_loss": 3.6426244710177533, + "tokens_seen": 1107755008 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005368746238716149, + "loss": 1.4744, + "theoretical_loss": 3.6426045065260633, + "tokens_seen": 1107820544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005368585757271816, + "loss": 1.4021, + "theoretical_loss": 3.642584543546061, + "tokens_seen": 1107886080 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005368425275827483, + "loss": 1.4745, + "theoretical_loss": 3.642564582077544, + "tokens_seen": 1107951616 + }, + { + "epoch": 0.34, + "learning_rate": 0.000536826479438315, + "loss": 1.4097, + "theoretical_loss": 3.6425446221203073, + "tokens_seen": 1108017152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005368104312938817, + "loss": 1.3831, + "theoretical_loss": 3.6425246636741475, + "tokens_seen": 1108082688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005367943831494484, + "loss": 1.3982, + "theoretical_loss": 3.6425047067388614, + "tokens_seen": 1108148224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005367783350050151, + "loss": 1.3431, + "theoretical_loss": 3.6424847513142447, + "tokens_seen": 1108213760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005367622868605817, + "loss": 1.4644, + "theoretical_loss": 3.6424647974000934, + "tokens_seen": 1108279296 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005367462387161485, + "loss": 1.2962, + "theoretical_loss": 3.642444844996205, + "tokens_seen": 1108344832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005367301905717151, + "loss": 1.3107, + "theoretical_loss": 3.6424248941023745, + "tokens_seen": 1108410368 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005367141424272819, + "loss": 1.3053, + "theoretical_loss": 3.642404944718399, + "tokens_seen": 1108475904 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005366980942828485, + "loss": 1.4671, + "theoretical_loss": 3.6423849968440756, + "tokens_seen": 1108541440 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005366820461384153, + "loss": 1.4127, + "theoretical_loss": 3.6423650504791993, + "tokens_seen": 1108606976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005366659979939819, + "loss": 1.4346, + "theoretical_loss": 3.6423451056235683, + "tokens_seen": 1108672512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005366499498495487, + "loss": 1.3316, + "theoretical_loss": 3.642325162276978, + "tokens_seen": 1108738048 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005366339017051153, + "loss": 1.3739, + "theoretical_loss": 3.642305220439226, + "tokens_seen": 1108803584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005366178535606821, + "loss": 1.3253, + "theoretical_loss": 3.6422852801101078, + "tokens_seen": 1108869120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005366018054162487, + "loss": 1.3901, + "theoretical_loss": 3.6422653412894217, + "tokens_seen": 1108934656 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005365857572718155, + "loss": 1.2944, + "theoretical_loss": 3.642245403976963, + "tokens_seen": 1109000192 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005365697091273821, + "loss": 1.3555, + "theoretical_loss": 3.642225468172529, + "tokens_seen": 1109065728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005365536609829489, + "loss": 1.3868, + "theoretical_loss": 3.6422055338759165, + "tokens_seen": 1109131264 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005365376128385155, + "loss": 1.3555, + "theoretical_loss": 3.642185601086923, + "tokens_seen": 1109196800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005365215646940823, + "loss": 1.4028, + "theoretical_loss": 3.642165669805344, + "tokens_seen": 1109262336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005365055165496489, + "loss": 1.3435, + "theoretical_loss": 3.642145740030978, + "tokens_seen": 1109327872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005364894684052157, + "loss": 1.3088, + "theoretical_loss": 3.6421258117636217, + "tokens_seen": 1109393408 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005364734202607823, + "loss": 1.2877, + "theoretical_loss": 3.6421058850030708, + "tokens_seen": 1109458944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005364573721163491, + "loss": 1.3197, + "theoretical_loss": 3.642085959749124, + "tokens_seen": 1109524480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005364413239719157, + "loss": 1.4337, + "theoretical_loss": 3.6420660360015775, + "tokens_seen": 1109590016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005364252758274825, + "loss": 1.3989, + "theoretical_loss": 3.6420461137602285, + "tokens_seen": 1109655552 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005364092276830491, + "loss": 1.41, + "theoretical_loss": 3.6420261930248747, + "tokens_seen": 1109721088 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005363931795386159, + "loss": 1.4657, + "theoretical_loss": 3.642006273795313, + "tokens_seen": 1109786624 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005363771313941825, + "loss": 1.2838, + "theoretical_loss": 3.64198635607134, + "tokens_seen": 1109852160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005363610832497493, + "loss": 1.3769, + "theoretical_loss": 3.6419664398527543, + "tokens_seen": 1109917696 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005363450351053159, + "loss": 1.413, + "theoretical_loss": 3.6419465251393524, + "tokens_seen": 1109983232 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005363289869608827, + "loss": 1.3616, + "theoretical_loss": 3.641926611930932, + "tokens_seen": 1110048768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005363129388164493, + "loss": 1.2939, + "theoretical_loss": 3.64190670022729, + "tokens_seen": 1110114304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005362968906720161, + "loss": 1.3579, + "theoretical_loss": 3.641886790028225, + "tokens_seen": 1110179840 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005362808425275827, + "loss": 1.3616, + "theoretical_loss": 3.6418668813335335, + "tokens_seen": 1110245376 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005362647943831495, + "loss": 1.3598, + "theoretical_loss": 3.6418469741430135, + "tokens_seen": 1110310912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005362487462387161, + "loss": 1.308, + "theoretical_loss": 3.641827068456462, + "tokens_seen": 1110376448 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005362326980942829, + "loss": 1.3328, + "theoretical_loss": 3.6418071642736773, + "tokens_seen": 1110441984 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005362166499498495, + "loss": 1.3164, + "theoretical_loss": 3.6417872615944566, + "tokens_seen": 1110507520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005362006018054163, + "loss": 1.4097, + "theoretical_loss": 3.641767360418598, + "tokens_seen": 1110573056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005361845536609829, + "loss": 1.4196, + "theoretical_loss": 3.641747460745899, + "tokens_seen": 1110638592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005361685055165497, + "loss": 1.2782, + "theoretical_loss": 3.6417275625761576, + "tokens_seen": 1110704128 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005361524573721163, + "loss": 1.3362, + "theoretical_loss": 3.6417076659091707, + "tokens_seen": 1110769664 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005361364092276831, + "loss": 1.3231, + "theoretical_loss": 3.6416877707447375, + "tokens_seen": 1110835200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005361203610832497, + "loss": 1.3188, + "theoretical_loss": 3.6416678770826554, + "tokens_seen": 1110900736 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005361043129388165, + "loss": 1.4302, + "theoretical_loss": 3.641647984922722, + "tokens_seen": 1110966272 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005360882647943831, + "loss": 1.2772, + "theoretical_loss": 3.6416280942647354, + "tokens_seen": 1111031808 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005360722166499499, + "loss": 1.4144, + "theoretical_loss": 3.6416082051084935, + "tokens_seen": 1111097344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005360561685055165, + "loss": 1.3272, + "theoretical_loss": 3.641588317453795, + "tokens_seen": 1111162880 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005360401203610833, + "loss": 1.3278, + "theoretical_loss": 3.641568431300437, + "tokens_seen": 1111228416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005360240722166499, + "loss": 1.2932, + "theoretical_loss": 3.6415485466482185, + "tokens_seen": 1111293952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005360080240722167, + "loss": 1.4051, + "theoretical_loss": 3.6415286634969375, + "tokens_seen": 1111359488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005359919759277833, + "loss": 1.2971, + "theoretical_loss": 3.6415087818463916, + "tokens_seen": 1111425024 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005359759277833501, + "loss": 1.303, + "theoretical_loss": 3.6414889016963796, + "tokens_seen": 1111490560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005359598796389167, + "loss": 1.2754, + "theoretical_loss": 3.6414690230467, + "tokens_seen": 1111556096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005359438314944835, + "loss": 1.4072, + "theoretical_loss": 3.6414491458971505, + "tokens_seen": 1111621632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005359277833500501, + "loss": 1.3584, + "theoretical_loss": 3.6414292702475297, + "tokens_seen": 1111687168 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005359117352056169, + "loss": 1.4359, + "theoretical_loss": 3.641409396097636, + "tokens_seen": 1111752704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005358956870611835, + "loss": 1.3398, + "theoretical_loss": 3.641389523447268, + "tokens_seen": 1111818240 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005358796389167503, + "loss": 1.3735, + "theoretical_loss": 3.6413696522962242, + "tokens_seen": 1111883776 + }, + { + "epoch": 0.34, + "learning_rate": 0.000535863590772317, + "loss": 1.3086, + "theoretical_loss": 3.641349782644303, + "tokens_seen": 1111949312 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005358475426278837, + "loss": 1.2525, + "theoretical_loss": 3.641329914491303, + "tokens_seen": 1112014848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005358314944834503, + "loss": 1.3499, + "theoretical_loss": 3.6413100478370226, + "tokens_seen": 1112080384 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005358154463390171, + "loss": 1.3117, + "theoretical_loss": 3.6412901826812605, + "tokens_seen": 1112145920 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005357993981945837, + "loss": 1.3835, + "theoretical_loss": 3.641270319023816, + "tokens_seen": 1112211456 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005357833500501505, + "loss": 1.3015, + "theoretical_loss": 3.6412504568644866, + "tokens_seen": 1112276992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005357673019057172, + "loss": 1.4119, + "theoretical_loss": 3.641230596203072, + "tokens_seen": 1112342528 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005357512537612839, + "loss": 1.2786, + "theoretical_loss": 3.6412107370393705, + "tokens_seen": 1112408064 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005357352056168507, + "loss": 1.3128, + "theoretical_loss": 3.6411908793731813, + "tokens_seen": 1112473600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005357191574724173, + "loss": 1.3677, + "theoretical_loss": 3.6411710232043033, + "tokens_seen": 1112539136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005357031093279841, + "loss": 1.3733, + "theoretical_loss": 3.641151168532535, + "tokens_seen": 1112604672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005356870611835507, + "loss": 1.4127, + "theoretical_loss": 3.6411313153576756, + "tokens_seen": 1112670208 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005356710130391175, + "loss": 1.3449, + "theoretical_loss": 3.6411114636795237, + "tokens_seen": 1112735744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005356549648946841, + "loss": 1.2621, + "theoretical_loss": 3.641091613497879, + "tokens_seen": 1112801280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005356389167502509, + "loss": 1.3397, + "theoretical_loss": 3.6410717648125406, + "tokens_seen": 1112866816 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005356228686058175, + "loss": 1.4282, + "theoretical_loss": 3.6410519176233067, + "tokens_seen": 1112932352 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005356068204613843, + "loss": 1.4523, + "theoretical_loss": 3.641032071929977, + "tokens_seen": 1112997888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005355907723169509, + "loss": 1.3401, + "theoretical_loss": 3.641012227732351, + "tokens_seen": 1113063424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005355747241725177, + "loss": 1.2824, + "theoretical_loss": 3.6409923850302275, + "tokens_seen": 1113128960 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005355586760280843, + "loss": 1.366, + "theoretical_loss": 3.640972543823406, + "tokens_seen": 1113194496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005355426278836511, + "loss": 1.4025, + "theoretical_loss": 3.6409527041116854, + "tokens_seen": 1113260032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005355265797392177, + "loss": 1.4499, + "theoretical_loss": 3.640932865894865, + "tokens_seen": 1113325568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005355105315947845, + "loss": 1.4383, + "theoretical_loss": 3.640913029172745, + "tokens_seen": 1113391104 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005354944834503511, + "loss": 1.3928, + "theoretical_loss": 3.640893193945124, + "tokens_seen": 1113456640 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005354784353059178, + "loss": 1.416, + "theoretical_loss": 3.6408733602118017, + "tokens_seen": 1113522176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005354623871614845, + "loss": 1.3317, + "theoretical_loss": 3.6408535279725776, + "tokens_seen": 1113587712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005354463390170512, + "loss": 1.3307, + "theoretical_loss": 3.640833697227251, + "tokens_seen": 1113653248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005354302908726179, + "loss": 1.4601, + "theoretical_loss": 3.640813867975622, + "tokens_seen": 1113718784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005354142427281846, + "loss": 1.3148, + "theoretical_loss": 3.6407940402174894, + "tokens_seen": 1113784320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005353981945837513, + "loss": 1.3995, + "theoretical_loss": 3.640774213952654, + "tokens_seen": 1113849856 + }, + { + "epoch": 0.34, + "learning_rate": 0.000535382146439318, + "loss": 1.3994, + "theoretical_loss": 3.6407543891809144, + "tokens_seen": 1113915392 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005353660982948846, + "loss": 1.4735, + "theoretical_loss": 3.640734565902071, + "tokens_seen": 1113980928 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005353500501504514, + "loss": 1.3982, + "theoretical_loss": 3.6407147441159236, + "tokens_seen": 1114046464 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 416569, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3156498670578003, + "objective/train/theoretical_loss": 3.640694923822271, + "objective/train/tokens_used": 1134572000, + "theoretical_loss": 3.640694923822271, + "tokens_seen": 1114112000 + }, + { + "epoch": 0.34, + "learning_rate": 0.000535334002006018, + "loss": 1.3156, + "theoretical_loss": 3.640694923822271, + "tokens_seen": 1114112000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005353179538615848, + "loss": 1.2673, + "theoretical_loss": 3.6406751050209145, + "tokens_seen": 1114177536 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005353019057171514, + "loss": 1.383, + "theoretical_loss": 3.6406552877116525, + "tokens_seen": 1114243072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005352858575727182, + "loss": 1.4454, + "theoretical_loss": 3.640635471894286, + "tokens_seen": 1114308608 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005352698094282848, + "loss": 1.3471, + "theoretical_loss": 3.640615657568614, + "tokens_seen": 1114374144 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005352537612838516, + "loss": 1.3559, + "theoretical_loss": 3.640595844734438, + "tokens_seen": 1114439680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005352377131394182, + "loss": 1.2936, + "theoretical_loss": 3.6405760333915564, + "tokens_seen": 1114505216 + }, + { + "epoch": 0.34, + "learning_rate": 0.000535221664994985, + "loss": 1.3848, + "theoretical_loss": 3.640556223539771, + "tokens_seen": 1114570752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005352056168505516, + "loss": 1.268, + "theoretical_loss": 3.6405364151788797, + "tokens_seen": 1114636288 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005351895687061184, + "loss": 1.3162, + "theoretical_loss": 3.640516608308684, + "tokens_seen": 1114701824 + }, + { + "epoch": 0.34, + "learning_rate": 0.000535173520561685, + "loss": 1.4906, + "theoretical_loss": 3.6404968029289844, + "tokens_seen": 1114767360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005351574724172518, + "loss": 1.3358, + "theoretical_loss": 3.6404769990395804, + "tokens_seen": 1114832896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005351414242728184, + "loss": 1.4759, + "theoretical_loss": 3.6404571966402726, + "tokens_seen": 1114898432 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005351253761283852, + "loss": 1.3611, + "theoretical_loss": 3.640437395730861, + "tokens_seen": 1114963968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005351093279839518, + "loss": 1.3159, + "theoretical_loss": 3.6404175963111465, + "tokens_seen": 1115029504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005350932798395186, + "loss": 1.3636, + "theoretical_loss": 3.640397798380929, + "tokens_seen": 1115095040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005350772316950852, + "loss": 1.4797, + "theoretical_loss": 3.640378001940009, + "tokens_seen": 1115160576 + }, + { + "epoch": 0.34, + "learning_rate": 0.000535061183550652, + "loss": 1.4735, + "theoretical_loss": 3.6403582069881866, + "tokens_seen": 1115226112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005350451354062186, + "loss": 1.3651, + "theoretical_loss": 3.640338413525263, + "tokens_seen": 1115291648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005350290872617854, + "loss": 1.4095, + "theoretical_loss": 3.640318621551039, + "tokens_seen": 1115357184 + }, + { + "epoch": 0.34, + "learning_rate": 0.000535013039117352, + "loss": 1.3452, + "theoretical_loss": 3.640298831065314, + "tokens_seen": 1115422720 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005349969909729188, + "loss": 1.435, + "theoretical_loss": 3.6402790420678888, + "tokens_seen": 1115488256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005349809428284854, + "loss": 1.4533, + "theoretical_loss": 3.640259254558565, + "tokens_seen": 1115553792 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005349648946840522, + "loss": 1.382, + "theoretical_loss": 3.6402394685371426, + "tokens_seen": 1115619328 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005349488465396188, + "loss": 1.4456, + "theoretical_loss": 3.640219684003422, + "tokens_seen": 1115684864 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005349327983951856, + "loss": 1.3801, + "theoretical_loss": 3.640199900957205, + "tokens_seen": 1115750400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005349167502507522, + "loss": 1.2718, + "theoretical_loss": 3.6401801193982917, + "tokens_seen": 1115815936 + }, + { + "epoch": 0.34, + "learning_rate": 0.000534900702106319, + "loss": 1.2542, + "theoretical_loss": 3.6401603393264828, + "tokens_seen": 1115881472 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005348846539618856, + "loss": 1.3393, + "theoretical_loss": 3.6401405607415795, + "tokens_seen": 1115947008 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005348686058174524, + "loss": 1.3298, + "theoretical_loss": 3.6401207836433827, + "tokens_seen": 1116012544 + }, + { + "epoch": 0.34, + "learning_rate": 0.000534852557673019, + "loss": 1.3606, + "theoretical_loss": 3.640101008031693, + "tokens_seen": 1116078080 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005348365095285858, + "loss": 1.4865, + "theoretical_loss": 3.6400812339063124, + "tokens_seen": 1116143616 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005348204613841524, + "loss": 1.2289, + "theoretical_loss": 3.6400614612670408, + "tokens_seen": 1116209152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005348044132397192, + "loss": 1.3863, + "theoretical_loss": 3.6400416901136796, + "tokens_seen": 1116274688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005347883650952858, + "loss": 1.3158, + "theoretical_loss": 3.6400219204460305, + "tokens_seen": 1116340224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005347723169508526, + "loss": 1.3845, + "theoretical_loss": 3.6400021522638935, + "tokens_seen": 1116405760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005347562688064192, + "loss": 1.3103, + "theoretical_loss": 3.6399823855670705, + "tokens_seen": 1116471296 + }, + { + "epoch": 0.34, + "learning_rate": 0.000534740220661986, + "loss": 1.3984, + "theoretical_loss": 3.6399626203553628, + "tokens_seen": 1116536832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005347241725175526, + "loss": 1.2736, + "theoretical_loss": 3.639942856628571, + "tokens_seen": 1116602368 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005347081243731194, + "loss": 1.3643, + "theoretical_loss": 3.6399230943864977, + "tokens_seen": 1116667904 + }, + { + "epoch": 0.34, + "learning_rate": 0.000534692076228686, + "loss": 1.4367, + "theoretical_loss": 3.639903333628943, + "tokens_seen": 1116733440 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005346760280842528, + "loss": 1.2781, + "theoretical_loss": 3.639883574355709, + "tokens_seen": 1116798976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005346599799398194, + "loss": 1.2751, + "theoretical_loss": 3.6398638165665966, + "tokens_seen": 1116864512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005346439317953862, + "loss": 1.3697, + "theoretical_loss": 3.6398440602614075, + "tokens_seen": 1116930048 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005346278836509528, + "loss": 1.4117, + "theoretical_loss": 3.6398243054399426, + "tokens_seen": 1116995584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005346118355065196, + "loss": 1.313, + "theoretical_loss": 3.639804552102005, + "tokens_seen": 1117061120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005345957873620862, + "loss": 1.4516, + "theoretical_loss": 3.6397848002473943, + "tokens_seen": 1117126656 + }, + { + "epoch": 0.34, + "learning_rate": 0.000534579739217653, + "loss": 1.3614, + "theoretical_loss": 3.6397650498759138, + "tokens_seen": 1117192192 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005345636910732196, + "loss": 1.5001, + "theoretical_loss": 3.639745300987364, + "tokens_seen": 1117257728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005345476429287864, + "loss": 1.3916, + "theoretical_loss": 3.639725553581547, + "tokens_seen": 1117323264 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005345315947843532, + "loss": 1.3054, + "theoretical_loss": 3.639705807658265, + "tokens_seen": 1117388800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005345155466399198, + "loss": 1.3713, + "theoretical_loss": 3.6396860632173187, + "tokens_seen": 1117454336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005344994984954866, + "loss": 1.3268, + "theoretical_loss": 3.6396663202585104, + "tokens_seen": 1117519872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005344834503510532, + "loss": 1.3096, + "theoretical_loss": 3.639646578781642, + "tokens_seen": 1117585408 + }, + { + "epoch": 0.34, + "learning_rate": 0.00053446740220662, + "loss": 1.2857, + "theoretical_loss": 3.6396268387865156, + "tokens_seen": 1117650944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005344513540621866, + "loss": 1.3031, + "theoretical_loss": 3.639607100272933, + "tokens_seen": 1117716480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005344353059177534, + "loss": 1.3131, + "theoretical_loss": 3.639587363240696, + "tokens_seen": 1117782016 + }, + { + "epoch": 0.34, + "learning_rate": 0.00053441925777332, + "loss": 1.3918, + "theoretical_loss": 3.6395676276896065, + "tokens_seen": 1117847552 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005344032096288868, + "loss": 1.406, + "theoretical_loss": 3.6395478936194663, + "tokens_seen": 1117913088 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005343871614844534, + "loss": 1.3954, + "theoretical_loss": 3.6395281610300776, + "tokens_seen": 1117978624 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005343711133400202, + "loss": 1.3761, + "theoretical_loss": 3.6395084299212432, + "tokens_seen": 1118044160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005343550651955868, + "loss": 1.3811, + "theoretical_loss": 3.6394887002927647, + "tokens_seen": 1118109696 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005343390170511536, + "loss": 1.274, + "theoretical_loss": 3.639468972144444, + "tokens_seen": 1118175232 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005343229689067202, + "loss": 1.4621, + "theoretical_loss": 3.639449245476084, + "tokens_seen": 1118240768 + }, + { + "epoch": 0.34, + "learning_rate": 0.000534306920762287, + "loss": 1.3572, + "theoretical_loss": 3.6394295202874862, + "tokens_seen": 1118306304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005342908726178536, + "loss": 1.3092, + "theoretical_loss": 3.6394097965784535, + "tokens_seen": 1118371840 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005342748244734204, + "loss": 1.3921, + "theoretical_loss": 3.6393900743487877, + "tokens_seen": 1118437376 + }, + { + "epoch": 0.34, + "learning_rate": 0.000534258776328987, + "loss": 1.382, + "theoretical_loss": 3.6393703535982915, + "tokens_seen": 1118502912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005342427281845538, + "loss": 1.3019, + "theoretical_loss": 3.6393506343267674, + "tokens_seen": 1118568448 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005342266800401204, + "loss": 1.4143, + "theoretical_loss": 3.6393309165340177, + "tokens_seen": 1118633984 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005342106318956872, + "loss": 1.4506, + "theoretical_loss": 3.6393112002198444, + "tokens_seen": 1118699520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005341945837512538, + "loss": 1.3551, + "theoretical_loss": 3.6392914853840512, + "tokens_seen": 1118765056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005341785356068206, + "loss": 1.3537, + "theoretical_loss": 3.639271772026439, + "tokens_seen": 1118830592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005341624874623872, + "loss": 1.3343, + "theoretical_loss": 3.6392520601468124, + "tokens_seen": 1118896128 + }, + { + "epoch": 0.34, + "learning_rate": 0.000534146439317954, + "loss": 1.3418, + "theoretical_loss": 3.6392323497449723, + "tokens_seen": 1118961664 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005341303911735206, + "loss": 1.3859, + "theoretical_loss": 3.6392126408207224, + "tokens_seen": 1119027200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005341143430290874, + "loss": 1.3559, + "theoretical_loss": 3.6391929333738644, + "tokens_seen": 1119092736 + }, + { + "epoch": 0.34, + "learning_rate": 0.000534098294884654, + "loss": 1.4003, + "theoretical_loss": 3.639173227404202, + "tokens_seen": 1119158272 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005340822467402207, + "loss": 1.4168, + "theoretical_loss": 3.639153522911538, + "tokens_seen": 1119223808 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005340661985957874, + "loss": 1.4425, + "theoretical_loss": 3.6391338198956746, + "tokens_seen": 1119289344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005340501504513541, + "loss": 1.3851, + "theoretical_loss": 3.639114118356415, + "tokens_seen": 1119354880 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005340341023069208, + "loss": 1.3425, + "theoretical_loss": 3.6390944182935616, + "tokens_seen": 1119420416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005340180541624875, + "loss": 1.2919, + "theoretical_loss": 3.639074719706919, + "tokens_seen": 1119485952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005340020060180542, + "loss": 1.3872, + "theoretical_loss": 3.6390550225962874, + "tokens_seen": 1119551488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005339859578736209, + "loss": 1.3686, + "theoretical_loss": 3.6390353269614724, + "tokens_seen": 1119617024 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005339699097291876, + "loss": 1.3951, + "theoretical_loss": 3.6390156328022756, + "tokens_seen": 1119682560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005339538615847543, + "loss": 1.3075, + "theoretical_loss": 3.638995940118501, + "tokens_seen": 1119748096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005339378134403209, + "loss": 1.4119, + "theoretical_loss": 3.6389762489099504, + "tokens_seen": 1119813632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005339217652958877, + "loss": 1.3106, + "theoretical_loss": 3.638956559176428, + "tokens_seen": 1119879168 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005339057171514543, + "loss": 1.3643, + "theoretical_loss": 3.638936870917737, + "tokens_seen": 1119944704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005338896690070211, + "loss": 1.2895, + "theoretical_loss": 3.63891718413368, + "tokens_seen": 1120010240 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005338736208625877, + "loss": 1.4278, + "theoretical_loss": 3.638897498824061, + "tokens_seen": 1120075776 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005338575727181545, + "loss": 1.4127, + "theoretical_loss": 3.6388778149886836, + "tokens_seen": 1120141312 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005338415245737211, + "loss": 1.4273, + "theoretical_loss": 3.63885813262735, + "tokens_seen": 1120206848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005338254764292879, + "loss": 1.3313, + "theoretical_loss": 3.638838451739864, + "tokens_seen": 1120272384 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005338094282848545, + "loss": 1.3325, + "theoretical_loss": 3.6388187723260286, + "tokens_seen": 1120337920 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005337933801404213, + "loss": 1.353, + "theoretical_loss": 3.6387990943856487, + "tokens_seen": 1120403456 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005337773319959879, + "loss": 1.3517, + "theoretical_loss": 3.638779417918527, + "tokens_seen": 1120468992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005337612838515547, + "loss": 1.3631, + "theoretical_loss": 3.6387597429244662, + "tokens_seen": 1120534528 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005337452357071213, + "loss": 1.4388, + "theoretical_loss": 3.638740069403271, + "tokens_seen": 1120600064 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 418846, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4220236539840698, + "objective/train/theoretical_loss": 3.638720397354745, + "objective/train/tokens_used": 1141125600, + "theoretical_loss": 3.638720397354745, + "tokens_seen": 1120665600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005337291875626881, + "loss": 1.422, + "theoretical_loss": 3.638720397354745, + "tokens_seen": 1120665600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005337131394182547, + "loss": 1.4216, + "theoretical_loss": 3.638700726778691, + "tokens_seen": 1120731136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005336970912738215, + "loss": 1.3476, + "theoretical_loss": 3.6386810576749133, + "tokens_seen": 1120796672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005336810431293881, + "loss": 1.3282, + "theoretical_loss": 3.638661390043216, + "tokens_seen": 1120862208 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005336649949849549, + "loss": 1.3414, + "theoretical_loss": 3.638641723883402, + "tokens_seen": 1120927744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005336489468405215, + "loss": 1.2866, + "theoretical_loss": 3.638622059195275, + "tokens_seen": 1120993280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005336328986960883, + "loss": 1.3447, + "theoretical_loss": 3.6386023959786398, + "tokens_seen": 1121058816 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005336168505516549, + "loss": 1.3197, + "theoretical_loss": 3.6385827342333, + "tokens_seen": 1121124352 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005336008024072217, + "loss": 1.4682, + "theoretical_loss": 3.638563073959059, + "tokens_seen": 1121189888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005335847542627883, + "loss": 1.3868, + "theoretical_loss": 3.638543415155721, + "tokens_seen": 1121255424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005335687061183551, + "loss": 1.3832, + "theoretical_loss": 3.6385237578230907, + "tokens_seen": 1121320960 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005335526579739217, + "loss": 1.337, + "theoretical_loss": 3.6385041019609705, + "tokens_seen": 1121386496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005335366098294885, + "loss": 1.4122, + "theoretical_loss": 3.6384844475691662, + "tokens_seen": 1121452032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005335205616850551, + "loss": 1.2603, + "theoretical_loss": 3.638464794647481, + "tokens_seen": 1121517568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005335045135406219, + "loss": 1.3956, + "theoretical_loss": 3.638445143195719, + "tokens_seen": 1121583104 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005334884653961885, + "loss": 1.441, + "theoretical_loss": 3.638425493213685, + "tokens_seen": 1121648640 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005334724172517553, + "loss": 1.3888, + "theoretical_loss": 3.6384058447011824, + "tokens_seen": 1121714176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005334563691073219, + "loss": 1.3796, + "theoretical_loss": 3.638386197658016, + "tokens_seen": 1121779712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005334403209628887, + "loss": 1.3599, + "theoretical_loss": 3.6383665520839896, + "tokens_seen": 1121845248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005334242728184553, + "loss": 1.2925, + "theoretical_loss": 3.638346907978908, + "tokens_seen": 1121910784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005334082246740221, + "loss": 1.3894, + "theoretical_loss": 3.638327265342576, + "tokens_seen": 1121976320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005333921765295887, + "loss": 1.3431, + "theoretical_loss": 3.6383076241747965, + "tokens_seen": 1122041856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005333761283851555, + "loss": 1.396, + "theoretical_loss": 3.638287984475375, + "tokens_seen": 1122107392 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005333600802407221, + "loss": 1.3793, + "theoretical_loss": 3.6382683462441165, + "tokens_seen": 1122172928 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005333440320962889, + "loss": 1.2587, + "theoretical_loss": 3.6382487094808242, + "tokens_seen": 1122238464 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005333279839518556, + "loss": 1.3793, + "theoretical_loss": 3.638229074185304, + "tokens_seen": 1122304000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005333119358074223, + "loss": 1.3316, + "theoretical_loss": 3.6382094403573593, + "tokens_seen": 1122369536 + }, + { + "epoch": 0.34, + "learning_rate": 0.000533295887662989, + "loss": 1.3682, + "theoretical_loss": 3.6381898079967954, + "tokens_seen": 1122435072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005332798395185557, + "loss": 1.3543, + "theoretical_loss": 3.6381701771034165, + "tokens_seen": 1122500608 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005332637913741224, + "loss": 1.3554, + "theoretical_loss": 3.6381505476770277, + "tokens_seen": 1122566144 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005332477432296891, + "loss": 1.3955, + "theoretical_loss": 3.638130919717434, + "tokens_seen": 1122631680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005332316950852559, + "loss": 1.2782, + "theoretical_loss": 3.6381112932244397, + "tokens_seen": 1122697216 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005332156469408225, + "loss": 1.3996, + "theoretical_loss": 3.6380916681978492, + "tokens_seen": 1122762752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005331995987963893, + "loss": 1.3721, + "theoretical_loss": 3.6380720446374686, + "tokens_seen": 1122828288 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005331835506519559, + "loss": 1.3567, + "theoretical_loss": 3.638052422543102, + "tokens_seen": 1122893824 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005331675025075227, + "loss": 1.4483, + "theoretical_loss": 3.6380328019145542, + "tokens_seen": 1122959360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005331514543630893, + "loss": 1.4153, + "theoretical_loss": 3.63801318275163, + "tokens_seen": 1123024896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005331354062186561, + "loss": 1.3635, + "theoretical_loss": 3.637993565054135, + "tokens_seen": 1123090432 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005331193580742227, + "loss": 1.335, + "theoretical_loss": 3.637973948821874, + "tokens_seen": 1123155968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005331033099297895, + "loss": 1.4378, + "theoretical_loss": 3.637954334054652, + "tokens_seen": 1123221504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005330872617853561, + "loss": 1.4303, + "theoretical_loss": 3.6379347207522748, + "tokens_seen": 1123287040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005330712136409229, + "loss": 1.4658, + "theoretical_loss": 3.6379151089145463, + "tokens_seen": 1123352576 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005330551654964895, + "loss": 1.3051, + "theoretical_loss": 3.6378954985412726, + "tokens_seen": 1123418112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005330391173520563, + "loss": 1.3318, + "theoretical_loss": 3.6378758896322587, + "tokens_seen": 1123483648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005330230692076229, + "loss": 1.3568, + "theoretical_loss": 3.6378562821873097, + "tokens_seen": 1123549184 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005330070210631897, + "loss": 1.3384, + "theoretical_loss": 3.6378366762062306, + "tokens_seen": 1123614720 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005329909729187563, + "loss": 1.4684, + "theoretical_loss": 3.637817071688828, + "tokens_seen": 1123680256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005329749247743231, + "loss": 1.3996, + "theoretical_loss": 3.637797468634906, + "tokens_seen": 1123745792 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005329588766298897, + "loss": 1.4241, + "theoretical_loss": 3.63777786704427, + "tokens_seen": 1123811328 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005329428284854565, + "loss": 1.4433, + "theoretical_loss": 3.6377582669167268, + "tokens_seen": 1123876864 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005329267803410231, + "loss": 1.425, + "theoretical_loss": 3.63773866825208, + "tokens_seen": 1123942400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005329107321965899, + "loss": 1.3044, + "theoretical_loss": 3.637719071050137, + "tokens_seen": 1124007936 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005328946840521565, + "loss": 1.2935, + "theoretical_loss": 3.6376994753107015, + "tokens_seen": 1124073472 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005328786359077233, + "loss": 1.3857, + "theoretical_loss": 3.6376798810335806, + "tokens_seen": 1124139008 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005328625877632899, + "loss": 1.5013, + "theoretical_loss": 3.6376602882185796, + "tokens_seen": 1124204544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005328465396188567, + "loss": 1.322, + "theoretical_loss": 3.6376406968655037, + "tokens_seen": 1124270080 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005328304914744233, + "loss": 1.4086, + "theoretical_loss": 3.6376211069741586, + "tokens_seen": 1124335616 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005328144433299901, + "loss": 1.3315, + "theoretical_loss": 3.6376015185443507, + "tokens_seen": 1124401152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005327983951855567, + "loss": 1.3911, + "theoretical_loss": 3.6375819315758853, + "tokens_seen": 1124466688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005327823470411235, + "loss": 1.4223, + "theoretical_loss": 3.6375623460685684, + "tokens_seen": 1124532224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005327662988966901, + "loss": 1.2633, + "theoretical_loss": 3.6375427620222056, + "tokens_seen": 1124597760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005327502507522569, + "loss": 1.378, + "theoretical_loss": 3.637523179436603, + "tokens_seen": 1124663296 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005327342026078235, + "loss": 1.3091, + "theoretical_loss": 3.6375035983115667, + "tokens_seen": 1124728832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005327181544633903, + "loss": 1.4311, + "theoretical_loss": 3.6374840186469024, + "tokens_seen": 1124794368 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005327021063189569, + "loss": 1.2737, + "theoretical_loss": 3.637464440442416, + "tokens_seen": 1124859904 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005326860581745236, + "loss": 1.3206, + "theoretical_loss": 3.6374448636979135, + "tokens_seen": 1124925440 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005326700100300903, + "loss": 1.3523, + "theoretical_loss": 3.6374252884132017, + "tokens_seen": 1124990976 + }, + { + "epoch": 0.34, + "learning_rate": 0.000532653961885657, + "loss": 1.3761, + "theoretical_loss": 3.637405714588086, + "tokens_seen": 1125056512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005326379137412237, + "loss": 1.3164, + "theoretical_loss": 3.6373861422223728, + "tokens_seen": 1125122048 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005326218655967904, + "loss": 1.4219, + "theoretical_loss": 3.637366571315868, + "tokens_seen": 1125187584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005326058174523571, + "loss": 1.3639, + "theoretical_loss": 3.6373470018683784, + "tokens_seen": 1125253120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005325897693079238, + "loss": 1.3299, + "theoretical_loss": 3.63732743387971, + "tokens_seen": 1125318656 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005325737211634905, + "loss": 1.4669, + "theoretical_loss": 3.637307867349669, + "tokens_seen": 1125384192 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005325576730190572, + "loss": 1.3164, + "theoretical_loss": 3.637288302278061, + "tokens_seen": 1125449728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005325416248746238, + "loss": 1.3889, + "theoretical_loss": 3.6372687386646945, + "tokens_seen": 1125515264 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005325255767301906, + "loss": 1.4088, + "theoretical_loss": 3.637249176509374, + "tokens_seen": 1125580800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005325095285857572, + "loss": 1.3709, + "theoretical_loss": 3.6372296158119064, + "tokens_seen": 1125646336 + }, + { + "epoch": 0.34, + "learning_rate": 0.000532493480441324, + "loss": 1.3342, + "theoretical_loss": 3.637210056572098, + "tokens_seen": 1125711872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005324774322968906, + "loss": 1.3767, + "theoretical_loss": 3.6371904987897565, + "tokens_seen": 1125777408 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005324613841524574, + "loss": 1.3146, + "theoretical_loss": 3.637170942464687, + "tokens_seen": 1125842944 + }, + { + "epoch": 0.34, + "learning_rate": 0.000532445336008024, + "loss": 1.4892, + "theoretical_loss": 3.6371513875966968, + "tokens_seen": 1125908480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005324292878635908, + "loss": 1.3595, + "theoretical_loss": 3.6371318341855927, + "tokens_seen": 1125974016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005324132397191574, + "loss": 1.3388, + "theoretical_loss": 3.6371122822311808, + "tokens_seen": 1126039552 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005323971915747242, + "loss": 1.2626, + "theoretical_loss": 3.6370927317332677, + "tokens_seen": 1126105088 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005323811434302908, + "loss": 1.4696, + "theoretical_loss": 3.6370731826916614, + "tokens_seen": 1126170624 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005323650952858576, + "loss": 1.31, + "theoretical_loss": 3.6370536351061675, + "tokens_seen": 1126236160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005323490471414242, + "loss": 1.3951, + "theoretical_loss": 3.637034088976593, + "tokens_seen": 1126301696 + }, + { + "epoch": 0.34, + "learning_rate": 0.000532332998996991, + "loss": 1.4153, + "theoretical_loss": 3.6370145443027453, + "tokens_seen": 1126367232 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005323169508525576, + "loss": 1.3841, + "theoretical_loss": 3.6369950010844305, + "tokens_seen": 1126432768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005323009027081244, + "loss": 1.4289, + "theoretical_loss": 3.636975459321456, + "tokens_seen": 1126498304 + }, + { + "epoch": 0.34, + "learning_rate": 0.000532284854563691, + "loss": 1.3573, + "theoretical_loss": 3.636955919013629, + "tokens_seen": 1126563840 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005322688064192578, + "loss": 1.3539, + "theoretical_loss": 3.636936380160756, + "tokens_seen": 1126629376 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005322527582748244, + "loss": 1.3869, + "theoretical_loss": 3.6369168427626444, + "tokens_seen": 1126694912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005322367101303912, + "loss": 1.2934, + "theoretical_loss": 3.636897306819101, + "tokens_seen": 1126760448 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005322206619859578, + "loss": 1.4425, + "theoretical_loss": 3.6368777723299326, + "tokens_seen": 1126825984 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005322046138415246, + "loss": 1.4384, + "theoretical_loss": 3.6368582392949476, + "tokens_seen": 1126891520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005321885656970912, + "loss": 1.3746, + "theoretical_loss": 3.6368387077139523, + "tokens_seen": 1126957056 + }, + { + "epoch": 0.34, + "learning_rate": 0.000532172517552658, + "loss": 1.4029, + "theoretical_loss": 3.6368191775867533, + "tokens_seen": 1127022592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005321564694082246, + "loss": 1.3307, + "theoretical_loss": 3.636799648913159, + "tokens_seen": 1127088128 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005321404212637914, + "loss": 1.3248, + "theoretical_loss": 3.6367801216929765, + "tokens_seen": 1127153664 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 421187, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3242814540863037, + "objective/train/theoretical_loss": 3.6367605959260123, + "objective/train/tokens_used": 1147679200, + "theoretical_loss": 3.6367605959260123, + "tokens_seen": 1127219200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005321243731193581, + "loss": 1.3243, + "theoretical_loss": 3.6367605959260123, + "tokens_seen": 1127219200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005321083249749248, + "loss": 1.4676, + "theoretical_loss": 3.636741071612075, + "tokens_seen": 1127284736 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005320922768304915, + "loss": 1.4104, + "theoretical_loss": 3.636721548750971, + "tokens_seen": 1127350272 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005320762286860582, + "loss": 1.3152, + "theoretical_loss": 3.6367020273425084, + "tokens_seen": 1127415808 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005320601805416249, + "loss": 1.3357, + "theoretical_loss": 3.636682507386494, + "tokens_seen": 1127481344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005320441323971916, + "loss": 1.3891, + "theoretical_loss": 3.636662988882736, + "tokens_seen": 1127546880 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005320280842527583, + "loss": 1.3352, + "theoretical_loss": 3.6366434718310416, + "tokens_seen": 1127612416 + }, + { + "epoch": 0.34, + "learning_rate": 0.000532012036108325, + "loss": 1.3956, + "theoretical_loss": 3.636623956231219, + "tokens_seen": 1127677952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005319959879638917, + "loss": 1.5484, + "theoretical_loss": 3.6366044420830743, + "tokens_seen": 1127743488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005319799398194584, + "loss": 1.3381, + "theoretical_loss": 3.6365849293864168, + "tokens_seen": 1127809024 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005319638916750251, + "loss": 1.4707, + "theoretical_loss": 3.6365654181410534, + "tokens_seen": 1127874560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005319478435305918, + "loss": 1.2865, + "theoretical_loss": 3.6365459083467915, + "tokens_seen": 1127940096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005319317953861585, + "loss": 1.3916, + "theoretical_loss": 3.63652640000344, + "tokens_seen": 1128005632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005319157472417252, + "loss": 1.4691, + "theoretical_loss": 3.636506893110806, + "tokens_seen": 1128071168 + }, + { + "epoch": 0.34, + "learning_rate": 0.000531899699097292, + "loss": 1.3846, + "theoretical_loss": 3.6364873876686974, + "tokens_seen": 1128136704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005318836509528586, + "loss": 1.407, + "theoretical_loss": 3.636467883676922, + "tokens_seen": 1128202240 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005318676028084254, + "loss": 1.3183, + "theoretical_loss": 3.636448381135288, + "tokens_seen": 1128267776 + }, + { + "epoch": 0.34, + "learning_rate": 0.000531851554663992, + "loss": 1.4489, + "theoretical_loss": 3.636428880043603, + "tokens_seen": 1128333312 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005318355065195588, + "loss": 1.4038, + "theoretical_loss": 3.636409380401675, + "tokens_seen": 1128398848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005318194583751254, + "loss": 1.2704, + "theoretical_loss": 3.636389882209313, + "tokens_seen": 1128464384 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005318034102306922, + "loss": 1.3512, + "theoretical_loss": 3.6363703854663236, + "tokens_seen": 1128529920 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005317873620862588, + "loss": 1.3607, + "theoretical_loss": 3.6363508901725154, + "tokens_seen": 1128595456 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005317713139418256, + "loss": 1.3426, + "theoretical_loss": 3.6363313963276975, + "tokens_seen": 1128660992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005317552657973922, + "loss": 1.4347, + "theoretical_loss": 3.636311903931677, + "tokens_seen": 1128726528 + }, + { + "epoch": 0.34, + "learning_rate": 0.000531739217652959, + "loss": 1.441, + "theoretical_loss": 3.6362924129842624, + "tokens_seen": 1128792064 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005317231695085256, + "loss": 1.3526, + "theoretical_loss": 3.6362729234852615, + "tokens_seen": 1128857600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005317071213640924, + "loss": 1.319, + "theoretical_loss": 3.636253435434484, + "tokens_seen": 1128923136 + }, + { + "epoch": 0.34, + "learning_rate": 0.000531691073219659, + "loss": 1.3552, + "theoretical_loss": 3.6362339488317366, + "tokens_seen": 1128988672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005316750250752258, + "loss": 1.3865, + "theoretical_loss": 3.6362144636768283, + "tokens_seen": 1129054208 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005316589769307924, + "loss": 1.4074, + "theoretical_loss": 3.6361949799695683, + "tokens_seen": 1129119744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005316429287863592, + "loss": 1.3897, + "theoretical_loss": 3.6361754977097633, + "tokens_seen": 1129185280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005316268806419258, + "loss": 1.5212, + "theoretical_loss": 3.6361560168972233, + "tokens_seen": 1129250816 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005316108324974926, + "loss": 1.4583, + "theoretical_loss": 3.6361365375317565, + "tokens_seen": 1129316352 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005315947843530592, + "loss": 1.4101, + "theoretical_loss": 3.6361170596131704, + "tokens_seen": 1129381888 + }, + { + "epoch": 0.34, + "learning_rate": 0.000531578736208626, + "loss": 1.4696, + "theoretical_loss": 3.6360975831412747, + "tokens_seen": 1129447424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005315626880641926, + "loss": 1.3767, + "theoretical_loss": 3.6360781081158775, + "tokens_seen": 1129512960 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005315466399197594, + "loss": 1.4161, + "theoretical_loss": 3.6360586345367882, + "tokens_seen": 1129578496 + }, + { + "epoch": 0.34, + "learning_rate": 0.000531530591775326, + "loss": 1.3555, + "theoretical_loss": 3.6360391624038146, + "tokens_seen": 1129644032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005315145436308928, + "loss": 1.3062, + "theoretical_loss": 3.6360196917167658, + "tokens_seen": 1129709568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005314984954864594, + "loss": 1.342, + "theoretical_loss": 3.6360002224754506, + "tokens_seen": 1129775104 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005314824473420262, + "loss": 1.3213, + "theoretical_loss": 3.6359807546796774, + "tokens_seen": 1129840640 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005314663991975928, + "loss": 1.4305, + "theoretical_loss": 3.6359612883292556, + "tokens_seen": 1129906176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005314503510531596, + "loss": 1.3854, + "theoretical_loss": 3.6359418234239937, + "tokens_seen": 1129971712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005314343029087262, + "loss": 1.339, + "theoretical_loss": 3.6359223599637005, + "tokens_seen": 1130037248 + }, + { + "epoch": 0.34, + "learning_rate": 0.000531418254764293, + "loss": 1.3633, + "theoretical_loss": 3.6359028979481853, + "tokens_seen": 1130102784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005314022066198596, + "loss": 1.4129, + "theoretical_loss": 3.635883437377257, + "tokens_seen": 1130168320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005313861584754264, + "loss": 1.3417, + "theoretical_loss": 3.6358639782507245, + "tokens_seen": 1130233856 + }, + { + "epoch": 0.34, + "learning_rate": 0.000531370110330993, + "loss": 1.3748, + "theoretical_loss": 3.635844520568397, + "tokens_seen": 1130299392 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005313540621865598, + "loss": 1.4058, + "theoretical_loss": 3.6358250643300836, + "tokens_seen": 1130364928 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005313380140421264, + "loss": 1.2984, + "theoretical_loss": 3.635805609535593, + "tokens_seen": 1130430464 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005313219658976932, + "loss": 1.3731, + "theoretical_loss": 3.635786156184735, + "tokens_seen": 1130496000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005313059177532598, + "loss": 1.3888, + "theoretical_loss": 3.635766704277318, + "tokens_seen": 1130561536 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005312898696088265, + "loss": 1.4552, + "theoretical_loss": 3.635747253813152, + "tokens_seen": 1130627072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005312738214643932, + "loss": 1.3274, + "theoretical_loss": 3.635727804792046, + "tokens_seen": 1130692608 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005312577733199599, + "loss": 1.283, + "theoretical_loss": 3.635708357213809, + "tokens_seen": 1130758144 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005312417251755266, + "loss": 1.4684, + "theoretical_loss": 3.635688911078251, + "tokens_seen": 1130823680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005312256770310933, + "loss": 1.2612, + "theoretical_loss": 3.635669466385181, + "tokens_seen": 1130889216 + }, + { + "epoch": 0.34, + "learning_rate": 0.00053120962888666, + "loss": 1.2422, + "theoretical_loss": 3.635650023134408, + "tokens_seen": 1130954752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005311935807422267, + "loss": 1.4358, + "theoretical_loss": 3.635630581325742, + "tokens_seen": 1131020288 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005311775325977934, + "loss": 1.3476, + "theoretical_loss": 3.635611140958993, + "tokens_seen": 1131085824 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005311614844533601, + "loss": 1.3838, + "theoretical_loss": 3.6355917020339685, + "tokens_seen": 1131151360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005311454363089267, + "loss": 1.3809, + "theoretical_loss": 3.635572264550481, + "tokens_seen": 1131216896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005311293881644935, + "loss": 1.4123, + "theoretical_loss": 3.635552828508337, + "tokens_seen": 1131282432 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005311133400200601, + "loss": 1.4071, + "theoretical_loss": 3.635533393907349, + "tokens_seen": 1131347968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005310972918756269, + "loss": 1.2988, + "theoretical_loss": 3.635513960747325, + "tokens_seen": 1131413504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005310812437311935, + "loss": 1.3727, + "theoretical_loss": 3.6354945290280742, + "tokens_seen": 1131479040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005310651955867603, + "loss": 1.3125, + "theoretical_loss": 3.6354750987494078, + "tokens_seen": 1131544576 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005310491474423269, + "loss": 1.3022, + "theoretical_loss": 3.635455669911135, + "tokens_seen": 1131610112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005310330992978937, + "loss": 1.3857, + "theoretical_loss": 3.635436242513065, + "tokens_seen": 1131675648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005310170511534603, + "loss": 1.3945, + "theoretical_loss": 3.6354168165550087, + "tokens_seen": 1131741184 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005310010030090271, + "loss": 1.4251, + "theoretical_loss": 3.6353973920367757, + "tokens_seen": 1131806720 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005309849548645937, + "loss": 1.3205, + "theoretical_loss": 3.635377968958175, + "tokens_seen": 1131872256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005309689067201605, + "loss": 1.3943, + "theoretical_loss": 3.6353585473190178, + "tokens_seen": 1131937792 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005309528585757271, + "loss": 1.3529, + "theoretical_loss": 3.6353391271191136, + "tokens_seen": 1132003328 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005309368104312939, + "loss": 1.387, + "theoretical_loss": 3.6353197083582725, + "tokens_seen": 1132068864 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005309207622868606, + "loss": 1.4444, + "theoretical_loss": 3.6353002910363035, + "tokens_seen": 1132134400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005309047141424273, + "loss": 1.3359, + "theoretical_loss": 3.6352808751530183, + "tokens_seen": 1132199936 + }, + { + "epoch": 0.34, + "learning_rate": 0.000530888665997994, + "loss": 1.4685, + "theoretical_loss": 3.6352614607082265, + "tokens_seen": 1132265472 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005308726178535607, + "loss": 1.249, + "theoretical_loss": 3.6352420477017384, + "tokens_seen": 1132331008 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005308565697091274, + "loss": 1.382, + "theoretical_loss": 3.6352226361333635, + "tokens_seen": 1132396544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005308405215646941, + "loss": 1.3581, + "theoretical_loss": 3.635203226002912, + "tokens_seen": 1132462080 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005308244734202608, + "loss": 1.3465, + "theoretical_loss": 3.6351838173101956, + "tokens_seen": 1132527616 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005308084252758275, + "loss": 1.4279, + "theoretical_loss": 3.6351644100550233, + "tokens_seen": 1132593152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005307923771313942, + "loss": 1.4729, + "theoretical_loss": 3.635145004237206, + "tokens_seen": 1132658688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005307763289869609, + "loss": 1.2839, + "theoretical_loss": 3.635125599856554, + "tokens_seen": 1132724224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005307602808425276, + "loss": 1.337, + "theoretical_loss": 3.6351061969128775, + "tokens_seen": 1132789760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005307442326980943, + "loss": 1.3216, + "theoretical_loss": 3.6350867954059876, + "tokens_seen": 1132855296 + }, + { + "epoch": 0.34, + "learning_rate": 0.000530728184553661, + "loss": 1.3168, + "theoretical_loss": 3.635067395335694, + "tokens_seen": 1132920832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005307121364092277, + "loss": 1.3536, + "theoretical_loss": 3.6350479967018074, + "tokens_seen": 1132986368 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005306960882647944, + "loss": 1.3034, + "theoretical_loss": 3.6350285995041385, + "tokens_seen": 1133051904 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005306800401203611, + "loss": 1.2992, + "theoretical_loss": 3.635009203742498, + "tokens_seen": 1133117440 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005306639919759278, + "loss": 1.386, + "theoretical_loss": 3.6349898094166964, + "tokens_seen": 1133182976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005306479438314945, + "loss": 1.3033, + "theoretical_loss": 3.634970416526545, + "tokens_seen": 1133248512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005306318956870612, + "loss": 1.385, + "theoretical_loss": 3.6349510250718535, + "tokens_seen": 1133314048 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005306158475426279, + "loss": 1.2855, + "theoretical_loss": 3.634931635052433, + "tokens_seen": 1133379584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005305997993981946, + "loss": 1.4257, + "theoretical_loss": 3.6349122464680947, + "tokens_seen": 1133445120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005305837512537613, + "loss": 1.28, + "theoretical_loss": 3.6348928593186494, + "tokens_seen": 1133510656 + }, + { + "epoch": 0.34, + "learning_rate": 0.000530567703109328, + "loss": 1.4011, + "theoretical_loss": 3.634873473603907, + "tokens_seen": 1133576192 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005305516549648947, + "loss": 1.3567, + "theoretical_loss": 3.63485408932368, + "tokens_seen": 1133641728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005305356068204614, + "loss": 1.3755, + "theoretical_loss": 3.6348347064777773, + "tokens_seen": 1133707264 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 423441, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.404093861579895, + "objective/train/theoretical_loss": 3.6348153250660116, + "objective/train/tokens_used": 1154232800, + "theoretical_loss": 3.6348153250660116, + "tokens_seen": 1133772800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005305195586760281, + "loss": 1.4041, + "theoretical_loss": 3.6348153250660116, + "tokens_seen": 1133772800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005305035105315948, + "loss": 1.3372, + "theoretical_loss": 3.634795945088193, + "tokens_seen": 1133838336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005304874623871615, + "loss": 1.3755, + "theoretical_loss": 3.634776566544133, + "tokens_seen": 1133903872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005304714142427283, + "loss": 1.4421, + "theoretical_loss": 3.6347571894336426, + "tokens_seen": 1133969408 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005304553660982949, + "loss": 1.2647, + "theoretical_loss": 3.6347378137565323, + "tokens_seen": 1134034944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005304393179538617, + "loss": 1.404, + "theoretical_loss": 3.634718439512614, + "tokens_seen": 1134100480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005304232698094283, + "loss": 1.3959, + "theoretical_loss": 3.6346990667016987, + "tokens_seen": 1134166016 + }, + { + "epoch": 0.34, + "learning_rate": 0.000530407221664995, + "loss": 1.449, + "theoretical_loss": 3.634679695323597, + "tokens_seen": 1134231552 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005303911735205617, + "loss": 1.4569, + "theoretical_loss": 3.634660325378121, + "tokens_seen": 1134297088 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005303751253761285, + "loss": 1.3266, + "theoretical_loss": 3.634640956865082, + "tokens_seen": 1134362624 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005303590772316951, + "loss": 1.3425, + "theoretical_loss": 3.634621589784291, + "tokens_seen": 1134428160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005303430290872619, + "loss": 1.3499, + "theoretical_loss": 3.6346022241355587, + "tokens_seen": 1134493696 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005303269809428285, + "loss": 1.3863, + "theoretical_loss": 3.634582859918697, + "tokens_seen": 1134559232 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005303109327983953, + "loss": 1.2896, + "theoretical_loss": 3.6345634971335183, + "tokens_seen": 1134624768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005302948846539619, + "loss": 1.3754, + "theoretical_loss": 3.6345441357798327, + "tokens_seen": 1134690304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005302788365095287, + "loss": 1.3117, + "theoretical_loss": 3.634524775857452, + "tokens_seen": 1134755840 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005302627883650953, + "loss": 1.4115, + "theoretical_loss": 3.6345054173661886, + "tokens_seen": 1134821376 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005302467402206621, + "loss": 1.3636, + "theoretical_loss": 3.6344860603058526, + "tokens_seen": 1134886912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005302306920762287, + "loss": 1.4226, + "theoretical_loss": 3.6344667046762567, + "tokens_seen": 1134952448 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005302146439317955, + "loss": 1.4046, + "theoretical_loss": 3.634447350477213, + "tokens_seen": 1135017984 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005301985957873621, + "loss": 1.1918, + "theoretical_loss": 3.6344279977085314, + "tokens_seen": 1135083520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005301825476429289, + "loss": 1.3438, + "theoretical_loss": 3.6344086463700256, + "tokens_seen": 1135149056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005301664994984955, + "loss": 1.2676, + "theoretical_loss": 3.6343892964615057, + "tokens_seen": 1135214592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005301504513540623, + "loss": 1.409, + "theoretical_loss": 3.6343699479827842, + "tokens_seen": 1135280128 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005301344032096289, + "loss": 1.444, + "theoretical_loss": 3.6343506009336735, + "tokens_seen": 1135345664 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005301183550651957, + "loss": 1.3108, + "theoretical_loss": 3.6343312553139846, + "tokens_seen": 1135411200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005301023069207623, + "loss": 1.363, + "theoretical_loss": 3.63431191112353, + "tokens_seen": 1135476736 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005300862587763291, + "loss": 1.3553, + "theoretical_loss": 3.6342925683621203, + "tokens_seen": 1135542272 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005300702106318957, + "loss": 1.4472, + "theoretical_loss": 3.6342732270295692, + "tokens_seen": 1135607808 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005300541624874625, + "loss": 1.3764, + "theoretical_loss": 3.6342538871256878, + "tokens_seen": 1135673344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005300381143430291, + "loss": 1.4549, + "theoretical_loss": 3.634234548650288, + "tokens_seen": 1135738880 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005300220661985959, + "loss": 1.3852, + "theoretical_loss": 3.6342152116031827, + "tokens_seen": 1135804416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005300060180541625, + "loss": 1.3842, + "theoretical_loss": 3.634195875984183, + "tokens_seen": 1135869952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005299899699097293, + "loss": 1.4286, + "theoretical_loss": 3.634176541793102, + "tokens_seen": 1135935488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005299739217652959, + "loss": 1.3034, + "theoretical_loss": 3.6341572090297514, + "tokens_seen": 1136001024 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005299578736208627, + "loss": 1.3209, + "theoretical_loss": 3.6341378776939433, + "tokens_seen": 1136066560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005299418254764293, + "loss": 1.3125, + "theoretical_loss": 3.63411854778549, + "tokens_seen": 1136132096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005299257773319961, + "loss": 1.3159, + "theoretical_loss": 3.634099219304203, + "tokens_seen": 1136197632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005299097291875627, + "loss": 1.3449, + "theoretical_loss": 3.6340798922498965, + "tokens_seen": 1136263168 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005298936810431294, + "loss": 1.4401, + "theoretical_loss": 3.6340605666223817, + "tokens_seen": 1136328704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005298776328986961, + "loss": 1.5107, + "theoretical_loss": 3.6340412424214708, + "tokens_seen": 1136394240 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005298615847542628, + "loss": 1.3261, + "theoretical_loss": 3.634021919646977, + "tokens_seen": 1136459776 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005298455366098295, + "loss": 1.4818, + "theoretical_loss": 3.6340025982987116, + "tokens_seen": 1136525312 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005298294884653962, + "loss": 1.3727, + "theoretical_loss": 3.633983278376488, + "tokens_seen": 1136590848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005298134403209629, + "loss": 1.3015, + "theoretical_loss": 3.6339639598801186, + "tokens_seen": 1136656384 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005297973921765296, + "loss": 1.4121, + "theoretical_loss": 3.633944642809416, + "tokens_seen": 1136721920 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005297813440320963, + "loss": 1.4199, + "theoretical_loss": 3.6339253271641923, + "tokens_seen": 1136787456 + }, + { + "epoch": 0.34, + "learning_rate": 0.000529765295887663, + "loss": 1.3873, + "theoretical_loss": 3.633906012944261, + "tokens_seen": 1136852992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005297492477432296, + "loss": 1.4896, + "theoretical_loss": 3.633886700149434, + "tokens_seen": 1136918528 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005297331995987964, + "loss": 1.3127, + "theoretical_loss": 3.6338673887795245, + "tokens_seen": 1136984064 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005297171514543631, + "loss": 1.3484, + "theoretical_loss": 3.633848078834345, + "tokens_seen": 1137049600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005297011033099298, + "loss": 1.4279, + "theoretical_loss": 3.633828770313708, + "tokens_seen": 1137115136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005296850551654965, + "loss": 1.4169, + "theoretical_loss": 3.6338094632174274, + "tokens_seen": 1137180672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005296690070210632, + "loss": 1.328, + "theoretical_loss": 3.633790157545315, + "tokens_seen": 1137246208 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005296529588766299, + "loss": 1.3762, + "theoretical_loss": 3.6337708532971833, + "tokens_seen": 1137311744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005296369107321966, + "loss": 1.372, + "theoretical_loss": 3.6337515504728466, + "tokens_seen": 1137377280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005296208625877633, + "loss": 1.4582, + "theoretical_loss": 3.633732249072117, + "tokens_seen": 1137442816 + }, + { + "epoch": 0.34, + "learning_rate": 0.00052960481444333, + "loss": 1.3682, + "theoretical_loss": 3.6337129490948072, + "tokens_seen": 1137508352 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005295887662988967, + "loss": 1.4549, + "theoretical_loss": 3.6336936505407316, + "tokens_seen": 1137573888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005295727181544634, + "loss": 1.3964, + "theoretical_loss": 3.633674353409702, + "tokens_seen": 1137639424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005295566700100301, + "loss": 1.3569, + "theoretical_loss": 3.6336550577015316, + "tokens_seen": 1137704960 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005295406218655968, + "loss": 1.4293, + "theoretical_loss": 3.633635763416034, + "tokens_seen": 1137770496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005295245737211635, + "loss": 1.4589, + "theoretical_loss": 3.633616470553022, + "tokens_seen": 1137836032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005295085255767302, + "loss": 1.3128, + "theoretical_loss": 3.633597179112309, + "tokens_seen": 1137901568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005294924774322969, + "loss": 1.386, + "theoretical_loss": 3.6335778890937083, + "tokens_seen": 1137967104 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005294764292878636, + "loss": 1.3712, + "theoretical_loss": 3.633558600497033, + "tokens_seen": 1138032640 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005294603811434303, + "loss": 1.4071, + "theoretical_loss": 3.633539313322096, + "tokens_seen": 1138098176 + }, + { + "epoch": 0.34, + "learning_rate": 0.000529444332998997, + "loss": 1.4422, + "theoretical_loss": 3.6335200275687116, + "tokens_seen": 1138163712 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005294282848545637, + "loss": 1.2571, + "theoretical_loss": 3.633500743236693, + "tokens_seen": 1138229248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005294122367101304, + "loss": 1.3057, + "theoretical_loss": 3.6334814603258527, + "tokens_seen": 1138294784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005293961885656971, + "loss": 1.4153, + "theoretical_loss": 3.6334621788360053, + "tokens_seen": 1138360320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005293801404212638, + "loss": 1.4567, + "theoretical_loss": 3.6334428987669636, + "tokens_seen": 1138425856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0005293640922768305, + "loss": 1.4039, + "theoretical_loss": 3.633423620118541, + "tokens_seen": 1138491392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005293480441323972, + "loss": 1.3875, + "theoretical_loss": 3.633404342890551, + "tokens_seen": 1138556928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005293319959879639, + "loss": 1.3644, + "theoretical_loss": 3.6333850670828083, + "tokens_seen": 1138622464 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005293159478435306, + "loss": 1.4958, + "theoretical_loss": 3.633365792695126, + "tokens_seen": 1138688000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005292998996990973, + "loss": 1.369, + "theoretical_loss": 3.6333465197273167, + "tokens_seen": 1138753536 + }, + { + "epoch": 0.35, + "learning_rate": 0.000529283851554664, + "loss": 1.438, + "theoretical_loss": 3.633327248179195, + "tokens_seen": 1138819072 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005292678034102307, + "loss": 1.4047, + "theoretical_loss": 3.633307978050575, + "tokens_seen": 1138884608 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005292517552657974, + "loss": 1.3743, + "theoretical_loss": 3.6332887093412696, + "tokens_seen": 1138950144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005292357071213641, + "loss": 1.4025, + "theoretical_loss": 3.6332694420510934, + "tokens_seen": 1139015680 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005292196589769308, + "loss": 1.3961, + "theoretical_loss": 3.6332501761798595, + "tokens_seen": 1139081216 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005292036108324975, + "loss": 1.401, + "theoretical_loss": 3.633230911727382, + "tokens_seen": 1139146752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005291875626880642, + "loss": 1.4397, + "theoretical_loss": 3.633211648693475, + "tokens_seen": 1139212288 + }, + { + "epoch": 0.35, + "learning_rate": 0.000529171514543631, + "loss": 1.3493, + "theoretical_loss": 3.633192387077953, + "tokens_seen": 1139277824 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005291554663991976, + "loss": 1.4053, + "theoretical_loss": 3.633173126880629, + "tokens_seen": 1139343360 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005291394182547643, + "loss": 1.3934, + "theoretical_loss": 3.633153868101317, + "tokens_seen": 1139408896 + }, + { + "epoch": 0.35, + "learning_rate": 0.000529123370110331, + "loss": 1.3261, + "theoretical_loss": 3.633134610739832, + "tokens_seen": 1139474432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005291073219658978, + "loss": 1.4124, + "theoretical_loss": 3.6331153547959874, + "tokens_seen": 1139539968 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005290912738214644, + "loss": 1.447, + "theoretical_loss": 3.633096100269597, + "tokens_seen": 1139605504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005290752256770312, + "loss": 1.5022, + "theoretical_loss": 3.633076847160476, + "tokens_seen": 1139671040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005290591775325978, + "loss": 1.4294, + "theoretical_loss": 3.633057595468437, + "tokens_seen": 1139736576 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005290431293881646, + "loss": 1.3671, + "theoretical_loss": 3.633038345193296, + "tokens_seen": 1139802112 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005290270812437312, + "loss": 1.3739, + "theoretical_loss": 3.6330190963348663, + "tokens_seen": 1139867648 + }, + { + "epoch": 0.35, + "learning_rate": 0.000529011033099298, + "loss": 1.4044, + "theoretical_loss": 3.632999848892962, + "tokens_seen": 1139933184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005289949849548646, + "loss": 1.4431, + "theoretical_loss": 3.6329806028673985, + "tokens_seen": 1139998720 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005289789368104314, + "loss": 1.4347, + "theoretical_loss": 3.6329613582579885, + "tokens_seen": 1140064256 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528962888665998, + "loss": 1.5042, + "theoretical_loss": 3.632942115064548, + "tokens_seen": 1140129792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005289468405215648, + "loss": 1.3348, + "theoretical_loss": 3.6329228732868906, + "tokens_seen": 1140195328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005289307923771314, + "loss": 1.3703, + "theoretical_loss": 3.6329036329248305, + "tokens_seen": 1140260864 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 425800, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4154813289642334, + "objective/train/theoretical_loss": 3.632884393978183, + "objective/train/tokens_used": 1160786400, + "theoretical_loss": 3.632884393978183, + "tokens_seen": 1140326400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005289147442326982, + "loss": 1.4155, + "theoretical_loss": 3.632884393978183, + "tokens_seen": 1140326400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005288986960882648, + "loss": 1.4578, + "theoretical_loss": 3.632865156446763, + "tokens_seen": 1140391936 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005288826479438316, + "loss": 1.5177, + "theoretical_loss": 3.6328459203303836, + "tokens_seen": 1140457472 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005288665997993982, + "loss": 1.3315, + "theoretical_loss": 3.63282668562886, + "tokens_seen": 1140523008 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528850551654965, + "loss": 1.4036, + "theoretical_loss": 3.632807452342007, + "tokens_seen": 1140588544 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005288345035105316, + "loss": 1.3994, + "theoretical_loss": 3.63278822046964, + "tokens_seen": 1140654080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005288184553660984, + "loss": 1.3047, + "theoretical_loss": 3.6327689900115727, + "tokens_seen": 1140719616 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528802407221665, + "loss": 1.3687, + "theoretical_loss": 3.63274976096762, + "tokens_seen": 1140785152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005287863590772318, + "loss": 1.3333, + "theoretical_loss": 3.6327305333375968, + "tokens_seen": 1140850688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005287703109327984, + "loss": 1.3696, + "theoretical_loss": 3.6327113071213177, + "tokens_seen": 1140916224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005287542627883652, + "loss": 1.3354, + "theoretical_loss": 3.632692082318598, + "tokens_seen": 1140981760 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005287382146439318, + "loss": 1.2717, + "theoretical_loss": 3.6326728589292525, + "tokens_seen": 1141047296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005287221664994986, + "loss": 1.3226, + "theoretical_loss": 3.632653636953096, + "tokens_seen": 1141112832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005287061183550652, + "loss": 1.3774, + "theoretical_loss": 3.6326344163899433, + "tokens_seen": 1141178368 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528690070210632, + "loss": 1.512, + "theoretical_loss": 3.63261519723961, + "tokens_seen": 1141243904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005286740220661986, + "loss": 1.376, + "theoretical_loss": 3.63259597950191, + "tokens_seen": 1141309440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005286579739217654, + "loss": 1.3781, + "theoretical_loss": 3.63257676317666, + "tokens_seen": 1141374976 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528641925777332, + "loss": 1.4394, + "theoretical_loss": 3.6325575482636734, + "tokens_seen": 1141440512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005286258776328988, + "loss": 1.2657, + "theoretical_loss": 3.632538334762766, + "tokens_seen": 1141506048 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005286098294884654, + "loss": 1.3466, + "theoretical_loss": 3.6325191226737537, + "tokens_seen": 1141571584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005285937813440322, + "loss": 1.301, + "theoretical_loss": 3.6324999119964505, + "tokens_seen": 1141637120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005285777331995988, + "loss": 1.3315, + "theoretical_loss": 3.632480702730673, + "tokens_seen": 1141702656 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005285616850551656, + "loss": 1.5219, + "theoretical_loss": 3.6324614948762344, + "tokens_seen": 1141768192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005285456369107322, + "loss": 1.293, + "theoretical_loss": 3.6324422884329524, + "tokens_seen": 1141833728 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528529588766299, + "loss": 1.4272, + "theoretical_loss": 3.6324230834006404, + "tokens_seen": 1141899264 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005285135406218656, + "loss": 1.2841, + "theoretical_loss": 3.632403879779115, + "tokens_seen": 1141964800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005284974924774324, + "loss": 1.4483, + "theoretical_loss": 3.632384677568191, + "tokens_seen": 1142030336 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528481444332999, + "loss": 1.2183, + "theoretical_loss": 3.632365476767684, + "tokens_seen": 1142095872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005284653961885657, + "loss": 1.3837, + "theoretical_loss": 3.63234627737741, + "tokens_seen": 1142161408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005284493480441324, + "loss": 1.3682, + "theoretical_loss": 3.6323270793971836, + "tokens_seen": 1142226944 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005284332998996991, + "loss": 1.3882, + "theoretical_loss": 3.632307882826821, + "tokens_seen": 1142292480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005284172517552658, + "loss": 1.3651, + "theoretical_loss": 3.6322886876661373, + "tokens_seen": 1142358016 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005284012036108325, + "loss": 1.4432, + "theoretical_loss": 3.632269493914949, + "tokens_seen": 1142423552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005283851554663992, + "loss": 1.359, + "theoretical_loss": 3.6322503015730705, + "tokens_seen": 1142489088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005283691073219659, + "loss": 1.3563, + "theoretical_loss": 3.6322311106403182, + "tokens_seen": 1142554624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005283530591775326, + "loss": 1.3642, + "theoretical_loss": 3.632211921116508, + "tokens_seen": 1142620160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005283370110330993, + "loss": 1.4548, + "theoretical_loss": 3.632192733001455, + "tokens_seen": 1142685696 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528320962888666, + "loss": 1.3801, + "theoretical_loss": 3.632173546294976, + "tokens_seen": 1142751232 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005283049147442327, + "loss": 1.335, + "theoretical_loss": 3.6321543609968856, + "tokens_seen": 1142816768 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005282888665997994, + "loss": 1.4076, + "theoretical_loss": 3.6321351771070005, + "tokens_seen": 1142882304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005282728184553661, + "loss": 1.2526, + "theoretical_loss": 3.6321159946251367, + "tokens_seen": 1142947840 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005282567703109328, + "loss": 1.3344, + "theoretical_loss": 3.6320968135511094, + "tokens_seen": 1143013376 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005282407221664995, + "loss": 1.4056, + "theoretical_loss": 3.632077633884735, + "tokens_seen": 1143078912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005282246740220662, + "loss": 1.32, + "theoretical_loss": 3.6320584556258297, + "tokens_seen": 1143144448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005282086258776329, + "loss": 1.2697, + "theoretical_loss": 3.6320392787742093, + "tokens_seen": 1143209984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005281925777331996, + "loss": 1.3624, + "theoretical_loss": 3.6320201033296895, + "tokens_seen": 1143275520 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005281765295887663, + "loss": 1.3654, + "theoretical_loss": 3.6320009292920874, + "tokens_seen": 1143341056 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528160481444333, + "loss": 1.3937, + "theoretical_loss": 3.6319817566612183, + "tokens_seen": 1143406592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005281444332998997, + "loss": 1.4662, + "theoretical_loss": 3.6319625854368986, + "tokens_seen": 1143472128 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005281283851554664, + "loss": 1.3763, + "theoretical_loss": 3.6319434156189443, + "tokens_seen": 1143537664 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005281123370110331, + "loss": 1.4362, + "theoretical_loss": 3.6319242472071718, + "tokens_seen": 1143603200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005280962888665998, + "loss": 1.3927, + "theoretical_loss": 3.631905080201398, + "tokens_seen": 1143668736 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005280802407221665, + "loss": 1.3272, + "theoretical_loss": 3.6318859146014377, + "tokens_seen": 1143734272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005280641925777332, + "loss": 1.3781, + "theoretical_loss": 3.6318667504071094, + "tokens_seen": 1143799808 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005280481444332999, + "loss": 1.3182, + "theoretical_loss": 3.6318475876182275, + "tokens_seen": 1143865344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005280320962888666, + "loss": 1.429, + "theoretical_loss": 3.631828426234609, + "tokens_seen": 1143930880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005280160481444333, + "loss": 1.4242, + "theoretical_loss": 3.631809266256071, + "tokens_seen": 1143996416 + }, + { + "epoch": 0.35, + "learning_rate": 0.000528, + "loss": 1.4694, + "theoretical_loss": 3.631790107682429, + "tokens_seen": 1144061952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005279839518555667, + "loss": 1.4266, + "theoretical_loss": 3.6317709505135003, + "tokens_seen": 1144127488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005279679037111334, + "loss": 1.4016, + "theoretical_loss": 3.631751794749101, + "tokens_seen": 1144193024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005279518555667001, + "loss": 1.383, + "theoretical_loss": 3.6317326403890484, + "tokens_seen": 1144258560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005279358074222668, + "loss": 1.4448, + "theoretical_loss": 3.6317134874331582, + "tokens_seen": 1144324096 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005279197592778335, + "loss": 1.457, + "theoretical_loss": 3.6316943358812477, + "tokens_seen": 1144389632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005279037111334002, + "loss": 1.3938, + "theoretical_loss": 3.631675185733133, + "tokens_seen": 1144455168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005278876629889669, + "loss": 1.3577, + "theoretical_loss": 3.6316560369886313, + "tokens_seen": 1144520704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005278716148445336, + "loss": 1.3605, + "theoretical_loss": 3.6316368896475586, + "tokens_seen": 1144586240 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005278555667001003, + "loss": 1.4155, + "theoretical_loss": 3.6316177437097332, + "tokens_seen": 1144651776 + }, + { + "epoch": 0.35, + "learning_rate": 0.000527839518555667, + "loss": 1.3711, + "theoretical_loss": 3.6315985991749704, + "tokens_seen": 1144717312 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005278234704112337, + "loss": 1.2947, + "theoretical_loss": 3.631579456043088, + "tokens_seen": 1144782848 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005278074222668004, + "loss": 1.3373, + "theoretical_loss": 3.631560314313903, + "tokens_seen": 1144848384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005277913741223671, + "loss": 1.357, + "theoretical_loss": 3.631541173987231, + "tokens_seen": 1144913920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005277753259779338, + "loss": 1.4053, + "theoretical_loss": 3.6315220350628907, + "tokens_seen": 1144979456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005277592778335005, + "loss": 1.4807, + "theoretical_loss": 3.631502897540698, + "tokens_seen": 1145044992 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005277432296890673, + "loss": 1.378, + "theoretical_loss": 3.6314837614204705, + "tokens_seen": 1145110528 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005277271815446339, + "loss": 1.2736, + "theoretical_loss": 3.6314646267020247, + "tokens_seen": 1145176064 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005277111334002007, + "loss": 1.3342, + "theoretical_loss": 3.6314454933851783, + "tokens_seen": 1145241600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005276950852557673, + "loss": 1.2851, + "theoretical_loss": 3.6314263614697477, + "tokens_seen": 1145307136 + }, + { + "epoch": 0.35, + "learning_rate": 0.000527679037111334, + "loss": 1.3728, + "theoretical_loss": 3.6314072309555505, + "tokens_seen": 1145372672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005276629889669007, + "loss": 1.3694, + "theoretical_loss": 3.631388101842404, + "tokens_seen": 1145438208 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005276469408224675, + "loss": 1.401, + "theoretical_loss": 3.6313689741301256, + "tokens_seen": 1145503744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005276308926780341, + "loss": 1.4914, + "theoretical_loss": 3.631349847818532, + "tokens_seen": 1145569280 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005276148445336009, + "loss": 1.3808, + "theoretical_loss": 3.6313307229074407, + "tokens_seen": 1145634816 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005275987963891675, + "loss": 1.3475, + "theoretical_loss": 3.6313115993966694, + "tokens_seen": 1145700352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005275827482447343, + "loss": 1.3886, + "theoretical_loss": 3.6312924772860358, + "tokens_seen": 1145765888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005275667001003009, + "loss": 1.3654, + "theoretical_loss": 3.631273356575356, + "tokens_seen": 1145831424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005275506519558677, + "loss": 1.418, + "theoretical_loss": 3.6312542372644483, + "tokens_seen": 1145896960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005275346038114343, + "loss": 1.4194, + "theoretical_loss": 3.6312351193531303, + "tokens_seen": 1145962496 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005275185556670011, + "loss": 1.3933, + "theoretical_loss": 3.6312160028412195, + "tokens_seen": 1146028032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005275025075225677, + "loss": 1.4395, + "theoretical_loss": 3.631196887728533, + "tokens_seen": 1146093568 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005274864593781345, + "loss": 1.4654, + "theoretical_loss": 3.6311777740148887, + "tokens_seen": 1146159104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005274704112337011, + "loss": 1.3589, + "theoretical_loss": 3.631158661700104, + "tokens_seen": 1146224640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005274543630892679, + "loss": 1.31, + "theoretical_loss": 3.631139550783997, + "tokens_seen": 1146290176 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005274383149448345, + "loss": 1.3667, + "theoretical_loss": 3.631120441266385, + "tokens_seen": 1146355712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005274222668004013, + "loss": 1.3592, + "theoretical_loss": 3.6311013331470856, + "tokens_seen": 1146421248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005274062186559679, + "loss": 1.4639, + "theoretical_loss": 3.631082226425917, + "tokens_seen": 1146486784 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005273901705115347, + "loss": 1.2897, + "theoretical_loss": 3.6310631211026965, + "tokens_seen": 1146552320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005273741223671013, + "loss": 1.4196, + "theoretical_loss": 3.6310440171772425, + "tokens_seen": 1146617856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005273580742226681, + "loss": 1.4442, + "theoretical_loss": 3.6310249146493723, + "tokens_seen": 1146683392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005273420260782347, + "loss": 1.4396, + "theoretical_loss": 3.631005813518904, + "tokens_seen": 1146748928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005273259779338015, + "loss": 1.3806, + "theoretical_loss": 3.6309867137856555, + "tokens_seen": 1146814464 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 428521, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3795771598815918, + "objective/train/theoretical_loss": 3.630967615449445, + "objective/train/tokens_used": 1167340000, + "theoretical_loss": 3.630967615449445, + "tokens_seen": 1146880000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005273099297893682, + "loss": 1.3796, + "theoretical_loss": 3.630967615449445, + "tokens_seen": 1146880000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005272938816449349, + "loss": 1.3816, + "theoretical_loss": 3.63094851851009, + "tokens_seen": 1146945536 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005272778335005016, + "loss": 1.4563, + "theoretical_loss": 3.6309294229674087, + "tokens_seen": 1147011072 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005272617853560683, + "loss": 1.3254, + "theoretical_loss": 3.63091032882122, + "tokens_seen": 1147076608 + }, + { + "epoch": 0.35, + "learning_rate": 0.000527245737211635, + "loss": 1.3232, + "theoretical_loss": 3.630891236071341, + "tokens_seen": 1147142144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005272296890672017, + "loss": 1.3149, + "theoretical_loss": 3.63087214471759, + "tokens_seen": 1147207680 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005272136409227683, + "loss": 1.4058, + "theoretical_loss": 3.6308530547597853, + "tokens_seen": 1147273216 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005271975927783351, + "loss": 1.4029, + "theoretical_loss": 3.6308339661977445, + "tokens_seen": 1147338752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005271815446339017, + "loss": 1.3872, + "theoretical_loss": 3.630814879031287, + "tokens_seen": 1147404288 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005271654964894685, + "loss": 1.3189, + "theoretical_loss": 3.6307957932602304, + "tokens_seen": 1147469824 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005271494483450351, + "loss": 1.3641, + "theoretical_loss": 3.630776708884393, + "tokens_seen": 1147535360 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005271334002006019, + "loss": 1.3648, + "theoretical_loss": 3.630757625903593, + "tokens_seen": 1147600896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005271173520561685, + "loss": 1.3454, + "theoretical_loss": 3.630738544317649, + "tokens_seen": 1147666432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005271013039117353, + "loss": 1.3726, + "theoretical_loss": 3.6307194641263796, + "tokens_seen": 1147731968 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005270852557673019, + "loss": 1.374, + "theoretical_loss": 3.630700385329603, + "tokens_seen": 1147797504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005270692076228686, + "loss": 1.3752, + "theoretical_loss": 3.6306813079271376, + "tokens_seen": 1147863040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005270531594784353, + "loss": 1.4384, + "theoretical_loss": 3.630662231918802, + "tokens_seen": 1147928576 + }, + { + "epoch": 0.35, + "learning_rate": 0.000527037111334002, + "loss": 1.298, + "theoretical_loss": 3.630643157304415, + "tokens_seen": 1147994112 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005270210631895687, + "loss": 1.3639, + "theoretical_loss": 3.6306240840837942, + "tokens_seen": 1148059648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005270050150451354, + "loss": 1.3431, + "theoretical_loss": 3.6306050122567592, + "tokens_seen": 1148125184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005269889669007021, + "loss": 1.4035, + "theoretical_loss": 3.6305859418231283, + "tokens_seen": 1148190720 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005269729187562688, + "loss": 1.4641, + "theoretical_loss": 3.6305668727827203, + "tokens_seen": 1148256256 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005269568706118355, + "loss": 1.2963, + "theoretical_loss": 3.630547805135354, + "tokens_seen": 1148321792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005269408224674022, + "loss": 1.3974, + "theoretical_loss": 3.630528738880847, + "tokens_seen": 1148387328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005269247743229689, + "loss": 1.4059, + "theoretical_loss": 3.63050967401902, + "tokens_seen": 1148452864 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005269087261785356, + "loss": 1.3806, + "theoretical_loss": 3.630490610549691, + "tokens_seen": 1148518400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005268926780341023, + "loss": 1.3465, + "theoretical_loss": 3.630471548472678, + "tokens_seen": 1148583936 + }, + { + "epoch": 0.35, + "learning_rate": 0.000526876629889669, + "loss": 1.3442, + "theoretical_loss": 3.6304524877878004, + "tokens_seen": 1148649472 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005268605817452357, + "loss": 1.3793, + "theoretical_loss": 3.630433428494878, + "tokens_seen": 1148715008 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005268445336008024, + "loss": 1.2879, + "theoretical_loss": 3.630414370593728, + "tokens_seen": 1148780544 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005268284854563691, + "loss": 1.3278, + "theoretical_loss": 3.6303953140841716, + "tokens_seen": 1148846080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005268124373119358, + "loss": 1.335, + "theoretical_loss": 3.6303762589660256, + "tokens_seen": 1148911616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005267963891675025, + "loss": 1.399, + "theoretical_loss": 3.6303572052391107, + "tokens_seen": 1148977152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005267803410230692, + "loss": 1.4131, + "theoretical_loss": 3.630338152903245, + "tokens_seen": 1149042688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005267642928786359, + "loss": 1.2918, + "theoretical_loss": 3.630319101958248, + "tokens_seen": 1149108224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005267482447342026, + "loss": 1.3794, + "theoretical_loss": 3.630300052403939, + "tokens_seen": 1149173760 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005267321965897693, + "loss": 1.4345, + "theoretical_loss": 3.6302810042401368, + "tokens_seen": 1149239296 + }, + { + "epoch": 0.35, + "learning_rate": 0.000526716148445336, + "loss": 1.3951, + "theoretical_loss": 3.6302619574666606, + "tokens_seen": 1149304832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005267001003009027, + "loss": 1.4103, + "theoretical_loss": 3.6302429120833297, + "tokens_seen": 1149370368 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005266840521564694, + "loss": 1.4591, + "theoretical_loss": 3.630223868089964, + "tokens_seen": 1149435904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005266680040120361, + "loss": 1.3702, + "theoretical_loss": 3.6302048254863823, + "tokens_seen": 1149501440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005266519558676028, + "loss": 1.3862, + "theoretical_loss": 3.6301857842724035, + "tokens_seen": 1149566976 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005266359077231695, + "loss": 1.4407, + "theoretical_loss": 3.6301667444478474, + "tokens_seen": 1149632512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005266198595787362, + "loss": 1.4366, + "theoretical_loss": 3.630147706012534, + "tokens_seen": 1149698048 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005266038114343029, + "loss": 1.3968, + "theoretical_loss": 3.6301286689662824, + "tokens_seen": 1149763584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005265877632898696, + "loss": 1.5306, + "theoretical_loss": 3.630109633308911, + "tokens_seen": 1149829120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005265717151454363, + "loss": 1.4009, + "theoretical_loss": 3.6300905990402415, + "tokens_seen": 1149894656 + }, + { + "epoch": 0.35, + "learning_rate": 0.000526555667001003, + "loss": 1.3858, + "theoretical_loss": 3.6300715661600913, + "tokens_seen": 1149960192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005265396188565697, + "loss": 1.3607, + "theoretical_loss": 3.630052534668281, + "tokens_seen": 1150025728 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005265235707121364, + "loss": 1.3483, + "theoretical_loss": 3.63003350456463, + "tokens_seen": 1150091264 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005265075225677031, + "loss": 1.3049, + "theoretical_loss": 3.630014475848958, + "tokens_seen": 1150156800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005264914744232698, + "loss": 1.4194, + "theoretical_loss": 3.6299954485210852, + "tokens_seen": 1150222336 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005264754262788365, + "loss": 1.3678, + "theoretical_loss": 3.629976422580831, + "tokens_seen": 1150287872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005264593781344032, + "loss": 1.5322, + "theoretical_loss": 3.629957398028014, + "tokens_seen": 1150353408 + }, + { + "epoch": 0.35, + "learning_rate": 0.00052644332998997, + "loss": 1.3398, + "theoretical_loss": 3.629938374862456, + "tokens_seen": 1150418944 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005264272818455366, + "loss": 1.3173, + "theoretical_loss": 3.6299193530839755, + "tokens_seen": 1150484480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005264112337011033, + "loss": 1.4315, + "theoretical_loss": 3.629900332692392, + "tokens_seen": 1150550016 + }, + { + "epoch": 0.35, + "learning_rate": 0.00052639518555667, + "loss": 1.3982, + "theoretical_loss": 3.629881313687527, + "tokens_seen": 1150615552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005263791374122368, + "loss": 1.5124, + "theoretical_loss": 3.6298622960691995, + "tokens_seen": 1150681088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005263630892678034, + "loss": 1.4298, + "theoretical_loss": 3.629843279837229, + "tokens_seen": 1150746624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005263470411233702, + "loss": 1.3975, + "theoretical_loss": 3.629824264991436, + "tokens_seen": 1150812160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005263309929789368, + "loss": 1.2989, + "theoretical_loss": 3.6298052515316406, + "tokens_seen": 1150877696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005263149448345036, + "loss": 1.4118, + "theoretical_loss": 3.6297862394576628, + "tokens_seen": 1150943232 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005262988966900702, + "loss": 1.3815, + "theoretical_loss": 3.629767228769323, + "tokens_seen": 1151008768 + }, + { + "epoch": 0.35, + "learning_rate": 0.000526282848545637, + "loss": 1.3542, + "theoretical_loss": 3.62974821946644, + "tokens_seen": 1151074304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005262668004012036, + "loss": 1.4166, + "theoretical_loss": 3.6297292115488355, + "tokens_seen": 1151139840 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005262507522567704, + "loss": 1.4608, + "theoretical_loss": 3.6297102050163286, + "tokens_seen": 1151205376 + }, + { + "epoch": 0.35, + "learning_rate": 0.000526234704112337, + "loss": 1.469, + "theoretical_loss": 3.6296911998687404, + "tokens_seen": 1151270912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005262186559679038, + "loss": 1.4067, + "theoretical_loss": 3.629672196105891, + "tokens_seen": 1151336448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005262026078234704, + "loss": 1.3345, + "theoretical_loss": 3.6296531937276004, + "tokens_seen": 1151401984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005261865596790372, + "loss": 1.2628, + "theoretical_loss": 3.629634192733689, + "tokens_seen": 1151467520 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005261705115346038, + "loss": 1.3314, + "theoretical_loss": 3.6296151931239766, + "tokens_seen": 1151533056 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005261544633901706, + "loss": 1.405, + "theoretical_loss": 3.629596194898285, + "tokens_seen": 1151598592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005261384152457372, + "loss": 1.4068, + "theoretical_loss": 3.6295771980564333, + "tokens_seen": 1151664128 + }, + { + "epoch": 0.35, + "learning_rate": 0.000526122367101304, + "loss": 1.482, + "theoretical_loss": 3.6295582025982425, + "tokens_seen": 1151729664 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005261063189568707, + "loss": 1.4559, + "theoretical_loss": 3.629539208523533, + "tokens_seen": 1151795200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005260902708124374, + "loss": 1.35, + "theoretical_loss": 3.6295202158321254, + "tokens_seen": 1151860736 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005260742226680041, + "loss": 1.3218, + "theoretical_loss": 3.6295012245238403, + "tokens_seen": 1151926272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005260581745235708, + "loss": 1.4638, + "theoretical_loss": 3.6294822345984983, + "tokens_seen": 1151991808 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005260421263791375, + "loss": 1.3755, + "theoretical_loss": 3.62946324605592, + "tokens_seen": 1152057344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005260260782347042, + "loss": 1.4236, + "theoretical_loss": 3.6294442588959255, + "tokens_seen": 1152122880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005260100300902709, + "loss": 1.3972, + "theoretical_loss": 3.6294252731183363, + "tokens_seen": 1152188416 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005259939819458376, + "loss": 1.5635, + "theoretical_loss": 3.629406288722973, + "tokens_seen": 1152253952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005259779338014043, + "loss": 1.41, + "theoretical_loss": 3.629387305709656, + "tokens_seen": 1152319488 + }, + { + "epoch": 0.35, + "learning_rate": 0.000525961885656971, + "loss": 1.25, + "theoretical_loss": 3.6293683240782064, + "tokens_seen": 1152385024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005259458375125377, + "loss": 1.393, + "theoretical_loss": 3.629349343828445, + "tokens_seen": 1152450560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005259297893681044, + "loss": 1.4461, + "theoretical_loss": 3.6293303649601922, + "tokens_seen": 1152516096 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005259137412236711, + "loss": 1.3679, + "theoretical_loss": 3.62931138747327, + "tokens_seen": 1152581632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005258976930792378, + "loss": 1.3961, + "theoretical_loss": 3.6292924113674983, + "tokens_seen": 1152647168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005258816449348045, + "loss": 1.4138, + "theoretical_loss": 3.629273436642698, + "tokens_seen": 1152712704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005258655967903712, + "loss": 1.3948, + "theoretical_loss": 3.6292544632986905, + "tokens_seen": 1152778240 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005258495486459379, + "loss": 1.4521, + "theoretical_loss": 3.6292354913352973, + "tokens_seen": 1152843776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005258335005015046, + "loss": 1.3928, + "theoretical_loss": 3.6292165207523386, + "tokens_seen": 1152909312 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005258174523570712, + "loss": 1.4373, + "theoretical_loss": 3.629197551549636, + "tokens_seen": 1152974848 + }, + { + "epoch": 0.35, + "learning_rate": 0.000525801404212638, + "loss": 1.3344, + "theoretical_loss": 3.6291785837270103, + "tokens_seen": 1153040384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005257853560682046, + "loss": 1.3886, + "theoretical_loss": 3.6291596172842833, + "tokens_seen": 1153105920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005257693079237714, + "loss": 1.4124, + "theoretical_loss": 3.6291406522212757, + "tokens_seen": 1153171456 + }, + { + "epoch": 0.35, + "learning_rate": 0.000525753259779338, + "loss": 1.3986, + "theoretical_loss": 3.629121688537808, + "tokens_seen": 1153236992 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005257372116349048, + "loss": 1.4598, + "theoretical_loss": 3.629102726233703, + "tokens_seen": 1153302528 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005257211634904714, + "loss": 1.4089, + "theoretical_loss": 3.6290837653087813, + "tokens_seen": 1153368064 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 430839, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4103814363479614, + "objective/train/theoretical_loss": 3.6290648057628636, + "objective/train/tokens_used": 1173893600, + "theoretical_loss": 3.6290648057628636, + "tokens_seen": 1153433600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005257051153460382, + "loss": 1.4104, + "theoretical_loss": 3.6290648057628636, + "tokens_seen": 1153433600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005256890672016048, + "loss": 1.4204, + "theoretical_loss": 3.6290458475957723, + "tokens_seen": 1153499136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005256730190571715, + "loss": 1.2892, + "theoretical_loss": 3.629026890807328, + "tokens_seen": 1153564672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005256569709127382, + "loss": 1.3903, + "theoretical_loss": 3.6290079353973526, + "tokens_seen": 1153630208 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005256409227683049, + "loss": 1.3751, + "theoretical_loss": 3.6289889813656675, + "tokens_seen": 1153695744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005256248746238716, + "loss": 1.4071, + "theoretical_loss": 3.628970028712094, + "tokens_seen": 1153761280 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005256088264794383, + "loss": 1.3593, + "theoretical_loss": 3.6289510774364535, + "tokens_seen": 1153826816 + }, + { + "epoch": 0.35, + "learning_rate": 0.000525592778335005, + "loss": 1.3827, + "theoretical_loss": 3.628932127538568, + "tokens_seen": 1153892352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005255767301905717, + "loss": 1.3501, + "theoretical_loss": 3.6289131790182587, + "tokens_seen": 1153957888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005255606820461384, + "loss": 1.3349, + "theoretical_loss": 3.628894231875348, + "tokens_seen": 1154023424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005255446339017051, + "loss": 1.4114, + "theoretical_loss": 3.6288752861096567, + "tokens_seen": 1154088960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005255285857572718, + "loss": 1.3158, + "theoretical_loss": 3.628856341721007, + "tokens_seen": 1154154496 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005255125376128385, + "loss": 1.4129, + "theoretical_loss": 3.62883739870922, + "tokens_seen": 1154220032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005254964894684052, + "loss": 1.4149, + "theoretical_loss": 3.628818457074118, + "tokens_seen": 1154285568 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005254804413239719, + "loss": 1.32, + "theoretical_loss": 3.6287995168155227, + "tokens_seen": 1154351104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005254643931795386, + "loss": 1.3252, + "theoretical_loss": 3.628780577933256, + "tokens_seen": 1154416640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005254483450351053, + "loss": 1.3947, + "theoretical_loss": 3.6287616404271397, + "tokens_seen": 1154482176 + }, + { + "epoch": 0.35, + "learning_rate": 0.000525432296890672, + "loss": 1.373, + "theoretical_loss": 3.6287427042969953, + "tokens_seen": 1154547712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005254162487462387, + "loss": 1.4732, + "theoretical_loss": 3.6287237695426455, + "tokens_seen": 1154613248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005254002006018054, + "loss": 1.3108, + "theoretical_loss": 3.6287048361639114, + "tokens_seen": 1154678784 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005253841524573721, + "loss": 1.4655, + "theoretical_loss": 3.6286859041606156, + "tokens_seen": 1154744320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005253681043129388, + "loss": 1.3853, + "theoretical_loss": 3.62866697353258, + "tokens_seen": 1154809856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005253520561685055, + "loss": 1.4006, + "theoretical_loss": 3.6286480442796267, + "tokens_seen": 1154875392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005253360080240722, + "loss": 1.3807, + "theoretical_loss": 3.628629116401578, + "tokens_seen": 1154940928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005253199598796389, + "loss": 1.2917, + "theoretical_loss": 3.6286101898982555, + "tokens_seen": 1155006464 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005253039117352056, + "loss": 1.3973, + "theoretical_loss": 3.628591264769481, + "tokens_seen": 1155072000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005252878635907723, + "loss": 1.36, + "theoretical_loss": 3.628572341015078, + "tokens_seen": 1155137536 + }, + { + "epoch": 0.35, + "learning_rate": 0.000525271815446339, + "loss": 1.3976, + "theoretical_loss": 3.6285534186348682, + "tokens_seen": 1155203072 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005252557673019057, + "loss": 1.4032, + "theoretical_loss": 3.628534497628673, + "tokens_seen": 1155268608 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005252397191574724, + "loss": 1.5535, + "theoretical_loss": 3.628515577996316, + "tokens_seen": 1155334144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005252236710130391, + "loss": 1.3508, + "theoretical_loss": 3.6284966597376185, + "tokens_seen": 1155399680 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005252076228686058, + "loss": 1.4375, + "theoretical_loss": 3.628477742852403, + "tokens_seen": 1155465216 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005251915747241725, + "loss": 1.4205, + "theoretical_loss": 3.6284588273404927, + "tokens_seen": 1155530752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005251755265797392, + "loss": 1.3448, + "theoretical_loss": 3.628439913201709, + "tokens_seen": 1155596288 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005251594784353059, + "loss": 1.3973, + "theoretical_loss": 3.6284210004358757, + "tokens_seen": 1155661824 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005251434302908726, + "loss": 1.2941, + "theoretical_loss": 3.6284020890428135, + "tokens_seen": 1155727360 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005251273821464393, + "loss": 1.3799, + "theoretical_loss": 3.6283831790223458, + "tokens_seen": 1155792896 + }, + { + "epoch": 0.35, + "learning_rate": 0.000525111334002006, + "loss": 1.4399, + "theoretical_loss": 3.6283642703742958, + "tokens_seen": 1155858432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005250952858575727, + "loss": 1.3201, + "theoretical_loss": 3.628345363098485, + "tokens_seen": 1155923968 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005250792377131394, + "loss": 1.3799, + "theoretical_loss": 3.6283264571947367, + "tokens_seen": 1155989504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005250631895687061, + "loss": 1.3865, + "theoretical_loss": 3.628307552662873, + "tokens_seen": 1156055040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005250471414242728, + "loss": 1.3282, + "theoretical_loss": 3.628288649502718, + "tokens_seen": 1156120576 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005250310932798395, + "loss": 1.3695, + "theoretical_loss": 3.6282697477140924, + "tokens_seen": 1156186112 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005250150451354063, + "loss": 1.3492, + "theoretical_loss": 3.6282508472968202, + "tokens_seen": 1156251648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005249989969909729, + "loss": 1.3687, + "theoretical_loss": 3.6282319482507237, + "tokens_seen": 1156317184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005249829488465397, + "loss": 1.3872, + "theoretical_loss": 3.628213050575626, + "tokens_seen": 1156382720 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005249669007021063, + "loss": 1.3624, + "theoretical_loss": 3.62819415427135, + "tokens_seen": 1156448256 + }, + { + "epoch": 0.35, + "learning_rate": 0.000524950852557673, + "loss": 1.3307, + "theoretical_loss": 3.628175259337719, + "tokens_seen": 1156513792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005249348044132397, + "loss": 1.3196, + "theoretical_loss": 3.628156365774555, + "tokens_seen": 1156579328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005249187562688065, + "loss": 1.3678, + "theoretical_loss": 3.628137473581681, + "tokens_seen": 1156644864 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005249027081243732, + "loss": 1.3128, + "theoretical_loss": 3.628118582758921, + "tokens_seen": 1156710400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005248866599799399, + "loss": 1.3795, + "theoretical_loss": 3.6280996933060967, + "tokens_seen": 1156775936 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005248706118355066, + "loss": 1.3838, + "theoretical_loss": 3.6280808052230316, + "tokens_seen": 1156841472 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005248545636910733, + "loss": 1.3662, + "theoretical_loss": 3.6280619185095495, + "tokens_seen": 1156907008 + }, + { + "epoch": 0.35, + "learning_rate": 0.00052483851554664, + "loss": 1.4064, + "theoretical_loss": 3.628043033165473, + "tokens_seen": 1156972544 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005248224674022067, + "loss": 1.3993, + "theoretical_loss": 3.6280241491906247, + "tokens_seen": 1157038080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005248064192577734, + "loss": 1.4194, + "theoretical_loss": 3.628005266584829, + "tokens_seen": 1157103616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005247903711133401, + "loss": 1.2949, + "theoretical_loss": 3.6279863853479077, + "tokens_seen": 1157169152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005247743229689068, + "loss": 1.2877, + "theoretical_loss": 3.6279675054796856, + "tokens_seen": 1157234688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005247582748244735, + "loss": 1.4079, + "theoretical_loss": 3.6279486269799843, + "tokens_seen": 1157300224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005247422266800402, + "loss": 1.3739, + "theoretical_loss": 3.6279297498486285, + "tokens_seen": 1157365760 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005247261785356069, + "loss": 1.329, + "theoretical_loss": 3.6279108740854413, + "tokens_seen": 1157431296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005247101303911736, + "loss": 1.4018, + "theoretical_loss": 3.6278919996902452, + "tokens_seen": 1157496832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005246940822467403, + "loss": 1.2931, + "theoretical_loss": 3.6278731266628643, + "tokens_seen": 1157562368 + }, + { + "epoch": 0.35, + "learning_rate": 0.000524678034102307, + "loss": 1.318, + "theoretical_loss": 3.6278542550031223, + "tokens_seen": 1157627904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005246619859578737, + "loss": 1.4043, + "theoretical_loss": 3.627835384710842, + "tokens_seen": 1157693440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005246459378134404, + "loss": 1.4346, + "theoretical_loss": 3.6278165157858475, + "tokens_seen": 1157758976 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005246298896690071, + "loss": 1.2976, + "theoretical_loss": 3.627797648227962, + "tokens_seen": 1157824512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005246138415245738, + "loss": 1.4459, + "theoretical_loss": 3.6277787820370087, + "tokens_seen": 1157890048 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005245977933801405, + "loss": 1.2885, + "theoretical_loss": 3.6277599172128125, + "tokens_seen": 1157955584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005245817452357072, + "loss": 1.3537, + "theoretical_loss": 3.6277410537551953, + "tokens_seen": 1158021120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005245656970912739, + "loss": 1.3384, + "theoretical_loss": 3.627722191663983, + "tokens_seen": 1158086656 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005245496489468406, + "loss": 1.3301, + "theoretical_loss": 3.6277033309389966, + "tokens_seen": 1158152192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005245336008024073, + "loss": 1.3708, + "theoretical_loss": 3.627684471580062, + "tokens_seen": 1158217728 + }, + { + "epoch": 0.35, + "learning_rate": 0.000524517552657974, + "loss": 1.3811, + "theoretical_loss": 3.627665613587002, + "tokens_seen": 1158283264 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005245015045135407, + "loss": 1.3411, + "theoretical_loss": 3.6276467569596402, + "tokens_seen": 1158348800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005244854563691074, + "loss": 1.3714, + "theoretical_loss": 3.6276279016978012, + "tokens_seen": 1158414336 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005244694082246741, + "loss": 1.3772, + "theoretical_loss": 3.6276090478013088, + "tokens_seen": 1158479872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005244533600802408, + "loss": 1.3174, + "theoretical_loss": 3.6275901952699865, + "tokens_seen": 1158545408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005244373119358075, + "loss": 1.3292, + "theoretical_loss": 3.627571344103658, + "tokens_seen": 1158610944 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005244212637913741, + "loss": 1.3295, + "theoretical_loss": 3.6275524943021478, + "tokens_seen": 1158676480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005244052156469409, + "loss": 1.4409, + "theoretical_loss": 3.62753364586528, + "tokens_seen": 1158742016 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005243891675025075, + "loss": 1.3394, + "theoretical_loss": 3.6275147987928777, + "tokens_seen": 1158807552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005243731193580743, + "loss": 1.333, + "theoretical_loss": 3.6274959530847664, + "tokens_seen": 1158873088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005243570712136409, + "loss": 1.3826, + "theoretical_loss": 3.6274771087407687, + "tokens_seen": 1158938624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005243410230692077, + "loss": 1.3827, + "theoretical_loss": 3.6274582657607097, + "tokens_seen": 1159004160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005243249749247743, + "loss": 1.2797, + "theoretical_loss": 3.6274394241444132, + "tokens_seen": 1159069696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005243089267803411, + "loss": 1.2169, + "theoretical_loss": 3.6274205838917037, + "tokens_seen": 1159135232 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005242928786359077, + "loss": 1.3469, + "theoretical_loss": 3.6274017450024054, + "tokens_seen": 1159200768 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005242768304914744, + "loss": 1.4264, + "theoretical_loss": 3.627382907476342, + "tokens_seen": 1159266304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005242607823470411, + "loss": 1.3354, + "theoretical_loss": 3.6273640713133384, + "tokens_seen": 1159331840 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005242447342026078, + "loss": 1.4108, + "theoretical_loss": 3.6273452365132184, + "tokens_seen": 1159397376 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005242286860581745, + "loss": 1.3271, + "theoretical_loss": 3.627326403075807, + "tokens_seen": 1159462912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005242126379137412, + "loss": 1.4366, + "theoretical_loss": 3.6273075710009284, + "tokens_seen": 1159528448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005241965897693079, + "loss": 1.2923, + "theoretical_loss": 3.6272887402884066, + "tokens_seen": 1159593984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005241805416248746, + "loss": 1.3058, + "theoretical_loss": 3.6272699109380664, + "tokens_seen": 1159659520 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005241644934804413, + "loss": 1.3749, + "theoretical_loss": 3.627251082949732, + "tokens_seen": 1159725056 + }, + { + "epoch": 0.35, + "learning_rate": 0.000524148445336008, + "loss": 1.5047, + "theoretical_loss": 3.6272322563232287, + "tokens_seen": 1159790592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005241323971915747, + "loss": 1.371, + "theoretical_loss": 3.6272134310583803, + "tokens_seen": 1159856128 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005241163490471414, + "loss": 1.3488, + "theoretical_loss": 3.627194607155011, + "tokens_seen": 1159921664 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 433221, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.457830786705017, + "objective/train/theoretical_loss": 3.6271757846129464, + "objective/train/tokens_used": 1180447200, + "theoretical_loss": 3.6271757846129464, + "tokens_seen": 1159987200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005241003009027081, + "loss": 1.4578, + "theoretical_loss": 3.6271757846129464, + "tokens_seen": 1159987200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005240842527582748, + "loss": 1.3967, + "theoretical_loss": 3.6271569634320113, + "tokens_seen": 1160052736 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005240682046138415, + "loss": 1.3772, + "theoretical_loss": 3.6271381436120294, + "tokens_seen": 1160118272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005240521564694082, + "loss": 1.3613, + "theoretical_loss": 3.627119325152826, + "tokens_seen": 1160183808 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005240361083249749, + "loss": 1.3709, + "theoretical_loss": 3.627100508054226, + "tokens_seen": 1160249344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005240200601805416, + "loss": 1.3854, + "theoretical_loss": 3.627081692316053, + "tokens_seen": 1160314880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005240040120361083, + "loss": 1.3656, + "theoretical_loss": 3.6270628779381333, + "tokens_seen": 1160380416 + }, + { + "epoch": 0.35, + "learning_rate": 0.000523987963891675, + "loss": 1.4466, + "theoretical_loss": 3.627044064920291, + "tokens_seen": 1160445952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005239719157472417, + "loss": 1.3799, + "theoretical_loss": 3.6270252532623513, + "tokens_seen": 1160511488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005239558676028084, + "loss": 1.5272, + "theoretical_loss": 3.627006442964139, + "tokens_seen": 1160577024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005239398194583751, + "loss": 1.3706, + "theoretical_loss": 3.626987634025479, + "tokens_seen": 1160642560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005239237713139418, + "loss": 1.2985, + "theoretical_loss": 3.626968826446196, + "tokens_seen": 1160708096 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005239077231695085, + "loss": 1.3859, + "theoretical_loss": 3.626950020226116, + "tokens_seen": 1160773632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005238916750250752, + "loss": 1.2852, + "theoretical_loss": 3.6269312153650626, + "tokens_seen": 1160839168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005238756268806419, + "loss": 1.3928, + "theoretical_loss": 3.6269124118628615, + "tokens_seen": 1160904704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005238595787362086, + "loss": 1.296, + "theoretical_loss": 3.626893609719338, + "tokens_seen": 1160970240 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005238435305917753, + "loss": 1.271, + "theoretical_loss": 3.6268748089343177, + "tokens_seen": 1161035776 + }, + { + "epoch": 0.35, + "learning_rate": 0.000523827482447342, + "loss": 1.3977, + "theoretical_loss": 3.6268560095076245, + "tokens_seen": 1161101312 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005238114343029087, + "loss": 1.3525, + "theoretical_loss": 3.626837211439085, + "tokens_seen": 1161166848 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005237953861584754, + "loss": 1.3921, + "theoretical_loss": 3.626818414728523, + "tokens_seen": 1161232384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005237793380140421, + "loss": 1.3625, + "theoretical_loss": 3.626799619375765, + "tokens_seen": 1161297920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005237632898696088, + "loss": 1.3479, + "theoretical_loss": 3.626780825380636, + "tokens_seen": 1161363456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005237472417251755, + "loss": 1.3432, + "theoretical_loss": 3.6267620327429606, + "tokens_seen": 1161428992 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005237311935807422, + "loss": 1.4137, + "theoretical_loss": 3.626743241462565, + "tokens_seen": 1161494528 + }, + { + "epoch": 0.35, + "learning_rate": 0.000523715145436309, + "loss": 1.3272, + "theoretical_loss": 3.6267244515392743, + "tokens_seen": 1161560064 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005236990972918757, + "loss": 1.3781, + "theoretical_loss": 3.626705662972914, + "tokens_seen": 1161625600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005236830491474423, + "loss": 1.4453, + "theoretical_loss": 3.6266868757633093, + "tokens_seen": 1161691136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005236670010030091, + "loss": 1.4473, + "theoretical_loss": 3.626668089910286, + "tokens_seen": 1161756672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005236509528585757, + "loss": 1.319, + "theoretical_loss": 3.62664930541367, + "tokens_seen": 1161822208 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005236349047141425, + "loss": 1.5111, + "theoretical_loss": 3.6266305222732855, + "tokens_seen": 1161887744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005236188565697092, + "loss": 1.4301, + "theoretical_loss": 3.6266117404889595, + "tokens_seen": 1161953280 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005236028084252759, + "loss": 1.2486, + "theoretical_loss": 3.626592960060517, + "tokens_seen": 1162018816 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005235867602808426, + "loss": 1.4479, + "theoretical_loss": 3.6265741809877836, + "tokens_seen": 1162084352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005235707121364093, + "loss": 1.3297, + "theoretical_loss": 3.6265554032705856, + "tokens_seen": 1162149888 + }, + { + "epoch": 0.35, + "learning_rate": 0.000523554663991976, + "loss": 1.3644, + "theoretical_loss": 3.626536626908748, + "tokens_seen": 1162215424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005235386158475427, + "loss": 1.3989, + "theoretical_loss": 3.626517851902097, + "tokens_seen": 1162280960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005235225677031094, + "loss": 1.3542, + "theoretical_loss": 3.6264990782504576, + "tokens_seen": 1162346496 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005235065195586761, + "loss": 1.3019, + "theoretical_loss": 3.626480305953657, + "tokens_seen": 1162412032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005234904714142428, + "loss": 1.3117, + "theoretical_loss": 3.6264615350115195, + "tokens_seen": 1162477568 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005234744232698095, + "loss": 1.3108, + "theoretical_loss": 3.6264427654238727, + "tokens_seen": 1162543104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005234583751253762, + "loss": 1.297, + "theoretical_loss": 3.6264239971905408, + "tokens_seen": 1162608640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005234423269809429, + "loss": 1.3144, + "theoretical_loss": 3.6264052303113505, + "tokens_seen": 1162674176 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005234262788365096, + "loss": 1.3804, + "theoretical_loss": 3.626386464786128, + "tokens_seen": 1162739712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005234102306920763, + "loss": 1.2975, + "theoretical_loss": 3.626367700614699, + "tokens_seen": 1162805248 + }, + { + "epoch": 0.35, + "learning_rate": 0.000523394182547643, + "loss": 1.4024, + "theoretical_loss": 3.6263489377968896, + "tokens_seen": 1162870784 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005233781344032097, + "loss": 1.4146, + "theoretical_loss": 3.6263301763325257, + "tokens_seen": 1162936320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005233620862587764, + "loss": 1.3418, + "theoretical_loss": 3.6263114162214345, + "tokens_seen": 1163001856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005233460381143431, + "loss": 1.3856, + "theoretical_loss": 3.6262926574634404, + "tokens_seen": 1163067392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005233299899699098, + "loss": 1.3283, + "theoretical_loss": 3.6262739000583704, + "tokens_seen": 1163132928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005233139418254765, + "loss": 1.4756, + "theoretical_loss": 3.626255144006051, + "tokens_seen": 1163198464 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005232978936810432, + "loss": 1.5107, + "theoretical_loss": 3.6262363893063085, + "tokens_seen": 1163264000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005232818455366099, + "loss": 1.4459, + "theoretical_loss": 3.626217635958968, + "tokens_seen": 1163329536 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005232657973921766, + "loss": 1.3832, + "theoretical_loss": 3.6261988839638573, + "tokens_seen": 1163395072 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005232497492477433, + "loss": 1.3663, + "theoretical_loss": 3.626180133320802, + "tokens_seen": 1163460608 + }, + { + "epoch": 0.35, + "learning_rate": 0.00052323370110331, + "loss": 1.3042, + "theoretical_loss": 3.6261613840296283, + "tokens_seen": 1163526144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005232176529588767, + "loss": 1.4429, + "theoretical_loss": 3.6261426360901625, + "tokens_seen": 1163591680 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005232016048144434, + "loss": 1.3127, + "theoretical_loss": 3.626123889502232, + "tokens_seen": 1163657216 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005231855566700101, + "loss": 1.3154, + "theoretical_loss": 3.6261051442656624, + "tokens_seen": 1163722752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005231695085255768, + "loss": 1.3107, + "theoretical_loss": 3.62608640038028, + "tokens_seen": 1163788288 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005231534603811435, + "loss": 1.2844, + "theoretical_loss": 3.6260676578459123, + "tokens_seen": 1163853824 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005231374122367102, + "loss": 1.4422, + "theoretical_loss": 3.626048916662385, + "tokens_seen": 1163919360 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005231213640922769, + "loss": 1.356, + "theoretical_loss": 3.6260301768295253, + "tokens_seen": 1163984896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005231053159478436, + "loss": 1.3198, + "theoretical_loss": 3.6260114383471587, + "tokens_seen": 1164050432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005230892678034103, + "loss": 1.3174, + "theoretical_loss": 3.625992701215113, + "tokens_seen": 1164115968 + }, + { + "epoch": 0.35, + "learning_rate": 0.000523073219658977, + "loss": 1.4578, + "theoretical_loss": 3.625973965433215, + "tokens_seen": 1164181504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005230571715145437, + "loss": 1.2782, + "theoretical_loss": 3.6259552310012904, + "tokens_seen": 1164247040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005230411233701104, + "loss": 1.3836, + "theoretical_loss": 3.6259364979191666, + "tokens_seen": 1164312576 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005230250752256771, + "loss": 1.3553, + "theoretical_loss": 3.62591776618667, + "tokens_seen": 1164378112 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005230090270812438, + "loss": 1.3139, + "theoretical_loss": 3.625899035803628, + "tokens_seen": 1164443648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005229929789368104, + "loss": 1.2855, + "theoretical_loss": 3.625880306769867, + "tokens_seen": 1164509184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005229769307923772, + "loss": 1.4523, + "theoretical_loss": 3.625861579085214, + "tokens_seen": 1164574720 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005229608826479438, + "loss": 1.5086, + "theoretical_loss": 3.625842852749496, + "tokens_seen": 1164640256 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005229448345035106, + "loss": 1.333, + "theoretical_loss": 3.62582412776254, + "tokens_seen": 1164705792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005229287863590772, + "loss": 1.4433, + "theoretical_loss": 3.625805404124172, + "tokens_seen": 1164771328 + }, + { + "epoch": 0.35, + "learning_rate": 0.000522912738214644, + "loss": 1.3392, + "theoretical_loss": 3.6257866818342204, + "tokens_seen": 1164836864 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005228966900702106, + "loss": 1.3856, + "theoretical_loss": 3.6257679608925115, + "tokens_seen": 1164902400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005228806419257773, + "loss": 1.4125, + "theoretical_loss": 3.6257492412988723, + "tokens_seen": 1164967936 + }, + { + "epoch": 0.35, + "learning_rate": 0.000522864593781344, + "loss": 1.3866, + "theoretical_loss": 3.6257305230531305, + "tokens_seen": 1165033472 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005228485456369107, + "loss": 1.4609, + "theoretical_loss": 3.6257118061551132, + "tokens_seen": 1165099008 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005228324974924774, + "loss": 1.3326, + "theoretical_loss": 3.6256930906046465, + "tokens_seen": 1165164544 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005228164493480441, + "loss": 1.4038, + "theoretical_loss": 3.625674376401559, + "tokens_seen": 1165230080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005228004012036108, + "loss": 1.3502, + "theoretical_loss": 3.6256556635456763, + "tokens_seen": 1165295616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005227843530591775, + "loss": 1.3399, + "theoretical_loss": 3.6256369520368272, + "tokens_seen": 1165361152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005227683049147442, + "loss": 1.3734, + "theoretical_loss": 3.6256182418748377, + "tokens_seen": 1165426688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005227522567703109, + "loss": 1.329, + "theoretical_loss": 3.6255995330595363, + "tokens_seen": 1165492224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005227362086258776, + "loss": 1.303, + "theoretical_loss": 3.6255808255907502, + "tokens_seen": 1165557760 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005227201604814443, + "loss": 1.3862, + "theoretical_loss": 3.625562119468306, + "tokens_seen": 1165623296 + }, + { + "epoch": 0.35, + "learning_rate": 0.000522704112337011, + "loss": 1.3145, + "theoretical_loss": 3.6255434146920313, + "tokens_seen": 1165688832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005226880641925777, + "loss": 1.4494, + "theoretical_loss": 3.6255247112617544, + "tokens_seen": 1165754368 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005226720160481444, + "loss": 1.3299, + "theoretical_loss": 3.625506009177302, + "tokens_seen": 1165819904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005226559679037111, + "loss": 1.3587, + "theoretical_loss": 3.625487308438502, + "tokens_seen": 1165885440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005226399197592778, + "loss": 1.3457, + "theoretical_loss": 3.625468609045181, + "tokens_seen": 1165950976 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005226238716148445, + "loss": 1.4533, + "theoretical_loss": 3.625449910997168, + "tokens_seen": 1166016512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005226078234704112, + "loss": 1.3921, + "theoretical_loss": 3.6254312142942897, + "tokens_seen": 1166082048 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005225917753259779, + "loss": 1.4375, + "theoretical_loss": 3.6254125189363737, + "tokens_seen": 1166147584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005225757271815446, + "loss": 1.4562, + "theoretical_loss": 3.6253938249232482, + "tokens_seen": 1166213120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005225596790371113, + "loss": 1.2958, + "theoretical_loss": 3.6253751322547405, + "tokens_seen": 1166278656 + }, + { + "epoch": 0.35, + "learning_rate": 0.000522543630892678, + "loss": 1.4297, + "theoretical_loss": 3.6253564409306787, + "tokens_seen": 1166344192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005225275827482447, + "loss": 1.488, + "theoretical_loss": 3.62533775095089, + "tokens_seen": 1166409728 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005225115346038114, + "loss": 1.2392, + "theoretical_loss": 3.625319062315203, + "tokens_seen": 1166475264 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 435569, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3418095111846924, + "objective/train/theoretical_loss": 3.625300375023445, + "objective/train/tokens_used": 1187000800, + "theoretical_loss": 3.625300375023445, + "tokens_seen": 1166540800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005224954864593782, + "loss": 1.3418, + "theoretical_loss": 3.625300375023445, + "tokens_seen": 1166540800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005224794383149448, + "loss": 1.3552, + "theoretical_loss": 3.625281689075444, + "tokens_seen": 1166606336 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005224633901705116, + "loss": 1.3691, + "theoretical_loss": 3.6252630044710275, + "tokens_seen": 1166671872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005224473420260782, + "loss": 1.3906, + "theoretical_loss": 3.625244321210024, + "tokens_seen": 1166737408 + }, + { + "epoch": 0.35, + "learning_rate": 0.000522431293881645, + "loss": 1.3692, + "theoretical_loss": 3.625225639292261, + "tokens_seen": 1166802944 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005224152457372116, + "loss": 1.435, + "theoretical_loss": 3.6252069587175666, + "tokens_seen": 1166868480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005223991975927784, + "loss": 1.3749, + "theoretical_loss": 3.6251882794857693, + "tokens_seen": 1166934016 + }, + { + "epoch": 0.35, + "learning_rate": 0.000522383149448345, + "loss": 1.3942, + "theoretical_loss": 3.6251696015966965, + "tokens_seen": 1166999552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005223671013039118, + "loss": 1.3588, + "theoretical_loss": 3.6251509250501766, + "tokens_seen": 1167065088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005223510531594784, + "loss": 1.3952, + "theoretical_loss": 3.625132249846038, + "tokens_seen": 1167130624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005223350050150452, + "loss": 1.4272, + "theoretical_loss": 3.625113575984108, + "tokens_seen": 1167196160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005223189568706118, + "loss": 1.407, + "theoretical_loss": 3.6250949034642153, + "tokens_seen": 1167261696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005223029087261786, + "loss": 1.3664, + "theoretical_loss": 3.6250762322861885, + "tokens_seen": 1167327232 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005222868605817452, + "loss": 1.3911, + "theoretical_loss": 3.6250575624498556, + "tokens_seen": 1167392768 + }, + { + "epoch": 0.35, + "learning_rate": 0.000522270812437312, + "loss": 1.4135, + "theoretical_loss": 3.625038893955044, + "tokens_seen": 1167458304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005222547642928787, + "loss": 1.3402, + "theoretical_loss": 3.625020226801584, + "tokens_seen": 1167523840 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005222387161484454, + "loss": 1.4366, + "theoretical_loss": 3.6250015609893014, + "tokens_seen": 1167589376 + }, + { + "epoch": 0.35, + "learning_rate": 0.000522222668004012, + "loss": 1.4342, + "theoretical_loss": 3.624982896518026, + "tokens_seen": 1167654912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005222066198595788, + "loss": 1.3607, + "theoretical_loss": 3.624964233387587, + "tokens_seen": 1167720448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005221905717151455, + "loss": 1.2909, + "theoretical_loss": 3.624945571597811, + "tokens_seen": 1167785984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005221745235707122, + "loss": 1.3614, + "theoretical_loss": 3.6249269111485276, + "tokens_seen": 1167851520 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005221584754262789, + "loss": 1.4168, + "theoretical_loss": 3.624908252039565, + "tokens_seen": 1167917056 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005221424272818456, + "loss": 1.3236, + "theoretical_loss": 3.6248895942707517, + "tokens_seen": 1167982592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005221263791374123, + "loss": 1.3617, + "theoretical_loss": 3.6248709378419166, + "tokens_seen": 1168048128 + }, + { + "epoch": 0.35, + "learning_rate": 0.000522110330992979, + "loss": 1.3242, + "theoretical_loss": 3.6248522827528875, + "tokens_seen": 1168113664 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005220942828485457, + "loss": 1.3784, + "theoretical_loss": 3.624833629003494, + "tokens_seen": 1168179200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005220782347041124, + "loss": 1.4763, + "theoretical_loss": 3.624814976593564, + "tokens_seen": 1168244736 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005220621865596791, + "loss": 1.3698, + "theoretical_loss": 3.6247963255229267, + "tokens_seen": 1168310272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005220461384152458, + "loss": 1.4092, + "theoretical_loss": 3.6247776757914103, + "tokens_seen": 1168375808 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005220300902708125, + "loss": 1.392, + "theoretical_loss": 3.624759027398844, + "tokens_seen": 1168441344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005220140421263792, + "loss": 1.3645, + "theoretical_loss": 3.624740380345057, + "tokens_seen": 1168506880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005219979939819459, + "loss": 1.4336, + "theoretical_loss": 3.6247217346298766, + "tokens_seen": 1168572416 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005219819458375126, + "loss": 1.3969, + "theoretical_loss": 3.624703090253133, + "tokens_seen": 1168637952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005219658976930793, + "loss": 1.477, + "theoretical_loss": 3.6246844472146544, + "tokens_seen": 1168703488 + }, + { + "epoch": 0.35, + "learning_rate": 0.000521949849548646, + "loss": 1.4557, + "theoretical_loss": 3.62466580551427, + "tokens_seen": 1168769024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005219338014042127, + "loss": 1.4389, + "theoretical_loss": 3.624647165151809, + "tokens_seen": 1168834560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005219177532597794, + "loss": 1.251, + "theoretical_loss": 3.6246285261270996, + "tokens_seen": 1168900096 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005219017051153461, + "loss": 1.3291, + "theoretical_loss": 3.624609888439972, + "tokens_seen": 1168965632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005218856569709128, + "loss": 1.2611, + "theoretical_loss": 3.6245912520902537, + "tokens_seen": 1169031168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005218696088264795, + "loss": 1.5587, + "theoretical_loss": 3.624572617077775, + "tokens_seen": 1169096704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005218535606820462, + "loss": 1.4261, + "theoretical_loss": 3.624553983402364, + "tokens_seen": 1169162240 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005218375125376129, + "loss": 1.375, + "theoretical_loss": 3.6245353510638507, + "tokens_seen": 1169227776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005218214643931796, + "loss": 1.4087, + "theoretical_loss": 3.624516720062064, + "tokens_seen": 1169293312 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005218054162487463, + "loss": 1.4713, + "theoretical_loss": 3.624498090396833, + "tokens_seen": 1169358848 + }, + { + "epoch": 0.35, + "learning_rate": 0.000521789368104313, + "loss": 1.3495, + "theoretical_loss": 3.6244794620679865, + "tokens_seen": 1169424384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005217733199598797, + "loss": 1.377, + "theoretical_loss": 3.6244608350753547, + "tokens_seen": 1169489920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005217572718154464, + "loss": 1.4015, + "theoretical_loss": 3.6244422094187665, + "tokens_seen": 1169555456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005217412236710131, + "loss": 1.3424, + "theoretical_loss": 3.6244235850980506, + "tokens_seen": 1169620992 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005217251755265798, + "loss": 1.4101, + "theoretical_loss": 3.624404962113037, + "tokens_seen": 1169686528 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005217091273821465, + "loss": 1.3303, + "theoretical_loss": 3.6243863404635546, + "tokens_seen": 1169752064 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005216930792377132, + "loss": 1.4005, + "theoretical_loss": 3.6243677201494338, + "tokens_seen": 1169817600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005216770310932799, + "loss": 1.3779, + "theoretical_loss": 3.6243491011705027, + "tokens_seen": 1169883136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005216609829488466, + "loss": 1.4334, + "theoretical_loss": 3.6243304835265917, + "tokens_seen": 1169948672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005216449348044133, + "loss": 1.4257, + "theoretical_loss": 3.62431186721753, + "tokens_seen": 1170014208 + }, + { + "epoch": 0.35, + "learning_rate": 0.00052162888665998, + "loss": 1.308, + "theoretical_loss": 3.624293252243147, + "tokens_seen": 1170079744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005216128385155467, + "loss": 1.3864, + "theoretical_loss": 3.6242746386032723, + "tokens_seen": 1170145280 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005215967903711133, + "loss": 1.3679, + "theoretical_loss": 3.624256026297736, + "tokens_seen": 1170210816 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005215807422266801, + "loss": 1.4892, + "theoretical_loss": 3.624237415326367, + "tokens_seen": 1170276352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005215646940822467, + "loss": 1.4393, + "theoretical_loss": 3.6242188056889955, + "tokens_seen": 1170341888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005215486459378135, + "loss": 1.4487, + "theoretical_loss": 3.6242001973854507, + "tokens_seen": 1170407424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005215325977933801, + "loss": 1.4195, + "theoretical_loss": 3.6241815904155628, + "tokens_seen": 1170472960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005215165496489469, + "loss": 1.3036, + "theoretical_loss": 3.6241629847791614, + "tokens_seen": 1170538496 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005215005015045135, + "loss": 1.426, + "theoretical_loss": 3.6241443804760762, + "tokens_seen": 1170604032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005214844533600803, + "loss": 1.3862, + "theoretical_loss": 3.624125777506137, + "tokens_seen": 1170669568 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005214684052156469, + "loss": 1.4652, + "theoretical_loss": 3.624107175869174, + "tokens_seen": 1170735104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005214523570712136, + "loss": 1.3108, + "theoretical_loss": 3.6240885755650165, + "tokens_seen": 1170800640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005214363089267803, + "loss": 1.3802, + "theoretical_loss": 3.6240699765934945, + "tokens_seen": 1170866176 + }, + { + "epoch": 0.35, + "learning_rate": 0.000521420260782347, + "loss": 1.4118, + "theoretical_loss": 3.624051378954438, + "tokens_seen": 1170931712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005214042126379137, + "loss": 1.4548, + "theoretical_loss": 3.624032782647678, + "tokens_seen": 1170997248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005213881644934804, + "loss": 1.4966, + "theoretical_loss": 3.6240141876730423, + "tokens_seen": 1171062784 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005213721163490471, + "loss": 1.5511, + "theoretical_loss": 3.6239955940303634, + "tokens_seen": 1171128320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005213560682046138, + "loss": 1.3228, + "theoretical_loss": 3.6239770017194695, + "tokens_seen": 1171193856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005213400200601805, + "loss": 1.3627, + "theoretical_loss": 3.6239584107401916, + "tokens_seen": 1171259392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005213239719157472, + "loss": 1.4738, + "theoretical_loss": 3.6239398210923595, + "tokens_seen": 1171324928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005213079237713139, + "loss": 1.4142, + "theoretical_loss": 3.6239212327758037, + "tokens_seen": 1171390464 + }, + { + "epoch": 0.35, + "learning_rate": 0.0005212918756268807, + "loss": 1.4766, + "theoretical_loss": 3.623902645790354, + "tokens_seen": 1171456000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005212758274824473, + "loss": 1.2887, + "theoretical_loss": 3.6238840601358406, + "tokens_seen": 1171521536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005212597793380141, + "loss": 1.4686, + "theoretical_loss": 3.6238654758120945, + "tokens_seen": 1171587072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005212437311935807, + "loss": 1.279, + "theoretical_loss": 3.6238468928189453, + "tokens_seen": 1171652608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005212276830491475, + "loss": 1.2981, + "theoretical_loss": 3.623828311156223, + "tokens_seen": 1171718144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005212116349047141, + "loss": 1.4431, + "theoretical_loss": 3.6238097308237585, + "tokens_seen": 1171783680 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005211955867602809, + "loss": 1.3883, + "theoretical_loss": 3.623791151821382, + "tokens_seen": 1171849216 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005211955867602809, + "loss": 1.4005, + "theoretical_loss": 3.623772574148924, + "tokens_seen": 1171914752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005211795386158475, + "loss": 1.3445, + "theoretical_loss": 3.623753997806215, + "tokens_seen": 1171980288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005211634904714143, + "loss": 1.4137, + "theoretical_loss": 3.6237354227930854, + "tokens_seen": 1172045824 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005211474423269809, + "loss": 1.4238, + "theoretical_loss": 3.6237168491093654, + "tokens_seen": 1172111360 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005211313941825477, + "loss": 1.4283, + "theoretical_loss": 3.6236982767548858, + "tokens_seen": 1172176896 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005211153460381143, + "loss": 1.424, + "theoretical_loss": 3.6236797057294776, + "tokens_seen": 1172242432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005210992978936811, + "loss": 1.3256, + "theoretical_loss": 3.623661136032971, + "tokens_seen": 1172307968 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005210832497492477, + "loss": 1.3694, + "theoretical_loss": 3.623642567665196, + "tokens_seen": 1172373504 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005210672016048145, + "loss": 1.3329, + "theoretical_loss": 3.623624000625984, + "tokens_seen": 1172439040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005210511534603811, + "loss": 1.2222, + "theoretical_loss": 3.6236054349151656, + "tokens_seen": 1172504576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005210351053159479, + "loss": 1.4164, + "theoretical_loss": 3.623586870532572, + "tokens_seen": 1172570112 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005210190571715145, + "loss": 1.3525, + "theoretical_loss": 3.623568307478032, + "tokens_seen": 1172635648 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005210030090270813, + "loss": 1.2727, + "theoretical_loss": 3.623549745751379, + "tokens_seen": 1172701184 + }, + { + "epoch": 0.36, + "learning_rate": 0.000520986960882648, + "loss": 1.4545, + "theoretical_loss": 3.623531185352442, + "tokens_seen": 1172766720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005209709127382147, + "loss": 1.423, + "theoretical_loss": 3.6235126262810526, + "tokens_seen": 1172832256 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005209548645937813, + "loss": 1.4742, + "theoretical_loss": 3.623494068537042, + "tokens_seen": 1172897792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005209388164493481, + "loss": 1.5037, + "theoretical_loss": 3.6234755121202396, + "tokens_seen": 1172963328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005209227683049147, + "loss": 1.3475, + "theoretical_loss": 3.623456957030478, + "tokens_seen": 1173028864 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 437925, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4428167343139648, + "objective/train/theoretical_loss": 3.623438403267587, + "objective/train/tokens_used": 1193554400, + "theoretical_loss": 3.623438403267587, + "tokens_seen": 1173094400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005209067201604815, + "loss": 1.4428, + "theoretical_loss": 3.623438403267587, + "tokens_seen": 1173094400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005208906720160482, + "loss": 1.5029, + "theoretical_loss": 3.6234198508313984, + "tokens_seen": 1173159936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005208746238716149, + "loss": 1.353, + "theoretical_loss": 3.623401299721743, + "tokens_seen": 1173225472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005208585757271816, + "loss": 1.4643, + "theoretical_loss": 3.6233827499384517, + "tokens_seen": 1173291008 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005208425275827483, + "loss": 1.4433, + "theoretical_loss": 3.6233642014813556, + "tokens_seen": 1173356544 + }, + { + "epoch": 0.36, + "learning_rate": 0.000520826479438315, + "loss": 1.423, + "theoretical_loss": 3.6233456543502855, + "tokens_seen": 1173422080 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005208104312938817, + "loss": 1.455, + "theoretical_loss": 3.623327108545074, + "tokens_seen": 1173487616 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005207943831494484, + "loss": 1.4343, + "theoretical_loss": 3.6233085640655505, + "tokens_seen": 1173553152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005207783350050151, + "loss": 1.4573, + "theoretical_loss": 3.623290020911547, + "tokens_seen": 1173618688 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005207622868605818, + "loss": 1.3753, + "theoretical_loss": 3.6232714790828946, + "tokens_seen": 1173684224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005207462387161485, + "loss": 1.3861, + "theoretical_loss": 3.6232529385794248, + "tokens_seen": 1173749760 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005207301905717152, + "loss": 1.4334, + "theoretical_loss": 3.6232343994009684, + "tokens_seen": 1173815296 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005207141424272819, + "loss": 1.3529, + "theoretical_loss": 3.623215861547358, + "tokens_seen": 1173880832 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005206980942828486, + "loss": 1.3712, + "theoretical_loss": 3.623197325018423, + "tokens_seen": 1173946368 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005206820461384153, + "loss": 1.4668, + "theoretical_loss": 3.6231787898139967, + "tokens_seen": 1174011904 + }, + { + "epoch": 0.36, + "learning_rate": 0.000520665997993982, + "loss": 1.3188, + "theoretical_loss": 3.623160255933909, + "tokens_seen": 1174077440 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005206499498495487, + "loss": 1.4571, + "theoretical_loss": 3.6231417233779926, + "tokens_seen": 1174142976 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005206339017051154, + "loss": 1.3682, + "theoretical_loss": 3.6231231921460783, + "tokens_seen": 1174208512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005206178535606821, + "loss": 1.3279, + "theoretical_loss": 3.6231046622379974, + "tokens_seen": 1174274048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005206018054162488, + "loss": 1.2943, + "theoretical_loss": 3.6230861336535822, + "tokens_seen": 1174339584 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005205857572718155, + "loss": 1.3576, + "theoretical_loss": 3.623067606392664, + "tokens_seen": 1174405120 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005205697091273822, + "loss": 1.4647, + "theoretical_loss": 3.6230490804550737, + "tokens_seen": 1174470656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005205536609829489, + "loss": 1.4574, + "theoretical_loss": 3.6230305558406446, + "tokens_seen": 1174536192 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005205376128385156, + "loss": 1.4082, + "theoretical_loss": 3.623012032549206, + "tokens_seen": 1174601728 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005205215646940823, + "loss": 1.4176, + "theoretical_loss": 3.622993510580592, + "tokens_seen": 1174667264 + }, + { + "epoch": 0.36, + "learning_rate": 0.000520505516549649, + "loss": 1.3671, + "theoretical_loss": 3.622974989934633, + "tokens_seen": 1174732800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005204894684052157, + "loss": 1.4287, + "theoretical_loss": 3.622956470611161, + "tokens_seen": 1174798336 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005204734202607824, + "loss": 1.5065, + "theoretical_loss": 3.622937952610008, + "tokens_seen": 1174863872 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005204573721163491, + "loss": 1.5148, + "theoretical_loss": 3.6229194359310055, + "tokens_seen": 1174929408 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005204413239719158, + "loss": 1.6243, + "theoretical_loss": 3.6229009205739855, + "tokens_seen": 1174994944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005204252758274825, + "loss": 1.361, + "theoretical_loss": 3.62288240653878, + "tokens_seen": 1175060480 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005204092276830492, + "loss": 1.5018, + "theoretical_loss": 3.6228638938252216, + "tokens_seen": 1175126016 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005203931795386159, + "loss": 1.5478, + "theoretical_loss": 3.6228453824331406, + "tokens_seen": 1175191552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005203771313941826, + "loss": 1.5864, + "theoretical_loss": 3.6228268723623698, + "tokens_seen": 1175257088 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005203610832497493, + "loss": 1.4002, + "theoretical_loss": 3.6228083636127417, + "tokens_seen": 1175322624 + }, + { + "epoch": 0.36, + "learning_rate": 0.000520345035105316, + "loss": 1.5267, + "theoretical_loss": 3.622789856184088, + "tokens_seen": 1175388160 + }, + { + "epoch": 0.36, + "learning_rate": 0.000520345035105316, + "loss": 1.6949, + "theoretical_loss": 3.6227713500762406, + "tokens_seen": 1175453696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005203289869608827, + "loss": 1.829, + "theoretical_loss": 3.622752845289032, + "tokens_seen": 1175519232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005203129388164494, + "loss": 1.5658, + "theoretical_loss": 3.6227343418222935, + "tokens_seen": 1175584768 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005202968906720161, + "loss": 1.7761, + "theoretical_loss": 3.622715839675858, + "tokens_seen": 1175650304 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005202808425275828, + "loss": 1.529, + "theoretical_loss": 3.622697338849558, + "tokens_seen": 1175715840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005202647943831495, + "loss": 1.4332, + "theoretical_loss": 3.6226788393432248, + "tokens_seen": 1175781376 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005202487462387162, + "loss": 1.6543, + "theoretical_loss": 3.6226603411566916, + "tokens_seen": 1175846912 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005202326980942829, + "loss": 1.6284, + "theoretical_loss": 3.62264184428979, + "tokens_seen": 1175912448 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005202166499498496, + "loss": 1.6184, + "theoretical_loss": 3.6226233487423523, + "tokens_seen": 1175977984 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005202006018054162, + "loss": 1.5158, + "theoretical_loss": 3.6226048545142113, + "tokens_seen": 1176043520 + }, + { + "epoch": 0.36, + "learning_rate": 0.000520184553660983, + "loss": 1.5403, + "theoretical_loss": 3.622586361605199, + "tokens_seen": 1176109056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005201685055165496, + "loss": 1.4262, + "theoretical_loss": 3.6225678700151485, + "tokens_seen": 1176174592 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005201524573721164, + "loss": 1.4368, + "theoretical_loss": 3.6225493797438917, + "tokens_seen": 1176240128 + }, + { + "epoch": 0.36, + "learning_rate": 0.000520136409227683, + "loss": 1.395, + "theoretical_loss": 3.6225308907912606, + "tokens_seen": 1176305664 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005201203610832498, + "loss": 1.3425, + "theoretical_loss": 3.6225124031570886, + "tokens_seen": 1176371200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005201043129388164, + "loss": 1.4735, + "theoretical_loss": 3.622493916841208, + "tokens_seen": 1176436736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005200882647943832, + "loss": 1.4935, + "theoretical_loss": 3.6224754318434504, + "tokens_seen": 1176502272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005200722166499498, + "loss": 1.3631, + "theoretical_loss": 3.62245694816365, + "tokens_seen": 1176567808 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005200561685055166, + "loss": 1.4877, + "theoretical_loss": 3.622438465801638, + "tokens_seen": 1176633344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005200401203610832, + "loss": 1.5666, + "theoretical_loss": 3.622419984757248, + "tokens_seen": 1176698880 + }, + { + "epoch": 0.36, + "learning_rate": 0.00052002407221665, + "loss": 1.3728, + "theoretical_loss": 3.6224015050303127, + "tokens_seen": 1176764416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005200080240722166, + "loss": 1.3202, + "theoretical_loss": 3.6223830266206645, + "tokens_seen": 1176829952 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005199919759277834, + "loss": 1.5186, + "theoretical_loss": 3.622364549528136, + "tokens_seen": 1176895488 + }, + { + "epoch": 0.36, + "learning_rate": 0.00051997592778335, + "loss": 1.4075, + "theoretical_loss": 3.62234607375256, + "tokens_seen": 1176961024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005199598796389168, + "loss": 1.3541, + "theoretical_loss": 3.6223275992937696, + "tokens_seen": 1177026560 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005199438314944834, + "loss": 1.4398, + "theoretical_loss": 3.6223091261515976, + "tokens_seen": 1177092096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005199277833500502, + "loss": 1.3622, + "theoretical_loss": 3.6222906543258766, + "tokens_seen": 1177157632 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005199117352056168, + "loss": 1.4558, + "theoretical_loss": 3.6222721838164396, + "tokens_seen": 1177223168 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005198956870611836, + "loss": 1.4102, + "theoretical_loss": 3.6222537146231195, + "tokens_seen": 1177288704 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005198796389167502, + "loss": 1.4887, + "theoretical_loss": 3.62223524674575, + "tokens_seen": 1177354240 + }, + { + "epoch": 0.36, + "learning_rate": 0.000519863590772317, + "loss": 1.446, + "theoretical_loss": 3.622216780184163, + "tokens_seen": 1177419776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005198475426278836, + "loss": 1.4307, + "theoretical_loss": 3.622198314938192, + "tokens_seen": 1177485312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005198314944834504, + "loss": 1.5301, + "theoretical_loss": 3.6221798510076706, + "tokens_seen": 1177550848 + }, + { + "epoch": 0.36, + "learning_rate": 0.000519815446339017, + "loss": 1.3633, + "theoretical_loss": 3.6221613883924304, + "tokens_seen": 1177616384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005197993981945838, + "loss": 1.487, + "theoretical_loss": 3.622142927092306, + "tokens_seen": 1177681920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005197833500501504, + "loss": 1.3718, + "theoretical_loss": 3.6221244671071298, + "tokens_seen": 1177747456 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005197673019057172, + "loss": 1.4807, + "theoretical_loss": 3.6221060084367354, + "tokens_seen": 1177812992 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005197512537612838, + "loss": 1.5191, + "theoretical_loss": 3.6220875510809556, + "tokens_seen": 1177878528 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005197352056168506, + "loss": 1.4827, + "theoretical_loss": 3.6220690950396235, + "tokens_seen": 1177944064 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005197191574724172, + "loss": 1.3823, + "theoretical_loss": 3.622050640312573, + "tokens_seen": 1178009600 + }, + { + "epoch": 0.36, + "learning_rate": 0.000519703109327984, + "loss": 1.3765, + "theoretical_loss": 3.622032186899637, + "tokens_seen": 1178075136 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005196870611835506, + "loss": 1.4424, + "theoretical_loss": 3.622013734800649, + "tokens_seen": 1178140672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005196710130391174, + "loss": 1.5181, + "theoretical_loss": 3.6219952840154424, + "tokens_seen": 1178206208 + }, + { + "epoch": 0.36, + "learning_rate": 0.000519654964894684, + "loss": 1.3913, + "theoretical_loss": 3.62197683454385, + "tokens_seen": 1178271744 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005196389167502508, + "loss": 1.5082, + "theoretical_loss": 3.6219583863857068, + "tokens_seen": 1178337280 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005196228686058174, + "loss": 1.3708, + "theoretical_loss": 3.6219399395408436, + "tokens_seen": 1178402816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005196068204613842, + "loss": 1.3191, + "theoretical_loss": 3.6219214940090962, + "tokens_seen": 1178468352 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005195907723169508, + "loss": 1.5285, + "theoretical_loss": 3.6219030497902978, + "tokens_seen": 1178533888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005195747241725176, + "loss": 1.5154, + "theoretical_loss": 3.6218846068842807, + "tokens_seen": 1178599424 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005195586760280842, + "loss": 1.4441, + "theoretical_loss": 3.6218661652908795, + "tokens_seen": 1178664960 + }, + { + "epoch": 0.36, + "learning_rate": 0.000519542627883651, + "loss": 1.4105, + "theoretical_loss": 3.6218477250099275, + "tokens_seen": 1178730496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005195265797392177, + "loss": 1.4615, + "theoretical_loss": 3.621829286041258, + "tokens_seen": 1178796032 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005195105315947844, + "loss": 1.5846, + "theoretical_loss": 3.6218108483847056, + "tokens_seen": 1178861568 + }, + { + "epoch": 0.36, + "learning_rate": 0.000519494483450351, + "loss": 1.3889, + "theoretical_loss": 3.6217924120401035, + "tokens_seen": 1178927104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005194784353059178, + "loss": 1.4404, + "theoretical_loss": 3.621773977007285, + "tokens_seen": 1178992640 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005194623871614845, + "loss": 1.3594, + "theoretical_loss": 3.6217555432860844, + "tokens_seen": 1179058176 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005194463390170512, + "loss": 1.3955, + "theoretical_loss": 3.621737110876335, + "tokens_seen": 1179123712 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005194302908726179, + "loss": 1.4397, + "theoretical_loss": 3.6217186797778713, + "tokens_seen": 1179189248 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005194142427281846, + "loss": 1.4353, + "theoretical_loss": 3.6217002499905266, + "tokens_seen": 1179254784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005193981945837513, + "loss": 1.5064, + "theoretical_loss": 3.6216818215141346, + "tokens_seen": 1179320320 + }, + { + "epoch": 0.36, + "learning_rate": 0.000519382146439318, + "loss": 1.351, + "theoretical_loss": 3.62166339434853, + "tokens_seen": 1179385856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005193660982948847, + "loss": 1.4955, + "theoretical_loss": 3.6216449684935466, + "tokens_seen": 1179451392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005193500501504514, + "loss": 1.4246, + "theoretical_loss": 3.6216265439490174, + "tokens_seen": 1179516928 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005193340020060181, + "loss": 1.3993, + "theoretical_loss": 3.621608120714777, + "tokens_seen": 1179582464 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 440655, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.362579107284546, + "objective/train/theoretical_loss": 3.6215896987906597, + "objective/train/tokens_used": 1200108000, + "theoretical_loss": 3.6215896987906597, + "tokens_seen": 1179648000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005193179538615848, + "loss": 1.3626, + "theoretical_loss": 3.6215896987906597, + "tokens_seen": 1179648000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005193019057171515, + "loss": 1.3607, + "theoretical_loss": 3.621571278176499, + "tokens_seen": 1179713536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005192858575727182, + "loss": 1.4552, + "theoretical_loss": 3.6215528588721297, + "tokens_seen": 1179779072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005192698094282849, + "loss": 1.375, + "theoretical_loss": 3.6215344408773857, + "tokens_seen": 1179844608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005192537612838516, + "loss": 1.3654, + "theoretical_loss": 3.6215160241921005, + "tokens_seen": 1179910144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005192377131394183, + "loss": 1.4954, + "theoretical_loss": 3.621497608816109, + "tokens_seen": 1179975680 + }, + { + "epoch": 0.36, + "learning_rate": 0.000519221664994985, + "loss": 1.4386, + "theoretical_loss": 3.621479194749245, + "tokens_seen": 1180041216 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005192056168505517, + "loss": 1.4826, + "theoretical_loss": 3.6214607819913427, + "tokens_seen": 1180106752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005191895687061184, + "loss": 1.4643, + "theoretical_loss": 3.621442370542237, + "tokens_seen": 1180172288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005191735205616851, + "loss": 1.4602, + "theoretical_loss": 3.6214239604017617, + "tokens_seen": 1180237824 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005191574724172518, + "loss": 1.3715, + "theoretical_loss": 3.621405551569751, + "tokens_seen": 1180303360 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005191414242728185, + "loss": 1.3783, + "theoretical_loss": 3.6213871440460395, + "tokens_seen": 1180368896 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005191253761283852, + "loss": 1.4018, + "theoretical_loss": 3.6213687378304615, + "tokens_seen": 1180434432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005191093279839519, + "loss": 1.3217, + "theoretical_loss": 3.6213503329228516, + "tokens_seen": 1180499968 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005190932798395186, + "loss": 1.4061, + "theoretical_loss": 3.621331929323044, + "tokens_seen": 1180565504 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005190772316950853, + "loss": 1.3126, + "theoretical_loss": 3.621313527030873, + "tokens_seen": 1180631040 + }, + { + "epoch": 0.36, + "learning_rate": 0.000519061183550652, + "loss": 1.4515, + "theoretical_loss": 3.621295126046174, + "tokens_seen": 1180696576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005190451354062187, + "loss": 1.4094, + "theoretical_loss": 3.6212767263687806, + "tokens_seen": 1180762112 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005190290872617854, + "loss": 1.4553, + "theoretical_loss": 3.621258327998528, + "tokens_seen": 1180827648 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005190130391173521, + "loss": 1.5231, + "theoretical_loss": 3.62123993093525, + "tokens_seen": 1180893184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005189969909729188, + "loss": 1.2812, + "theoretical_loss": 3.621221535178782, + "tokens_seen": 1180958720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005189809428284855, + "loss": 1.476, + "theoretical_loss": 3.6212031407289587, + "tokens_seen": 1181024256 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005189648946840522, + "loss": 1.3703, + "theoretical_loss": 3.621184747585614, + "tokens_seen": 1181089792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005189488465396189, + "loss": 1.4658, + "theoretical_loss": 3.621166355748583, + "tokens_seen": 1181155328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005189327983951856, + "loss": 1.349, + "theoretical_loss": 3.621147965217701, + "tokens_seen": 1181220864 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005189167502507523, + "loss": 1.4338, + "theoretical_loss": 3.621129575992802, + "tokens_seen": 1181286400 + }, + { + "epoch": 0.36, + "learning_rate": 0.000518900702106319, + "loss": 1.3312, + "theoretical_loss": 3.6211111880737215, + "tokens_seen": 1181351936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005188846539618857, + "loss": 1.4763, + "theoretical_loss": 3.6210928014602937, + "tokens_seen": 1181417472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005188686058174524, + "loss": 1.3625, + "theoretical_loss": 3.621074416152354, + "tokens_seen": 1181483008 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005188525576730191, + "loss": 1.4346, + "theoretical_loss": 3.621056032149737, + "tokens_seen": 1181548544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005188365095285858, + "loss": 1.3502, + "theoretical_loss": 3.6210376494522776, + "tokens_seen": 1181614080 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005188204613841525, + "loss": 1.5139, + "theoretical_loss": 3.621019268059811, + "tokens_seen": 1181679616 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005188044132397191, + "loss": 1.3841, + "theoretical_loss": 3.621000887972172, + "tokens_seen": 1181745152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005187883650952859, + "loss": 1.3822, + "theoretical_loss": 3.6209825091891954, + "tokens_seen": 1181810688 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005187723169508525, + "loss": 1.4865, + "theoretical_loss": 3.620964131710717, + "tokens_seen": 1181876224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005187562688064193, + "loss": 1.4539, + "theoretical_loss": 3.620945755536571, + "tokens_seen": 1181941760 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005187402206619859, + "loss": 1.3335, + "theoretical_loss": 3.620927380666593, + "tokens_seen": 1182007296 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005187241725175527, + "loss": 1.4021, + "theoretical_loss": 3.620909007100618, + "tokens_seen": 1182072832 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005187081243731193, + "loss": 1.3735, + "theoretical_loss": 3.6208906348384815, + "tokens_seen": 1182138368 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005186920762286861, + "loss": 1.4509, + "theoretical_loss": 3.6208722638800177, + "tokens_seen": 1182203904 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005186760280842527, + "loss": 1.4254, + "theoretical_loss": 3.620853894225063, + "tokens_seen": 1182269440 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005186599799398195, + "loss": 1.4476, + "theoretical_loss": 3.620835525873453, + "tokens_seen": 1182334976 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005186439317953861, + "loss": 1.3198, + "theoretical_loss": 3.6208171588250213, + "tokens_seen": 1182400512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005186278836509529, + "loss": 1.3894, + "theoretical_loss": 3.6207987930796044, + "tokens_seen": 1182466048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005186118355065195, + "loss": 1.3511, + "theoretical_loss": 3.620780428637037, + "tokens_seen": 1182531584 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005185957873620863, + "loss": 1.3876, + "theoretical_loss": 3.6207620654971553, + "tokens_seen": 1182597120 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005185797392176529, + "loss": 1.3333, + "theoretical_loss": 3.620743703659794, + "tokens_seen": 1182662656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005185636910732197, + "loss": 1.3942, + "theoretical_loss": 3.620725343124789, + "tokens_seen": 1182728192 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005185476429287863, + "loss": 1.437, + "theoretical_loss": 3.620706983891975, + "tokens_seen": 1182793728 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005185315947843531, + "loss": 1.5087, + "theoretical_loss": 3.620688625961189, + "tokens_seen": 1182859264 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005185155466399197, + "loss": 1.355, + "theoretical_loss": 3.6206702693322654, + "tokens_seen": 1182924800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005184994984954865, + "loss": 1.4422, + "theoretical_loss": 3.6206519140050393, + "tokens_seen": 1182990336 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005184834503510531, + "loss": 1.324, + "theoretical_loss": 3.620633559979348, + "tokens_seen": 1183055872 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005184674022066199, + "loss": 1.358, + "theoretical_loss": 3.6206152072550255, + "tokens_seen": 1183121408 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005184513540621865, + "loss": 1.3968, + "theoretical_loss": 3.620596855831908, + "tokens_seen": 1183186944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005184353059177533, + "loss": 1.3691, + "theoretical_loss": 3.620578505709831, + "tokens_seen": 1183252480 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005184192577733199, + "loss": 1.3177, + "theoretical_loss": 3.6205601568886308, + "tokens_seen": 1183318016 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005184032096288867, + "loss": 1.4622, + "theoretical_loss": 3.6205418093681425, + "tokens_seen": 1183383552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005183871614844533, + "loss": 1.3584, + "theoretical_loss": 3.6205234631482024, + "tokens_seen": 1183449088 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005183711133400201, + "loss": 1.3804, + "theoretical_loss": 3.6205051182286456, + "tokens_seen": 1183514624 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005183550651955867, + "loss": 1.4295, + "theoretical_loss": 3.620486774609309, + "tokens_seen": 1183580160 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005183390170511535, + "loss": 1.2962, + "theoretical_loss": 3.620468432290027, + "tokens_seen": 1183645696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005183229689067201, + "loss": 1.3385, + "theoretical_loss": 3.620450091270637, + "tokens_seen": 1183711232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005183069207622869, + "loss": 1.5993, + "theoretical_loss": 3.620431751550974, + "tokens_seen": 1183776768 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005182908726178535, + "loss": 1.3925, + "theoretical_loss": 3.6204134131308736, + "tokens_seen": 1183842304 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005182748244734203, + "loss": 1.4818, + "theoretical_loss": 3.620395076010173, + "tokens_seen": 1183907840 + }, + { + "epoch": 0.36, + "learning_rate": 0.000518258776328987, + "loss": 1.4509, + "theoretical_loss": 3.6203767401887075, + "tokens_seen": 1183973376 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005182427281845537, + "loss": 1.3939, + "theoretical_loss": 3.6203584056663134, + "tokens_seen": 1184038912 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005182266800401203, + "loss": 1.5077, + "theoretical_loss": 3.620340072442826, + "tokens_seen": 1184104448 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005182106318956871, + "loss": 1.4883, + "theoretical_loss": 3.620321740518082, + "tokens_seen": 1184169984 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005181945837512537, + "loss": 1.2309, + "theoretical_loss": 3.6203034098919176, + "tokens_seen": 1184235520 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005181785356068205, + "loss": 1.394, + "theoretical_loss": 3.6202850805641695, + "tokens_seen": 1184301056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005181624874623871, + "loss": 1.442, + "theoretical_loss": 3.6202667525346723, + "tokens_seen": 1184366592 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005181464393179539, + "loss": 1.414, + "theoretical_loss": 3.6202484258032634, + "tokens_seen": 1184432128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005181303911735206, + "loss": 1.3614, + "theoretical_loss": 3.6202301003697785, + "tokens_seen": 1184497664 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005181143430290873, + "loss": 1.4078, + "theoretical_loss": 3.620211776234055, + "tokens_seen": 1184563200 + }, + { + "epoch": 0.36, + "learning_rate": 0.000518098294884654, + "loss": 1.3734, + "theoretical_loss": 3.6201934533959275, + "tokens_seen": 1184628736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005180822467402207, + "loss": 1.3415, + "theoretical_loss": 3.620175131855234, + "tokens_seen": 1184694272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005180661985957874, + "loss": 1.4248, + "theoretical_loss": 3.6201568116118095, + "tokens_seen": 1184759808 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005180501504513541, + "loss": 1.363, + "theoretical_loss": 3.6201384926654914, + "tokens_seen": 1184825344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005180341023069208, + "loss": 1.4731, + "theoretical_loss": 3.620120175016115, + "tokens_seen": 1184890880 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005180180541624875, + "loss": 1.5139, + "theoretical_loss": 3.620101858663518, + "tokens_seen": 1184956416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005180020060180542, + "loss": 1.3262, + "theoretical_loss": 3.620083543607536, + "tokens_seen": 1185021952 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005179859578736209, + "loss": 1.4694, + "theoretical_loss": 3.620065229848006, + "tokens_seen": 1185087488 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005179699097291876, + "loss": 1.3311, + "theoretical_loss": 3.620046917384764, + "tokens_seen": 1185153024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005179538615847543, + "loss": 1.3742, + "theoretical_loss": 3.620028606217647, + "tokens_seen": 1185218560 + }, + { + "epoch": 0.36, + "learning_rate": 0.000517937813440321, + "loss": 1.411, + "theoretical_loss": 3.620010296346492, + "tokens_seen": 1185284096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005179217652958877, + "loss": 1.3966, + "theoretical_loss": 3.619991987771135, + "tokens_seen": 1185349632 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005179057171514544, + "loss": 1.4269, + "theoretical_loss": 3.619973680491413, + "tokens_seen": 1185415168 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005178896690070211, + "loss": 1.3383, + "theoretical_loss": 3.619955374507162, + "tokens_seen": 1185480704 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005178736208625878, + "loss": 1.3285, + "theoretical_loss": 3.61993706981822, + "tokens_seen": 1185546240 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005178575727181545, + "loss": 1.3684, + "theoretical_loss": 3.6199187664244223, + "tokens_seen": 1185611776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005178415245737212, + "loss": 1.3121, + "theoretical_loss": 3.6199004643256067, + "tokens_seen": 1185677312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005178254764292879, + "loss": 1.3473, + "theoretical_loss": 3.6198821635216096, + "tokens_seen": 1185742848 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005178094282848546, + "loss": 1.3844, + "theoretical_loss": 3.6198638640122676, + "tokens_seen": 1185808384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005177933801404213, + "loss": 1.3941, + "theoretical_loss": 3.6198455657974185, + "tokens_seen": 1185873920 + }, + { + "epoch": 0.36, + "learning_rate": 0.000517777331995988, + "loss": 1.3645, + "theoretical_loss": 3.6198272688768984, + "tokens_seen": 1185939456 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005177612838515547, + "loss": 1.5115, + "theoretical_loss": 3.619808973250544, + "tokens_seen": 1186004992 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005177452357071214, + "loss": 1.4152, + "theoretical_loss": 3.6197906789181933, + "tokens_seen": 1186070528 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005177291875626881, + "loss": 1.4372, + "theoretical_loss": 3.619772385879682, + "tokens_seen": 1186136064 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 442936, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3713927268981934, + "objective/train/theoretical_loss": 3.6197540941348487, + "objective/train/tokens_used": 1206661600, + "theoretical_loss": 3.6197540941348487, + "tokens_seen": 1186201600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005177131394182549, + "loss": 1.3714, + "theoretical_loss": 3.6197540941348487, + "tokens_seen": 1186201600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005176970912738215, + "loss": 1.3981, + "theoretical_loss": 3.619735803683529, + "tokens_seen": 1186267136 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005176810431293883, + "loss": 1.5116, + "theoretical_loss": 3.61971751452556, + "tokens_seen": 1186332672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005176649949849549, + "loss": 1.5091, + "theoretical_loss": 3.6196992266607797, + "tokens_seen": 1186398208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005176489468405217, + "loss": 1.3583, + "theoretical_loss": 3.619680940089025, + "tokens_seen": 1186463744 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005176328986960883, + "loss": 1.3644, + "theoretical_loss": 3.6196626548101327, + "tokens_seen": 1186529280 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005176168505516551, + "loss": 1.3759, + "theoretical_loss": 3.6196443708239405, + "tokens_seen": 1186594816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005176008024072217, + "loss": 1.4095, + "theoretical_loss": 3.6196260881302846, + "tokens_seen": 1186660352 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005175847542627885, + "loss": 1.3471, + "theoretical_loss": 3.619607806729004, + "tokens_seen": 1186725888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005175687061183551, + "loss": 1.5671, + "theoretical_loss": 3.619589526619934, + "tokens_seen": 1186791424 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005175526579739219, + "loss": 1.4225, + "theoretical_loss": 3.6195712478029134, + "tokens_seen": 1186856960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005175366098294885, + "loss": 1.3393, + "theoretical_loss": 3.6195529702777787, + "tokens_seen": 1186922496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005175205616850552, + "loss": 1.5232, + "theoretical_loss": 3.6195346940443676, + "tokens_seen": 1186988032 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005175045135406219, + "loss": 1.399, + "theoretical_loss": 3.619516419102518, + "tokens_seen": 1187053568 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005174884653961886, + "loss": 1.3378, + "theoretical_loss": 3.619498145452066, + "tokens_seen": 1187119104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005174724172517553, + "loss": 1.5187, + "theoretical_loss": 3.6194798730928506, + "tokens_seen": 1187184640 + }, + { + "epoch": 0.36, + "learning_rate": 0.000517456369107322, + "loss": 1.4637, + "theoretical_loss": 3.6194616020247077, + "tokens_seen": 1187250176 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005174403209628887, + "loss": 1.383, + "theoretical_loss": 3.619443332247476, + "tokens_seen": 1187315712 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005174242728184554, + "loss": 1.4014, + "theoretical_loss": 3.6194250637609926, + "tokens_seen": 1187381248 + }, + { + "epoch": 0.36, + "learning_rate": 0.000517408224674022, + "loss": 1.4008, + "theoretical_loss": 3.6194067965650953, + "tokens_seen": 1187446784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005173921765295888, + "loss": 1.5344, + "theoretical_loss": 3.619388530659622, + "tokens_seen": 1187512320 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005173761283851554, + "loss": 1.3875, + "theoretical_loss": 3.6193702660444096, + "tokens_seen": 1187577856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005173600802407222, + "loss": 1.4081, + "theoretical_loss": 3.619352002719296, + "tokens_seen": 1187643392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005173440320962888, + "loss": 1.4457, + "theoretical_loss": 3.619333740684119, + "tokens_seen": 1187708928 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005173279839518556, + "loss": 1.431, + "theoretical_loss": 3.6193154799387166, + "tokens_seen": 1187774464 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005173119358074222, + "loss": 1.3997, + "theoretical_loss": 3.6192972204829257, + "tokens_seen": 1187840000 + }, + { + "epoch": 0.36, + "learning_rate": 0.000517295887662989, + "loss": 1.3051, + "theoretical_loss": 3.619278962316585, + "tokens_seen": 1187905536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005172798395185556, + "loss": 1.2913, + "theoretical_loss": 3.619260705439532, + "tokens_seen": 1187971072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005172637913741224, + "loss": 1.4338, + "theoretical_loss": 3.619242449851604, + "tokens_seen": 1188036608 + }, + { + "epoch": 0.36, + "learning_rate": 0.000517247743229689, + "loss": 1.3016, + "theoretical_loss": 3.61922419555264, + "tokens_seen": 1188102144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005172316950852558, + "loss": 1.468, + "theoretical_loss": 3.619205942542477, + "tokens_seen": 1188167680 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005172156469408224, + "loss": 1.3503, + "theoretical_loss": 3.6191876908209526, + "tokens_seen": 1188233216 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005171995987963892, + "loss": 1.3445, + "theoretical_loss": 3.619169440387906, + "tokens_seen": 1188298752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005171835506519558, + "loss": 1.4162, + "theoretical_loss": 3.6191511912431746, + "tokens_seen": 1188364288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005171675025075226, + "loss": 1.3811, + "theoretical_loss": 3.619132943386596, + "tokens_seen": 1188429824 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005171514543630892, + "loss": 1.3478, + "theoretical_loss": 3.6191146968180083, + "tokens_seen": 1188495360 + }, + { + "epoch": 0.36, + "learning_rate": 0.000517135406218656, + "loss": 1.5712, + "theoretical_loss": 3.61909645153725, + "tokens_seen": 1188560896 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005171193580742226, + "loss": 1.3675, + "theoretical_loss": 3.6190782075441597, + "tokens_seen": 1188626432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005171033099297894, + "loss": 1.2456, + "theoretical_loss": 3.6190599648385744, + "tokens_seen": 1188691968 + }, + { + "epoch": 0.36, + "learning_rate": 0.000517087261785356, + "loss": 1.3298, + "theoretical_loss": 3.6190417234203327, + "tokens_seen": 1188757504 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005170712136409228, + "loss": 1.4293, + "theoretical_loss": 3.6190234832892734, + "tokens_seen": 1188823040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005170551654964894, + "loss": 1.4617, + "theoretical_loss": 3.6190052444452334, + "tokens_seen": 1188888576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005170391173520562, + "loss": 1.2724, + "theoretical_loss": 3.6189870068880525, + "tokens_seen": 1188954112 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005170230692076228, + "loss": 1.3796, + "theoretical_loss": 3.6189687706175677, + "tokens_seen": 1189019648 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005170070210631896, + "loss": 1.3906, + "theoretical_loss": 3.618950535633618, + "tokens_seen": 1189085184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005169909729187562, + "loss": 1.3477, + "theoretical_loss": 3.6189323019360415, + "tokens_seen": 1189150720 + }, + { + "epoch": 0.36, + "learning_rate": 0.000516974924774323, + "loss": 1.416, + "theoretical_loss": 3.6189140695246764, + "tokens_seen": 1189216256 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005169588766298896, + "loss": 1.3515, + "theoretical_loss": 3.6188958383993617, + "tokens_seen": 1189281792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005169428284854564, + "loss": 1.3059, + "theoretical_loss": 3.618877608559935, + "tokens_seen": 1189347328 + }, + { + "epoch": 0.36, + "learning_rate": 0.000516926780341023, + "loss": 1.2535, + "theoretical_loss": 3.6188593800062354, + "tokens_seen": 1189412864 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005169107321965898, + "loss": 1.373, + "theoretical_loss": 3.618841152738101, + "tokens_seen": 1189478400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005168946840521564, + "loss": 1.3467, + "theoretical_loss": 3.618822926755371, + "tokens_seen": 1189543936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005168786359077232, + "loss": 1.2571, + "theoretical_loss": 3.618804702057883, + "tokens_seen": 1189609472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005168625877632898, + "loss": 1.4709, + "theoretical_loss": 3.6187864786454758, + "tokens_seen": 1189675008 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005168465396188566, + "loss": 1.3001, + "theoretical_loss": 3.6187682565179884, + "tokens_seen": 1189740544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005168304914744232, + "loss": 1.4543, + "theoretical_loss": 3.6187500356752595, + "tokens_seen": 1189806080 + }, + { + "epoch": 0.36, + "learning_rate": 0.00051681444332999, + "loss": 1.3061, + "theoretical_loss": 3.618731816117127, + "tokens_seen": 1189871616 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005167983951855566, + "loss": 1.3717, + "theoretical_loss": 3.6187135978434304, + "tokens_seen": 1189937152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005167823470411234, + "loss": 1.2994, + "theoretical_loss": 3.6186953808540085, + "tokens_seen": 1190002688 + }, + { + "epoch": 0.36, + "learning_rate": 0.00051676629889669, + "loss": 1.3811, + "theoretical_loss": 3.618677165148699, + "tokens_seen": 1190068224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005167502507522568, + "loss": 1.3926, + "theoretical_loss": 3.6186589507273412, + "tokens_seen": 1190133760 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005167342026078235, + "loss": 1.4964, + "theoretical_loss": 3.618640737589775, + "tokens_seen": 1190199296 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005167181544633902, + "loss": 1.3404, + "theoretical_loss": 3.6186225257358373, + "tokens_seen": 1190264832 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005167021063189569, + "loss": 1.4628, + "theoretical_loss": 3.618604315165368, + "tokens_seen": 1190330368 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005166860581745236, + "loss": 1.3521, + "theoretical_loss": 3.6185861058782063, + "tokens_seen": 1190395904 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005166700100300903, + "loss": 1.3827, + "theoretical_loss": 3.6185678978741906, + "tokens_seen": 1190461440 + }, + { + "epoch": 0.36, + "learning_rate": 0.000516653961885657, + "loss": 1.3187, + "theoretical_loss": 3.6185496911531603, + "tokens_seen": 1190526976 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005166379137412237, + "loss": 1.3905, + "theoretical_loss": 3.6185314857149535, + "tokens_seen": 1190592512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005166218655967904, + "loss": 1.3552, + "theoretical_loss": 3.61851328155941, + "tokens_seen": 1190658048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005166058174523571, + "loss": 1.3564, + "theoretical_loss": 3.6184950786863688, + "tokens_seen": 1190723584 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005165897693079238, + "loss": 1.3963, + "theoretical_loss": 3.6184768770956683, + "tokens_seen": 1190789120 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005165737211634905, + "loss": 1.3654, + "theoretical_loss": 3.6184586767871485, + "tokens_seen": 1190854656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005165576730190572, + "loss": 1.3521, + "theoretical_loss": 3.6184404777606485, + "tokens_seen": 1190920192 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005165416248746239, + "loss": 1.4025, + "theoretical_loss": 3.6184222800160057, + "tokens_seen": 1190985728 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005165255767301906, + "loss": 1.412, + "theoretical_loss": 3.6184040835530618, + "tokens_seen": 1191051264 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005165095285857574, + "loss": 1.4754, + "theoretical_loss": 3.6183858883716544, + "tokens_seen": 1191116800 + }, + { + "epoch": 0.36, + "learning_rate": 0.000516493480441324, + "loss": 1.3623, + "theoretical_loss": 3.6183676944716234, + "tokens_seen": 1191182336 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005164774322968908, + "loss": 1.3409, + "theoretical_loss": 3.6183495018528076, + "tokens_seen": 1191247872 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005164613841524574, + "loss": 1.289, + "theoretical_loss": 3.618331310515047, + "tokens_seen": 1191313408 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005164453360080242, + "loss": 1.221, + "theoretical_loss": 3.61831312045818, + "tokens_seen": 1191378944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005164292878635908, + "loss": 1.3637, + "theoretical_loss": 3.6182949316820467, + "tokens_seen": 1191444480 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005164132397191576, + "loss": 1.3728, + "theoretical_loss": 3.6182767441864856, + "tokens_seen": 1191510016 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005163971915747242, + "loss": 1.3939, + "theoretical_loss": 3.6182585579713376, + "tokens_seen": 1191575552 + }, + { + "epoch": 0.36, + "learning_rate": 0.000516381143430291, + "loss": 1.3726, + "theoretical_loss": 3.618240373036441, + "tokens_seen": 1191641088 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005163650952858576, + "loss": 1.4135, + "theoretical_loss": 3.618222189381635, + "tokens_seen": 1191706624 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005163490471414244, + "loss": 1.4685, + "theoretical_loss": 3.6182040070067605, + "tokens_seen": 1191772160 + }, + { + "epoch": 0.36, + "learning_rate": 0.000516332998996991, + "loss": 1.4498, + "theoretical_loss": 3.618185825911656, + "tokens_seen": 1191837696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005163169508525578, + "loss": 1.5509, + "theoretical_loss": 3.6181676460961607, + "tokens_seen": 1191903232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005163009027081244, + "loss": 1.4495, + "theoretical_loss": 3.618149467560115, + "tokens_seen": 1191968768 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005162848545636912, + "loss": 1.4142, + "theoretical_loss": 3.6181312903033587, + "tokens_seen": 1192034304 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005162688064192578, + "loss": 1.3122, + "theoretical_loss": 3.6181131143257304, + "tokens_seen": 1192099840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005162527582748246, + "loss": 1.4759, + "theoretical_loss": 3.6180949396270705, + "tokens_seen": 1192165376 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005162367101303912, + "loss": 1.4818, + "theoretical_loss": 3.618076766207219, + "tokens_seen": 1192230912 + }, + { + "epoch": 0.36, + "learning_rate": 0.000516220661985958, + "loss": 1.4199, + "theoretical_loss": 3.6180585940660146, + "tokens_seen": 1192296448 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005162046138415246, + "loss": 1.6053, + "theoretical_loss": 3.618040423203298, + "tokens_seen": 1192361984 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005161885656970914, + "loss": 1.5464, + "theoretical_loss": 3.618022253618909, + "tokens_seen": 1192427520 + }, + { + "epoch": 0.36, + "learning_rate": 0.000516172517552658, + "loss": 1.371, + "theoretical_loss": 3.6180040853126862, + "tokens_seen": 1192493056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005161564694082248, + "loss": 1.3982, + "theoretical_loss": 3.6179859182844707, + "tokens_seen": 1192558592 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005161404212637914, + "loss": 1.3586, + "theoretical_loss": 3.6179677525341027, + "tokens_seen": 1192624128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005161243731193581, + "loss": 1.3322, + "theoretical_loss": 3.6179495880614203, + "tokens_seen": 1192689664 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 445326, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3727704286575317, + "objective/train/theoretical_loss": 3.6179314248662653, + "objective/train/tokens_used": 1213215200, + "theoretical_loss": 3.6179314248662653, + "tokens_seen": 1192755200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005161083249749248, + "loss": 1.3728, + "theoretical_loss": 3.6179314248662653, + "tokens_seen": 1192755200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005160922768304915, + "loss": 1.2939, + "theoretical_loss": 3.617913262948477, + "tokens_seen": 1192820736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005160762286860582, + "loss": 1.3699, + "theoretical_loss": 3.6178951023078945, + "tokens_seen": 1192886272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005160601805416249, + "loss": 1.3708, + "theoretical_loss": 3.6178769429443594, + "tokens_seen": 1192951808 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005160441323971916, + "loss": 1.3813, + "theoretical_loss": 3.6178587848577104, + "tokens_seen": 1193017344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005160280842527583, + "loss": 1.4361, + "theoretical_loss": 3.6178406280477886, + "tokens_seen": 1193082880 + }, + { + "epoch": 0.36, + "learning_rate": 0.000516012036108325, + "loss": 1.3714, + "theoretical_loss": 3.6178224725144332, + "tokens_seen": 1193148416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005159959879638917, + "loss": 1.5637, + "theoretical_loss": 3.6178043182574857, + "tokens_seen": 1193213952 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005159799398194583, + "loss": 1.3846, + "theoretical_loss": 3.617786165276785, + "tokens_seen": 1193279488 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005159638916750251, + "loss": 1.4255, + "theoretical_loss": 3.617768013572171, + "tokens_seen": 1193345024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005159478435305917, + "loss": 1.5366, + "theoretical_loss": 3.6177498631434846, + "tokens_seen": 1193410560 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005159317953861585, + "loss": 1.3325, + "theoretical_loss": 3.6177317139905667, + "tokens_seen": 1193476096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005159157472417251, + "loss": 1.4048, + "theoretical_loss": 3.617713566113257, + "tokens_seen": 1193541632 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005158996990972919, + "loss": 1.4334, + "theoretical_loss": 3.617695419511395, + "tokens_seen": 1193607168 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005158836509528585, + "loss": 1.4401, + "theoretical_loss": 3.617677274184822, + "tokens_seen": 1193672704 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005158676028084253, + "loss": 1.4731, + "theoretical_loss": 3.6176591301333785, + "tokens_seen": 1193738240 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005158515546639919, + "loss": 1.3793, + "theoretical_loss": 3.6176409873569044, + "tokens_seen": 1193803776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005158355065195587, + "loss": 1.3539, + "theoretical_loss": 3.61762284585524, + "tokens_seen": 1193869312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005158194583751253, + "loss": 1.3803, + "theoretical_loss": 3.617604705628226, + "tokens_seen": 1193934848 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005158034102306921, + "loss": 1.3335, + "theoretical_loss": 3.617586566675703, + "tokens_seen": 1194000384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005157873620862587, + "loss": 1.4143, + "theoretical_loss": 3.6175684289975116, + "tokens_seen": 1194065920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005157713139418255, + "loss": 1.3136, + "theoretical_loss": 3.6175502925934917, + "tokens_seen": 1194131456 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005157552657973921, + "loss": 1.3241, + "theoretical_loss": 3.6175321574634847, + "tokens_seen": 1194196992 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005157392176529589, + "loss": 1.3348, + "theoretical_loss": 3.6175140236073307, + "tokens_seen": 1194262528 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005157231695085255, + "loss": 1.3139, + "theoretical_loss": 3.6174958910248707, + "tokens_seen": 1194328064 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005157071213640923, + "loss": 1.3932, + "theoretical_loss": 3.6174777597159444, + "tokens_seen": 1194393600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005156910732196589, + "loss": 1.3691, + "theoretical_loss": 3.6174596296803934, + "tokens_seen": 1194459136 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005156750250752257, + "loss": 1.3933, + "theoretical_loss": 3.6174415009180585, + "tokens_seen": 1194524672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005156589769307923, + "loss": 1.3013, + "theoretical_loss": 3.61742337342878, + "tokens_seen": 1194590208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005156429287863591, + "loss": 1.4073, + "theoretical_loss": 3.6174052472123983, + "tokens_seen": 1194655744 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005156268806419257, + "loss": 1.3229, + "theoretical_loss": 3.617387122268755, + "tokens_seen": 1194721280 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005156108324974925, + "loss": 1.2123, + "theoretical_loss": 3.6173689985976907, + "tokens_seen": 1194786816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005155947843530591, + "loss": 1.4739, + "theoretical_loss": 3.617350876199046, + "tokens_seen": 1194852352 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005155787362086259, + "loss": 1.3076, + "theoretical_loss": 3.6173327550726624, + "tokens_seen": 1194917888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005155626880641925, + "loss": 1.2393, + "theoretical_loss": 3.6173146352183796, + "tokens_seen": 1194983424 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005155466399197593, + "loss": 1.3974, + "theoretical_loss": 3.6172965166360402, + "tokens_seen": 1195048960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005155305917753259, + "loss": 1.3517, + "theoretical_loss": 3.6172783993254836, + "tokens_seen": 1195114496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005155145436308927, + "loss": 1.3545, + "theoretical_loss": 3.6172602832865515, + "tokens_seen": 1195180032 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005154984954864593, + "loss": 1.2423, + "theoretical_loss": 3.617242168519085, + "tokens_seen": 1195245568 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005154824473420261, + "loss": 1.3504, + "theoretical_loss": 3.617224055022925, + "tokens_seen": 1195311104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005154663991975927, + "loss": 1.2781, + "theoretical_loss": 3.617205942797913, + "tokens_seen": 1195376640 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005154503510531595, + "loss": 1.3687, + "theoretical_loss": 3.6171878318438893, + "tokens_seen": 1195442176 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005154343029087261, + "loss": 1.3069, + "theoretical_loss": 3.6171697221606953, + "tokens_seen": 1195507712 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005154182547642929, + "loss": 1.3191, + "theoretical_loss": 3.617151613748173, + "tokens_seen": 1195573248 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005154022066198596, + "loss": 1.4188, + "theoretical_loss": 3.6171335066061623, + "tokens_seen": 1195638784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005153861584754263, + "loss": 1.4264, + "theoretical_loss": 3.6171154007345057, + "tokens_seen": 1195704320 + }, + { + "epoch": 0.36, + "learning_rate": 0.000515370110330993, + "loss": 1.2362, + "theoretical_loss": 3.617097296133043, + "tokens_seen": 1195769856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005153540621865597, + "loss": 1.2405, + "theoretical_loss": 3.617079192801617, + "tokens_seen": 1195835392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005153380140421264, + "loss": 1.5178, + "theoretical_loss": 3.6170610907400684, + "tokens_seen": 1195900928 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005153219658976931, + "loss": 1.4033, + "theoretical_loss": 3.617042989948238, + "tokens_seen": 1195966464 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005153059177532599, + "loss": 1.314, + "theoretical_loss": 3.617024890425968, + "tokens_seen": 1196032000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005152898696088265, + "loss": 1.2934, + "theoretical_loss": 3.617006792173099, + "tokens_seen": 1196097536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005152738214643933, + "loss": 1.5188, + "theoretical_loss": 3.616988695189473, + "tokens_seen": 1196163072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005152577733199599, + "loss": 1.3939, + "theoretical_loss": 3.6169705994749317, + "tokens_seen": 1196228608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005152417251755267, + "loss": 1.343, + "theoretical_loss": 3.6169525050293156, + "tokens_seen": 1196294144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005152256770310933, + "loss": 1.4617, + "theoretical_loss": 3.6169344118524673, + "tokens_seen": 1196359680 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005152096288866601, + "loss": 1.2447, + "theoretical_loss": 3.616916319944228, + "tokens_seen": 1196425216 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005151935807422267, + "loss": 1.4626, + "theoretical_loss": 3.6168982293044385, + "tokens_seen": 1196490752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005151775325977935, + "loss": 1.478, + "theoretical_loss": 3.6168801399329413, + "tokens_seen": 1196556288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005151614844533601, + "loss": 1.4129, + "theoretical_loss": 3.6168620518295778, + "tokens_seen": 1196621824 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005151454363089269, + "loss": 1.3259, + "theoretical_loss": 3.6168439649941897, + "tokens_seen": 1196687360 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005151293881644935, + "loss": 1.3122, + "theoretical_loss": 3.616825879426618, + "tokens_seen": 1196752896 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005151133400200603, + "loss": 1.3562, + "theoretical_loss": 3.6168077951267055, + "tokens_seen": 1196818432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005150972918756269, + "loss": 1.3796, + "theoretical_loss": 3.6167897120942936, + "tokens_seen": 1196883968 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005150812437311937, + "loss": 1.3033, + "theoretical_loss": 3.6167716303292234, + "tokens_seen": 1196949504 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005150651955867603, + "loss": 1.4483, + "theoretical_loss": 3.616753549831337, + "tokens_seen": 1197015040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005150491474423271, + "loss": 1.504, + "theoretical_loss": 3.616735470600477, + "tokens_seen": 1197080576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005150330992978937, + "loss": 1.4366, + "theoretical_loss": 3.6167173926364846, + "tokens_seen": 1197146112 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005150170511534605, + "loss": 1.351, + "theoretical_loss": 3.6166993159392016, + "tokens_seen": 1197211648 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005150010030090271, + "loss": 1.4072, + "theoretical_loss": 3.61668124050847, + "tokens_seen": 1197277184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005149849548645939, + "loss": 1.2802, + "theoretical_loss": 3.6166631663441318, + "tokens_seen": 1197342720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005149689067201605, + "loss": 1.3054, + "theoretical_loss": 3.616645093446029, + "tokens_seen": 1197408256 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005149528585757273, + "loss": 1.2776, + "theoretical_loss": 3.6166270218140033, + "tokens_seen": 1197473792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005149368104312939, + "loss": 1.2992, + "theoretical_loss": 3.6166089514478976, + "tokens_seen": 1197539328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005149207622868607, + "loss": 1.3459, + "theoretical_loss": 3.6165908823475528, + "tokens_seen": 1197604864 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005149047141424273, + "loss": 1.3215, + "theoretical_loss": 3.6165728145128115, + "tokens_seen": 1197670400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005148886659979941, + "loss": 1.2881, + "theoretical_loss": 3.616554747943516, + "tokens_seen": 1197735936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005148726178535607, + "loss": 1.3406, + "theoretical_loss": 3.616536682639508, + "tokens_seen": 1197801472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005148565697091275, + "loss": 1.3337, + "theoretical_loss": 3.6165186186006304, + "tokens_seen": 1197867008 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005148405215646941, + "loss": 1.2754, + "theoretical_loss": 3.6165005558267245, + "tokens_seen": 1197932544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005148244734202609, + "loss": 1.354, + "theoretical_loss": 3.616482494317633, + "tokens_seen": 1197998080 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005148084252758275, + "loss": 1.3207, + "theoretical_loss": 3.6164644340731984, + "tokens_seen": 1198063616 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005147923771313943, + "loss": 1.2963, + "theoretical_loss": 3.616446375093262, + "tokens_seen": 1198129152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005147763289869609, + "loss": 1.3272, + "theoretical_loss": 3.6164283173776672, + "tokens_seen": 1198194688 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005147602808425277, + "loss": 1.3673, + "theoretical_loss": 3.616410260926256, + "tokens_seen": 1198260224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005147442326980943, + "loss": 1.3682, + "theoretical_loss": 3.6163922057388707, + "tokens_seen": 1198325760 + }, + { + "epoch": 0.36, + "learning_rate": 0.000514728184553661, + "loss": 1.338, + "theoretical_loss": 3.6163741518153536, + "tokens_seen": 1198391296 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005147121364092277, + "loss": 1.2458, + "theoretical_loss": 3.6163560991555466, + "tokens_seen": 1198456832 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005146960882647944, + "loss": 1.3576, + "theoretical_loss": 3.6163380477592932, + "tokens_seen": 1198522368 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005146800401203611, + "loss": 1.3366, + "theoretical_loss": 3.6163199976264355, + "tokens_seen": 1198587904 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005146639919759278, + "loss": 1.2164, + "theoretical_loss": 3.6163019487568158, + "tokens_seen": 1198653440 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005146479438314945, + "loss": 1.3945, + "theoretical_loss": 3.6162839011502763, + "tokens_seen": 1198718976 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005146318956870612, + "loss": 1.4475, + "theoretical_loss": 3.6162658548066604, + "tokens_seen": 1198784512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005146158475426279, + "loss": 1.4561, + "theoretical_loss": 3.6162478097258104, + "tokens_seen": 1198850048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005145997993981946, + "loss": 1.2794, + "theoretical_loss": 3.6162297659075686, + "tokens_seen": 1198915584 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005145837512537612, + "loss": 1.3897, + "theoretical_loss": 3.616211723351778, + "tokens_seen": 1198981120 + }, + { + "epoch": 0.36, + "learning_rate": 0.000514567703109328, + "loss": 1.3955, + "theoretical_loss": 3.616193682058281, + "tokens_seen": 1199046656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005145516549648946, + "loss": 1.3375, + "theoretical_loss": 3.6161756420269207, + "tokens_seen": 1199112192 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005145356068204614, + "loss": 1.2683, + "theoretical_loss": 3.6161576032575393, + "tokens_seen": 1199177728 + }, + { + "epoch": 0.36, + "learning_rate": 0.000514519558676028, + "loss": 1.377, + "theoretical_loss": 3.6161395657499797, + "tokens_seen": 1199243264 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 447636, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.344670057296753, + "objective/train/theoretical_loss": 3.6161215295040847, + "objective/train/tokens_used": 1219768800, + "theoretical_loss": 3.6161215295040847, + "tokens_seen": 1199308800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005145035105315948, + "loss": 1.3447, + "theoretical_loss": 3.6161215295040847, + "tokens_seen": 1199308800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005144874623871614, + "loss": 1.3569, + "theoretical_loss": 3.6161034945196975, + "tokens_seen": 1199374336 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005144714142427282, + "loss": 1.3937, + "theoretical_loss": 3.6160854607966604, + "tokens_seen": 1199439872 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005144553660982948, + "loss": 1.2506, + "theoretical_loss": 3.6160674283348166, + "tokens_seen": 1199505408 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005144393179538616, + "loss": 1.3619, + "theoretical_loss": 3.616049397134009, + "tokens_seen": 1199570944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005144232698094282, + "loss": 1.3653, + "theoretical_loss": 3.6160313671940805, + "tokens_seen": 1199636480 + }, + { + "epoch": 0.36, + "learning_rate": 0.000514407221664995, + "loss": 1.3525, + "theoretical_loss": 3.6160133385148736, + "tokens_seen": 1199702016 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005143911735205616, + "loss": 1.103, + "theoretical_loss": 3.615995311096232, + "tokens_seen": 1199767552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005143751253761284, + "loss": 1.353, + "theoretical_loss": 3.6159772849379985, + "tokens_seen": 1199833088 + }, + { + "epoch": 0.36, + "learning_rate": 0.000514359077231695, + "loss": 1.3905, + "theoretical_loss": 3.6159592600400163, + "tokens_seen": 1199898624 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005143430290872618, + "loss": 1.2776, + "theoretical_loss": 3.6159412364021275, + "tokens_seen": 1199964160 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005143269809428284, + "loss": 1.3529, + "theoretical_loss": 3.6159232140241766, + "tokens_seen": 1200029696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005143109327983952, + "loss": 1.3878, + "theoretical_loss": 3.6159051929060055, + "tokens_seen": 1200095232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005142948846539618, + "loss": 1.3373, + "theoretical_loss": 3.615887173047458, + "tokens_seen": 1200160768 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005142788365095286, + "loss": 1.2653, + "theoretical_loss": 3.6158691544483776, + "tokens_seen": 1200226304 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005142627883650952, + "loss": 1.4508, + "theoretical_loss": 3.6158511371086064, + "tokens_seen": 1200291840 + }, + { + "epoch": 0.36, + "learning_rate": 0.000514246740220662, + "loss": 1.3538, + "theoretical_loss": 3.615833121027989, + "tokens_seen": 1200357376 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005142306920762286, + "loss": 1.3886, + "theoretical_loss": 3.6158151062063673, + "tokens_seen": 1200422912 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005142146439317954, + "loss": 1.2998, + "theoretical_loss": 3.6157970926435854, + "tokens_seen": 1200488448 + }, + { + "epoch": 0.36, + "learning_rate": 0.000514198595787362, + "loss": 1.4366, + "theoretical_loss": 3.615779080339487, + "tokens_seen": 1200553984 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005141825476429288, + "loss": 1.2981, + "theoretical_loss": 3.615761069293914, + "tokens_seen": 1200619520 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005141664994984954, + "loss": 1.3693, + "theoretical_loss": 3.6157430595067113, + "tokens_seen": 1200685056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005141504513540622, + "loss": 1.3256, + "theoretical_loss": 3.615725050977722, + "tokens_seen": 1200750592 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005141344032096288, + "loss": 1.2619, + "theoretical_loss": 3.6157070437067884, + "tokens_seen": 1200816128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005141183550651956, + "loss": 1.3202, + "theoretical_loss": 3.6156890376937554, + "tokens_seen": 1200881664 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005141023069207624, + "loss": 1.3684, + "theoretical_loss": 3.615671032938466, + "tokens_seen": 1200947200 + }, + { + "epoch": 0.36, + "learning_rate": 0.000514086258776329, + "loss": 1.337, + "theoretical_loss": 3.615653029440763, + "tokens_seen": 1201012736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005140702106318958, + "loss": 1.3344, + "theoretical_loss": 3.615635027200491, + "tokens_seen": 1201078272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005140541624874624, + "loss": 1.4457, + "theoretical_loss": 3.6156170262174934, + "tokens_seen": 1201143808 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005140381143430292, + "loss": 1.2638, + "theoretical_loss": 3.615599026491613, + "tokens_seen": 1201209344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005140220661985958, + "loss": 1.383, + "theoretical_loss": 3.615581028022694, + "tokens_seen": 1201274880 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005140060180541626, + "loss": 1.3202, + "theoretical_loss": 3.61556303081058, + "tokens_seen": 1201340416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005139899699097292, + "loss": 1.4487, + "theoretical_loss": 3.6155450348551152, + "tokens_seen": 1201405952 + }, + { + "epoch": 0.36, + "learning_rate": 0.000513973921765296, + "loss": 1.3527, + "theoretical_loss": 3.6155270401561426, + "tokens_seen": 1201471488 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005139578736208626, + "loss": 1.2265, + "theoretical_loss": 3.615509046713506, + "tokens_seen": 1201537024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005139418254764294, + "loss": 1.2858, + "theoretical_loss": 3.615491054527049, + "tokens_seen": 1201602560 + }, + { + "epoch": 0.36, + "learning_rate": 0.000513925777331996, + "loss": 1.4191, + "theoretical_loss": 3.615473063596616, + "tokens_seen": 1201668096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005139097291875628, + "loss": 1.3372, + "theoretical_loss": 3.6154550739220506, + "tokens_seen": 1201733632 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005138936810431294, + "loss": 1.3714, + "theoretical_loss": 3.6154370855031965, + "tokens_seen": 1201799168 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005138776328986962, + "loss": 1.3224, + "theoretical_loss": 3.6154190983398977, + "tokens_seen": 1201864704 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005138615847542628, + "loss": 1.2942, + "theoretical_loss": 3.6154011124319982, + "tokens_seen": 1201930240 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005138455366098296, + "loss": 1.3391, + "theoretical_loss": 3.615383127779342, + "tokens_seen": 1201995776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005138294884653962, + "loss": 1.2574, + "theoretical_loss": 3.6153651443817725, + "tokens_seen": 1202061312 + }, + { + "epoch": 0.36, + "learning_rate": 0.000513813440320963, + "loss": 1.2692, + "theoretical_loss": 3.615347162239134, + "tokens_seen": 1202126848 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005137973921765296, + "loss": 1.3313, + "theoretical_loss": 3.615329181351271, + "tokens_seen": 1202192384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005137813440320964, + "loss": 1.3857, + "theoretical_loss": 3.6153112017180264, + "tokens_seen": 1202257920 + }, + { + "epoch": 0.36, + "learning_rate": 0.000513765295887663, + "loss": 1.3604, + "theoretical_loss": 3.615293223339246, + "tokens_seen": 1202323456 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005137492477432298, + "loss": 1.3419, + "theoretical_loss": 3.6152752462147726, + "tokens_seen": 1202388992 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005137331995987964, + "loss": 1.3215, + "theoretical_loss": 3.6152572703444505, + "tokens_seen": 1202454528 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005137171514543632, + "loss": 1.3028, + "theoretical_loss": 3.6152392957281236, + "tokens_seen": 1202520064 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005137011033099298, + "loss": 1.4154, + "theoretical_loss": 3.6152213223656373, + "tokens_seen": 1202585600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005136850551654966, + "loss": 1.3399, + "theoretical_loss": 3.6152033502568344, + "tokens_seen": 1202651136 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005136690070210632, + "loss": 1.3774, + "theoretical_loss": 3.6151853794015603, + "tokens_seen": 1202716672 + }, + { + "epoch": 0.36, + "learning_rate": 0.00051365295887663, + "loss": 1.3511, + "theoretical_loss": 3.615167409799658, + "tokens_seen": 1202782208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005136369107321966, + "loss": 1.4236, + "theoretical_loss": 3.6151494414509733, + "tokens_seen": 1202847744 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005136208625877634, + "loss": 1.2626, + "theoretical_loss": 3.6151314743553495, + "tokens_seen": 1202913280 + }, + { + "epoch": 0.36, + "learning_rate": 0.00051360481444333, + "loss": 1.3314, + "theoretical_loss": 3.615113508512631, + "tokens_seen": 1202978816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005135887662988968, + "loss": 1.3153, + "theoretical_loss": 3.615095543922662, + "tokens_seen": 1203044352 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005135727181544634, + "loss": 1.4373, + "theoretical_loss": 3.6150775805852877, + "tokens_seen": 1203109888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005135566700100302, + "loss": 1.389, + "theoretical_loss": 3.6150596185003527, + "tokens_seen": 1203175424 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005135406218655968, + "loss": 1.3606, + "theoretical_loss": 3.6150416576677005, + "tokens_seen": 1203240960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005135245737211636, + "loss": 1.3579, + "theoretical_loss": 3.6150236980871755, + "tokens_seen": 1203306496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005135085255767302, + "loss": 1.3216, + "theoretical_loss": 3.6150057397586233, + "tokens_seen": 1203372032 + }, + { + "epoch": 0.36, + "learning_rate": 0.000513492477432297, + "loss": 1.4349, + "theoretical_loss": 3.6149877826818875, + "tokens_seen": 1203437568 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005134764292878636, + "loss": 1.362, + "theoretical_loss": 3.6149698268568136, + "tokens_seen": 1203503104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005134603811434304, + "loss": 1.4336, + "theoretical_loss": 3.614951872283245, + "tokens_seen": 1203568640 + }, + { + "epoch": 0.36, + "learning_rate": 0.000513444332998997, + "loss": 1.5446, + "theoretical_loss": 3.614933918961027, + "tokens_seen": 1203634176 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005134282848545638, + "loss": 1.3746, + "theoretical_loss": 3.614915966890005, + "tokens_seen": 1203699712 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005134122367101304, + "loss": 1.2867, + "theoretical_loss": 3.614898016070022, + "tokens_seen": 1203765248 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005133961885656972, + "loss": 1.2292, + "theoretical_loss": 3.614880066500924, + "tokens_seen": 1203830784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005133801404212638, + "loss": 1.2721, + "theoretical_loss": 3.6148621181825558, + "tokens_seen": 1203896320 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005133640922768306, + "loss": 1.345, + "theoretical_loss": 3.614844171114761, + "tokens_seen": 1203961856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005133480441323972, + "loss": 1.4352, + "theoretical_loss": 3.614826225297386, + "tokens_seen": 1204027392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005133319959879639, + "loss": 1.2752, + "theoretical_loss": 3.614808280730274, + "tokens_seen": 1204092928 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005133159478435306, + "loss": 1.3784, + "theoretical_loss": 3.614790337413271, + "tokens_seen": 1204158464 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005132998996990973, + "loss": 1.3438, + "theoretical_loss": 3.6147723953462214, + "tokens_seen": 1204224000 + }, + { + "epoch": 0.36, + "learning_rate": 0.000513283851554664, + "loss": 1.3465, + "theoretical_loss": 3.61475445452897, + "tokens_seen": 1204289536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005132678034102307, + "loss": 1.3385, + "theoretical_loss": 3.614736514961362, + "tokens_seen": 1204355072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005132517552657974, + "loss": 1.3752, + "theoretical_loss": 3.6147185766432433, + "tokens_seen": 1204420608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0005132357071213641, + "loss": 1.2697, + "theoretical_loss": 3.6147006395744565, + "tokens_seen": 1204486144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005132196589769308, + "loss": 1.3649, + "theoretical_loss": 3.6146827037548492, + "tokens_seen": 1204551680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005132036108324975, + "loss": 1.3273, + "theoretical_loss": 3.6146647691842646, + "tokens_seen": 1204617216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005131875626880641, + "loss": 1.3027, + "theoretical_loss": 3.6146468358625485, + "tokens_seen": 1204682752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005131715145436309, + "loss": 1.4179, + "theoretical_loss": 3.614628903789546, + "tokens_seen": 1204748288 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005131554663991975, + "loss": 1.2765, + "theoretical_loss": 3.614610972965102, + "tokens_seen": 1204813824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005131394182547643, + "loss": 1.4689, + "theoretical_loss": 3.6145930433890623, + "tokens_seen": 1204879360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005131233701103309, + "loss": 1.3666, + "theoretical_loss": 3.6145751150612715, + "tokens_seen": 1204944896 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005131073219658977, + "loss": 1.4836, + "theoretical_loss": 3.6145571879815748, + "tokens_seen": 1205010432 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005130912738214643, + "loss": 1.2944, + "theoretical_loss": 3.614539262149818, + "tokens_seen": 1205075968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005130752256770311, + "loss": 1.3396, + "theoretical_loss": 3.6145213375658454, + "tokens_seen": 1205141504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005130591775325977, + "loss": 1.3665, + "theoretical_loss": 3.614503414229503, + "tokens_seen": 1205207040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005130431293881645, + "loss": 1.2628, + "theoretical_loss": 3.614485492140636, + "tokens_seen": 1205272576 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005130270812437311, + "loss": 1.271, + "theoretical_loss": 3.61446757129909, + "tokens_seen": 1205338112 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005130110330992979, + "loss": 1.2272, + "theoretical_loss": 3.6144496517047098, + "tokens_seen": 1205403648 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005129949849548645, + "loss": 1.4757, + "theoretical_loss": 3.6144317333573412, + "tokens_seen": 1205469184 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005129789368104313, + "loss": 1.4922, + "theoretical_loss": 3.6144138162568296, + "tokens_seen": 1205534720 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005129628886659979, + "loss": 1.3725, + "theoretical_loss": 3.6143959004030206, + "tokens_seen": 1205600256 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005129468405215647, + "loss": 1.3052, + "theoretical_loss": 3.614377985795759, + "tokens_seen": 1205665792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005129307923771313, + "loss": 1.2023, + "theoretical_loss": 3.6143600724348914, + "tokens_seen": 1205731328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005129147442326981, + "loss": 1.3916, + "theoretical_loss": 3.614342160320262, + "tokens_seen": 1205796864 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 449979, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3932790756225586, + "objective/train/theoretical_loss": 3.614324249451718, + "objective/train/tokens_used": 1226322400, + "theoretical_loss": 3.614324249451718, + "tokens_seen": 1205862400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005128986960882648, + "loss": 1.3933, + "theoretical_loss": 3.614324249451718, + "tokens_seen": 1205862400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005128826479438315, + "loss": 1.324, + "theoretical_loss": 3.6143063398291035, + "tokens_seen": 1205927936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005128665997993983, + "loss": 1.5281, + "theoretical_loss": 3.6142884314522647, + "tokens_seen": 1205993472 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005128505516549649, + "loss": 1.4008, + "theoretical_loss": 3.6142705243210473, + "tokens_seen": 1206059008 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005128345035105317, + "loss": 1.3898, + "theoretical_loss": 3.614252618435297, + "tokens_seen": 1206124544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005128184553660983, + "loss": 1.4352, + "theoretical_loss": 3.6142347137948603, + "tokens_seen": 1206190080 + }, + { + "epoch": 0.37, + "learning_rate": 0.000512802407221665, + "loss": 1.3508, + "theoretical_loss": 3.614216810399581, + "tokens_seen": 1206255616 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005127863590772317, + "loss": 1.3035, + "theoretical_loss": 3.614198908249307, + "tokens_seen": 1206321152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005127703109327985, + "loss": 1.2496, + "theoretical_loss": 3.6141810073438823, + "tokens_seen": 1206386688 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005127542627883651, + "loss": 1.2472, + "theoretical_loss": 3.6141631076831544, + "tokens_seen": 1206452224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005127382146439319, + "loss": 1.558, + "theoretical_loss": 3.6141452092669675, + "tokens_seen": 1206517760 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005127221664994985, + "loss": 1.3121, + "theoretical_loss": 3.6141273120951682, + "tokens_seen": 1206583296 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005127061183550653, + "loss": 1.4056, + "theoretical_loss": 3.6141094161676026, + "tokens_seen": 1206648832 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005126900702106319, + "loss": 1.4209, + "theoretical_loss": 3.614091521484117, + "tokens_seen": 1206714368 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005126740220661987, + "loss": 1.3715, + "theoretical_loss": 3.6140736280445562, + "tokens_seen": 1206779904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005126579739217653, + "loss": 1.3043, + "theoretical_loss": 3.6140557358487673, + "tokens_seen": 1206845440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005126419257773321, + "loss": 1.3479, + "theoretical_loss": 3.614037844896595, + "tokens_seen": 1206910976 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005126258776328987, + "loss": 1.3219, + "theoretical_loss": 3.614019955187887, + "tokens_seen": 1206976512 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005126098294884655, + "loss": 1.2744, + "theoretical_loss": 3.614002066722488, + "tokens_seen": 1207042048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005125937813440321, + "loss": 1.4586, + "theoretical_loss": 3.6139841795002448, + "tokens_seen": 1207107584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005125777331995989, + "loss": 1.2821, + "theoretical_loss": 3.6139662935210035, + "tokens_seen": 1207173120 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005125616850551655, + "loss": 1.3641, + "theoretical_loss": 3.6139484087846094, + "tokens_seen": 1207238656 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005125456369107323, + "loss": 1.3374, + "theoretical_loss": 3.61393052529091, + "tokens_seen": 1207304192 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005125295887662989, + "loss": 1.3901, + "theoretical_loss": 3.6139126430397512, + "tokens_seen": 1207369728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005125135406218657, + "loss": 1.3309, + "theoretical_loss": 3.613894762030978, + "tokens_seen": 1207435264 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005124974924774323, + "loss": 1.3813, + "theoretical_loss": 3.613876882264438, + "tokens_seen": 1207500800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005124814443329991, + "loss": 1.3257, + "theoretical_loss": 3.613859003739977, + "tokens_seen": 1207566336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005124653961885657, + "loss": 1.413, + "theoretical_loss": 3.6138411264574413, + "tokens_seen": 1207631872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005124493480441325, + "loss": 1.3139, + "theoretical_loss": 3.6138232504166776, + "tokens_seen": 1207697408 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005124332998996991, + "loss": 1.3551, + "theoretical_loss": 3.613805375617532, + "tokens_seen": 1207762944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005124172517552659, + "loss": 1.2477, + "theoretical_loss": 3.61378750205985, + "tokens_seen": 1207828480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005124012036108325, + "loss": 1.4634, + "theoretical_loss": 3.613769629743479, + "tokens_seen": 1207894016 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005123851554663993, + "loss": 1.2122, + "theoretical_loss": 3.6137517586682666, + "tokens_seen": 1207959552 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005123691073219659, + "loss": 1.3138, + "theoretical_loss": 3.613733888834057, + "tokens_seen": 1208025088 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005123530591775327, + "loss": 1.3162, + "theoretical_loss": 3.6137160202406977, + "tokens_seen": 1208090624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005123370110330993, + "loss": 1.2247, + "theoretical_loss": 3.6136981528880354, + "tokens_seen": 1208156160 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005123209628886661, + "loss": 1.4182, + "theoretical_loss": 3.6136802867759164, + "tokens_seen": 1208221696 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005123049147442327, + "loss": 1.3628, + "theoretical_loss": 3.6136624219041877, + "tokens_seen": 1208287232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005122888665997995, + "loss": 1.3782, + "theoretical_loss": 3.613644558272695, + "tokens_seen": 1208352768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005122728184553661, + "loss": 1.2983, + "theoretical_loss": 3.613626695881286, + "tokens_seen": 1208418304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005122567703109329, + "loss": 1.2728, + "theoretical_loss": 3.6136088347298068, + "tokens_seen": 1208483840 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005122407221664995, + "loss": 1.3636, + "theoretical_loss": 3.613590974818104, + "tokens_seen": 1208549376 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005122246740220663, + "loss": 1.4548, + "theoretical_loss": 3.613573116146025, + "tokens_seen": 1208614912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005122086258776329, + "loss": 1.4042, + "theoretical_loss": 3.613555258713416, + "tokens_seen": 1208680448 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005121925777331997, + "loss": 1.3027, + "theoretical_loss": 3.6135374025201235, + "tokens_seen": 1208745984 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005121765295887663, + "loss": 1.3504, + "theoretical_loss": 3.6135195475659945, + "tokens_seen": 1208811520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005121604814443331, + "loss": 1.3821, + "theoretical_loss": 3.6135016938508766, + "tokens_seen": 1208877056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005121444332998997, + "loss": 1.4599, + "theoretical_loss": 3.6134838413746158, + "tokens_seen": 1208942592 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005121283851554665, + "loss": 1.4473, + "theoretical_loss": 3.6134659901370587, + "tokens_seen": 1209008128 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005121123370110331, + "loss": 1.4117, + "theoretical_loss": 3.613448140138053, + "tokens_seen": 1209073664 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005120962888665999, + "loss": 1.3911, + "theoretical_loss": 3.613430291377446, + "tokens_seen": 1209139200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005120802407221665, + "loss": 1.4174, + "theoretical_loss": 3.613412443855083, + "tokens_seen": 1209204736 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005120641925777333, + "loss": 1.4236, + "theoretical_loss": 3.613394597570813, + "tokens_seen": 1209270272 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005120481444332999, + "loss": 1.4132, + "theoretical_loss": 3.6133767525244815, + "tokens_seen": 1209335808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005120320962888667, + "loss": 1.3545, + "theoretical_loss": 3.613358908715936, + "tokens_seen": 1209401344 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005120160481444333, + "loss": 1.3348, + "theoretical_loss": 3.6133410661450243, + "tokens_seen": 1209466880 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005120000000000001, + "loss": 1.2872, + "theoretical_loss": 3.6133232248115927, + "tokens_seen": 1209532416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005119839518555667, + "loss": 1.4174, + "theoretical_loss": 3.6133053847154883, + "tokens_seen": 1209597952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005119679037111335, + "loss": 1.3576, + "theoretical_loss": 3.6132875458565588, + "tokens_seen": 1209663488 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005119518555667001, + "loss": 1.3178, + "theoretical_loss": 3.6132697082346508, + "tokens_seen": 1209729024 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005119358074222668, + "loss": 1.3236, + "theoretical_loss": 3.613251871849612, + "tokens_seen": 1209794560 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005119197592778335, + "loss": 1.4219, + "theoretical_loss": 3.613234036701289, + "tokens_seen": 1209860096 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005119037111334002, + "loss": 1.3403, + "theoretical_loss": 3.6132162027895296, + "tokens_seen": 1209925632 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005118876629889669, + "loss": 1.4164, + "theoretical_loss": 3.613198370114181, + "tokens_seen": 1209991168 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005118716148445336, + "loss": 1.4305, + "theoretical_loss": 3.613180538675091, + "tokens_seen": 1210056704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005118555667001003, + "loss": 1.3811, + "theoretical_loss": 3.613162708472106, + "tokens_seen": 1210122240 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511839518555667, + "loss": 1.36, + "theoretical_loss": 3.6131448795050733, + "tokens_seen": 1210187776 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005118234704112337, + "loss": 1.4411, + "theoretical_loss": 3.6131270517738416, + "tokens_seen": 1210253312 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005118074222668004, + "loss": 1.3893, + "theoretical_loss": 3.6131092252782575, + "tokens_seen": 1210318848 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511791374122367, + "loss": 1.2106, + "theoretical_loss": 3.6130914000181686, + "tokens_seen": 1210384384 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005117753259779338, + "loss": 1.3517, + "theoretical_loss": 3.613073575993422, + "tokens_seen": 1210449920 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005117592778335004, + "loss": 1.3524, + "theoretical_loss": 3.6130557532038656, + "tokens_seen": 1210515456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005117432296890672, + "loss": 1.2986, + "theoretical_loss": 3.6130379316493464, + "tokens_seen": 1210580992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005117271815446338, + "loss": 1.4303, + "theoretical_loss": 3.613020111329713, + "tokens_seen": 1210646528 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005117111334002006, + "loss": 1.3423, + "theoretical_loss": 3.613002292244812, + "tokens_seen": 1210712064 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005116950852557673, + "loss": 1.3132, + "theoretical_loss": 3.6129844743944917, + "tokens_seen": 1210777600 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511679037111334, + "loss": 1.4236, + "theoretical_loss": 3.6129666577785993, + "tokens_seen": 1210843136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005116629889669007, + "loss": 1.371, + "theoretical_loss": 3.6129488423969827, + "tokens_seen": 1210908672 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005116469408224674, + "loss": 1.2875, + "theoretical_loss": 3.6129310282494895, + "tokens_seen": 1210974208 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005116308926780341, + "loss": 1.3725, + "theoretical_loss": 3.6129132153359675, + "tokens_seen": 1211039744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005116148445336008, + "loss": 1.4045, + "theoretical_loss": 3.6128954036562644, + "tokens_seen": 1211105280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005115987963891675, + "loss": 1.3339, + "theoretical_loss": 3.6128775932102277, + "tokens_seen": 1211170816 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005115827482447342, + "loss": 1.2382, + "theoretical_loss": 3.612859783997706, + "tokens_seen": 1211236352 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511566700100301, + "loss": 1.3204, + "theoretical_loss": 3.6128419760185464, + "tokens_seen": 1211301888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005115506519558676, + "loss": 1.4165, + "theoretical_loss": 3.612824169272597, + "tokens_seen": 1211367424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005115346038114343, + "loss": 1.4058, + "theoretical_loss": 3.612806363759706, + "tokens_seen": 1211432960 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511518555667001, + "loss": 1.3108, + "theoretical_loss": 3.6127885594797204, + "tokens_seen": 1211498496 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005115025075225678, + "loss": 1.337, + "theoretical_loss": 3.612770756432489, + "tokens_seen": 1211564032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005114864593781344, + "loss": 1.3194, + "theoretical_loss": 3.61275295461786, + "tokens_seen": 1211629568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005114704112337012, + "loss": 1.2531, + "theoretical_loss": 3.61273515403568, + "tokens_seen": 1211695104 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005114543630892678, + "loss": 1.3836, + "theoretical_loss": 3.612717354685798, + "tokens_seen": 1211760640 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005114383149448346, + "loss": 1.4171, + "theoretical_loss": 3.612699556568063, + "tokens_seen": 1211826176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005114222668004012, + "loss": 1.3362, + "theoretical_loss": 3.612681759682321, + "tokens_seen": 1211891712 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511406218655968, + "loss": 1.3938, + "theoretical_loss": 3.612663964028422, + "tokens_seen": 1211957248 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005113901705115346, + "loss": 1.3532, + "theoretical_loss": 3.6126461696062124, + "tokens_seen": 1212022784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005113741223671014, + "loss": 1.302, + "theoretical_loss": 3.6126283764155422, + "tokens_seen": 1212088320 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511358074222668, + "loss": 1.484, + "theoretical_loss": 3.612610584456258, + "tokens_seen": 1212153856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005113420260782348, + "loss": 1.3405, + "theoretical_loss": 3.6125927937282087, + "tokens_seen": 1212219392 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005113259779338014, + "loss": 1.2725, + "theoretical_loss": 3.612575004231243, + "tokens_seen": 1212284928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005113099297893682, + "loss": 1.3569, + "theoretical_loss": 3.612557215965208, + "tokens_seen": 1212350464 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 452247, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3285186290740967, + "objective/train/theoretical_loss": 3.6125394289299533, + "objective/train/tokens_used": 1232876000, + "theoretical_loss": 3.6125394289299533, + "tokens_seen": 1212416000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005112938816449348, + "loss": 1.3285, + "theoretical_loss": 3.6125394289299533, + "tokens_seen": 1212416000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005112778335005016, + "loss": 1.4339, + "theoretical_loss": 3.6125216431253264, + "tokens_seen": 1212481536 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005112617853560682, + "loss": 1.3698, + "theoretical_loss": 3.6125038585511753, + "tokens_seen": 1212547072 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511245737211635, + "loss": 1.4013, + "theoretical_loss": 3.6124860752073493, + "tokens_seen": 1212612608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005112296890672016, + "loss": 1.3235, + "theoretical_loss": 3.6124682930936967, + "tokens_seen": 1212678144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005112136409227684, + "loss": 1.2711, + "theoretical_loss": 3.6124505122100654, + "tokens_seen": 1212743680 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511197592778335, + "loss": 1.323, + "theoretical_loss": 3.6124327325563037, + "tokens_seen": 1212809216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005111815446339018, + "loss": 1.3481, + "theoretical_loss": 3.6124149541322605, + "tokens_seen": 1212874752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005111654964894684, + "loss": 1.4589, + "theoretical_loss": 3.612397176937785, + "tokens_seen": 1212940288 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005111494483450352, + "loss": 1.3239, + "theoretical_loss": 3.612379400972724, + "tokens_seen": 1213005824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005111334002006018, + "loss": 1.377, + "theoretical_loss": 3.612361626236928, + "tokens_seen": 1213071360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005111173520561686, + "loss": 1.3878, + "theoretical_loss": 3.612343852730244, + "tokens_seen": 1213136896 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005111013039117352, + "loss": 1.3788, + "theoretical_loss": 3.6123260804525215, + "tokens_seen": 1213202432 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511085255767302, + "loss": 1.3356, + "theoretical_loss": 3.612308309403609, + "tokens_seen": 1213267968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005110692076228686, + "loss": 1.35, + "theoretical_loss": 3.6122905395833547, + "tokens_seen": 1213333504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005110531594784354, + "loss": 1.3785, + "theoretical_loss": 3.6122727709916083, + "tokens_seen": 1213399040 + }, + { + "epoch": 0.37, + "learning_rate": 0.000511037111334002, + "loss": 1.2913, + "theoretical_loss": 3.612255003628217, + "tokens_seen": 1213464576 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005110210631895688, + "loss": 1.3385, + "theoretical_loss": 3.6122372374930314, + "tokens_seen": 1213530112 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005110050150451354, + "loss": 1.3208, + "theoretical_loss": 3.612219472585899, + "tokens_seen": 1213595648 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005109889669007022, + "loss": 1.2324, + "theoretical_loss": 3.6122017089066687, + "tokens_seen": 1213661184 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005109729187562688, + "loss": 1.287, + "theoretical_loss": 3.6121839464551897, + "tokens_seen": 1213726720 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005109568706118356, + "loss": 1.3556, + "theoretical_loss": 3.6121661852313105, + "tokens_seen": 1213792256 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005109408224674022, + "loss": 1.4278, + "theoretical_loss": 3.6121484252348806, + "tokens_seen": 1213857792 + }, + { + "epoch": 0.37, + "learning_rate": 0.000510924774322969, + "loss": 1.3068, + "theoretical_loss": 3.6121306664657484, + "tokens_seen": 1213923328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005109087261785356, + "loss": 1.3704, + "theoretical_loss": 3.6121129089237627, + "tokens_seen": 1213988864 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005108926780341024, + "loss": 1.4599, + "theoretical_loss": 3.612095152608773, + "tokens_seen": 1214054400 + }, + { + "epoch": 0.37, + "learning_rate": 0.000510876629889669, + "loss": 1.3881, + "theoretical_loss": 3.612077397520628, + "tokens_seen": 1214119936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005108605817452358, + "loss": 1.3307, + "theoretical_loss": 3.6120596436591765, + "tokens_seen": 1214185472 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005108445336008024, + "loss": 1.3402, + "theoretical_loss": 3.6120418910242678, + "tokens_seen": 1214251008 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005108284854563692, + "loss": 1.406, + "theoretical_loss": 3.612024139615751, + "tokens_seen": 1214316544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005108124373119358, + "loss": 1.4624, + "theoretical_loss": 3.612006389433475, + "tokens_seen": 1214382080 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005107963891675026, + "loss": 1.3723, + "theoretical_loss": 3.611988640477289, + "tokens_seen": 1214447616 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005107803410230692, + "loss": 1.3604, + "theoretical_loss": 3.6119708927470424, + "tokens_seen": 1214513152 + }, + { + "epoch": 0.37, + "learning_rate": 0.000510764292878636, + "loss": 1.2729, + "theoretical_loss": 3.611953146242584, + "tokens_seen": 1214578688 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005107482447342026, + "loss": 1.4767, + "theoretical_loss": 3.6119354009637634, + "tokens_seen": 1214644224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005107321965897694, + "loss": 1.3563, + "theoretical_loss": 3.6119176569104297, + "tokens_seen": 1214709760 + }, + { + "epoch": 0.37, + "learning_rate": 0.000510716148445336, + "loss": 1.2529, + "theoretical_loss": 3.611899914082432, + "tokens_seen": 1214775296 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005107001003009028, + "loss": 1.3961, + "theoretical_loss": 3.6118821724796195, + "tokens_seen": 1214840832 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005106840521564694, + "loss": 1.2674, + "theoretical_loss": 3.6118644321018416, + "tokens_seen": 1214906368 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005106680040120362, + "loss": 1.4125, + "theoretical_loss": 3.611846692948948, + "tokens_seen": 1214971904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005106519558676028, + "loss": 1.2882, + "theoretical_loss": 3.6118289550207874, + "tokens_seen": 1215037440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005106359077231696, + "loss": 1.2875, + "theoretical_loss": 3.61181121831721, + "tokens_seen": 1215102976 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005106198595787362, + "loss": 1.3734, + "theoretical_loss": 3.6117934828380642, + "tokens_seen": 1215168512 + }, + { + "epoch": 0.37, + "learning_rate": 0.000510603811434303, + "loss": 1.43, + "theoretical_loss": 3.6117757485832005, + "tokens_seen": 1215234048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005105877632898696, + "loss": 1.3515, + "theoretical_loss": 3.611758015552468, + "tokens_seen": 1215299584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005105717151454364, + "loss": 1.4771, + "theoretical_loss": 3.6117402837457155, + "tokens_seen": 1215365120 + }, + { + "epoch": 0.37, + "learning_rate": 0.000510555667001003, + "loss": 1.3368, + "theoretical_loss": 3.6117225531627932, + "tokens_seen": 1215430656 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005105396188565698, + "loss": 1.3991, + "theoretical_loss": 3.6117048238035507, + "tokens_seen": 1215496192 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005105235707121364, + "loss": 1.2898, + "theoretical_loss": 3.611687095667838, + "tokens_seen": 1215561728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005105075225677031, + "loss": 1.2827, + "theoretical_loss": 3.6116693687555035, + "tokens_seen": 1215627264 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005104914744232698, + "loss": 1.3789, + "theoretical_loss": 3.6116516430663976, + "tokens_seen": 1215692800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005104754262788365, + "loss": 1.3492, + "theoretical_loss": 3.6116339186003694, + "tokens_seen": 1215758336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005104593781344032, + "loss": 1.3514, + "theoretical_loss": 3.6116161953572696, + "tokens_seen": 1215823872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005104433299899699, + "loss": 1.4058, + "theoretical_loss": 3.6115984733369473, + "tokens_seen": 1215889408 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005104272818455366, + "loss": 1.2881, + "theoretical_loss": 3.6115807525392523, + "tokens_seen": 1215954944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005104112337011033, + "loss": 1.1863, + "theoretical_loss": 3.611563032964034, + "tokens_seen": 1216020480 + }, + { + "epoch": 0.37, + "learning_rate": 0.00051039518555667, + "loss": 1.3315, + "theoretical_loss": 3.611545314611143, + "tokens_seen": 1216086016 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005103791374122367, + "loss": 1.3564, + "theoretical_loss": 3.6115275974804284, + "tokens_seen": 1216151552 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005103630892678034, + "loss": 1.3664, + "theoretical_loss": 3.61150988157174, + "tokens_seen": 1216217088 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005103470411233701, + "loss": 1.3732, + "theoretical_loss": 3.6114921668849282, + "tokens_seen": 1216282624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005103309929789368, + "loss": 1.3661, + "theoretical_loss": 3.6114744534198424, + "tokens_seen": 1216348160 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005103149448345035, + "loss": 1.3685, + "theoretical_loss": 3.6114567411763328, + "tokens_seen": 1216413696 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005102988966900702, + "loss": 1.3365, + "theoretical_loss": 3.6114390301542496, + "tokens_seen": 1216479232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005102828485456369, + "loss": 1.3269, + "theoretical_loss": 3.6114213203534424, + "tokens_seen": 1216544768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005102668004012036, + "loss": 1.3754, + "theoretical_loss": 3.611403611773761, + "tokens_seen": 1216610304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005102507522567703, + "loss": 1.4201, + "theoretical_loss": 3.611385904415056, + "tokens_seen": 1216675840 + }, + { + "epoch": 0.37, + "learning_rate": 0.000510234704112337, + "loss": 1.2799, + "theoretical_loss": 3.611368198277177, + "tokens_seen": 1216741376 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005102186559679037, + "loss": 1.3677, + "theoretical_loss": 3.611350493359974, + "tokens_seen": 1216806912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005102026078234704, + "loss": 1.4332, + "theoretical_loss": 3.6113327896632983, + "tokens_seen": 1216872448 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005101865596790371, + "loss": 1.1932, + "theoretical_loss": 3.6113150871869983, + "tokens_seen": 1216937984 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005101705115346038, + "loss": 1.4411, + "theoretical_loss": 3.611297385930925, + "tokens_seen": 1217003520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005101544633901705, + "loss": 1.4206, + "theoretical_loss": 3.611279685894929, + "tokens_seen": 1217069056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005101384152457373, + "loss": 1.3184, + "theoretical_loss": 3.6112619870788594, + "tokens_seen": 1217134592 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005101223671013039, + "loss": 1.3455, + "theoretical_loss": 3.611244289482568, + "tokens_seen": 1217200128 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005101063189568707, + "loss": 1.3366, + "theoretical_loss": 3.6112265931059033, + "tokens_seen": 1217265664 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005100902708124373, + "loss": 1.3613, + "theoretical_loss": 3.6112088979487167, + "tokens_seen": 1217331200 + }, + { + "epoch": 0.37, + "learning_rate": 0.000510074222668004, + "loss": 1.3714, + "theoretical_loss": 3.611191204010858, + "tokens_seen": 1217396736 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005100581745235707, + "loss": 1.3857, + "theoretical_loss": 3.6111735112921783, + "tokens_seen": 1217462272 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005100421263791375, + "loss": 1.3566, + "theoretical_loss": 3.6111558197925273, + "tokens_seen": 1217527808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005100260782347041, + "loss": 1.3171, + "theoretical_loss": 3.6111381295117555, + "tokens_seen": 1217593344 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005100100300902709, + "loss": 1.4091, + "theoretical_loss": 3.6111204404497137, + "tokens_seen": 1217658880 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005099939819458375, + "loss": 1.3606, + "theoretical_loss": 3.611102752606252, + "tokens_seen": 1217724416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005099779338014043, + "loss": 1.4125, + "theoretical_loss": 3.6110850659812206, + "tokens_seen": 1217789952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005099618856569709, + "loss": 1.3603, + "theoretical_loss": 3.611067380574471, + "tokens_seen": 1217855488 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005099458375125377, + "loss": 1.2804, + "theoretical_loss": 3.6110496963858525, + "tokens_seen": 1217921024 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005099297893681043, + "loss": 1.2868, + "theoretical_loss": 3.6110320134152163, + "tokens_seen": 1217986560 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005099137412236711, + "loss": 1.3486, + "theoretical_loss": 3.611014331662413, + "tokens_seen": 1218052096 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005098976930792377, + "loss": 1.293, + "theoretical_loss": 3.610996651127293, + "tokens_seen": 1218117632 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005098816449348045, + "loss": 1.3092, + "theoretical_loss": 3.6109789718097076, + "tokens_seen": 1218183168 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005098655967903711, + "loss": 1.3241, + "theoretical_loss": 3.610961293709506, + "tokens_seen": 1218248704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005098495486459379, + "loss": 1.3143, + "theoretical_loss": 3.6109436168265407, + "tokens_seen": 1218314240 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005098335005015045, + "loss": 1.2352, + "theoretical_loss": 3.6109259411606613, + "tokens_seen": 1218379776 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005098174523570713, + "loss": 1.3496, + "theoretical_loss": 3.6109082667117187, + "tokens_seen": 1218445312 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005098014042126379, + "loss": 1.3108, + "theoretical_loss": 3.6108905934795636, + "tokens_seen": 1218510848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005097853560682047, + "loss": 1.3925, + "theoretical_loss": 3.610872921464047, + "tokens_seen": 1218576384 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005097693079237713, + "loss": 1.3437, + "theoretical_loss": 3.61085525066502, + "tokens_seen": 1218641920 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005097532597793381, + "loss": 1.3041, + "theoretical_loss": 3.610837581082333, + "tokens_seen": 1218707456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005097372116349047, + "loss": 1.4162, + "theoretical_loss": 3.610819912715837, + "tokens_seen": 1218772992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005097211634904715, + "loss": 1.3938, + "theoretical_loss": 3.6108022455653828, + "tokens_seen": 1218838528 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005097051153460381, + "loss": 1.3083, + "theoretical_loss": 3.6107845796308213, + "tokens_seen": 1218904064 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 454856, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3784759044647217, + "objective/train/theoretical_loss": 3.6107669149120034, + "objective/train/tokens_used": 1239429600, + "theoretical_loss": 3.6107669149120034, + "tokens_seen": 1218969600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005096890672016049, + "loss": 1.3785, + "theoretical_loss": 3.6107669149120034, + "tokens_seen": 1218969600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005096730190571715, + "loss": 1.2953, + "theoretical_loss": 3.6107492514087802, + "tokens_seen": 1219035136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005096569709127383, + "loss": 1.3006, + "theoretical_loss": 3.610731589121003, + "tokens_seen": 1219100672 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005096409227683049, + "loss": 1.4216, + "theoretical_loss": 3.610713928048523, + "tokens_seen": 1219166208 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005096248746238717, + "loss": 1.3039, + "theoretical_loss": 3.61069626819119, + "tokens_seen": 1219231744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005096088264794383, + "loss": 1.3284, + "theoretical_loss": 3.6106786095488563, + "tokens_seen": 1219297280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005095927783350051, + "loss": 1.3267, + "theoretical_loss": 3.610660952121373, + "tokens_seen": 1219362816 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005095767301905717, + "loss": 1.3436, + "theoretical_loss": 3.61064329590859, + "tokens_seen": 1219428352 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005095606820461385, + "loss": 1.3815, + "theoretical_loss": 3.61062564091036, + "tokens_seen": 1219493888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005095446339017051, + "loss": 1.3099, + "theoretical_loss": 3.6106079871265333, + "tokens_seen": 1219559424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005095285857572719, + "loss": 1.4211, + "theoretical_loss": 3.6105903345569614, + "tokens_seen": 1219624960 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005095125376128385, + "loss": 1.2515, + "theoretical_loss": 3.6105726832014953, + "tokens_seen": 1219690496 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005094964894684053, + "loss": 1.3941, + "theoretical_loss": 3.6105550330599865, + "tokens_seen": 1219756032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005094804413239719, + "loss": 1.375, + "theoretical_loss": 3.6105373841322863, + "tokens_seen": 1219821568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005094643931795387, + "loss": 1.2955, + "theoretical_loss": 3.610519736418246, + "tokens_seen": 1219887104 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005094483450351053, + "loss": 1.3279, + "theoretical_loss": 3.6105020899177167, + "tokens_seen": 1219952640 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005094322968906721, + "loss": 1.4145, + "theoretical_loss": 3.6104844446305497, + "tokens_seen": 1220018176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005094162487462387, + "loss": 1.359, + "theoretical_loss": 3.6104668005565976, + "tokens_seen": 1220083712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005094002006018055, + "loss": 1.3153, + "theoretical_loss": 3.61044915769571, + "tokens_seen": 1220149248 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005093841524573721, + "loss": 1.2464, + "theoretical_loss": 3.61043151604774, + "tokens_seen": 1220214784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005093681043129389, + "loss": 1.4338, + "theoretical_loss": 3.6104138756125375, + "tokens_seen": 1220280320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005093520561685055, + "loss": 1.3469, + "theoretical_loss": 3.6103962363899553, + "tokens_seen": 1220345856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005093360080240723, + "loss": 1.3032, + "theoretical_loss": 3.610378598379844, + "tokens_seen": 1220411392 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005093199598796389, + "loss": 1.3318, + "theoretical_loss": 3.610360961582056, + "tokens_seen": 1220476928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005093039117352057, + "loss": 1.2993, + "theoretical_loss": 3.6103433259964426, + "tokens_seen": 1220542464 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005092878635907724, + "loss": 1.3874, + "theoretical_loss": 3.610325691622855, + "tokens_seen": 1220608000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005092718154463391, + "loss": 1.3125, + "theoretical_loss": 3.610308058461145, + "tokens_seen": 1220673536 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005092557673019057, + "loss": 1.3865, + "theoretical_loss": 3.610290426511165, + "tokens_seen": 1220739072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005092397191574725, + "loss": 1.4934, + "theoretical_loss": 3.6102727957727656, + "tokens_seen": 1220804608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005092236710130391, + "loss": 1.1852, + "theoretical_loss": 3.610255166245799, + "tokens_seen": 1220870144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005092076228686059, + "loss": 1.3606, + "theoretical_loss": 3.610237537930117, + "tokens_seen": 1220935680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005091915747241725, + "loss": 1.2603, + "theoretical_loss": 3.6102199108255713, + "tokens_seen": 1221001216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005091755265797393, + "loss": 1.2832, + "theoretical_loss": 3.6102022849320132, + "tokens_seen": 1221066752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005091594784353059, + "loss": 1.2876, + "theoretical_loss": 3.6101846602492955, + "tokens_seen": 1221132288 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005091434302908727, + "loss": 1.4466, + "theoretical_loss": 3.61016703677727, + "tokens_seen": 1221197824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005091273821464393, + "loss": 1.3709, + "theoretical_loss": 3.6101494145157873, + "tokens_seen": 1221263360 + }, + { + "epoch": 0.37, + "learning_rate": 0.000509111334002006, + "loss": 1.2116, + "theoretical_loss": 3.6101317934647, + "tokens_seen": 1221328896 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005090952858575727, + "loss": 1.3835, + "theoretical_loss": 3.610114173623861, + "tokens_seen": 1221394432 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005090792377131394, + "loss": 1.4493, + "theoretical_loss": 3.6100965549931203, + "tokens_seen": 1221459968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005090631895687061, + "loss": 1.3734, + "theoretical_loss": 3.610078937572332, + "tokens_seen": 1221525504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005090471414242728, + "loss": 1.3489, + "theoretical_loss": 3.6100613213613464, + "tokens_seen": 1221591040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005090310932798395, + "loss": 1.3938, + "theoretical_loss": 3.610043706360016, + "tokens_seen": 1221656576 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005090150451354062, + "loss": 1.3172, + "theoretical_loss": 3.6100260925681935, + "tokens_seen": 1221722112 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005089989969909729, + "loss": 1.3536, + "theoretical_loss": 3.61000847998573, + "tokens_seen": 1221787648 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005089829488465396, + "loss": 1.3207, + "theoretical_loss": 3.6099908686124786, + "tokens_seen": 1221853184 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005089669007021063, + "loss": 1.4137, + "theoretical_loss": 3.6099732584482904, + "tokens_seen": 1221918720 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508950852557673, + "loss": 1.349, + "theoretical_loss": 3.6099556494930187, + "tokens_seen": 1221984256 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005089348044132397, + "loss": 1.312, + "theoretical_loss": 3.6099380417465143, + "tokens_seen": 1222049792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005089187562688064, + "loss": 1.385, + "theoretical_loss": 3.6099204352086307, + "tokens_seen": 1222115328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005089027081243731, + "loss": 1.3663, + "theoretical_loss": 3.6099028298792195, + "tokens_seen": 1222180864 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005088866599799398, + "loss": 1.2938, + "theoretical_loss": 3.6098852257581333, + "tokens_seen": 1222246400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005088706118355065, + "loss": 1.2664, + "theoretical_loss": 3.609867622845224, + "tokens_seen": 1222311936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005088545636910732, + "loss": 1.3179, + "theoretical_loss": 3.6098500211403435, + "tokens_seen": 1222377472 + }, + { + "epoch": 0.37, + "learning_rate": 0.00050883851554664, + "loss": 1.2728, + "theoretical_loss": 3.609832420643345, + "tokens_seen": 1222443008 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005088224674022066, + "loss": 1.3619, + "theoretical_loss": 3.609814821354081, + "tokens_seen": 1222508544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005088064192577733, + "loss": 1.3143, + "theoretical_loss": 3.6097972232724027, + "tokens_seen": 1222574080 + }, + { + "epoch": 0.37, + "learning_rate": 0.00050879037111334, + "loss": 1.2835, + "theoretical_loss": 3.6097796263981636, + "tokens_seen": 1222639616 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005087743229689068, + "loss": 1.3286, + "theoretical_loss": 3.6097620307312157, + "tokens_seen": 1222705152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005087582748244734, + "loss": 1.3393, + "theoretical_loss": 3.609744436271412, + "tokens_seen": 1222770688 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005087422266800402, + "loss": 1.3639, + "theoretical_loss": 3.609726843018604, + "tokens_seen": 1222836224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005087261785356068, + "loss": 1.2787, + "theoretical_loss": 3.609709250972645, + "tokens_seen": 1222901760 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005087101303911736, + "loss": 1.4277, + "theoretical_loss": 3.6096916601333873, + "tokens_seen": 1222967296 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005086940822467402, + "loss": 1.3713, + "theoretical_loss": 3.6096740705006836, + "tokens_seen": 1223032832 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508678034102307, + "loss": 1.3548, + "theoretical_loss": 3.609656482074386, + "tokens_seen": 1223098368 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005086619859578736, + "loss": 1.4388, + "theoretical_loss": 3.609638894854348, + "tokens_seen": 1223163904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005086459378134404, + "loss": 1.3509, + "theoretical_loss": 3.6096213088404214, + "tokens_seen": 1223229440 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508629889669007, + "loss": 1.4621, + "theoretical_loss": 3.6096037240324597, + "tokens_seen": 1223294976 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005086138415245738, + "loss": 1.3335, + "theoretical_loss": 3.6095861404303147, + "tokens_seen": 1223360512 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005085977933801404, + "loss": 1.3223, + "theoretical_loss": 3.609568558033839, + "tokens_seen": 1223426048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005085817452357072, + "loss": 1.3595, + "theoretical_loss": 3.609550976842887, + "tokens_seen": 1223491584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005085656970912738, + "loss": 1.2777, + "theoretical_loss": 3.6095333968573096, + "tokens_seen": 1223557120 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005085496489468406, + "loss": 1.2758, + "theoretical_loss": 3.609515818076961, + "tokens_seen": 1223622656 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005085336008024072, + "loss": 1.3207, + "theoretical_loss": 3.609498240501693, + "tokens_seen": 1223688192 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508517552657974, + "loss": 1.3889, + "theoretical_loss": 3.6094806641313593, + "tokens_seen": 1223753728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005085015045135406, + "loss": 1.3693, + "theoretical_loss": 3.609463088965812, + "tokens_seen": 1223819264 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005084854563691074, + "loss": 1.3772, + "theoretical_loss": 3.609445515004904, + "tokens_seen": 1223884800 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508469408224674, + "loss": 1.4867, + "theoretical_loss": 3.609427942248489, + "tokens_seen": 1223950336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005084533600802408, + "loss": 1.3661, + "theoretical_loss": 3.60941037069642, + "tokens_seen": 1224015872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005084373119358074, + "loss": 1.4277, + "theoretical_loss": 3.609392800348549, + "tokens_seen": 1224081408 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005084212637913742, + "loss": 1.4364, + "theoretical_loss": 3.609375231204729, + "tokens_seen": 1224146944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005084052156469408, + "loss": 1.2869, + "theoretical_loss": 3.6093576632648148, + "tokens_seen": 1224212480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005083891675025076, + "loss": 1.4071, + "theoretical_loss": 3.6093400965286575, + "tokens_seen": 1224278016 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005083731193580742, + "loss": 1.3725, + "theoretical_loss": 3.609322530996111, + "tokens_seen": 1224343552 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508357071213641, + "loss": 1.352, + "theoretical_loss": 3.609304966667028, + "tokens_seen": 1224409088 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005083410230692076, + "loss": 1.3969, + "theoretical_loss": 3.6092874035412628, + "tokens_seen": 1224474624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005083249749247744, + "loss": 1.2494, + "theoretical_loss": 3.6092698416186675, + "tokens_seen": 1224540160 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508308926780341, + "loss": 1.3441, + "theoretical_loss": 3.609252280899095, + "tokens_seen": 1224605696 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005082928786359078, + "loss": 1.3419, + "theoretical_loss": 3.6092347213823994, + "tokens_seen": 1224671232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005082768304914744, + "loss": 1.2938, + "theoretical_loss": 3.6092171630684335, + "tokens_seen": 1224736768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005082607823470412, + "loss": 1.403, + "theoretical_loss": 3.609199605957051, + "tokens_seen": 1224802304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005082447342026078, + "loss": 1.4334, + "theoretical_loss": 3.6091820500481044, + "tokens_seen": 1224867840 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005082286860581746, + "loss": 1.3556, + "theoretical_loss": 3.6091644953414477, + "tokens_seen": 1224933376 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005082126379137412, + "loss": 1.3048, + "theoretical_loss": 3.6091469418369337, + "tokens_seen": 1224998912 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508196589769308, + "loss": 1.3771, + "theoretical_loss": 3.6091293895344165, + "tokens_seen": 1225064448 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005081805416248746, + "loss": 1.3591, + "theoretical_loss": 3.609111838433749, + "tokens_seen": 1225129984 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005081644934804414, + "loss": 1.312, + "theoretical_loss": 3.6090942885347848, + "tokens_seen": 1225195520 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508148445336008, + "loss": 1.3225, + "theoretical_loss": 3.609076739837377, + "tokens_seen": 1225261056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005081323971915748, + "loss": 1.365, + "theoretical_loss": 3.6090591923413795, + "tokens_seen": 1225326592 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005081163490471414, + "loss": 1.2664, + "theoretical_loss": 3.609041646046646, + "tokens_seen": 1225392128 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005081003009027082, + "loss": 1.4068, + "theoretical_loss": 3.609024100953029, + "tokens_seen": 1225457664 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 457300, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3350735902786255, + "objective/train/theoretical_loss": 3.6090065570603826, + "objective/train/tokens_used": 1245983200, + "theoretical_loss": 3.6090065570603826, + "tokens_seen": 1225523200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005080842527582749, + "loss": 1.3351, + "theoretical_loss": 3.6090065570603826, + "tokens_seen": 1225523200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005080682046138416, + "loss": 1.437, + "theoretical_loss": 3.608989014368561, + "tokens_seen": 1225588736 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005080521564694083, + "loss": 1.3752, + "theoretical_loss": 3.608971472877417, + "tokens_seen": 1225654272 + }, + { + "epoch": 0.37, + "learning_rate": 0.000508036108324975, + "loss": 1.4172, + "theoretical_loss": 3.6089539325868047, + "tokens_seen": 1225719808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005080200601805417, + "loss": 1.4234, + "theoretical_loss": 3.6089363934965775, + "tokens_seen": 1225785344 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005080040120361084, + "loss": 1.4016, + "theoretical_loss": 3.608918855606589, + "tokens_seen": 1225850880 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005079879638916751, + "loss": 1.3281, + "theoretical_loss": 3.6089013189166934, + "tokens_seen": 1225916416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005079719157472418, + "loss": 1.3941, + "theoretical_loss": 3.608883783426744, + "tokens_seen": 1225981952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005079558676028085, + "loss": 1.4265, + "theoretical_loss": 3.6088662491365944, + "tokens_seen": 1226047488 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005079398194583752, + "loss": 1.3333, + "theoretical_loss": 3.608848716046099, + "tokens_seen": 1226113024 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005079237713139419, + "loss": 1.4318, + "theoretical_loss": 3.6088311841551106, + "tokens_seen": 1226178560 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005079077231695086, + "loss": 1.3655, + "theoretical_loss": 3.608813653463484, + "tokens_seen": 1226244096 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005078916750250753, + "loss": 1.3079, + "theoretical_loss": 3.6087961239710724, + "tokens_seen": 1226309632 + }, + { + "epoch": 0.37, + "learning_rate": 0.000507875626880642, + "loss": 1.286, + "theoretical_loss": 3.6087785956777303, + "tokens_seen": 1226375168 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005078595787362086, + "loss": 1.4543, + "theoretical_loss": 3.608761068583312, + "tokens_seen": 1226440704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005078435305917754, + "loss": 1.3885, + "theoretical_loss": 3.6087435426876695, + "tokens_seen": 1226506240 + }, + { + "epoch": 0.37, + "learning_rate": 0.000507827482447342, + "loss": 1.3076, + "theoretical_loss": 3.6087260179906586, + "tokens_seen": 1226571776 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005078114343029088, + "loss": 1.4322, + "theoretical_loss": 3.6087084944921326, + "tokens_seen": 1226637312 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005077953861584754, + "loss": 1.3772, + "theoretical_loss": 3.608690972191946, + "tokens_seen": 1226702848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005077793380140422, + "loss": 1.3858, + "theoretical_loss": 3.608673451089952, + "tokens_seen": 1226768384 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005077632898696088, + "loss": 1.3143, + "theoretical_loss": 3.6086559311860054, + "tokens_seen": 1226833920 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005077472417251756, + "loss": 1.5332, + "theoretical_loss": 3.6086384124799595, + "tokens_seen": 1226899456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005077311935807422, + "loss": 1.401, + "theoretical_loss": 3.6086208949716694, + "tokens_seen": 1226964992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005077151454363089, + "loss": 1.2803, + "theoretical_loss": 3.608603378660989, + "tokens_seen": 1227030528 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005076990972918756, + "loss": 1.3379, + "theoretical_loss": 3.6085858635477717, + "tokens_seen": 1227096064 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005076830491474423, + "loss": 1.4015, + "theoretical_loss": 3.608568349631872, + "tokens_seen": 1227161600 + }, + { + "epoch": 0.37, + "learning_rate": 0.000507667001003009, + "loss": 1.3219, + "theoretical_loss": 3.6085508369131447, + "tokens_seen": 1227227136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005076509528585757, + "loss": 1.3404, + "theoretical_loss": 3.608533325391444, + "tokens_seen": 1227292672 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005076349047141424, + "loss": 1.3439, + "theoretical_loss": 3.6085158150666232, + "tokens_seen": 1227358208 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005076188565697091, + "loss": 1.3839, + "theoretical_loss": 3.6084983059385376, + "tokens_seen": 1227423744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005076028084252758, + "loss": 1.3109, + "theoretical_loss": 3.608480798007041, + "tokens_seen": 1227489280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005075867602808425, + "loss": 1.3939, + "theoretical_loss": 3.6084632912719874, + "tokens_seen": 1227554816 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005075707121364092, + "loss": 1.347, + "theoretical_loss": 3.6084457857332324, + "tokens_seen": 1227620352 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005075546639919759, + "loss": 1.4168, + "theoretical_loss": 3.60842828139063, + "tokens_seen": 1227685888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005075386158475426, + "loss": 1.375, + "theoretical_loss": 3.608410778244033, + "tokens_seen": 1227751424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005075225677031093, + "loss": 1.3847, + "theoretical_loss": 3.608393276293298, + "tokens_seen": 1227816960 + }, + { + "epoch": 0.37, + "learning_rate": 0.000507506519558676, + "loss": 1.3846, + "theoretical_loss": 3.6083757755382786, + "tokens_seen": 1227882496 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005074904714142427, + "loss": 1.3701, + "theoretical_loss": 3.608358275978829, + "tokens_seen": 1227948032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005074744232698094, + "loss": 1.3872, + "theoretical_loss": 3.608340777614804, + "tokens_seen": 1228013568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005074583751253761, + "loss": 1.4768, + "theoretical_loss": 3.6083232804460583, + "tokens_seen": 1228079104 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005074423269809428, + "loss": 1.3811, + "theoretical_loss": 3.6083057844724467, + "tokens_seen": 1228144640 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005074262788365095, + "loss": 1.2859, + "theoretical_loss": 3.6082882896938226, + "tokens_seen": 1228210176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005074102306920762, + "loss": 1.3585, + "theoretical_loss": 3.6082707961100424, + "tokens_seen": 1228275712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005073941825476429, + "loss": 1.3851, + "theoretical_loss": 3.608253303720959, + "tokens_seen": 1228341248 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005073781344032097, + "loss": 1.2852, + "theoretical_loss": 3.6082358125264284, + "tokens_seen": 1228406784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005073620862587763, + "loss": 1.3504, + "theoretical_loss": 3.6082183225263043, + "tokens_seen": 1228472320 + }, + { + "epoch": 0.37, + "learning_rate": 0.000507346038114343, + "loss": 1.3436, + "theoretical_loss": 3.608200833720443, + "tokens_seen": 1228537856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005073299899699097, + "loss": 1.3062, + "theoretical_loss": 3.608183346108697, + "tokens_seen": 1228603392 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005073139418254765, + "loss": 1.2957, + "theoretical_loss": 3.6081658596909225, + "tokens_seen": 1228668928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005072978936810431, + "loss": 1.3833, + "theoretical_loss": 3.6081483744669747, + "tokens_seen": 1228734464 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005072818455366099, + "loss": 1.3352, + "theoretical_loss": 3.6081308904367067, + "tokens_seen": 1228800000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005072657973921765, + "loss": 1.3325, + "theoretical_loss": 3.6081134075999755, + "tokens_seen": 1228865536 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005072497492477433, + "loss": 1.2911, + "theoretical_loss": 3.608095925956634, + "tokens_seen": 1228931072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005072337011033099, + "loss": 1.3652, + "theoretical_loss": 3.6080784455065387, + "tokens_seen": 1228996608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005072176529588767, + "loss": 1.2738, + "theoretical_loss": 3.6080609662495435, + "tokens_seen": 1229062144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005072016048144433, + "loss": 1.3131, + "theoretical_loss": 3.608043488185504, + "tokens_seen": 1229127680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005071855566700101, + "loss": 1.2257, + "theoretical_loss": 3.6080260113142746, + "tokens_seen": 1229193216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005071695085255767, + "loss": 1.3493, + "theoretical_loss": 3.6080085356357103, + "tokens_seen": 1229258752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005071534603811435, + "loss": 1.4119, + "theoretical_loss": 3.607991061149667, + "tokens_seen": 1229324288 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005071374122367101, + "loss": 1.2278, + "theoretical_loss": 3.607973587855999, + "tokens_seen": 1229389824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005071213640922769, + "loss": 1.3173, + "theoretical_loss": 3.607956115754561, + "tokens_seen": 1229455360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005071053159478435, + "loss": 1.3242, + "theoretical_loss": 3.607938644845209, + "tokens_seen": 1229520896 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005070892678034103, + "loss": 1.2386, + "theoretical_loss": 3.6079211751277978, + "tokens_seen": 1229586432 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005070732196589769, + "loss": 1.4014, + "theoretical_loss": 3.607903706602183, + "tokens_seen": 1229651968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005070571715145437, + "loss": 1.4037, + "theoretical_loss": 3.6078862392682187, + "tokens_seen": 1229717504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005070411233701103, + "loss": 1.4239, + "theoretical_loss": 3.607868773125761, + "tokens_seen": 1229783040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005070250752256771, + "loss": 1.4254, + "theoretical_loss": 3.607851308174665, + "tokens_seen": 1229848576 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005070090270812437, + "loss": 1.2953, + "theoretical_loss": 3.6078338444147855, + "tokens_seen": 1229914112 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005069929789368105, + "loss": 1.3288, + "theoretical_loss": 3.6078163818459785, + "tokens_seen": 1229979648 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005069769307923771, + "loss": 1.3822, + "theoretical_loss": 3.607798920468099, + "tokens_seen": 1230045184 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005069608826479439, + "loss": 1.2026, + "theoretical_loss": 3.607781460281002, + "tokens_seen": 1230110720 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005069448345035105, + "loss": 1.4626, + "theoretical_loss": 3.6077640012845427, + "tokens_seen": 1230176256 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005069287863590773, + "loss": 1.3302, + "theoretical_loss": 3.6077465434785774, + "tokens_seen": 1230241792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005069127382146439, + "loss": 1.3917, + "theoretical_loss": 3.607729086862961, + "tokens_seen": 1230307328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005068966900702107, + "loss": 1.3246, + "theoretical_loss": 3.6077116314375486, + "tokens_seen": 1230372864 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005068806419257774, + "loss": 1.3418, + "theoretical_loss": 3.6076941772021964, + "tokens_seen": 1230438400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005068645937813441, + "loss": 1.3819, + "theoretical_loss": 3.6076767241567596, + "tokens_seen": 1230503936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005068485456369108, + "loss": 1.479, + "theoretical_loss": 3.6076592723010936, + "tokens_seen": 1230569472 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005068324974924775, + "loss": 1.3826, + "theoretical_loss": 3.6076418216350534, + "tokens_seen": 1230635008 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005068164493480442, + "loss": 1.2879, + "theoretical_loss": 3.6076243721584955, + "tokens_seen": 1230700544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005068004012036109, + "loss": 1.3685, + "theoretical_loss": 3.607606923871275, + "tokens_seen": 1230766080 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005067843530591776, + "loss": 1.2373, + "theoretical_loss": 3.6075894767732475, + "tokens_seen": 1230831616 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005067683049147443, + "loss": 1.3825, + "theoretical_loss": 3.607572030864269, + "tokens_seen": 1230897152 + }, + { + "epoch": 0.37, + "learning_rate": 0.000506752256770311, + "loss": 1.5321, + "theoretical_loss": 3.607554586144195, + "tokens_seen": 1230962688 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005067362086258777, + "loss": 1.3898, + "theoretical_loss": 3.607537142612881, + "tokens_seen": 1231028224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005067201604814444, + "loss": 1.4058, + "theoretical_loss": 3.6075197002701826, + "tokens_seen": 1231093760 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005067041123370111, + "loss": 1.3538, + "theoretical_loss": 3.6075022591159556, + "tokens_seen": 1231159296 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005066880641925778, + "loss": 1.4199, + "theoretical_loss": 3.6074848191500566, + "tokens_seen": 1231224832 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005066720160481445, + "loss": 1.3416, + "theoretical_loss": 3.60746738037234, + "tokens_seen": 1231290368 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005066559679037112, + "loss": 1.4344, + "theoretical_loss": 3.6074499427826625, + "tokens_seen": 1231355904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005066399197592779, + "loss": 1.3167, + "theoretical_loss": 3.6074325063808796, + "tokens_seen": 1231421440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005066238716148446, + "loss": 1.3212, + "theoretical_loss": 3.607415071166848, + "tokens_seen": 1231486976 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005066078234704113, + "loss": 1.4235, + "theoretical_loss": 3.6073976371404224, + "tokens_seen": 1231552512 + }, + { + "epoch": 0.37, + "learning_rate": 0.000506591775325978, + "loss": 1.3559, + "theoretical_loss": 3.6073802043014593, + "tokens_seen": 1231618048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005065757271815447, + "loss": 1.4384, + "theoretical_loss": 3.6073627726498145, + "tokens_seen": 1231683584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005065596790371114, + "loss": 1.2602, + "theoretical_loss": 3.6073453421853445, + "tokens_seen": 1231749120 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005065436308926781, + "loss": 1.302, + "theoretical_loss": 3.607327912907904, + "tokens_seen": 1231814656 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005065275827482448, + "loss": 1.2903, + "theoretical_loss": 3.6073104848173503, + "tokens_seen": 1231880192 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005065115346038115, + "loss": 1.3756, + "theoretical_loss": 3.607293057913539, + "tokens_seen": 1231945728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005064954864593782, + "loss": 1.3473, + "theoretical_loss": 3.607275632196326, + "tokens_seen": 1232011264 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 460068, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.335837721824646, + "objective/train/theoretical_loss": 3.6072582076655673, + "objective/train/tokens_used": 1252536800, + "theoretical_loss": 3.6072582076655673, + "tokens_seen": 1232076800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005064794383149449, + "loss": 1.3358, + "theoretical_loss": 3.6072582076655673, + "tokens_seen": 1232076800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005064633901705115, + "loss": 1.3527, + "theoretical_loss": 3.60724078432112, + "tokens_seen": 1232142336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005064473420260783, + "loss": 1.3171, + "theoretical_loss": 3.6072233621628387, + "tokens_seen": 1232207872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005064312938816449, + "loss": 1.37, + "theoretical_loss": 3.607205941190581, + "tokens_seen": 1232273408 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005064152457372117, + "loss": 1.4756, + "theoretical_loss": 3.607188521404202, + "tokens_seen": 1232338944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005063991975927783, + "loss": 1.338, + "theoretical_loss": 3.6071711028035587, + "tokens_seen": 1232404480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005063831494483451, + "loss": 1.31, + "theoretical_loss": 3.6071536853885062, + "tokens_seen": 1232470016 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005063671013039117, + "loss": 1.3823, + "theoretical_loss": 3.6071362691589024, + "tokens_seen": 1232535552 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005063510531594785, + "loss": 1.3183, + "theoretical_loss": 3.607118854114603, + "tokens_seen": 1232601088 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005063350050150451, + "loss": 1.3258, + "theoretical_loss": 3.6071014402554633, + "tokens_seen": 1232666624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005063189568706118, + "loss": 1.3963, + "theoretical_loss": 3.6070840275813403, + "tokens_seen": 1232732160 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005063029087261785, + "loss": 1.4304, + "theoretical_loss": 3.607066616092091, + "tokens_seen": 1232797696 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005062868605817452, + "loss": 1.4711, + "theoretical_loss": 3.607049205787571, + "tokens_seen": 1232863232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005062708124373119, + "loss": 1.3895, + "theoretical_loss": 3.6070317966676377, + "tokens_seen": 1232928768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005062547642928786, + "loss": 1.2783, + "theoretical_loss": 3.6070143887321455, + "tokens_seen": 1232994304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005062387161484453, + "loss": 1.3786, + "theoretical_loss": 3.6069969819809535, + "tokens_seen": 1233059840 + }, + { + "epoch": 0.37, + "learning_rate": 0.000506222668004012, + "loss": 1.315, + "theoretical_loss": 3.606979576413916, + "tokens_seen": 1233125376 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005062066198595787, + "loss": 1.3367, + "theoretical_loss": 3.606962172030891, + "tokens_seen": 1233190912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005061905717151454, + "loss": 1.3102, + "theoretical_loss": 3.606944768831734, + "tokens_seen": 1233256448 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005061745235707121, + "loss": 1.2803, + "theoretical_loss": 3.6069273668163024, + "tokens_seen": 1233321984 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005061584754262788, + "loss": 1.3037, + "theoretical_loss": 3.6069099659844523, + "tokens_seen": 1233387520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005061424272818455, + "loss": 1.2984, + "theoretical_loss": 3.6068925663360405, + "tokens_seen": 1233453056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005061263791374122, + "loss": 1.3704, + "theoretical_loss": 3.606875167870923, + "tokens_seen": 1233518592 + }, + { + "epoch": 0.37, + "learning_rate": 0.000506110330992979, + "loss": 1.3424, + "theoretical_loss": 3.6068577705889577, + "tokens_seen": 1233584128 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005060942828485456, + "loss": 1.3501, + "theoretical_loss": 3.606840374490001, + "tokens_seen": 1233649664 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005060782347041123, + "loss": 1.3631, + "theoretical_loss": 3.606822979573908, + "tokens_seen": 1233715200 + }, + { + "epoch": 0.37, + "learning_rate": 0.000506062186559679, + "loss": 1.38, + "theoretical_loss": 3.6068055858405375, + "tokens_seen": 1233780736 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005060461384152457, + "loss": 1.4103, + "theoretical_loss": 3.606788193289745, + "tokens_seen": 1233846272 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005060300902708124, + "loss": 1.2991, + "theoretical_loss": 3.606770801921388, + "tokens_seen": 1233911808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005060140421263792, + "loss": 1.4374, + "theoretical_loss": 3.6067534117353235, + "tokens_seen": 1233977344 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005059979939819458, + "loss": 1.3946, + "theoretical_loss": 3.6067360227314067, + "tokens_seen": 1234042880 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005059819458375126, + "loss": 1.2657, + "theoretical_loss": 3.606718634909497, + "tokens_seen": 1234108416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005059658976930792, + "loss": 1.347, + "theoretical_loss": 3.606701248269449, + "tokens_seen": 1234173952 + }, + { + "epoch": 0.37, + "learning_rate": 0.000505949849548646, + "loss": 1.3629, + "theoretical_loss": 3.6066838628111215, + "tokens_seen": 1234239488 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005059338014042126, + "loss": 1.3472, + "theoretical_loss": 3.6066664785343696, + "tokens_seen": 1234305024 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005059177532597794, + "loss": 1.4157, + "theoretical_loss": 3.6066490954390513, + "tokens_seen": 1234370560 + }, + { + "epoch": 0.37, + "learning_rate": 0.000505901705115346, + "loss": 1.3828, + "theoretical_loss": 3.606631713525024, + "tokens_seen": 1234436096 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005058856569709128, + "loss": 1.3882, + "theoretical_loss": 3.6066143327921436, + "tokens_seen": 1234501632 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005058696088264794, + "loss": 1.3137, + "theoretical_loss": 3.606596953240268, + "tokens_seen": 1234567168 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005058535606820462, + "loss": 1.2917, + "theoretical_loss": 3.6065795748692544, + "tokens_seen": 1234632704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005058375125376128, + "loss": 1.4203, + "theoretical_loss": 3.606562197678959, + "tokens_seen": 1234698240 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005058214643931796, + "loss": 1.3377, + "theoretical_loss": 3.6065448216692393, + "tokens_seen": 1234763776 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005058054162487462, + "loss": 1.3018, + "theoretical_loss": 3.606527446839953, + "tokens_seen": 1234829312 + }, + { + "epoch": 0.37, + "learning_rate": 0.000505789368104313, + "loss": 1.4158, + "theoretical_loss": 3.6065100731909565, + "tokens_seen": 1234894848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005057733199598796, + "loss": 1.2532, + "theoretical_loss": 3.6064927007221073, + "tokens_seen": 1234960384 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005057572718154464, + "loss": 1.3614, + "theoretical_loss": 3.6064753294332625, + "tokens_seen": 1235025920 + }, + { + "epoch": 0.37, + "learning_rate": 0.000505741223671013, + "loss": 1.2612, + "theoretical_loss": 3.6064579593242794, + "tokens_seen": 1235091456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005057251755265798, + "loss": 1.287, + "theoretical_loss": 3.6064405903950156, + "tokens_seen": 1235156992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005057091273821464, + "loss": 1.3206, + "theoretical_loss": 3.6064232226453283, + "tokens_seen": 1235222528 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005056930792377132, + "loss": 1.3481, + "theoretical_loss": 3.606405856075074, + "tokens_seen": 1235288064 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005056770310932799, + "loss": 1.4377, + "theoretical_loss": 3.606388490684111, + "tokens_seen": 1235353600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005056609829488466, + "loss": 1.3247, + "theoretical_loss": 3.606371126472296, + "tokens_seen": 1235419136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005056449348044133, + "loss": 1.4221, + "theoretical_loss": 3.606353763439487, + "tokens_seen": 1235484672 + }, + { + "epoch": 0.37, + "learning_rate": 0.00050562888665998, + "loss": 1.3322, + "theoretical_loss": 3.6063364015855415, + "tokens_seen": 1235550208 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005056128385155467, + "loss": 1.3508, + "theoretical_loss": 3.606319040910316, + "tokens_seen": 1235615744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005055967903711134, + "loss": 1.4019, + "theoretical_loss": 3.606301681413669, + "tokens_seen": 1235681280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005055807422266801, + "loss": 1.3079, + "theoretical_loss": 3.6062843230954567, + "tokens_seen": 1235746816 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005055646940822468, + "loss": 1.3191, + "theoretical_loss": 3.606266965955538, + "tokens_seen": 1235812352 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005055486459378135, + "loss": 1.3769, + "theoretical_loss": 3.6062496099937698, + "tokens_seen": 1235877888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005055325977933802, + "loss": 1.3773, + "theoretical_loss": 3.60623225521001, + "tokens_seen": 1235943424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005055165496489469, + "loss": 1.3992, + "theoretical_loss": 3.6062149016041154, + "tokens_seen": 1236008960 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005055005015045136, + "loss": 1.4037, + "theoretical_loss": 3.606197549175944, + "tokens_seen": 1236074496 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005054844533600803, + "loss": 1.3766, + "theoretical_loss": 3.606180197925354, + "tokens_seen": 1236140032 + }, + { + "epoch": 0.37, + "learning_rate": 0.000505468405215647, + "loss": 1.2718, + "theoretical_loss": 3.6061628478522025, + "tokens_seen": 1236205568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005054523570712137, + "loss": 1.3836, + "theoretical_loss": 3.606145498956347, + "tokens_seen": 1236271104 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005054363089267804, + "loss": 1.3166, + "theoretical_loss": 3.6061281512376464, + "tokens_seen": 1236336640 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005054202607823471, + "loss": 1.3164, + "theoretical_loss": 3.6061108046959567, + "tokens_seen": 1236402176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005054042126379138, + "loss": 1.3924, + "theoretical_loss": 3.6060934593311367, + "tokens_seen": 1236467712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005053881644934805, + "loss": 1.3718, + "theoretical_loss": 3.6060761151430443, + "tokens_seen": 1236533248 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005053721163490472, + "loss": 1.3858, + "theoretical_loss": 3.6060587721315365, + "tokens_seen": 1236598784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005053560682046139, + "loss": 1.4222, + "theoretical_loss": 3.6060414302964716, + "tokens_seen": 1236664320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005053400200601806, + "loss": 1.4606, + "theoretical_loss": 3.6060240896377085, + "tokens_seen": 1236729856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005053239719157473, + "loss": 1.4027, + "theoretical_loss": 3.606006750155103, + "tokens_seen": 1236795392 + }, + { + "epoch": 0.37, + "learning_rate": 0.000505307923771314, + "loss": 1.3666, + "theoretical_loss": 3.6059894118485145, + "tokens_seen": 1236860928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005052918756268807, + "loss": 1.3833, + "theoretical_loss": 3.6059720747178003, + "tokens_seen": 1236926464 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005052758274824474, + "loss": 1.4258, + "theoretical_loss": 3.605954738762819, + "tokens_seen": 1236992000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005052597793380141, + "loss": 1.3171, + "theoretical_loss": 3.605937403983428, + "tokens_seen": 1237057536 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005052437311935808, + "loss": 1.3714, + "theoretical_loss": 3.6059200703794856, + "tokens_seen": 1237123072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005052276830491475, + "loss": 1.3088, + "theoretical_loss": 3.6059027379508493, + "tokens_seen": 1237188608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005052116349047142, + "loss": 1.3192, + "theoretical_loss": 3.605885406697378, + "tokens_seen": 1237254144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005051955867602809, + "loss": 1.273, + "theoretical_loss": 3.605868076618929, + "tokens_seen": 1237319680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005051795386158476, + "loss": 1.2666, + "theoretical_loss": 3.605850747715361, + "tokens_seen": 1237385216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0005051634904714143, + "loss": 1.311, + "theoretical_loss": 3.6058334199865314, + "tokens_seen": 1237450752 + }, + { + "epoch": 0.38, + "learning_rate": 0.000505147442326981, + "loss": 1.3258, + "theoretical_loss": 3.605816093432299, + "tokens_seen": 1237516288 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005051313941825477, + "loss": 1.4274, + "theoretical_loss": 3.605798768052522, + "tokens_seen": 1237581824 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005051153460381144, + "loss": 1.2756, + "theoretical_loss": 3.6057814438470586, + "tokens_seen": 1237647360 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005050992978936811, + "loss": 1.2523, + "theoretical_loss": 3.6057641208157665, + "tokens_seen": 1237712896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005050832497492478, + "loss": 1.4099, + "theoretical_loss": 3.6057467989585046, + "tokens_seen": 1237778432 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005050672016048145, + "loss": 1.4207, + "theoretical_loss": 3.605729478275131, + "tokens_seen": 1237843968 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005050511534603812, + "loss": 1.4413, + "theoretical_loss": 3.6057121587655034, + "tokens_seen": 1237909504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005050351053159478, + "loss": 1.4589, + "theoretical_loss": 3.6056948404294804, + "tokens_seen": 1237975040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005050190571715146, + "loss": 1.3253, + "theoretical_loss": 3.6056775232669214, + "tokens_seen": 1238040576 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005050030090270812, + "loss": 1.3161, + "theoretical_loss": 3.6056602072776833, + "tokens_seen": 1238106112 + }, + { + "epoch": 0.38, + "learning_rate": 0.000504986960882648, + "loss": 1.3269, + "theoretical_loss": 3.6056428924616255, + "tokens_seen": 1238171648 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005049709127382146, + "loss": 1.4037, + "theoretical_loss": 3.605625578818606, + "tokens_seen": 1238237184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005049548645937814, + "loss": 1.2834, + "theoretical_loss": 3.605608266348483, + "tokens_seen": 1238302720 + }, + { + "epoch": 0.38, + "learning_rate": 0.000504938816449348, + "loss": 1.3971, + "theoretical_loss": 3.6055909550511154, + "tokens_seen": 1238368256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005049227683049147, + "loss": 1.3118, + "theoretical_loss": 3.605573644926362, + "tokens_seen": 1238433792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005049067201604814, + "loss": 1.4223, + "theoretical_loss": 3.6055563359740805, + "tokens_seen": 1238499328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005048906720160481, + "loss": 1.312, + "theoretical_loss": 3.60553902819413, + "tokens_seen": 1238564864 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 462318, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.295304775238037, + "objective/train/theoretical_loss": 3.605521721586369, + "objective/train/tokens_used": 1259090400, + "theoretical_loss": 3.605521721586369, + "tokens_seen": 1238630400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005048746238716148, + "loss": 1.2953, + "theoretical_loss": 3.605521721586369, + "tokens_seen": 1238630400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005048585757271815, + "loss": 1.4212, + "theoretical_loss": 3.605504416150656, + "tokens_seen": 1238695936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005048425275827482, + "loss": 1.414, + "theoretical_loss": 3.6054871118868492, + "tokens_seen": 1238761472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005048264794383149, + "loss": 1.3824, + "theoretical_loss": 3.6054698087948083, + "tokens_seen": 1238827008 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005048104312938816, + "loss": 1.407, + "theoretical_loss": 3.605452506874391, + "tokens_seen": 1238892544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005047943831494483, + "loss": 1.2508, + "theoretical_loss": 3.6054352061254566, + "tokens_seen": 1238958080 + }, + { + "epoch": 0.38, + "learning_rate": 0.000504778335005015, + "loss": 1.3121, + "theoretical_loss": 3.6054179065478635, + "tokens_seen": 1239023616 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005047622868605817, + "loss": 1.353, + "theoretical_loss": 3.605400608141471, + "tokens_seen": 1239089152 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005047462387161484, + "loss": 1.3226, + "theoretical_loss": 3.6053833109061366, + "tokens_seen": 1239154688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005047301905717151, + "loss": 1.3639, + "theoretical_loss": 3.605366014841721, + "tokens_seen": 1239220224 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005047141424272818, + "loss": 1.3573, + "theoretical_loss": 3.605348719948081, + "tokens_seen": 1239285760 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005046980942828485, + "loss": 1.3478, + "theoretical_loss": 3.605331426225076, + "tokens_seen": 1239351296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005046820461384152, + "loss": 1.3731, + "theoretical_loss": 3.6053141336725663, + "tokens_seen": 1239416832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005046659979939819, + "loss": 1.4025, + "theoretical_loss": 3.605296842290409, + "tokens_seen": 1239482368 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005046499498495487, + "loss": 1.3454, + "theoretical_loss": 3.605279552078464, + "tokens_seen": 1239547904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005046339017051153, + "loss": 1.3125, + "theoretical_loss": 3.6052622630365896, + "tokens_seen": 1239613440 + }, + { + "epoch": 0.38, + "learning_rate": 0.000504617853560682, + "loss": 1.3304, + "theoretical_loss": 3.605244975164646, + "tokens_seen": 1239678976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005046018054162487, + "loss": 1.3788, + "theoretical_loss": 3.6052276884624908, + "tokens_seen": 1239744512 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005045857572718155, + "loss": 1.2645, + "theoretical_loss": 3.605210402929983, + "tokens_seen": 1239810048 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005045697091273821, + "loss": 1.4503, + "theoretical_loss": 3.6051931185669828, + "tokens_seen": 1239875584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005045536609829489, + "loss": 1.3796, + "theoretical_loss": 3.6051758353733487, + "tokens_seen": 1239941120 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005045376128385155, + "loss": 1.3866, + "theoretical_loss": 3.6051585533489394, + "tokens_seen": 1240006656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005045215646940823, + "loss": 1.409, + "theoretical_loss": 3.605141272493615, + "tokens_seen": 1240072192 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005045055165496489, + "loss": 1.2781, + "theoretical_loss": 3.6051239928072336, + "tokens_seen": 1240137728 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005044894684052157, + "loss": 1.2644, + "theoretical_loss": 3.6051067142896542, + "tokens_seen": 1240203264 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005044734202607824, + "loss": 1.4656, + "theoretical_loss": 3.6050894369407374, + "tokens_seen": 1240268800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005044573721163491, + "loss": 1.415, + "theoretical_loss": 3.605072160760341, + "tokens_seen": 1240334336 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005044413239719158, + "loss": 1.4159, + "theoretical_loss": 3.605054885748325, + "tokens_seen": 1240399872 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005044252758274825, + "loss": 1.4115, + "theoretical_loss": 3.605037611904548, + "tokens_seen": 1240465408 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005044092276830492, + "loss": 1.423, + "theoretical_loss": 3.6050203392288704, + "tokens_seen": 1240530944 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005043931795386159, + "loss": 1.3354, + "theoretical_loss": 3.6050030677211504, + "tokens_seen": 1240596480 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005043771313941826, + "loss": 1.4362, + "theoretical_loss": 3.604985797381248, + "tokens_seen": 1240662016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005043610832497493, + "loss": 1.3856, + "theoretical_loss": 3.604968528209022, + "tokens_seen": 1240727552 + }, + { + "epoch": 0.38, + "learning_rate": 0.000504345035105316, + "loss": 1.3495, + "theoretical_loss": 3.6049512602043325, + "tokens_seen": 1240793088 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005043289869608827, + "loss": 1.3793, + "theoretical_loss": 3.604933993367038, + "tokens_seen": 1240858624 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005043129388164494, + "loss": 1.3111, + "theoretical_loss": 3.6049167276969984, + "tokens_seen": 1240924160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005042968906720161, + "loss": 1.3667, + "theoretical_loss": 3.604899463194074, + "tokens_seen": 1240989696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005042808425275828, + "loss": 1.4395, + "theoretical_loss": 3.604882199858123, + "tokens_seen": 1241055232 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005042647943831495, + "loss": 1.2835, + "theoretical_loss": 3.6048649376890047, + "tokens_seen": 1241120768 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005042487462387162, + "loss": 1.3282, + "theoretical_loss": 3.60484767668658, + "tokens_seen": 1241186304 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005042326980942829, + "loss": 1.2336, + "theoretical_loss": 3.6048304168507075, + "tokens_seen": 1241251840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005042166499498496, + "loss": 1.3637, + "theoretical_loss": 3.604813158181247, + "tokens_seen": 1241317376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005042006018054163, + "loss": 1.4167, + "theoretical_loss": 3.6047959006780586, + "tokens_seen": 1241382912 + }, + { + "epoch": 0.38, + "learning_rate": 0.000504184553660983, + "loss": 1.2663, + "theoretical_loss": 3.6047786443410006, + "tokens_seen": 1241448448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005041685055165497, + "loss": 1.2504, + "theoretical_loss": 3.6047613891699335, + "tokens_seen": 1241513984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005041524573721164, + "loss": 1.2216, + "theoretical_loss": 3.604744135164718, + "tokens_seen": 1241579520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005041364092276831, + "loss": 1.2835, + "theoretical_loss": 3.6047268823252114, + "tokens_seen": 1241645056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005041203610832498, + "loss": 1.4778, + "theoretical_loss": 3.604709630651276, + "tokens_seen": 1241710592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005041043129388165, + "loss": 1.3585, + "theoretical_loss": 3.6046923801427697, + "tokens_seen": 1241776128 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005040882647943832, + "loss": 1.2742, + "theoretical_loss": 3.604675130799553, + "tokens_seen": 1241841664 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005040722166499499, + "loss": 1.3234, + "theoretical_loss": 3.6046578826214852, + "tokens_seen": 1241907200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005040561685055166, + "loss": 1.2365, + "theoretical_loss": 3.604640635608427, + "tokens_seen": 1241972736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005040401203610833, + "loss": 1.3462, + "theoretical_loss": 3.6046233897602376, + "tokens_seen": 1242038272 + }, + { + "epoch": 0.38, + "learning_rate": 0.00050402407221665, + "loss": 1.4403, + "theoretical_loss": 3.604606145076777, + "tokens_seen": 1242103808 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005040080240722167, + "loss": 1.33, + "theoretical_loss": 3.6045889015579045, + "tokens_seen": 1242169344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005039919759277834, + "loss": 1.4073, + "theoretical_loss": 3.6045716592034815, + "tokens_seen": 1242234880 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005039759277833501, + "loss": 1.4206, + "theoretical_loss": 3.6045544180133664, + "tokens_seen": 1242300416 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005039598796389168, + "loss": 1.4498, + "theoretical_loss": 3.6045371779874205, + "tokens_seen": 1242365952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005039438314944835, + "loss": 1.3868, + "theoretical_loss": 3.6045199391255025, + "tokens_seen": 1242431488 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005039277833500502, + "loss": 1.4382, + "theoretical_loss": 3.6045027014274735, + "tokens_seen": 1242497024 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005039117352056169, + "loss": 1.3744, + "theoretical_loss": 3.6044854648931928, + "tokens_seen": 1242562560 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005038956870611836, + "loss": 1.3677, + "theoretical_loss": 3.6044682295225203, + "tokens_seen": 1242628096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005038796389167503, + "loss": 1.3868, + "theoretical_loss": 3.604450995315317, + "tokens_seen": 1242693632 + }, + { + "epoch": 0.38, + "learning_rate": 0.000503863590772317, + "loss": 1.2956, + "theoretical_loss": 3.604433762271442, + "tokens_seen": 1242759168 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005038475426278837, + "loss": 1.3589, + "theoretical_loss": 3.6044165303907567, + "tokens_seen": 1242824704 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005038314944834504, + "loss": 1.4071, + "theoretical_loss": 3.60439929967312, + "tokens_seen": 1242890240 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005038154463390171, + "loss": 1.443, + "theoretical_loss": 3.6043820701183926, + "tokens_seen": 1242955776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005037993981945838, + "loss": 1.3698, + "theoretical_loss": 3.6043648417264347, + "tokens_seen": 1243021312 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005037833500501505, + "loss": 1.3462, + "theoretical_loss": 3.6043476144971063, + "tokens_seen": 1243086848 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005037673019057172, + "loss": 1.3757, + "theoretical_loss": 3.6043303884302684, + "tokens_seen": 1243152384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005037512537612839, + "loss": 1.383, + "theoretical_loss": 3.604313163525781, + "tokens_seen": 1243217920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005037352056168506, + "loss": 1.3088, + "theoretical_loss": 3.604295939783503, + "tokens_seen": 1243283456 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005037191574724173, + "loss": 1.2931, + "theoretical_loss": 3.6042787172032966, + "tokens_seen": 1243348992 + }, + { + "epoch": 0.38, + "learning_rate": 0.000503703109327984, + "loss": 1.2515, + "theoretical_loss": 3.604261495785021, + "tokens_seen": 1243414528 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005036870611835507, + "loss": 1.3094, + "theoretical_loss": 3.6042442755285373, + "tokens_seen": 1243480064 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005036710130391174, + "loss": 1.2695, + "theoretical_loss": 3.6042270564337056, + "tokens_seen": 1243545600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005036549648946841, + "loss": 1.3465, + "theoretical_loss": 3.6042098385003865, + "tokens_seen": 1243611136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005036389167502507, + "loss": 1.2467, + "theoretical_loss": 3.6041926217284397, + "tokens_seen": 1243676672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005036228686058175, + "loss": 1.3276, + "theoretical_loss": 3.6041754061177267, + "tokens_seen": 1243742208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005036068204613841, + "loss": 1.4472, + "theoretical_loss": 3.604158191668107, + "tokens_seen": 1243807744 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005035907723169509, + "loss": 1.3864, + "theoretical_loss": 3.6041409783794425, + "tokens_seen": 1243873280 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005035747241725175, + "loss": 1.2525, + "theoretical_loss": 3.6041237662515924, + "tokens_seen": 1243938816 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005035586760280843, + "loss": 1.4654, + "theoretical_loss": 3.6041065552844174, + "tokens_seen": 1244004352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005035426278836509, + "loss": 1.2846, + "theoretical_loss": 3.604089345477779, + "tokens_seen": 1244069888 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005035265797392177, + "loss": 1.2922, + "theoretical_loss": 3.6040721368315367, + "tokens_seen": 1244135424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005035105315947843, + "loss": 1.3719, + "theoretical_loss": 3.604054929345552, + "tokens_seen": 1244200960 + }, + { + "epoch": 0.38, + "learning_rate": 0.000503494483450351, + "loss": 1.3033, + "theoretical_loss": 3.604037723019686, + "tokens_seen": 1244266496 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005034784353059177, + "loss": 1.3545, + "theoretical_loss": 3.6040205178537974, + "tokens_seen": 1244332032 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005034623871614844, + "loss": 1.3911, + "theoretical_loss": 3.6040033138477487, + "tokens_seen": 1244397568 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005034463390170511, + "loss": 1.3636, + "theoretical_loss": 3.6039861110014004, + "tokens_seen": 1244463104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005034302908726178, + "loss": 1.3968, + "theoretical_loss": 3.6039689093146126, + "tokens_seen": 1244528640 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005034142427281845, + "loss": 1.319, + "theoretical_loss": 3.6039517087872466, + "tokens_seen": 1244594176 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005033981945837512, + "loss": 1.2961, + "theoretical_loss": 3.603934509419163, + "tokens_seen": 1244659712 + }, + { + "epoch": 0.38, + "learning_rate": 0.000503382146439318, + "loss": 1.3385, + "theoretical_loss": 3.603917311210223, + "tokens_seen": 1244725248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005033660982948846, + "loss": 1.4564, + "theoretical_loss": 3.603900114160287, + "tokens_seen": 1244790784 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005033500501504513, + "loss": 1.3608, + "theoretical_loss": 3.6038829182692154, + "tokens_seen": 1244856320 + }, + { + "epoch": 0.38, + "learning_rate": 0.000503334002006018, + "loss": 1.4231, + "theoretical_loss": 3.6038657235368703, + "tokens_seen": 1244921856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005033179538615847, + "loss": 1.4522, + "theoretical_loss": 3.603848529963112, + "tokens_seen": 1244987392 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005033019057171515, + "loss": 1.3742, + "theoretical_loss": 3.6038313375478013, + "tokens_seen": 1245052928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005032858575727182, + "loss": 1.3972, + "theoretical_loss": 3.6038141462907998, + "tokens_seen": 1245118464 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 464634, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4943995475769043, + "objective/train/theoretical_loss": 3.6037969561919674, + "objective/train/tokens_used": 1265644000, + "theoretical_loss": 3.6037969561919674, + "tokens_seen": 1245184000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005032698094282849, + "loss": 1.4944, + "theoretical_loss": 3.6037969561919674, + "tokens_seen": 1245184000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005032537612838516, + "loss": 1.3277, + "theoretical_loss": 3.6037797672511664, + "tokens_seen": 1245249536 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005032377131394183, + "loss": 1.3919, + "theoretical_loss": 3.603762579468257, + "tokens_seen": 1245315072 + }, + { + "epoch": 0.38, + "learning_rate": 0.000503221664994985, + "loss": 1.474, + "theoretical_loss": 3.6037453928431007, + "tokens_seen": 1245380608 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005032056168505517, + "loss": 1.371, + "theoretical_loss": 3.6037282073755583, + "tokens_seen": 1245446144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005031895687061184, + "loss": 1.4123, + "theoretical_loss": 3.6037110230654914, + "tokens_seen": 1245511680 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005031735205616851, + "loss": 1.3928, + "theoretical_loss": 3.6036938399127605, + "tokens_seen": 1245577216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005031574724172518, + "loss": 1.405, + "theoretical_loss": 3.6036766579172266, + "tokens_seen": 1245642752 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005031414242728185, + "loss": 1.3704, + "theoretical_loss": 3.603659477078752, + "tokens_seen": 1245708288 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005031253761283852, + "loss": 1.3355, + "theoretical_loss": 3.6036422973971973, + "tokens_seen": 1245773824 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005031093279839519, + "loss": 1.299, + "theoretical_loss": 3.603625118872423, + "tokens_seen": 1245839360 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005030932798395186, + "loss": 1.3058, + "theoretical_loss": 3.6036079415042916, + "tokens_seen": 1245904896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005030772316950853, + "loss": 1.3353, + "theoretical_loss": 3.603590765292664, + "tokens_seen": 1245970432 + }, + { + "epoch": 0.38, + "learning_rate": 0.000503061183550652, + "loss": 1.3139, + "theoretical_loss": 3.603573590237401, + "tokens_seen": 1246035968 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005030451354062187, + "loss": 1.3014, + "theoretical_loss": 3.603556416338365, + "tokens_seen": 1246101504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005030290872617854, + "loss": 1.4509, + "theoretical_loss": 3.6035392435954154, + "tokens_seen": 1246167040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005030130391173521, + "loss": 1.4033, + "theoretical_loss": 3.603522072008416, + "tokens_seen": 1246232576 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005029969909729188, + "loss": 1.3634, + "theoretical_loss": 3.6035049015772267, + "tokens_seen": 1246298112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005029809428284855, + "loss": 1.369, + "theoretical_loss": 3.6034877323017094, + "tokens_seen": 1246363648 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005029648946840522, + "loss": 1.374, + "theoretical_loss": 3.603470564181725, + "tokens_seen": 1246429184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005029488465396189, + "loss": 1.3122, + "theoretical_loss": 3.6034533972171356, + "tokens_seen": 1246494720 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005029327983951856, + "loss": 1.3474, + "theoretical_loss": 3.6034362314078026, + "tokens_seen": 1246560256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005029167502507523, + "loss": 1.2811, + "theoretical_loss": 3.6034190667535877, + "tokens_seen": 1246625792 + }, + { + "epoch": 0.38, + "learning_rate": 0.000502900702106319, + "loss": 1.3505, + "theoretical_loss": 3.603401903254352, + "tokens_seen": 1246691328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005028846539618857, + "loss": 1.4226, + "theoretical_loss": 3.6033847409099575, + "tokens_seen": 1246756864 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005028686058174524, + "loss": 1.3709, + "theoretical_loss": 3.603367579720265, + "tokens_seen": 1246822400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005028525576730191, + "loss": 1.3745, + "theoretical_loss": 3.603350419685137, + "tokens_seen": 1246887936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005028365095285858, + "loss": 1.3058, + "theoretical_loss": 3.603333260804435, + "tokens_seen": 1246953472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005028204613841525, + "loss": 1.3611, + "theoretical_loss": 3.6033161030780203, + "tokens_seen": 1247019008 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005028044132397192, + "loss": 1.474, + "theoretical_loss": 3.603298946505755, + "tokens_seen": 1247084544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005027883650952859, + "loss": 1.4178, + "theoretical_loss": 3.6032817910875004, + "tokens_seen": 1247150080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005027723169508526, + "loss": 1.4281, + "theoretical_loss": 3.603264636823119, + "tokens_seen": 1247215616 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005027562688064193, + "loss": 1.285, + "theoretical_loss": 3.6032474837124715, + "tokens_seen": 1247281152 + }, + { + "epoch": 0.38, + "learning_rate": 0.000502740220661986, + "loss": 1.2615, + "theoretical_loss": 3.6032303317554204, + "tokens_seen": 1247346688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005027241725175527, + "loss": 1.3609, + "theoretical_loss": 3.603213180951827, + "tokens_seen": 1247412224 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005027081243731194, + "loss": 1.3491, + "theoretical_loss": 3.603196031301554, + "tokens_seen": 1247477760 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005026920762286861, + "loss": 1.4183, + "theoretical_loss": 3.603178882804462, + "tokens_seen": 1247543296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005026760280842528, + "loss": 1.3208, + "theoretical_loss": 3.6031617354604144, + "tokens_seen": 1247608832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005026599799398195, + "loss": 1.3522, + "theoretical_loss": 3.6031445892692724, + "tokens_seen": 1247674368 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005026439317953862, + "loss": 1.3008, + "theoretical_loss": 3.603127444230897, + "tokens_seen": 1247739904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005026278836509529, + "loss": 1.4172, + "theoretical_loss": 3.603110300345152, + "tokens_seen": 1247805440 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005026118355065196, + "loss": 1.4505, + "theoretical_loss": 3.603093157611898, + "tokens_seen": 1247870976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005025957873620863, + "loss": 1.3176, + "theoretical_loss": 3.603076016030997, + "tokens_seen": 1247936512 + }, + { + "epoch": 0.38, + "learning_rate": 0.000502579739217653, + "loss": 1.3978, + "theoretical_loss": 3.6030588756023123, + "tokens_seen": 1248002048 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005025636910732197, + "loss": 1.3083, + "theoretical_loss": 3.603041736325704, + "tokens_seen": 1248067584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005025476429287864, + "loss": 1.3577, + "theoretical_loss": 3.603024598201036, + "tokens_seen": 1248133120 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005025315947843531, + "loss": 1.3429, + "theoretical_loss": 3.6030074612281693, + "tokens_seen": 1248198656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005025155466399198, + "loss": 1.4168, + "theoretical_loss": 3.6029903254069664, + "tokens_seen": 1248264192 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005024994984954865, + "loss": 1.3391, + "theoretical_loss": 3.6029731907372895, + "tokens_seen": 1248329728 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005024834503510532, + "loss": 1.3574, + "theoretical_loss": 3.602956057219001, + "tokens_seen": 1248395264 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005024674022066199, + "loss": 1.3458, + "theoretical_loss": 3.602938924851962, + "tokens_seen": 1248460800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005024513540621866, + "loss": 1.4994, + "theoretical_loss": 3.6029217936360363, + "tokens_seen": 1248526336 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005024353059177533, + "loss": 1.4154, + "theoretical_loss": 3.602904663571085, + "tokens_seen": 1248591872 + }, + { + "epoch": 0.38, + "learning_rate": 0.00050241925777332, + "loss": 1.2558, + "theoretical_loss": 3.602887534656971, + "tokens_seen": 1248657408 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005024032096288867, + "loss": 1.4074, + "theoretical_loss": 3.6028704068935555, + "tokens_seen": 1248722944 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005023871614844534, + "loss": 1.3047, + "theoretical_loss": 3.6028532802807023, + "tokens_seen": 1248788480 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005023711133400201, + "loss": 1.3484, + "theoretical_loss": 3.602836154818273, + "tokens_seen": 1248854016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005023550651955868, + "loss": 1.3573, + "theoretical_loss": 3.6028190305061303, + "tokens_seen": 1248919552 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005023390170511535, + "loss": 1.2839, + "theoretical_loss": 3.6028019073441357, + "tokens_seen": 1248985088 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005023229689067202, + "loss": 1.4861, + "theoretical_loss": 3.602784785332152, + "tokens_seen": 1249050624 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005023069207622869, + "loss": 1.2985, + "theoretical_loss": 3.6027676644700426, + "tokens_seen": 1249116160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005022908726178536, + "loss": 1.3887, + "theoretical_loss": 3.6027505447576686, + "tokens_seen": 1249181696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005022748244734203, + "loss": 1.3211, + "theoretical_loss": 3.6027334261948933, + "tokens_seen": 1249247232 + }, + { + "epoch": 0.38, + "learning_rate": 0.000502258776328987, + "loss": 1.4253, + "theoretical_loss": 3.6027163087815786, + "tokens_seen": 1249312768 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005022427281845536, + "loss": 1.223, + "theoretical_loss": 3.602699192517588, + "tokens_seen": 1249378304 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005022266800401204, + "loss": 1.247, + "theoretical_loss": 3.6026820774027826, + "tokens_seen": 1249443840 + }, + { + "epoch": 0.38, + "learning_rate": 0.000502210631895687, + "loss": 1.31, + "theoretical_loss": 3.6026649634370265, + "tokens_seen": 1249509376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005021945837512538, + "loss": 1.311, + "theoretical_loss": 3.6026478506201816, + "tokens_seen": 1249574912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005021785356068204, + "loss": 1.3549, + "theoretical_loss": 3.6026307389521097, + "tokens_seen": 1249640448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005021624874623872, + "loss": 1.3598, + "theoretical_loss": 3.602613628432675, + "tokens_seen": 1249705984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005021464393179538, + "loss": 1.305, + "theoretical_loss": 3.6025965190617395, + "tokens_seen": 1249771520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005021303911735206, + "loss": 1.306, + "theoretical_loss": 3.6025794108391658, + "tokens_seen": 1249837056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005021143430290872, + "loss": 1.3886, + "theoretical_loss": 3.602562303764816, + "tokens_seen": 1249902592 + }, + { + "epoch": 0.38, + "learning_rate": 0.000502098294884654, + "loss": 1.3523, + "theoretical_loss": 3.6025451978385536, + "tokens_seen": 1249968128 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005020822467402206, + "loss": 1.2703, + "theoretical_loss": 3.6025280930602417, + "tokens_seen": 1250033664 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005020661985957874, + "loss": 1.3021, + "theoretical_loss": 3.6025109894297427, + "tokens_seen": 1250099200 + }, + { + "epoch": 0.38, + "learning_rate": 0.000502050150451354, + "loss": 1.3018, + "theoretical_loss": 3.602493886946919, + "tokens_seen": 1250164736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005020341023069208, + "loss": 1.2462, + "theoretical_loss": 3.6024767856116338, + "tokens_seen": 1250230272 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005020180541624874, + "loss": 1.3531, + "theoretical_loss": 3.60245968542375, + "tokens_seen": 1250295808 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005020020060180542, + "loss": 1.3802, + "theoretical_loss": 3.6024425863831304, + "tokens_seen": 1250361344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005019859578736208, + "loss": 1.3155, + "theoretical_loss": 3.6024254884896374, + "tokens_seen": 1250426880 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005019699097291876, + "loss": 1.4032, + "theoretical_loss": 3.6024083917431353, + "tokens_seen": 1250492416 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005019538615847542, + "loss": 1.2596, + "theoretical_loss": 3.602391296143485, + "tokens_seen": 1250557952 + }, + { + "epoch": 0.38, + "learning_rate": 0.000501937813440321, + "loss": 1.2961, + "theoretical_loss": 3.6023742016905516, + "tokens_seen": 1250623488 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005019217652958877, + "loss": 1.4271, + "theoretical_loss": 3.602357108384197, + "tokens_seen": 1250689024 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005019057171514544, + "loss": 1.2855, + "theoretical_loss": 3.602340016224284, + "tokens_seen": 1250754560 + }, + { + "epoch": 0.38, + "learning_rate": 0.000501889669007021, + "loss": 1.3072, + "theoretical_loss": 3.6023229252106765, + "tokens_seen": 1250820096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005018736208625878, + "loss": 1.3988, + "theoretical_loss": 3.6023058353432367, + "tokens_seen": 1250885632 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005018575727181545, + "loss": 1.3874, + "theoretical_loss": 3.602288746621828, + "tokens_seen": 1250951168 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005018415245737212, + "loss": 1.4641, + "theoretical_loss": 3.602271659046314, + "tokens_seen": 1251016704 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005018254764292879, + "loss": 1.4418, + "theoretical_loss": 3.602254572616557, + "tokens_seen": 1251082240 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005018094282848546, + "loss": 1.3561, + "theoretical_loss": 3.602237487332421, + "tokens_seen": 1251147776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005017933801404213, + "loss": 1.2997, + "theoretical_loss": 3.6022204031937686, + "tokens_seen": 1251213312 + }, + { + "epoch": 0.38, + "learning_rate": 0.000501777331995988, + "loss": 1.2848, + "theoretical_loss": 3.602203320200463, + "tokens_seen": 1251278848 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005017612838515547, + "loss": 1.359, + "theoretical_loss": 3.6021862383523677, + "tokens_seen": 1251344384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005017452357071214, + "loss": 1.3009, + "theoretical_loss": 3.602169157649346, + "tokens_seen": 1251409920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005017291875626881, + "loss": 1.3855, + "theoretical_loss": 3.602152078091261, + "tokens_seen": 1251475456 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005017131394182548, + "loss": 1.3381, + "theoretical_loss": 3.6021349996779763, + "tokens_seen": 1251540992 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005016970912738215, + "loss": 1.2761, + "theoretical_loss": 3.6021179224093545, + "tokens_seen": 1251606528 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005016810431293882, + "loss": 1.3626, + "theoretical_loss": 3.6021008462852593, + "tokens_seen": 1251672064 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 466920, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3522955179214478, + "objective/train/theoretical_loss": 3.6020837713055545, + "objective/train/tokens_used": 1272197600, + "theoretical_loss": 3.6020837713055545, + "tokens_seen": 1251737600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005016649949849549, + "loss": 1.3523, + "theoretical_loss": 3.6020837713055545, + "tokens_seen": 1251737600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005016489468405216, + "loss": 1.3247, + "theoretical_loss": 3.602066697470103, + "tokens_seen": 1251803136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005016328986960883, + "loss": 1.3308, + "theoretical_loss": 3.6020496247787688, + "tokens_seen": 1251868672 + }, + { + "epoch": 0.38, + "learning_rate": 0.000501616850551655, + "loss": 1.3528, + "theoretical_loss": 3.602032553231415, + "tokens_seen": 1251934208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005016008024072217, + "loss": 1.3771, + "theoretical_loss": 3.6020154828279045, + "tokens_seen": 1251999744 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005015847542627884, + "loss": 1.3367, + "theoretical_loss": 3.6019984135681016, + "tokens_seen": 1252065280 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005015687061183551, + "loss": 1.4154, + "theoretical_loss": 3.601981345451869, + "tokens_seen": 1252130816 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005015526579739218, + "loss": 1.2914, + "theoretical_loss": 3.6019642784790715, + "tokens_seen": 1252196352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005015366098294885, + "loss": 1.3742, + "theoretical_loss": 3.6019472126495717, + "tokens_seen": 1252261888 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005015205616850552, + "loss": 1.5392, + "theoretical_loss": 3.601930147963234, + "tokens_seen": 1252327424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005015045135406219, + "loss": 1.3756, + "theoretical_loss": 3.6019130844199205, + "tokens_seen": 1252392960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005014884653961886, + "loss": 1.3993, + "theoretical_loss": 3.601896022019496, + "tokens_seen": 1252458496 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005014724172517553, + "loss": 1.4101, + "theoretical_loss": 3.601878960761824, + "tokens_seen": 1252524032 + }, + { + "epoch": 0.38, + "learning_rate": 0.000501456369107322, + "loss": 1.2763, + "theoretical_loss": 3.6018619006467683, + "tokens_seen": 1252589568 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005014403209628887, + "loss": 1.2389, + "theoretical_loss": 3.601844841674192, + "tokens_seen": 1252655104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005014242728184554, + "loss": 1.3459, + "theoretical_loss": 3.6018277838439596, + "tokens_seen": 1252720640 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005014082246740221, + "loss": 1.4651, + "theoretical_loss": 3.601810727155934, + "tokens_seen": 1252786176 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005013921765295888, + "loss": 1.3933, + "theoretical_loss": 3.6017936716099794, + "tokens_seen": 1252851712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005013761283851555, + "loss": 1.333, + "theoretical_loss": 3.60177661720596, + "tokens_seen": 1252917248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005013600802407222, + "loss": 1.3965, + "theoretical_loss": 3.601759563943739, + "tokens_seen": 1252982784 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005013440320962889, + "loss": 1.2648, + "theoretical_loss": 3.601742511823181, + "tokens_seen": 1253048320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005013279839518556, + "loss": 1.5542, + "theoretical_loss": 3.6017254608441487, + "tokens_seen": 1253113856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005013119358074223, + "loss": 1.2782, + "theoretical_loss": 3.6017084110065065, + "tokens_seen": 1253179392 + }, + { + "epoch": 0.38, + "learning_rate": 0.000501295887662989, + "loss": 1.2666, + "theoretical_loss": 3.601691362310119, + "tokens_seen": 1253244928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005012798395185557, + "loss": 1.4584, + "theoretical_loss": 3.6016743147548493, + "tokens_seen": 1253310464 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005012637913741224, + "loss": 1.3877, + "theoretical_loss": 3.6016572683405617, + "tokens_seen": 1253376000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005012477432296891, + "loss": 1.4322, + "theoretical_loss": 3.60164022306712, + "tokens_seen": 1253441536 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005012316950852558, + "loss": 1.3283, + "theoretical_loss": 3.6016231789343887, + "tokens_seen": 1253507072 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005012156469408225, + "loss": 1.3224, + "theoretical_loss": 3.601606135942231, + "tokens_seen": 1253572608 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005011995987963892, + "loss": 1.4504, + "theoretical_loss": 3.601589094090512, + "tokens_seen": 1253638144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005011835506519559, + "loss": 1.3613, + "theoretical_loss": 3.6015720533790945, + "tokens_seen": 1253703680 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005011675025075226, + "loss": 1.421, + "theoretical_loss": 3.601555013807844, + "tokens_seen": 1253769216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005011514543630893, + "loss": 1.5193, + "theoretical_loss": 3.6015379753766235, + "tokens_seen": 1253834752 + }, + { + "epoch": 0.38, + "learning_rate": 0.000501135406218656, + "loss": 1.379, + "theoretical_loss": 3.6015209380852973, + "tokens_seen": 1253900288 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005011193580742227, + "loss": 1.4393, + "theoretical_loss": 3.60150390193373, + "tokens_seen": 1253965824 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005011033099297894, + "loss": 1.3073, + "theoretical_loss": 3.6014868669217854, + "tokens_seen": 1254031360 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005010872617853561, + "loss": 1.518, + "theoretical_loss": 3.6014698330493284, + "tokens_seen": 1254096896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005010712136409228, + "loss": 1.4067, + "theoretical_loss": 3.6014528003162223, + "tokens_seen": 1254162432 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005010551654964895, + "loss": 1.3509, + "theoretical_loss": 3.6014357687223324, + "tokens_seen": 1254227968 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005010391173520562, + "loss": 1.4841, + "theoretical_loss": 3.6014187382675216, + "tokens_seen": 1254293504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005010230692076229, + "loss": 1.3344, + "theoretical_loss": 3.6014017089516557, + "tokens_seen": 1254359040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005010070210631896, + "loss": 1.4701, + "theoretical_loss": 3.601384680774598, + "tokens_seen": 1254424576 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005009909729187563, + "loss": 1.434, + "theoretical_loss": 3.6013676537362134, + "tokens_seen": 1254490112 + }, + { + "epoch": 0.38, + "learning_rate": 0.000500974924774323, + "loss": 1.2267, + "theoretical_loss": 3.601350627836366, + "tokens_seen": 1254555648 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005009588766298897, + "loss": 1.3254, + "theoretical_loss": 3.60133360307492, + "tokens_seen": 1254621184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005009428284854564, + "loss": 1.242, + "theoretical_loss": 3.6013165794517406, + "tokens_seen": 1254686720 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005009267803410231, + "loss": 1.4128, + "theoretical_loss": 3.6012995569666915, + "tokens_seen": 1254752256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005009107321965898, + "loss": 1.3238, + "theoretical_loss": 3.601282535619638, + "tokens_seen": 1254817792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005008946840521565, + "loss": 1.2797, + "theoretical_loss": 3.601265515410443, + "tokens_seen": 1254883328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005008786359077232, + "loss": 1.3448, + "theoretical_loss": 3.6012484963389726, + "tokens_seen": 1254948864 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005008625877632899, + "loss": 1.4044, + "theoretical_loss": 3.60123147840509, + "tokens_seen": 1255014400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005008465396188565, + "loss": 1.4772, + "theoretical_loss": 3.601214461608661, + "tokens_seen": 1255079936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005008304914744233, + "loss": 1.4631, + "theoretical_loss": 3.60119744594955, + "tokens_seen": 1255145472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005008144433299899, + "loss": 1.2951, + "theoretical_loss": 3.601180431427621, + "tokens_seen": 1255211008 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005007983951855567, + "loss": 1.3871, + "theoretical_loss": 3.6011634180427397, + "tokens_seen": 1255276544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005007823470411233, + "loss": 1.3125, + "theoretical_loss": 3.6011464057947693, + "tokens_seen": 1255342080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005007662988966901, + "loss": 1.3851, + "theoretical_loss": 3.601129394683575, + "tokens_seen": 1255407616 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005007502507522567, + "loss": 1.4146, + "theoretical_loss": 3.6011123847090225, + "tokens_seen": 1255473152 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005007342026078235, + "loss": 1.34, + "theoretical_loss": 3.601095375870975, + "tokens_seen": 1255538688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005007181544633901, + "loss": 1.2836, + "theoretical_loss": 3.601078368169298, + "tokens_seen": 1255604224 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005007021063189569, + "loss": 1.2926, + "theoretical_loss": 3.6010613616038567, + "tokens_seen": 1255669760 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005006860581745235, + "loss": 1.4525, + "theoretical_loss": 3.6010443561745156, + "tokens_seen": 1255735296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005006700100300903, + "loss": 1.3249, + "theoretical_loss": 3.6010273518811386, + "tokens_seen": 1255800832 + }, + { + "epoch": 0.38, + "learning_rate": 0.000500653961885657, + "loss": 1.4953, + "theoretical_loss": 3.601010348723592, + "tokens_seen": 1255866368 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005006379137412237, + "loss": 1.4817, + "theoretical_loss": 3.600993346701739, + "tokens_seen": 1255931904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005006218655967903, + "loss": 1.4163, + "theoretical_loss": 3.6009763458154462, + "tokens_seen": 1255997440 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005006058174523571, + "loss": 1.3763, + "theoretical_loss": 3.6009593460645775, + "tokens_seen": 1256062976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005005897693079237, + "loss": 1.261, + "theoretical_loss": 3.6009423474489983, + "tokens_seen": 1256128512 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005005737211634905, + "loss": 1.4189, + "theoretical_loss": 3.6009253499685734, + "tokens_seen": 1256194048 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005005576730190571, + "loss": 1.4134, + "theoretical_loss": 3.6009083536231676, + "tokens_seen": 1256259584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005005416248746239, + "loss": 1.4498, + "theoretical_loss": 3.6008913584126456, + "tokens_seen": 1256325120 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005005255767301906, + "loss": 1.3224, + "theoretical_loss": 3.6008743643368737, + "tokens_seen": 1256390656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005005095285857573, + "loss": 1.3193, + "theoretical_loss": 3.6008573713957155, + "tokens_seen": 1256456192 + }, + { + "epoch": 0.38, + "learning_rate": 0.000500493480441324, + "loss": 1.4187, + "theoretical_loss": 3.6008403795890365, + "tokens_seen": 1256521728 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005004774322968907, + "loss": 1.3855, + "theoretical_loss": 3.6008233889167025, + "tokens_seen": 1256587264 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005004613841524574, + "loss": 1.3769, + "theoretical_loss": 3.6008063993785777, + "tokens_seen": 1256652800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005004453360080241, + "loss": 1.4571, + "theoretical_loss": 3.600789410974528, + "tokens_seen": 1256718336 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005004292878635908, + "loss": 1.5854, + "theoretical_loss": 3.600772423704418, + "tokens_seen": 1256783872 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005004132397191575, + "loss": 1.3782, + "theoretical_loss": 3.6007554375681132, + "tokens_seen": 1256849408 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005003971915747242, + "loss": 1.3488, + "theoretical_loss": 3.600738452565478, + "tokens_seen": 1256914944 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005003811434302909, + "loss": 1.3177, + "theoretical_loss": 3.6007214686963795, + "tokens_seen": 1256980480 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005003650952858576, + "loss": 1.4246, + "theoretical_loss": 3.600704485960681, + "tokens_seen": 1257046016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005003490471414243, + "loss": 1.3737, + "theoretical_loss": 3.600687504358249, + "tokens_seen": 1257111552 + }, + { + "epoch": 0.38, + "learning_rate": 0.000500332998996991, + "loss": 1.3447, + "theoretical_loss": 3.6006705238889483, + "tokens_seen": 1257177088 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005003169508525577, + "loss": 1.4386, + "theoretical_loss": 3.600653544552644, + "tokens_seen": 1257242624 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005003009027081244, + "loss": 1.35, + "theoretical_loss": 3.600636566349202, + "tokens_seen": 1257308160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005002848545636911, + "loss": 1.3323, + "theoretical_loss": 3.6006195892784874, + "tokens_seen": 1257373696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005002688064192578, + "loss": 1.4146, + "theoretical_loss": 3.6006026133403655, + "tokens_seen": 1257439232 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005002527582748245, + "loss": 1.2936, + "theoretical_loss": 3.6005856385347013, + "tokens_seen": 1257504768 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005002367101303912, + "loss": 1.3107, + "theoretical_loss": 3.6005686648613615, + "tokens_seen": 1257570304 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005002206619859579, + "loss": 1.5092, + "theoretical_loss": 3.6005516923202108, + "tokens_seen": 1257635840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005002046138415246, + "loss": 1.4233, + "theoretical_loss": 3.6005347209111145, + "tokens_seen": 1257701376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005001885656970913, + "loss": 1.4873, + "theoretical_loss": 3.600517750633938, + "tokens_seen": 1257766912 + }, + { + "epoch": 0.38, + "learning_rate": 0.000500172517552658, + "loss": 1.4322, + "theoretical_loss": 3.600500781488548, + "tokens_seen": 1257832448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005001564694082247, + "loss": 1.4331, + "theoretical_loss": 3.600483813474809, + "tokens_seen": 1257897984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005001404212637914, + "loss": 1.3066, + "theoretical_loss": 3.600466846592586, + "tokens_seen": 1257963520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005001243731193581, + "loss": 1.2841, + "theoretical_loss": 3.6004498808417464, + "tokens_seen": 1258029056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005001083249749248, + "loss": 1.3524, + "theoretical_loss": 3.6004329162221542, + "tokens_seen": 1258094592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005000922768304915, + "loss": 1.3055, + "theoretical_loss": 3.600415952733676, + "tokens_seen": 1258160128 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005000762286860582, + "loss": 1.3891, + "theoretical_loss": 3.600398990376177, + "tokens_seen": 1258225664 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 469219, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4047744274139404, + "objective/train/theoretical_loss": 3.600382029149523, + "objective/train/tokens_used": 1278751200, + "theoretical_loss": 3.600382029149523, + "tokens_seen": 1258291200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005000601805416249, + "loss": 1.4048, + "theoretical_loss": 3.600382029149523, + "tokens_seen": 1258291200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005000441323971916, + "loss": 1.4145, + "theoretical_loss": 3.60036506905358, + "tokens_seen": 1258356736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0005000280842527583, + "loss": 1.2372, + "theoretical_loss": 3.600348110088213, + "tokens_seen": 1258422272 + }, + { + "epoch": 0.38, + "learning_rate": 0.000500012036108325, + "loss": 1.2913, + "theoretical_loss": 3.600331152253289, + "tokens_seen": 1258487808 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004999959879638917, + "loss": 1.3503, + "theoretical_loss": 3.6003141955486724, + "tokens_seen": 1258553344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004999799398194584, + "loss": 1.4265, + "theoretical_loss": 3.6002972399742297, + "tokens_seen": 1258618880 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004999638916750251, + "loss": 1.4208, + "theoretical_loss": 3.600280285529827, + "tokens_seen": 1258684416 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004999478435305918, + "loss": 1.3216, + "theoretical_loss": 3.6002633322153295, + "tokens_seen": 1258749952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004999317953861585, + "loss": 1.4289, + "theoretical_loss": 3.6002463800306037, + "tokens_seen": 1258815488 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004999157472417252, + "loss": 1.3726, + "theoretical_loss": 3.6002294289755152, + "tokens_seen": 1258881024 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004998996990972919, + "loss": 1.397, + "theoretical_loss": 3.6002124790499295, + "tokens_seen": 1258946560 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004998836509528586, + "loss": 1.3213, + "theoretical_loss": 3.600195530253713, + "tokens_seen": 1259012096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004998676028084253, + "loss": 1.427, + "theoretical_loss": 3.6001785825867323, + "tokens_seen": 1259077632 + }, + { + "epoch": 0.38, + "learning_rate": 0.000499851554663992, + "loss": 1.4166, + "theoretical_loss": 3.6001616360488526, + "tokens_seen": 1259143168 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004998355065195587, + "loss": 1.2778, + "theoretical_loss": 3.6001446906399397, + "tokens_seen": 1259208704 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004998194583751254, + "loss": 1.3433, + "theoretical_loss": 3.60012774635986, + "tokens_seen": 1259274240 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004998034102306921, + "loss": 1.402, + "theoretical_loss": 3.60011080320848, + "tokens_seen": 1259339776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004997873620862588, + "loss": 1.345, + "theoretical_loss": 3.600093861185665, + "tokens_seen": 1259405312 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004997713139418255, + "loss": 1.4157, + "theoretical_loss": 3.600076920291281, + "tokens_seen": 1259470848 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004997552657973922, + "loss": 1.3997, + "theoretical_loss": 3.600059980525195, + "tokens_seen": 1259536384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004997392176529589, + "loss": 1.3191, + "theoretical_loss": 3.600043041887273, + "tokens_seen": 1259601920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004997231695085256, + "loss": 1.2851, + "theoretical_loss": 3.600026104377381, + "tokens_seen": 1259667456 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004997071213640923, + "loss": 1.3574, + "theoretical_loss": 3.6000091679953847, + "tokens_seen": 1259732992 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004996910732196591, + "loss": 1.3203, + "theoretical_loss": 3.5999922327411507, + "tokens_seen": 1259798528 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004996750250752257, + "loss": 1.3665, + "theoretical_loss": 3.5999752986145452, + "tokens_seen": 1259864064 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004996589769307925, + "loss": 1.3376, + "theoretical_loss": 3.599958365615435, + "tokens_seen": 1259929600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004996429287863591, + "loss": 1.4065, + "theoretical_loss": 3.5999414337436852, + "tokens_seen": 1259995136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004996268806419259, + "loss": 1.4472, + "theoretical_loss": 3.5999245029991633, + "tokens_seen": 1260060672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004996108324974925, + "loss": 1.4231, + "theoretical_loss": 3.599907573381735, + "tokens_seen": 1260126208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004995947843530592, + "loss": 1.4522, + "theoretical_loss": 3.599890644891267, + "tokens_seen": 1260191744 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004995787362086259, + "loss": 1.3942, + "theoretical_loss": 3.5998737175276254, + "tokens_seen": 1260257280 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004995626880641926, + "loss": 1.3016, + "theoretical_loss": 3.5998567912906765, + "tokens_seen": 1260322816 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004995466399197593, + "loss": 1.4314, + "theoretical_loss": 3.599839866180287, + "tokens_seen": 1260388352 + }, + { + "epoch": 0.38, + "learning_rate": 0.000499530591775326, + "loss": 1.3509, + "theoretical_loss": 3.5998229421963233, + "tokens_seen": 1260453888 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004995145436308927, + "loss": 1.3804, + "theoretical_loss": 3.5998060193386516, + "tokens_seen": 1260519424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004994984954864594, + "loss": 1.3267, + "theoretical_loss": 3.599789097607139, + "tokens_seen": 1260584960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004994824473420261, + "loss": 1.4271, + "theoretical_loss": 3.5997721770016518, + "tokens_seen": 1260650496 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004994663991975928, + "loss": 1.4118, + "theoretical_loss": 3.599755257522056, + "tokens_seen": 1260716032 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004994503510531594, + "loss": 1.3108, + "theoretical_loss": 3.5997383391682187, + "tokens_seen": 1260781568 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004994343029087262, + "loss": 1.4955, + "theoretical_loss": 3.599721421940006, + "tokens_seen": 1260847104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004994182547642928, + "loss": 1.4285, + "theoretical_loss": 3.599704505837286, + "tokens_seen": 1260912640 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004994022066198596, + "loss": 1.2906, + "theoretical_loss": 3.599687590859923, + "tokens_seen": 1260978176 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004993861584754262, + "loss": 1.4403, + "theoretical_loss": 3.599670677007785, + "tokens_seen": 1261043712 + }, + { + "epoch": 0.38, + "learning_rate": 0.000499370110330993, + "loss": 1.3372, + "theoretical_loss": 3.5996537642807387, + "tokens_seen": 1261109248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004993540621865596, + "loss": 1.4395, + "theoretical_loss": 3.5996368526786506, + "tokens_seen": 1261174784 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004993380140421264, + "loss": 1.3157, + "theoretical_loss": 3.5996199422013877, + "tokens_seen": 1261240320 + }, + { + "epoch": 0.38, + "learning_rate": 0.000499321965897693, + "loss": 1.4359, + "theoretical_loss": 3.599603032848816, + "tokens_seen": 1261305856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004993059177532598, + "loss": 1.3179, + "theoretical_loss": 3.599586124620803, + "tokens_seen": 1261371392 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004992898696088264, + "loss": 1.4381, + "theoretical_loss": 3.5995692175172147, + "tokens_seen": 1261436928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004992738214643932, + "loss": 1.4161, + "theoretical_loss": 3.599552311537919, + "tokens_seen": 1261502464 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004992577733199598, + "loss": 1.3407, + "theoretical_loss": 3.599535406682782, + "tokens_seen": 1261568000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004992417251755266, + "loss": 1.4378, + "theoretical_loss": 3.5995185029516708, + "tokens_seen": 1261633536 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004992256770310932, + "loss": 1.3633, + "theoretical_loss": 3.5995016003444524, + "tokens_seen": 1261699072 + }, + { + "epoch": 0.38, + "learning_rate": 0.00049920962888666, + "loss": 1.3965, + "theoretical_loss": 3.5994846988609934, + "tokens_seen": 1261764608 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004991935807422266, + "loss": 1.3417, + "theoretical_loss": 3.59946779850116, + "tokens_seen": 1261830144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004991775325977934, + "loss": 1.3917, + "theoretical_loss": 3.599450899264821, + "tokens_seen": 1261895680 + }, + { + "epoch": 0.38, + "learning_rate": 0.00049916148445336, + "loss": 1.3633, + "theoretical_loss": 3.599434001151842, + "tokens_seen": 1261961216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004991454363089268, + "loss": 1.2752, + "theoretical_loss": 3.5994171041620904, + "tokens_seen": 1262026752 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004991293881644935, + "loss": 1.3872, + "theoretical_loss": 3.5994002082954335, + "tokens_seen": 1262092288 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004991133400200602, + "loss": 1.3337, + "theoretical_loss": 3.599383313551737, + "tokens_seen": 1262157824 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004990972918756269, + "loss": 1.3516, + "theoretical_loss": 3.59936641993087, + "tokens_seen": 1262223360 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004990812437311936, + "loss": 1.3292, + "theoretical_loss": 3.599349527432698, + "tokens_seen": 1262288896 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004990651955867603, + "loss": 1.2523, + "theoretical_loss": 3.5993326360570888, + "tokens_seen": 1262354432 + }, + { + "epoch": 0.38, + "learning_rate": 0.000499049147442327, + "loss": 1.3071, + "theoretical_loss": 3.599315745803909, + "tokens_seen": 1262419968 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004990330992978937, + "loss": 1.482, + "theoretical_loss": 3.599298856673027, + "tokens_seen": 1262485504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004990170511534604, + "loss": 1.4289, + "theoretical_loss": 3.5992819686643083, + "tokens_seen": 1262551040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004990010030090271, + "loss": 1.3368, + "theoretical_loss": 3.5992650817776215, + "tokens_seen": 1262616576 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004989849548645938, + "loss": 1.3564, + "theoretical_loss": 3.599248196012833, + "tokens_seen": 1262682112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004989689067201605, + "loss": 1.2852, + "theoretical_loss": 3.59923131136981, + "tokens_seen": 1262747648 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004989528585757272, + "loss": 1.3465, + "theoretical_loss": 3.5992144278484206, + "tokens_seen": 1262813184 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004989368104312939, + "loss": 1.3968, + "theoretical_loss": 3.599197545448531, + "tokens_seen": 1262878720 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004989207622868606, + "loss": 1.3304, + "theoretical_loss": 3.5991806641700093, + "tokens_seen": 1262944256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004989047141424273, + "loss": 1.4434, + "theoretical_loss": 3.599163784012722, + "tokens_seen": 1263009792 + }, + { + "epoch": 0.38, + "learning_rate": 0.000498888665997994, + "loss": 1.2785, + "theoretical_loss": 3.5991469049765374, + "tokens_seen": 1263075328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004988726178535607, + "loss": 1.5067, + "theoretical_loss": 3.5991300270613227, + "tokens_seen": 1263140864 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004988565697091274, + "loss": 1.3279, + "theoretical_loss": 3.5991131502669447, + "tokens_seen": 1263206400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004988405215646941, + "loss": 1.3307, + "theoretical_loss": 3.599096274593272, + "tokens_seen": 1263271936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004988244734202608, + "loss": 1.4218, + "theoretical_loss": 3.5990794000401705, + "tokens_seen": 1263337472 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004988084252758275, + "loss": 1.3191, + "theoretical_loss": 3.5990625266075087, + "tokens_seen": 1263403008 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004987923771313942, + "loss": 1.3418, + "theoretical_loss": 3.5990456542951534, + "tokens_seen": 1263468544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004987763289869609, + "loss": 1.3498, + "theoretical_loss": 3.5990287831029733, + "tokens_seen": 1263534080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004987602808425276, + "loss": 1.3824, + "theoretical_loss": 3.5990119130308345, + "tokens_seen": 1263599616 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004987442326980943, + "loss": 1.4307, + "theoretical_loss": 3.5989950440786056, + "tokens_seen": 1263665152 + }, + { + "epoch": 0.38, + "learning_rate": 0.000498728184553661, + "loss": 1.3456, + "theoretical_loss": 3.5989781762461535, + "tokens_seen": 1263730688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004987121364092277, + "loss": 1.2969, + "theoretical_loss": 3.5989613095333466, + "tokens_seen": 1263796224 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004986960882647944, + "loss": 1.2862, + "theoretical_loss": 3.5989444439400518, + "tokens_seen": 1263861760 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004986800401203611, + "loss": 1.4465, + "theoretical_loss": 3.598927579466137, + "tokens_seen": 1263927296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004986639919759278, + "loss": 1.4093, + "theoretical_loss": 3.5989107161114697, + "tokens_seen": 1263992832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004986479438314945, + "loss": 1.3977, + "theoretical_loss": 3.598893853875918, + "tokens_seen": 1264058368 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004986318956870612, + "loss": 1.3801, + "theoretical_loss": 3.598876992759349, + "tokens_seen": 1264123904 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004986158475426279, + "loss": 1.3726, + "theoretical_loss": 3.598860132761631, + "tokens_seen": 1264189440 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004985997993981946, + "loss": 1.4458, + "theoretical_loss": 3.5988432738826317, + "tokens_seen": 1264254976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004985837512537613, + "loss": 1.4334, + "theoretical_loss": 3.5988264161222183, + "tokens_seen": 1264320512 + }, + { + "epoch": 0.38, + "learning_rate": 0.000498567703109328, + "loss": 1.3064, + "theoretical_loss": 3.5988095594802596, + "tokens_seen": 1264386048 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004985516549648947, + "loss": 1.4765, + "theoretical_loss": 3.5987927039566228, + "tokens_seen": 1264451584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004985356068204614, + "loss": 1.3005, + "theoretical_loss": 3.5987758495511755, + "tokens_seen": 1264517120 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004985195586760281, + "loss": 1.3715, + "theoretical_loss": 3.5987589962637863, + "tokens_seen": 1264582656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004985035105315948, + "loss": 1.4514, + "theoretical_loss": 3.5987421440943224, + "tokens_seen": 1264648192 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004984874623871616, + "loss": 1.3729, + "theoretical_loss": 3.5987252930426523, + "tokens_seen": 1264713728 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004984714142427282, + "loss": 1.4293, + "theoretical_loss": 3.5987084431086434, + "tokens_seen": 1264779264 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 471458, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3350255489349365, + "objective/train/theoretical_loss": 3.5986915942921636, + "objective/train/tokens_used": 1285304800, + "theoretical_loss": 3.5986915942921636, + "tokens_seen": 1264844800 + }, + { + "epoch": 0.38, + "learning_rate": 0.000498455366098295, + "loss": 1.335, + "theoretical_loss": 3.5986915942921636, + "tokens_seen": 1264844800 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004984393179538616, + "loss": 1.418, + "theoretical_loss": 3.598674746593082, + "tokens_seen": 1264910336 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004984232698094284, + "loss": 1.3695, + "theoretical_loss": 3.598657900011265, + "tokens_seen": 1264975872 + }, + { + "epoch": 0.38, + "learning_rate": 0.000498407221664995, + "loss": 1.3953, + "theoretical_loss": 3.598641054546582, + "tokens_seen": 1265041408 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004983911735205618, + "loss": 1.3459, + "theoretical_loss": 3.5986242101989006, + "tokens_seen": 1265106944 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004983751253761284, + "loss": 1.3766, + "theoretical_loss": 3.5986073669680883, + "tokens_seen": 1265172480 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004983590772316952, + "loss": 1.2846, + "theoretical_loss": 3.598590524854014, + "tokens_seen": 1265238016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004983430290872618, + "loss": 1.2862, + "theoretical_loss": 3.5985736838565456, + "tokens_seen": 1265303552 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004983269809428286, + "loss": 1.3138, + "theoretical_loss": 3.598556843975551, + "tokens_seen": 1265369088 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004983109327983952, + "loss": 1.504, + "theoretical_loss": 3.5985400052108982, + "tokens_seen": 1265434624 + }, + { + "epoch": 0.38, + "learning_rate": 0.000498294884653962, + "loss": 1.2758, + "theoretical_loss": 3.598523167562456, + "tokens_seen": 1265500160 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004982788365095286, + "loss": 1.4413, + "theoretical_loss": 3.5985063310300918, + "tokens_seen": 1265565696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004982627883650954, + "loss": 1.3382, + "theoretical_loss": 3.5984894956136753, + "tokens_seen": 1265631232 + }, + { + "epoch": 0.38, + "learning_rate": 0.000498246740220662, + "loss": 1.2997, + "theoretical_loss": 3.598472661313073, + "tokens_seen": 1265696768 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004982306920762288, + "loss": 1.4011, + "theoretical_loss": 3.598455828128154, + "tokens_seen": 1265762304 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004982146439317954, + "loss": 1.4017, + "theoretical_loss": 3.598438996058787, + "tokens_seen": 1265827840 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004981985957873622, + "loss": 1.3439, + "theoretical_loss": 3.5984221651048394, + "tokens_seen": 1265893376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004981825476429288, + "loss": 1.3763, + "theoretical_loss": 3.5984053352661807, + "tokens_seen": 1265958912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004981664994984955, + "loss": 1.3942, + "theoretical_loss": 3.598388506542678, + "tokens_seen": 1266024448 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004981504513540622, + "loss": 1.4165, + "theoretical_loss": 3.5983716789342, + "tokens_seen": 1266089984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004981344032096289, + "loss": 1.3337, + "theoretical_loss": 3.598354852440616, + "tokens_seen": 1266155520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004981183550651956, + "loss": 1.4478, + "theoretical_loss": 3.5983380270617937, + "tokens_seen": 1266221056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004981023069207623, + "loss": 1.304, + "theoretical_loss": 3.5983212027976013, + "tokens_seen": 1266286592 + }, + { + "epoch": 0.38, + "learning_rate": 0.000498086258776329, + "loss": 1.4393, + "theoretical_loss": 3.5983043796479084, + "tokens_seen": 1266352128 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004980702106318957, + "loss": 1.4848, + "theoretical_loss": 3.598287557612582, + "tokens_seen": 1266417664 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004980541624874624, + "loss": 1.3745, + "theoretical_loss": 3.5982707366914917, + "tokens_seen": 1266483200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004980381143430291, + "loss": 1.3508, + "theoretical_loss": 3.598253916884506, + "tokens_seen": 1266548736 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004980220661985957, + "loss": 1.5242, + "theoretical_loss": 3.5982370981914924, + "tokens_seen": 1266614272 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004980060180541625, + "loss": 1.4397, + "theoretical_loss": 3.5982202806123205, + "tokens_seen": 1266679808 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004979899699097291, + "loss": 1.5028, + "theoretical_loss": 3.5982034641468594, + "tokens_seen": 1266745344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004979739217652959, + "loss": 1.3761, + "theoretical_loss": 3.598186648794976, + "tokens_seen": 1266810880 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004979578736208625, + "loss": 1.4121, + "theoretical_loss": 3.5981698345565407, + "tokens_seen": 1266876416 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004979418254764293, + "loss": 1.3375, + "theoretical_loss": 3.5981530214314215, + "tokens_seen": 1266941952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004979257773319959, + "loss": 1.3865, + "theoretical_loss": 3.5981362094194864, + "tokens_seen": 1267007488 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004979097291875627, + "loss": 1.3083, + "theoretical_loss": 3.5981193985206055, + "tokens_seen": 1267073024 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004978936810431293, + "loss": 1.2944, + "theoretical_loss": 3.5981025887346467, + "tokens_seen": 1267138560 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004978776328986961, + "loss": 1.392, + "theoretical_loss": 3.5980857800614787, + "tokens_seen": 1267204096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004978615847542627, + "loss": 1.3873, + "theoretical_loss": 3.5980689725009705, + "tokens_seen": 1267269632 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004978455366098295, + "loss": 1.48, + "theoretical_loss": 3.598052166052991, + "tokens_seen": 1267335168 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004978294884653961, + "loss": 1.4681, + "theoretical_loss": 3.5980353607174087, + "tokens_seen": 1267400704 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004978134403209629, + "loss": 1.3321, + "theoretical_loss": 3.5980185564940927, + "tokens_seen": 1267466240 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004977973921765296, + "loss": 1.4598, + "theoretical_loss": 3.598001753382912, + "tokens_seen": 1267531776 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004977813440320963, + "loss": 1.2531, + "theoretical_loss": 3.5979849513837356, + "tokens_seen": 1267597312 + }, + { + "epoch": 0.38, + "learning_rate": 0.000497765295887663, + "loss": 1.4418, + "theoretical_loss": 3.597968150496432, + "tokens_seen": 1267662848 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004977492477432297, + "loss": 1.3662, + "theoretical_loss": 3.5979513507208702, + "tokens_seen": 1267728384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004977331995987964, + "loss": 1.3477, + "theoretical_loss": 3.5979345520569193, + "tokens_seen": 1267793920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004977171514543631, + "loss": 1.3803, + "theoretical_loss": 3.597917754504448, + "tokens_seen": 1267859456 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004977011033099298, + "loss": 1.33, + "theoretical_loss": 3.5979009580633265, + "tokens_seen": 1267924992 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004976850551654965, + "loss": 1.3024, + "theoretical_loss": 3.5978841627334224, + "tokens_seen": 1267990528 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004976690070210632, + "loss": 1.3274, + "theoretical_loss": 3.5978673685146054, + "tokens_seen": 1268056064 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004976529588766299, + "loss": 1.3525, + "theoretical_loss": 3.5978505754067442, + "tokens_seen": 1268121600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004976369107321966, + "loss": 1.3623, + "theoretical_loss": 3.5978337834097083, + "tokens_seen": 1268187136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004976208625877633, + "loss": 1.3931, + "theoretical_loss": 3.5978169925233665, + "tokens_seen": 1268252672 + }, + { + "epoch": 0.38, + "learning_rate": 0.00049760481444333, + "loss": 1.4012, + "theoretical_loss": 3.597800202747589, + "tokens_seen": 1268318208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004975887662988967, + "loss": 1.3646, + "theoretical_loss": 3.5977834140822433, + "tokens_seen": 1268383744 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004975727181544634, + "loss": 1.3727, + "theoretical_loss": 3.5977666265272, + "tokens_seen": 1268449280 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004975566700100301, + "loss": 1.3114, + "theoretical_loss": 3.5977498400823267, + "tokens_seen": 1268514816 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004975406218655968, + "loss": 1.4912, + "theoretical_loss": 3.5977330547474944, + "tokens_seen": 1268580352 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004975245737211635, + "loss": 1.3298, + "theoretical_loss": 3.5977162705225716, + "tokens_seen": 1268645888 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004975085255767302, + "loss": 1.4212, + "theoretical_loss": 3.5976994874074277, + "tokens_seen": 1268711424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004974924774322969, + "loss": 1.4649, + "theoretical_loss": 3.5976827054019314, + "tokens_seen": 1268776960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004974764292878636, + "loss": 1.3212, + "theoretical_loss": 3.597665924505953, + "tokens_seen": 1268842496 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004974603811434303, + "loss": 1.3809, + "theoretical_loss": 3.5976491447193606, + "tokens_seen": 1268908032 + }, + { + "epoch": 0.38, + "learning_rate": 0.000497444332998997, + "loss": 1.3838, + "theoretical_loss": 3.597632366042025, + "tokens_seen": 1268973568 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004974282848545637, + "loss": 1.3743, + "theoretical_loss": 3.597615588473815, + "tokens_seen": 1269039104 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004974122367101304, + "loss": 1.2475, + "theoretical_loss": 3.597598812014599, + "tokens_seen": 1269104640 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004973961885656971, + "loss": 1.3288, + "theoretical_loss": 3.5975820366642486, + "tokens_seen": 1269170176 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004973801404212638, + "loss": 1.2875, + "theoretical_loss": 3.5975652624226315, + "tokens_seen": 1269235712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004973640922768305, + "loss": 1.3022, + "theoretical_loss": 3.5975484892896175, + "tokens_seen": 1269301248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004973480441323972, + "loss": 1.3333, + "theoretical_loss": 3.597531717265076, + "tokens_seen": 1269366784 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004973319959879639, + "loss": 1.4133, + "theoretical_loss": 3.5975149463488774, + "tokens_seen": 1269432320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004973159478435306, + "loss": 1.3213, + "theoretical_loss": 3.59749817654089, + "tokens_seen": 1269497856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004972998996990973, + "loss": 1.3547, + "theoretical_loss": 3.5974814078409847, + "tokens_seen": 1269563392 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004972838515546641, + "loss": 1.2791, + "theoretical_loss": 3.5974646402490302, + "tokens_seen": 1269628928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004972678034102307, + "loss": 1.3205, + "theoretical_loss": 3.597447873764896, + "tokens_seen": 1269694464 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004972517552657975, + "loss": 1.3689, + "theoretical_loss": 3.5974311083884523, + "tokens_seen": 1269760000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004972357071213641, + "loss": 1.4532, + "theoretical_loss": 3.597414344119569, + "tokens_seen": 1269825536 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004972196589769309, + "loss": 1.438, + "theoretical_loss": 3.5973975809581145, + "tokens_seen": 1269891072 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004972036108324975, + "loss": 1.4829, + "theoretical_loss": 3.597380818903959, + "tokens_seen": 1269956608 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004971875626880643, + "loss": 1.2755, + "theoretical_loss": 3.5973640579569732, + "tokens_seen": 1270022144 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004971715145436309, + "loss": 1.3843, + "theoretical_loss": 3.5973472981170262, + "tokens_seen": 1270087680 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004971554663991977, + "loss": 1.2679, + "theoretical_loss": 3.5973305393839876, + "tokens_seen": 1270153216 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004971394182547643, + "loss": 1.438, + "theoretical_loss": 3.5973137817577268, + "tokens_seen": 1270218752 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004971233701103311, + "loss": 1.1991, + "theoretical_loss": 3.5972970252381145, + "tokens_seen": 1270284288 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004971073219658977, + "loss": 1.2715, + "theoretical_loss": 3.59728026982502, + "tokens_seen": 1270349824 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004970912738214645, + "loss": 1.2978, + "theoretical_loss": 3.5972635155183132, + "tokens_seen": 1270415360 + }, + { + "epoch": 0.38, + "learning_rate": 0.0004970752256770311, + "loss": 1.3815, + "theoretical_loss": 3.597246762317864, + "tokens_seen": 1270480896 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004970591775325979, + "loss": 1.5817, + "theoretical_loss": 3.5972300102235426, + "tokens_seen": 1270546432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004970431293881645, + "loss": 1.3906, + "theoretical_loss": 3.597213259235219, + "tokens_seen": 1270611968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004970270812437313, + "loss": 1.465, + "theoretical_loss": 3.5971965093527625, + "tokens_seen": 1270677504 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004970110330992979, + "loss": 1.3721, + "theoretical_loss": 3.597179760576043, + "tokens_seen": 1270743040 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004969949849548647, + "loss": 1.4789, + "theoretical_loss": 3.5971630129049315, + "tokens_seen": 1270808576 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004969789368104313, + "loss": 1.3995, + "theoretical_loss": 3.597146266339297, + "tokens_seen": 1270874112 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004969628886659981, + "loss": 1.3109, + "theoretical_loss": 3.59712952087901, + "tokens_seen": 1270939648 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004969468405215647, + "loss": 1.2993, + "theoretical_loss": 3.5971127765239403, + "tokens_seen": 1271005184 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004969307923771315, + "loss": 1.4228, + "theoretical_loss": 3.597096033273958, + "tokens_seen": 1271070720 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004969147442326981, + "loss": 1.4068, + "theoretical_loss": 3.5970792911289333, + "tokens_seen": 1271136256 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004968986960882649, + "loss": 1.317, + "theoretical_loss": 3.5970625500887365, + "tokens_seen": 1271201792 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004968826479438315, + "loss": 1.4629, + "theoretical_loss": 3.5970458101532374, + "tokens_seen": 1271267328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004968665997993983, + "loss": 1.4053, + "theoretical_loss": 3.5970290713223063, + "tokens_seen": 1271332864 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 473695, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3491677045822144, + "objective/train/theoretical_loss": 3.5970123335958135, + "objective/train/tokens_used": 1291858400, + "theoretical_loss": 3.5970123335958135, + "tokens_seen": 1271398400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004968505516549649, + "loss": 1.3492, + "theoretical_loss": 3.5970123335958135, + "tokens_seen": 1271398400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004968345035105317, + "loss": 1.3438, + "theoretical_loss": 3.5969955969736294, + "tokens_seen": 1271463936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004968184553660983, + "loss": 1.3913, + "theoretical_loss": 3.5969788614556237, + "tokens_seen": 1271529472 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004968024072216651, + "loss": 1.327, + "theoretical_loss": 3.596962127041667, + "tokens_seen": 1271595008 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004967863590772317, + "loss": 1.3474, + "theoretical_loss": 3.5969453937316294, + "tokens_seen": 1271660544 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004967703109327984, + "loss": 1.3904, + "theoretical_loss": 3.5969286615253813, + "tokens_seen": 1271726080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004967542627883651, + "loss": 1.3482, + "theoretical_loss": 3.5969119304227926, + "tokens_seen": 1271791616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004967382146439318, + "loss": 1.352, + "theoretical_loss": 3.5968952004237345, + "tokens_seen": 1271857152 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004967221664994985, + "loss": 1.2914, + "theoretical_loss": 3.5968784715280764, + "tokens_seen": 1271922688 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004967061183550652, + "loss": 1.2612, + "theoretical_loss": 3.5968617437356896, + "tokens_seen": 1271988224 + }, + { + "epoch": 0.39, + "learning_rate": 0.000496690070210632, + "loss": 1.5396, + "theoretical_loss": 3.5968450170464434, + "tokens_seen": 1272053760 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004966740220661986, + "loss": 1.3917, + "theoretical_loss": 3.596828291460209, + "tokens_seen": 1272119296 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004966579739217653, + "loss": 1.2537, + "theoretical_loss": 3.596811566976857, + "tokens_seen": 1272184832 + }, + { + "epoch": 0.39, + "learning_rate": 0.000496641925777332, + "loss": 1.3132, + "theoretical_loss": 3.596794843596258, + "tokens_seen": 1272250368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004966258776328986, + "loss": 1.3191, + "theoretical_loss": 3.5967781213182817, + "tokens_seen": 1272315904 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004966098294884654, + "loss": 1.3603, + "theoretical_loss": 3.5967614001427988, + "tokens_seen": 1272381440 + }, + { + "epoch": 0.39, + "learning_rate": 0.000496593781344032, + "loss": 1.3198, + "theoretical_loss": 3.59674468006968, + "tokens_seen": 1272446976 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004965777331995988, + "loss": 1.4708, + "theoretical_loss": 3.5967279610987957, + "tokens_seen": 1272512512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004965616850551654, + "loss": 1.2803, + "theoretical_loss": 3.596711243230017, + "tokens_seen": 1272578048 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004965456369107322, + "loss": 1.3516, + "theoretical_loss": 3.5966945264632137, + "tokens_seen": 1272643584 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004965295887662988, + "loss": 1.3027, + "theoretical_loss": 3.5966778107982575, + "tokens_seen": 1272709120 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004965135406218656, + "loss": 1.4837, + "theoretical_loss": 3.596661096235018, + "tokens_seen": 1272774656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004964974924774322, + "loss": 1.4093, + "theoretical_loss": 3.5966443827733663, + "tokens_seen": 1272840192 + }, + { + "epoch": 0.39, + "learning_rate": 0.000496481444332999, + "loss": 1.4871, + "theoretical_loss": 3.596627670413173, + "tokens_seen": 1272905728 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004964653961885656, + "loss": 1.2895, + "theoretical_loss": 3.596610959154309, + "tokens_seen": 1272971264 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004964493480441324, + "loss": 1.3435, + "theoretical_loss": 3.5965942489966447, + "tokens_seen": 1273036800 + }, + { + "epoch": 0.39, + "learning_rate": 0.000496433299899699, + "loss": 1.3647, + "theoretical_loss": 3.5965775399400512, + "tokens_seen": 1273102336 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004964172517552658, + "loss": 1.3647, + "theoretical_loss": 3.596560831984399, + "tokens_seen": 1273167872 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004964012036108325, + "loss": 1.2928, + "theoretical_loss": 3.596544125129559, + "tokens_seen": 1273233408 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004963851554663992, + "loss": 1.2998, + "theoretical_loss": 3.596527419375402, + "tokens_seen": 1273298944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004963691073219659, + "loss": 1.4863, + "theoretical_loss": 3.5965107147217994, + "tokens_seen": 1273364480 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004963530591775326, + "loss": 1.3955, + "theoretical_loss": 3.5964940111686206, + "tokens_seen": 1273430016 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004963370110330993, + "loss": 1.2934, + "theoretical_loss": 3.5964773087157385, + "tokens_seen": 1273495552 + }, + { + "epoch": 0.39, + "learning_rate": 0.000496320962888666, + "loss": 1.4437, + "theoretical_loss": 3.5964606073630216, + "tokens_seen": 1273561088 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004963049147442327, + "loss": 1.3929, + "theoretical_loss": 3.596443907110343, + "tokens_seen": 1273626624 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004962888665997994, + "loss": 1.3615, + "theoretical_loss": 3.5964272079575728, + "tokens_seen": 1273692160 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004962728184553661, + "loss": 1.489, + "theoretical_loss": 3.5964105099045813, + "tokens_seen": 1273757696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004962567703109328, + "loss": 1.3537, + "theoretical_loss": 3.596393812951241, + "tokens_seen": 1273823232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004962407221664995, + "loss": 1.3661, + "theoretical_loss": 3.5963771170974215, + "tokens_seen": 1273888768 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004962246740220662, + "loss": 1.3323, + "theoretical_loss": 3.596360422342995, + "tokens_seen": 1273954304 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004962086258776329, + "loss": 1.4022, + "theoretical_loss": 3.596343728687831, + "tokens_seen": 1274019840 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004961925777331996, + "loss": 1.2281, + "theoretical_loss": 3.596327036131802, + "tokens_seen": 1274085376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004961765295887663, + "loss": 1.3758, + "theoretical_loss": 3.5963103446747784, + "tokens_seen": 1274150912 + }, + { + "epoch": 0.39, + "learning_rate": 0.000496160481444333, + "loss": 1.1818, + "theoretical_loss": 3.596293654316632, + "tokens_seen": 1274216448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004961444332998997, + "loss": 1.3608, + "theoretical_loss": 3.5962769650572337, + "tokens_seen": 1274281984 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004961283851554664, + "loss": 1.3214, + "theoretical_loss": 3.5962602768964533, + "tokens_seen": 1274347520 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004961123370110331, + "loss": 1.3969, + "theoretical_loss": 3.596243589834164, + "tokens_seen": 1274413056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004960962888665998, + "loss": 1.2977, + "theoretical_loss": 3.596226903870236, + "tokens_seen": 1274478592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004960802407221666, + "loss": 1.5205, + "theoretical_loss": 3.596210219004541, + "tokens_seen": 1274544128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004960641925777332, + "loss": 1.4572, + "theoretical_loss": 3.5961935352369494, + "tokens_seen": 1274609664 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004960481444333, + "loss": 1.443, + "theoretical_loss": 3.5961768525673334, + "tokens_seen": 1274675200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004960320962888666, + "loss": 1.4889, + "theoretical_loss": 3.5961601709955637, + "tokens_seen": 1274740736 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004960160481444334, + "loss": 1.4062, + "theoretical_loss": 3.596143490521512, + "tokens_seen": 1274806272 + }, + { + "epoch": 0.39, + "learning_rate": 0.000496, + "loss": 1.4052, + "theoretical_loss": 3.5961268111450493, + "tokens_seen": 1274871808 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004959839518555668, + "loss": 1.2934, + "theoretical_loss": 3.5961101328660474, + "tokens_seen": 1274937344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004959679037111334, + "loss": 1.3701, + "theoretical_loss": 3.596093455684377, + "tokens_seen": 1275002880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004959518555667002, + "loss": 1.1738, + "theoretical_loss": 3.59607677959991, + "tokens_seen": 1275068416 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004959358074222668, + "loss": 1.4161, + "theoretical_loss": 3.596060104612518, + "tokens_seen": 1275133952 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004959197592778336, + "loss": 1.2693, + "theoretical_loss": 3.5960434307220717, + "tokens_seen": 1275199488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004959037111334002, + "loss": 1.377, + "theoretical_loss": 3.5960267579284437, + "tokens_seen": 1275265024 + }, + { + "epoch": 0.39, + "learning_rate": 0.000495887662988967, + "loss": 1.2522, + "theoretical_loss": 3.5960100862315043, + "tokens_seen": 1275330560 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004958716148445336, + "loss": 1.3311, + "theoretical_loss": 3.5959934156311255, + "tokens_seen": 1275396096 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004958555667001004, + "loss": 1.2938, + "theoretical_loss": 3.595976746127179, + "tokens_seen": 1275461632 + }, + { + "epoch": 0.39, + "learning_rate": 0.000495839518555667, + "loss": 1.3297, + "theoretical_loss": 3.5959600777195364, + "tokens_seen": 1275527168 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004958234704112338, + "loss": 1.2215, + "theoretical_loss": 3.5959434104080694, + "tokens_seen": 1275592704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004958074222668004, + "loss": 1.3114, + "theoretical_loss": 3.5959267441926483, + "tokens_seen": 1275658240 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004957913741223672, + "loss": 1.3586, + "theoretical_loss": 3.5959100790731466, + "tokens_seen": 1275723776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004957753259779338, + "loss": 1.3912, + "theoretical_loss": 3.595893415049435, + "tokens_seen": 1275789312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004957592778335006, + "loss": 1.3315, + "theoretical_loss": 3.5958767521213852, + "tokens_seen": 1275854848 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004957432296890672, + "loss": 1.3705, + "theoretical_loss": 3.595860090288869, + "tokens_seen": 1275920384 + }, + { + "epoch": 0.39, + "learning_rate": 0.000495727181544634, + "loss": 1.2685, + "theoretical_loss": 3.595843429551758, + "tokens_seen": 1275985920 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004957111334002006, + "loss": 1.4993, + "theoretical_loss": 3.595826769909924, + "tokens_seen": 1276051456 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004956950852557674, + "loss": 1.2649, + "theoretical_loss": 3.595810111363238, + "tokens_seen": 1276116992 + }, + { + "epoch": 0.39, + "learning_rate": 0.000495679037111334, + "loss": 1.2025, + "theoretical_loss": 3.5957934539115732, + "tokens_seen": 1276182528 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004956629889669008, + "loss": 1.2312, + "theoretical_loss": 3.5957767975548007, + "tokens_seen": 1276248064 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004956469408224674, + "loss": 1.2867, + "theoretical_loss": 3.595760142292792, + "tokens_seen": 1276313600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004956308926780342, + "loss": 1.247, + "theoretical_loss": 3.5957434881254198, + "tokens_seen": 1276379136 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004956148445336008, + "loss": 1.3473, + "theoretical_loss": 3.595726835052555, + "tokens_seen": 1276444672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004955987963891676, + "loss": 1.4787, + "theoretical_loss": 3.5957101830740696, + "tokens_seen": 1276510208 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004955827482447342, + "loss": 1.4871, + "theoretical_loss": 3.595693532189836, + "tokens_seen": 1276575744 + }, + { + "epoch": 0.39, + "learning_rate": 0.000495566700100301, + "loss": 1.3456, + "theoretical_loss": 3.595676882399726, + "tokens_seen": 1276641280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004955506519558676, + "loss": 1.2841, + "theoretical_loss": 3.5956602337036117, + "tokens_seen": 1276706816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004955346038114344, + "loss": 1.3812, + "theoretical_loss": 3.5956435861013643, + "tokens_seen": 1276772352 + }, + { + "epoch": 0.39, + "learning_rate": 0.000495518555667001, + "loss": 1.3972, + "theoretical_loss": 3.595626939592856, + "tokens_seen": 1276837888 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004955025075225678, + "loss": 1.2866, + "theoretical_loss": 3.59561029417796, + "tokens_seen": 1276903424 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004954864593781344, + "loss": 1.3645, + "theoretical_loss": 3.595593649856547, + "tokens_seen": 1276968960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004954704112337012, + "loss": 1.442, + "theoretical_loss": 3.5955770066284893, + "tokens_seen": 1277034496 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004954543630892678, + "loss": 1.3073, + "theoretical_loss": 3.595560364493659, + "tokens_seen": 1277100032 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004954383149448346, + "loss": 1.403, + "theoretical_loss": 3.595543723451929, + "tokens_seen": 1277165568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004954222668004012, + "loss": 1.3379, + "theoretical_loss": 3.5955270835031703, + "tokens_seen": 1277231104 + }, + { + "epoch": 0.39, + "learning_rate": 0.000495406218655968, + "loss": 1.3769, + "theoretical_loss": 3.595510444647256, + "tokens_seen": 1277296640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004953901705115346, + "loss": 1.3228, + "theoretical_loss": 3.5954938068840576, + "tokens_seen": 1277362176 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004953741223671013, + "loss": 1.4071, + "theoretical_loss": 3.5954771702134476, + "tokens_seen": 1277427712 + }, + { + "epoch": 0.39, + "learning_rate": 0.000495358074222668, + "loss": 1.3599, + "theoretical_loss": 3.5954605346352975, + "tokens_seen": 1277493248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004953420260782347, + "loss": 1.3706, + "theoretical_loss": 3.5954439001494802, + "tokens_seen": 1277558784 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004953259779338014, + "loss": 1.4196, + "theoretical_loss": 3.595427266755868, + "tokens_seen": 1277624320 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004953099297893681, + "loss": 1.3393, + "theoretical_loss": 3.595410634454333, + "tokens_seen": 1277689856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004952938816449348, + "loss": 1.3317, + "theoretical_loss": 3.5953940032447473, + "tokens_seen": 1277755392 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004952778335005015, + "loss": 1.4129, + "theoretical_loss": 3.5953773731269836, + "tokens_seen": 1277820928 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004952617853560683, + "loss": 1.338, + "theoretical_loss": 3.595360744100914, + "tokens_seen": 1277886464 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 476248, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.263485312461853, + "objective/train/theoretical_loss": 3.595344116166411, + "objective/train/tokens_used": 1298412000, + "theoretical_loss": 3.595344116166411, + "tokens_seen": 1277952000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004952457372116349, + "loss": 1.2635, + "theoretical_loss": 3.595344116166411, + "tokens_seen": 1277952000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004952296890672015, + "loss": 1.3897, + "theoretical_loss": 3.5953274893233464, + "tokens_seen": 1278017536 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004952136409227683, + "loss": 1.4093, + "theoretical_loss": 3.595310863571594, + "tokens_seen": 1278083072 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004951975927783349, + "loss": 1.318, + "theoretical_loss": 3.5952942389110243, + "tokens_seen": 1278148608 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004951815446339017, + "loss": 1.4358, + "theoretical_loss": 3.5952776153415114, + "tokens_seen": 1278214144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004951654964894683, + "loss": 1.3142, + "theoretical_loss": 3.5952609928629267, + "tokens_seen": 1278279680 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004951494483450351, + "loss": 1.5437, + "theoretical_loss": 3.5952443714751428, + "tokens_seen": 1278345216 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004951334002006017, + "loss": 1.3365, + "theoretical_loss": 3.5952277511780326, + "tokens_seen": 1278410752 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004951173520561685, + "loss": 1.3241, + "theoretical_loss": 3.5952111319714684, + "tokens_seen": 1278476288 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004951013039117351, + "loss": 1.3019, + "theoretical_loss": 3.5951945138553234, + "tokens_seen": 1278541824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004950852557673019, + "loss": 1.315, + "theoretical_loss": 3.595177896829469, + "tokens_seen": 1278607360 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004950692076228685, + "loss": 1.3915, + "theoretical_loss": 3.5951612808937785, + "tokens_seen": 1278672896 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004950531594784353, + "loss": 1.3409, + "theoretical_loss": 3.5951446660481245, + "tokens_seen": 1278738432 + }, + { + "epoch": 0.39, + "learning_rate": 0.000495037111334002, + "loss": 1.3322, + "theoretical_loss": 3.5951280522923796, + "tokens_seen": 1278803968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004950210631895687, + "loss": 1.4499, + "theoretical_loss": 3.5951114396264163, + "tokens_seen": 1278869504 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004950050150451354, + "loss": 1.3438, + "theoretical_loss": 3.595094828050107, + "tokens_seen": 1278935040 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004949889669007021, + "loss": 1.3639, + "theoretical_loss": 3.5950782175633256, + "tokens_seen": 1279000576 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004949729187562688, + "loss": 1.4491, + "theoretical_loss": 3.5950616081659437, + "tokens_seen": 1279066112 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004949568706118355, + "loss": 1.4069, + "theoretical_loss": 3.5950449998578335, + "tokens_seen": 1279131648 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004949408224674022, + "loss": 1.376, + "theoretical_loss": 3.5950283926388695, + "tokens_seen": 1279197184 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004949247743229689, + "loss": 1.3988, + "theoretical_loss": 3.5950117865089233, + "tokens_seen": 1279262720 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004949087261785356, + "loss": 1.3799, + "theoretical_loss": 3.594995181467868, + "tokens_seen": 1279328256 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004948926780341023, + "loss": 1.3201, + "theoretical_loss": 3.594978577515576, + "tokens_seen": 1279393792 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004948766298896691, + "loss": 1.388, + "theoretical_loss": 3.5949619746519206, + "tokens_seen": 1279459328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004948605817452357, + "loss": 1.3498, + "theoretical_loss": 3.594945372876775, + "tokens_seen": 1279524864 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004948445336008025, + "loss": 1.3523, + "theoretical_loss": 3.594928772190011, + "tokens_seen": 1279590400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004948284854563691, + "loss": 1.4539, + "theoretical_loss": 3.5949121725915028, + "tokens_seen": 1279655936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004948124373119359, + "loss": 1.2654, + "theoretical_loss": 3.5948955740811224, + "tokens_seen": 1279721472 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004947963891675025, + "loss": 1.3486, + "theoretical_loss": 3.5948789766587437, + "tokens_seen": 1279787008 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004947803410230693, + "loss": 1.4039, + "theoretical_loss": 3.594862380324238, + "tokens_seen": 1279852544 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004947642928786359, + "loss": 1.4316, + "theoretical_loss": 3.5948457850774798, + "tokens_seen": 1279918080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004947482447342027, + "loss": 1.315, + "theoretical_loss": 3.5948291909183423, + "tokens_seen": 1279983616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004947321965897693, + "loss": 1.2807, + "theoretical_loss": 3.5948125978466967, + "tokens_seen": 1280049152 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004947161484453361, + "loss": 1.448, + "theoretical_loss": 3.594796005862418, + "tokens_seen": 1280114688 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004947001003009027, + "loss": 1.3034, + "theoretical_loss": 3.5947794149653785, + "tokens_seen": 1280180224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004946840521564695, + "loss": 1.3152, + "theoretical_loss": 3.5947628251554518, + "tokens_seen": 1280245760 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004946680040120361, + "loss": 1.3952, + "theoretical_loss": 3.59474623643251, + "tokens_seen": 1280311296 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004946519558676029, + "loss": 1.3727, + "theoretical_loss": 3.5947296487964264, + "tokens_seen": 1280376832 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004946359077231695, + "loss": 1.3376, + "theoretical_loss": 3.594713062247075, + "tokens_seen": 1280442368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004946198595787363, + "loss": 1.458, + "theoretical_loss": 3.594696476784329, + "tokens_seen": 1280507904 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004946038114343029, + "loss": 1.3934, + "theoretical_loss": 3.5946798924080605, + "tokens_seen": 1280573440 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004945877632898697, + "loss": 1.2969, + "theoretical_loss": 3.5946633091181437, + "tokens_seen": 1280638976 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004945717151454363, + "loss": 1.3171, + "theoretical_loss": 3.5946467269144513, + "tokens_seen": 1280704512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004945556670010031, + "loss": 1.353, + "theoretical_loss": 3.5946301457968577, + "tokens_seen": 1280770048 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004945396188565697, + "loss": 1.3925, + "theoretical_loss": 3.5946135657652345, + "tokens_seen": 1280835584 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004945235707121365, + "loss": 1.3093, + "theoretical_loss": 3.5945969868194556, + "tokens_seen": 1280901120 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004945075225677031, + "loss": 1.2736, + "theoretical_loss": 3.594580408959395, + "tokens_seen": 1280966656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004944914744232699, + "loss": 1.3474, + "theoretical_loss": 3.5945638321849254, + "tokens_seen": 1281032192 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004944754262788365, + "loss": 1.2756, + "theoretical_loss": 3.594547256495921, + "tokens_seen": 1281097728 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004944593781344033, + "loss": 1.2609, + "theoretical_loss": 3.594530681892254, + "tokens_seen": 1281163264 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004944433299899699, + "loss": 1.3709, + "theoretical_loss": 3.594514108373799, + "tokens_seen": 1281228800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004944272818455367, + "loss": 1.4764, + "theoretical_loss": 3.5944975359404285, + "tokens_seen": 1281294336 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004944112337011033, + "loss": 1.4049, + "theoretical_loss": 3.594480964592016, + "tokens_seen": 1281359872 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004943951855566701, + "loss": 1.4034, + "theoretical_loss": 3.594464394328436, + "tokens_seen": 1281425408 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004943791374122367, + "loss": 1.3672, + "theoretical_loss": 3.594447825149561, + "tokens_seen": 1281490944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004943630892678035, + "loss": 1.3152, + "theoretical_loss": 3.5944312570552643, + "tokens_seen": 1281556480 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004943470411233701, + "loss": 1.3568, + "theoretical_loss": 3.594414690045421, + "tokens_seen": 1281622016 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004943309929789369, + "loss": 1.4287, + "theoretical_loss": 3.594398124119903, + "tokens_seen": 1281687552 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004943149448345035, + "loss": 1.4132, + "theoretical_loss": 3.5943815592785846, + "tokens_seen": 1281753088 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004942988966900703, + "loss": 1.3725, + "theoretical_loss": 3.5943649955213397, + "tokens_seen": 1281818624 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004942828485456369, + "loss": 1.3785, + "theoretical_loss": 3.594348432848041, + "tokens_seen": 1281884160 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004942668004012037, + "loss": 1.3698, + "theoretical_loss": 3.5943318712585635, + "tokens_seen": 1281949696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004942507522567703, + "loss": 1.3445, + "theoretical_loss": 3.59431531075278, + "tokens_seen": 1282015232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004942347041123371, + "loss": 1.3416, + "theoretical_loss": 3.594298751330564, + "tokens_seen": 1282080768 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004942186559679037, + "loss": 1.404, + "theoretical_loss": 3.5942821929917894, + "tokens_seen": 1282146304 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004942026078234705, + "loss": 1.3686, + "theoretical_loss": 3.5942656357363307, + "tokens_seen": 1282211840 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004941865596790371, + "loss": 1.2993, + "theoretical_loss": 3.5942490795640607, + "tokens_seen": 1282277376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004941705115346039, + "loss": 1.3628, + "theoretical_loss": 3.5942325244748536, + "tokens_seen": 1282342912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004941544633901705, + "loss": 1.2315, + "theoretical_loss": 3.594215970468583, + "tokens_seen": 1282408448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004941384152457373, + "loss": 1.3056, + "theoretical_loss": 3.594199417545123, + "tokens_seen": 1282473984 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004941223671013039, + "loss": 1.413, + "theoretical_loss": 3.5941828657043473, + "tokens_seen": 1282539520 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004941063189568707, + "loss": 1.4293, + "theoretical_loss": 3.5941663149461296, + "tokens_seen": 1282605056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004940902708124373, + "loss": 1.4052, + "theoretical_loss": 3.594149765270344, + "tokens_seen": 1282670592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004940742226680041, + "loss": 1.3821, + "theoretical_loss": 3.5941332166768647, + "tokens_seen": 1282736128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004940581745235707, + "loss": 1.3779, + "theoretical_loss": 3.594116669165565, + "tokens_seen": 1282801664 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004940421263791375, + "loss": 1.2671, + "theoretical_loss": 3.594100122736319, + "tokens_seen": 1282867200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004940260782347041, + "loss": 1.311, + "theoretical_loss": 3.594083577389001, + "tokens_seen": 1282932736 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004940100300902709, + "loss": 1.5339, + "theoretical_loss": 3.594067033123485, + "tokens_seen": 1282998272 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004939939819458375, + "loss": 1.4133, + "theoretical_loss": 3.5940504899396446, + "tokens_seen": 1283063808 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004939779338014042, + "loss": 1.43, + "theoretical_loss": 3.594033947837354, + "tokens_seen": 1283129344 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493961885656971, + "loss": 1.377, + "theoretical_loss": 3.5940174068164876, + "tokens_seen": 1283194880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004939458375125376, + "loss": 1.3817, + "theoretical_loss": 3.594000866876919, + "tokens_seen": 1283260416 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004939297893681043, + "loss": 1.3446, + "theoretical_loss": 3.5939843280185224, + "tokens_seen": 1283325952 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493913741223671, + "loss": 1.3003, + "theoretical_loss": 3.593967790241172, + "tokens_seen": 1283391488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004938976930792378, + "loss": 1.3724, + "theoretical_loss": 3.593951253544742, + "tokens_seen": 1283457024 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004938816449348044, + "loss": 1.3862, + "theoretical_loss": 3.593934717929107, + "tokens_seen": 1283522560 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004938655967903712, + "loss": 1.3385, + "theoretical_loss": 3.5939181833941403, + "tokens_seen": 1283588096 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004938495486459378, + "loss": 1.4509, + "theoretical_loss": 3.593901649939716, + "tokens_seen": 1283653632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004938335005015044, + "loss": 1.4164, + "theoretical_loss": 3.5938851175657094, + "tokens_seen": 1283719168 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004938174523570712, + "loss": 1.3659, + "theoretical_loss": 3.593868586271994, + "tokens_seen": 1283784704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004938014042126378, + "loss": 1.3485, + "theoretical_loss": 3.593852056058444, + "tokens_seen": 1283850240 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004937853560682046, + "loss": 1.263, + "theoretical_loss": 3.5938355269249347, + "tokens_seen": 1283915776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004937693079237712, + "loss": 1.4489, + "theoretical_loss": 3.593818998871339, + "tokens_seen": 1283981312 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493753259779338, + "loss": 1.3382, + "theoretical_loss": 3.593802471897532, + "tokens_seen": 1284046848 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004937372116349046, + "loss": 1.3296, + "theoretical_loss": 3.593785946003388, + "tokens_seen": 1284112384 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004937211634904714, + "loss": 1.3472, + "theoretical_loss": 3.593769421188781, + "tokens_seen": 1284177920 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493705115346038, + "loss": 1.3727, + "theoretical_loss": 3.5937528974535864, + "tokens_seen": 1284243456 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004936890672016048, + "loss": 1.3808, + "theoretical_loss": 3.593736374797677, + "tokens_seen": 1284308992 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004936730190571716, + "loss": 1.3203, + "theoretical_loss": 3.5937198532209287, + "tokens_seen": 1284374528 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004936569709127382, + "loss": 1.3046, + "theoretical_loss": 3.593703332723215, + "tokens_seen": 1284440064 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 478540, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4209537506103516, + "objective/train/theoretical_loss": 3.5936868133044113, + "objective/train/tokens_used": 1304965600, + "theoretical_loss": 3.5936868133044113, + "tokens_seen": 1284505600 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493640922768305, + "loss": 1.421, + "theoretical_loss": 3.5936868133044113, + "tokens_seen": 1284505600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004936248746238716, + "loss": 1.2472, + "theoretical_loss": 3.593670294964391, + "tokens_seen": 1284571136 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004936088264794384, + "loss": 1.3696, + "theoretical_loss": 3.5936537777030293, + "tokens_seen": 1284636672 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493592778335005, + "loss": 1.4212, + "theoretical_loss": 3.593637261520201, + "tokens_seen": 1284702208 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004935767301905718, + "loss": 1.3935, + "theoretical_loss": 3.59362074641578, + "tokens_seen": 1284767744 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004935606820461384, + "loss": 1.2954, + "theoretical_loss": 3.593604232389641, + "tokens_seen": 1284833280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004935446339017052, + "loss": 1.3741, + "theoretical_loss": 3.593587719441659, + "tokens_seen": 1284898816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004935285857572718, + "loss": 1.3289, + "theoretical_loss": 3.593571207571708, + "tokens_seen": 1284964352 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004935125376128386, + "loss": 1.2893, + "theoretical_loss": 3.593554696779664, + "tokens_seen": 1285029888 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004934964894684052, + "loss": 1.3174, + "theoretical_loss": 3.5935381870653997, + "tokens_seen": 1285095424 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493480441323972, + "loss": 1.35, + "theoretical_loss": 3.5935216784287913, + "tokens_seen": 1285160960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004934643931795386, + "loss": 1.4525, + "theoretical_loss": 3.5935051708697126, + "tokens_seen": 1285226496 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004934483450351054, + "loss": 1.3435, + "theoretical_loss": 3.5934886643880395, + "tokens_seen": 1285292032 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493432296890672, + "loss": 1.3071, + "theoretical_loss": 3.593472158983645, + "tokens_seen": 1285357568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004934162487462388, + "loss": 1.4038, + "theoretical_loss": 3.5934556546564056, + "tokens_seen": 1285423104 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004934002006018054, + "loss": 1.3461, + "theoretical_loss": 3.593439151406195, + "tokens_seen": 1285488640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004933841524573722, + "loss": 1.3556, + "theoretical_loss": 3.5934226492328882, + "tokens_seen": 1285554176 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004933681043129388, + "loss": 1.4077, + "theoretical_loss": 3.593406148136361, + "tokens_seen": 1285619712 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004933520561685056, + "loss": 1.3818, + "theoretical_loss": 3.593389648116487, + "tokens_seen": 1285685248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004933360080240722, + "loss": 1.3641, + "theoretical_loss": 3.593373149173141, + "tokens_seen": 1285750784 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493319959879639, + "loss": 1.3852, + "theoretical_loss": 3.5933566513061987, + "tokens_seen": 1285816320 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004933039117352056, + "loss": 1.3889, + "theoretical_loss": 3.593340154515535, + "tokens_seen": 1285881856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004932878635907724, + "loss": 1.2848, + "theoretical_loss": 3.5933236588010242, + "tokens_seen": 1285947392 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493271815446339, + "loss": 1.3029, + "theoretical_loss": 3.5933071641625416, + "tokens_seen": 1286012928 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004932557673019058, + "loss": 1.2787, + "theoretical_loss": 3.5932906705999628, + "tokens_seen": 1286078464 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004932397191574724, + "loss": 1.3427, + "theoretical_loss": 3.5932741781131616, + "tokens_seen": 1286144000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004932236710130392, + "loss": 1.3498, + "theoretical_loss": 3.593257686702014, + "tokens_seen": 1286209536 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004932076228686058, + "loss": 1.4051, + "theoretical_loss": 3.5932411963663946, + "tokens_seen": 1286275072 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004931915747241726, + "loss": 1.3569, + "theoretical_loss": 3.5932247071061783, + "tokens_seen": 1286340608 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004931755265797392, + "loss": 1.3509, + "theoretical_loss": 3.593208218921241, + "tokens_seen": 1286406144 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493159478435306, + "loss": 1.4739, + "theoretical_loss": 3.593191731811457, + "tokens_seen": 1286471680 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004931434302908726, + "loss": 1.4331, + "theoretical_loss": 3.5931752457767008, + "tokens_seen": 1286537216 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004931273821464394, + "loss": 1.2933, + "theoretical_loss": 3.5931587608168494, + "tokens_seen": 1286602752 + }, + { + "epoch": 0.39, + "learning_rate": 0.000493111334002006, + "loss": 1.5395, + "theoretical_loss": 3.5931422769317765, + "tokens_seen": 1286668288 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004930952858575728, + "loss": 1.2784, + "theoretical_loss": 3.593125794121358, + "tokens_seen": 1286733824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004930792377131394, + "loss": 1.3766, + "theoretical_loss": 3.5931093123854687, + "tokens_seen": 1286799360 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004930631895687062, + "loss": 1.459, + "theoretical_loss": 3.5930928317239843, + "tokens_seen": 1286864896 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004930471414242728, + "loss": 1.3436, + "theoretical_loss": 3.5930763521367797, + "tokens_seen": 1286930432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004930310932798396, + "loss": 1.4635, + "theoretical_loss": 3.59305987362373, + "tokens_seen": 1286995968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004930150451354062, + "loss": 1.314, + "theoretical_loss": 3.5930433961847106, + "tokens_seen": 1287061504 + }, + { + "epoch": 0.39, + "learning_rate": 0.000492998996990973, + "loss": 1.3625, + "theoretical_loss": 3.5930269198195974, + "tokens_seen": 1287127040 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004929829488465396, + "loss": 1.2586, + "theoretical_loss": 3.593010444528265, + "tokens_seen": 1287192576 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004929669007021064, + "loss": 1.2186, + "theoretical_loss": 3.592993970310589, + "tokens_seen": 1287258112 + }, + { + "epoch": 0.39, + "learning_rate": 0.000492950852557673, + "loss": 1.2852, + "theoretical_loss": 3.5929774971664448, + "tokens_seen": 1287323648 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004929348044132398, + "loss": 1.3423, + "theoretical_loss": 3.5929610250957076, + "tokens_seen": 1287389184 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004929187562688064, + "loss": 1.3779, + "theoretical_loss": 3.5929445540982536, + "tokens_seen": 1287454720 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004929027081243732, + "loss": 1.3524, + "theoretical_loss": 3.5929280841739573, + "tokens_seen": 1287520256 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004928866599799398, + "loss": 1.3232, + "theoretical_loss": 3.592911615322694, + "tokens_seen": 1287585792 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004928706118355066, + "loss": 1.3059, + "theoretical_loss": 3.5928951475443403, + "tokens_seen": 1287651328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004928545636910732, + "loss": 1.2953, + "theoretical_loss": 3.592878680838771, + "tokens_seen": 1287716864 + }, + { + "epoch": 0.39, + "learning_rate": 0.00049283851554664, + "loss": 1.3703, + "theoretical_loss": 3.592862215205862, + "tokens_seen": 1287782400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004928224674022066, + "loss": 1.4124, + "theoretical_loss": 3.592845750645488, + "tokens_seen": 1287847936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004928064192577734, + "loss": 1.3586, + "theoretical_loss": 3.5928292871575254, + "tokens_seen": 1287913472 + }, + { + "epoch": 0.39, + "learning_rate": 0.00049279037111334, + "loss": 1.3814, + "theoretical_loss": 3.5928128247418494, + "tokens_seen": 1287979008 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004927743229689068, + "loss": 1.364, + "theoretical_loss": 3.5927963633983357, + "tokens_seen": 1288044544 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004927582748244734, + "loss": 1.3805, + "theoretical_loss": 3.59277990312686, + "tokens_seen": 1288110080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004927422266800402, + "loss": 1.3886, + "theoretical_loss": 3.592763443927298, + "tokens_seen": 1288175616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004927261785356068, + "loss": 1.3708, + "theoretical_loss": 3.592746985799525, + "tokens_seen": 1288241152 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004927101303911736, + "loss": 1.3697, + "theoretical_loss": 3.5927305287434175, + "tokens_seen": 1288306688 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004926940822467402, + "loss": 1.4746, + "theoretical_loss": 3.59271407275885, + "tokens_seen": 1288372224 + }, + { + "epoch": 0.39, + "learning_rate": 0.000492678034102307, + "loss": 1.3392, + "theoretical_loss": 3.5926976178456993, + "tokens_seen": 1288437760 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004926619859578736, + "loss": 1.3031, + "theoretical_loss": 3.5926811640038405, + "tokens_seen": 1288503296 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004926459378134404, + "loss": 1.2424, + "theoretical_loss": 3.5926647112331493, + "tokens_seen": 1288568832 + }, + { + "epoch": 0.39, + "learning_rate": 0.000492629889669007, + "loss": 1.3595, + "theoretical_loss": 3.5926482595335023, + "tokens_seen": 1288634368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004926138415245738, + "loss": 1.3804, + "theoretical_loss": 3.5926318089047746, + "tokens_seen": 1288699904 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004925977933801404, + "loss": 1.2816, + "theoretical_loss": 3.5926153593468424, + "tokens_seen": 1288765440 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004925817452357071, + "loss": 1.3394, + "theoretical_loss": 3.5925989108595813, + "tokens_seen": 1288830976 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004925656970912738, + "loss": 1.2069, + "theoretical_loss": 3.5925824634428674, + "tokens_seen": 1288896512 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004925496489468405, + "loss": 1.3472, + "theoretical_loss": 3.5925660170965763, + "tokens_seen": 1288962048 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004925336008024073, + "loss": 1.4291, + "theoretical_loss": 3.5925495718205838, + "tokens_seen": 1289027584 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004925175526579739, + "loss": 1.342, + "theoretical_loss": 3.5925331276147663, + "tokens_seen": 1289093120 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004925015045135407, + "loss": 1.3867, + "theoretical_loss": 3.5925166844789995, + "tokens_seen": 1289158656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004924854563691073, + "loss": 1.2583, + "theoretical_loss": 3.59250024241316, + "tokens_seen": 1289224192 + }, + { + "epoch": 0.39, + "learning_rate": 0.000492469408224674, + "loss": 1.3895, + "theoretical_loss": 3.5924838014171225, + "tokens_seen": 1289289728 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004924533600802407, + "loss": 1.3211, + "theoretical_loss": 3.592467361490764, + "tokens_seen": 1289355264 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004924373119358075, + "loss": 1.3643, + "theoretical_loss": 3.5924509226339607, + "tokens_seen": 1289420800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004924212637913741, + "loss": 1.3129, + "theoretical_loss": 3.5924344848465877, + "tokens_seen": 1289486336 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004924052156469409, + "loss": 1.3428, + "theoretical_loss": 3.592418048128522, + "tokens_seen": 1289551872 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004923891675025075, + "loss": 1.4367, + "theoretical_loss": 3.592401612479639, + "tokens_seen": 1289617408 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004923731193580743, + "loss": 1.3106, + "theoretical_loss": 3.5923851778998155, + "tokens_seen": 1289682944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004923570712136409, + "loss": 1.3348, + "theoretical_loss": 3.5923687443889274, + "tokens_seen": 1289748480 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004923410230692077, + "loss": 1.3648, + "theoretical_loss": 3.5923523119468506, + "tokens_seen": 1289814016 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004923249749247743, + "loss": 1.3541, + "theoretical_loss": 3.5923358805734615, + "tokens_seen": 1289879552 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004923089267803411, + "loss": 1.354, + "theoretical_loss": 3.5923194502686364, + "tokens_seen": 1289945088 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004922928786359077, + "loss": 1.3486, + "theoretical_loss": 3.592303021032251, + "tokens_seen": 1290010624 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004922768304914745, + "loss": 1.3862, + "theoretical_loss": 3.5922865928641823, + "tokens_seen": 1290076160 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004922607823470411, + "loss": 1.2639, + "theoretical_loss": 3.592270165764306, + "tokens_seen": 1290141696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004922447342026079, + "loss": 1.3605, + "theoretical_loss": 3.5922537397324987, + "tokens_seen": 1290207232 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004922286860581745, + "loss": 1.3373, + "theoretical_loss": 3.592237314768637, + "tokens_seen": 1290272768 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004922126379137413, + "loss": 1.3199, + "theoretical_loss": 3.592220890872596, + "tokens_seen": 1290338304 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004921965897693079, + "loss": 1.2507, + "theoretical_loss": 3.5922044680442533, + "tokens_seen": 1290403840 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004921805416248747, + "loss": 1.3538, + "theoretical_loss": 3.5921880462834848, + "tokens_seen": 1290469376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004921644934804413, + "loss": 1.3094, + "theoretical_loss": 3.592171625590167, + "tokens_seen": 1290534912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004921484453360081, + "loss": 1.3063, + "theoretical_loss": 3.5921552059641764, + "tokens_seen": 1290600448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004921323971915747, + "loss": 1.3566, + "theoretical_loss": 3.592138787405389, + "tokens_seen": 1290665984 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004921163490471415, + "loss": 1.2796, + "theoretical_loss": 3.5921223699136817, + "tokens_seen": 1290731520 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004921003009027081, + "loss": 1.3301, + "theoretical_loss": 3.592105953488931, + "tokens_seen": 1290797056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004920842527582749, + "loss": 1.347, + "theoretical_loss": 3.592089538131013, + "tokens_seen": 1290862592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004920682046138415, + "loss": 1.3202, + "theoretical_loss": 3.5920731238398043, + "tokens_seen": 1290928128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004920521564694083, + "loss": 1.3471, + "theoretical_loss": 3.5920567106151817, + "tokens_seen": 1290993664 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 480625, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4643183946609497, + "objective/train/theoretical_loss": 3.5920402984570217, + "objective/train/tokens_used": 1311519200, + "theoretical_loss": 3.5920402984570217, + "tokens_seen": 1291059200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004920361083249749, + "loss": 1.4643, + "theoretical_loss": 3.5920402984570217, + "tokens_seen": 1291059200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004920200601805417, + "loss": 1.3528, + "theoretical_loss": 3.5920238873652006, + "tokens_seen": 1291124736 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004920040120361083, + "loss": 1.2517, + "theoretical_loss": 3.592007477339595, + "tokens_seen": 1291190272 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004919879638916751, + "loss": 1.2815, + "theoretical_loss": 3.591991068380082, + "tokens_seen": 1291255808 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004919719157472417, + "loss": 1.3056, + "theoretical_loss": 3.591974660486538, + "tokens_seen": 1291321344 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004919558676028085, + "loss": 1.2199, + "theoretical_loss": 3.5919582536588397, + "tokens_seen": 1291386880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004919398194583751, + "loss": 1.4008, + "theoretical_loss": 3.591941847896863, + "tokens_seen": 1291452416 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004919237713139419, + "loss": 1.3112, + "theoretical_loss": 3.591925443200486, + "tokens_seen": 1291517952 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004919077231695085, + "loss": 1.3115, + "theoretical_loss": 3.5919090395695843, + "tokens_seen": 1291583488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004918916750250753, + "loss": 1.2809, + "theoretical_loss": 3.5918926370040354, + "tokens_seen": 1291649024 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004918756268806419, + "loss": 1.3181, + "theoretical_loss": 3.5918762355037153, + "tokens_seen": 1291714560 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004918595787362087, + "loss": 1.3428, + "theoretical_loss": 3.5918598350685014, + "tokens_seen": 1291780096 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004918435305917753, + "loss": 1.3644, + "theoretical_loss": 3.5918434356982702, + "tokens_seen": 1291845632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004918274824473421, + "loss": 1.3212, + "theoretical_loss": 3.591827037392899, + "tokens_seen": 1291911168 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004918114343029087, + "loss": 1.3207, + "theoretical_loss": 3.5918106401522634, + "tokens_seen": 1291976704 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004917953861584755, + "loss": 1.3842, + "theoretical_loss": 3.5917942439762416, + "tokens_seen": 1292042240 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004917793380140421, + "loss": 1.2428, + "theoretical_loss": 3.59177784886471, + "tokens_seen": 1292107776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004917632898696089, + "loss": 1.4121, + "theoretical_loss": 3.5917614548175454, + "tokens_seen": 1292173312 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004917472417251755, + "loss": 1.3623, + "theoretical_loss": 3.591745061834625, + "tokens_seen": 1292238848 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004917311935807423, + "loss": 1.3316, + "theoretical_loss": 3.5917286699158253, + "tokens_seen": 1292304384 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004917151454363089, + "loss": 1.2605, + "theoretical_loss": 3.5917122790610234, + "tokens_seen": 1292369920 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004916990972918757, + "loss": 1.2514, + "theoretical_loss": 3.5916958892700968, + "tokens_seen": 1292435456 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004916830491474423, + "loss": 1.2794, + "theoretical_loss": 3.591679500542922, + "tokens_seen": 1292500992 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004916670010030091, + "loss": 1.3711, + "theoretical_loss": 3.5916631128793757, + "tokens_seen": 1292566528 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004916509528585757, + "loss": 1.3103, + "theoretical_loss": 3.591646726279336, + "tokens_seen": 1292632064 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004916349047141425, + "loss": 1.4212, + "theoretical_loss": 3.5916303407426797, + "tokens_seen": 1292697600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004916188565697091, + "loss": 1.304, + "theoretical_loss": 3.5916139562692826, + "tokens_seen": 1292763136 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004916028084252759, + "loss": 1.3502, + "theoretical_loss": 3.5915975728590235, + "tokens_seen": 1292828672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004915867602808425, + "loss": 1.3548, + "theoretical_loss": 3.5915811905117785, + "tokens_seen": 1292894208 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004915707121364093, + "loss": 1.2576, + "theoretical_loss": 3.5915648092274246, + "tokens_seen": 1292959744 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004915546639919759, + "loss": 1.4002, + "theoretical_loss": 3.59154842900584, + "tokens_seen": 1293025280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004915386158475427, + "loss": 1.2633, + "theoretical_loss": 3.5915320498469008, + "tokens_seen": 1293090816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004915225677031093, + "loss": 1.2777, + "theoretical_loss": 3.5915156717504853, + "tokens_seen": 1293156352 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004915065195586761, + "loss": 1.3947, + "theoretical_loss": 3.5914992947164697, + "tokens_seen": 1293221888 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004914904714142427, + "loss": 1.2215, + "theoretical_loss": 3.591482918744732, + "tokens_seen": 1293287424 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004914744232698095, + "loss": 1.4277, + "theoretical_loss": 3.591466543835149, + "tokens_seen": 1293352960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004914583751253761, + "loss": 1.5229, + "theoretical_loss": 3.5914501699875983, + "tokens_seen": 1293418496 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004914423269809429, + "loss": 1.3043, + "theoretical_loss": 3.5914337972019563, + "tokens_seen": 1293484032 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004914262788365095, + "loss": 1.4133, + "theoretical_loss": 3.591417425478102, + "tokens_seen": 1293549568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004914102306920763, + "loss": 1.3894, + "theoretical_loss": 3.591401054815911, + "tokens_seen": 1293615104 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004913941825476429, + "loss": 1.4074, + "theoretical_loss": 3.591384685215263, + "tokens_seen": 1293680640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004913781344032097, + "loss": 1.3511, + "theoretical_loss": 3.5913683166760326, + "tokens_seen": 1293746176 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004913620862587763, + "loss": 1.2483, + "theoretical_loss": 3.591351949198099, + "tokens_seen": 1293811712 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004913460381143431, + "loss": 1.2926, + "theoretical_loss": 3.591335582781339, + "tokens_seen": 1293877248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004913299899699097, + "loss": 1.2872, + "theoretical_loss": 3.5913192174256303, + "tokens_seen": 1293942784 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004913139418254765, + "loss": 1.3523, + "theoretical_loss": 3.5913028531308506, + "tokens_seen": 1294008320 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004912978936810431, + "loss": 1.4087, + "theoretical_loss": 3.591286489896877, + "tokens_seen": 1294073856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004912818455366099, + "loss": 1.4452, + "theoretical_loss": 3.591270127723587, + "tokens_seen": 1294139392 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004912657973921765, + "loss": 1.3128, + "theoretical_loss": 3.5912537666108584, + "tokens_seen": 1294204928 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004912497492477433, + "loss": 1.2451, + "theoretical_loss": 3.591237406558568, + "tokens_seen": 1294270464 + }, + { + "epoch": 0.39, + "learning_rate": 0.00049123370110331, + "loss": 1.3471, + "theoretical_loss": 3.591221047566594, + "tokens_seen": 1294336000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004912176529588767, + "loss": 1.2983, + "theoretical_loss": 3.591204689634815, + "tokens_seen": 1294401536 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004912016048144433, + "loss": 1.4174, + "theoretical_loss": 3.5911883327631067, + "tokens_seen": 1294467072 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004911855566700101, + "loss": 1.331, + "theoretical_loss": 3.5911719769513484, + "tokens_seen": 1294532608 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004911695085255767, + "loss": 1.376, + "theoretical_loss": 3.591155622199416, + "tokens_seen": 1294598144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004911534603811434, + "loss": 1.2616, + "theoretical_loss": 3.591139268507189, + "tokens_seen": 1294663680 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004911374122367102, + "loss": 1.4057, + "theoretical_loss": 3.5911229158745437, + "tokens_seen": 1294729216 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004911213640922768, + "loss": 1.2627, + "theoretical_loss": 3.5911065643013593, + "tokens_seen": 1294794752 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004911053159478436, + "loss": 1.2486, + "theoretical_loss": 3.5910902137875116, + "tokens_seen": 1294860288 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004910892678034102, + "loss": 1.3808, + "theoretical_loss": 3.59107386433288, + "tokens_seen": 1294925824 + }, + { + "epoch": 0.39, + "learning_rate": 0.000491073219658977, + "loss": 1.3547, + "theoretical_loss": 3.591057515937342, + "tokens_seen": 1294991360 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004910571715145436, + "loss": 1.3029, + "theoretical_loss": 3.591041168600774, + "tokens_seen": 1295056896 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004910411233701104, + "loss": 1.3477, + "theoretical_loss": 3.591024822323056, + "tokens_seen": 1295122432 + }, + { + "epoch": 0.39, + "learning_rate": 0.000491025075225677, + "loss": 1.4745, + "theoretical_loss": 3.5910084771040642, + "tokens_seen": 1295187968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004910090270812438, + "loss": 1.3426, + "theoretical_loss": 3.590992132943677, + "tokens_seen": 1295253504 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004909929789368104, + "loss": 1.372, + "theoretical_loss": 3.5909757898417722, + "tokens_seen": 1295319040 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004909769307923772, + "loss": 1.3924, + "theoretical_loss": 3.590959447798228, + "tokens_seen": 1295384576 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004909608826479438, + "loss": 1.3367, + "theoretical_loss": 3.5909431068129223, + "tokens_seen": 1295450112 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004909448345035106, + "loss": 1.3793, + "theoretical_loss": 3.5909267668857323, + "tokens_seen": 1295515648 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004909287863590772, + "loss": 1.437, + "theoretical_loss": 3.590910428016537, + "tokens_seen": 1295581184 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490912738214644, + "loss": 1.4451, + "theoretical_loss": 3.5908940902052136, + "tokens_seen": 1295646720 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004908966900702106, + "loss": 1.3559, + "theoretical_loss": 3.5908777534516405, + "tokens_seen": 1295712256 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004908806419257774, + "loss": 1.4423, + "theoretical_loss": 3.5908614177556957, + "tokens_seen": 1295777792 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490864593781344, + "loss": 1.3462, + "theoretical_loss": 3.590845083117257, + "tokens_seen": 1295843328 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004908485456369108, + "loss": 1.3865, + "theoretical_loss": 3.590828749536203, + "tokens_seen": 1295908864 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004908324974924774, + "loss": 1.4513, + "theoretical_loss": 3.590812417012411, + "tokens_seen": 1295974400 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004908164493480442, + "loss": 1.4306, + "theoretical_loss": 3.59079608554576, + "tokens_seen": 1296039936 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004908004012036108, + "loss": 1.3125, + "theoretical_loss": 3.5907797551361273, + "tokens_seen": 1296105472 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004907843530591776, + "loss": 1.3303, + "theoretical_loss": 3.5907634257833916, + "tokens_seen": 1296171008 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004907683049147442, + "loss": 1.3886, + "theoretical_loss": 3.5907470974874305, + "tokens_seen": 1296236544 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490752256770311, + "loss": 1.3693, + "theoretical_loss": 3.590730770248123, + "tokens_seen": 1296302080 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004907362086258776, + "loss": 1.3521, + "theoretical_loss": 3.5907144440653465, + "tokens_seen": 1296367616 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004907201604814444, + "loss": 1.377, + "theoretical_loss": 3.5906981189389793, + "tokens_seen": 1296433152 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490704112337011, + "loss": 1.3457, + "theoretical_loss": 3.5906817948689005, + "tokens_seen": 1296498688 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004906880641925778, + "loss": 1.2977, + "theoretical_loss": 3.5906654718549875, + "tokens_seen": 1296564224 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004906720160481444, + "loss": 1.255, + "theoretical_loss": 3.590649149897119, + "tokens_seen": 1296629760 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004906559679037112, + "loss": 1.3712, + "theoretical_loss": 3.590632828995173, + "tokens_seen": 1296695296 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004906399197592778, + "loss": 1.3481, + "theoretical_loss": 3.5906165091490285, + "tokens_seen": 1296760832 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004906238716148446, + "loss": 1.4315, + "theoretical_loss": 3.590600190358563, + "tokens_seen": 1296826368 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004906078234704112, + "loss": 1.2039, + "theoretical_loss": 3.5905838726236556, + "tokens_seen": 1296891904 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490591775325978, + "loss": 1.38, + "theoretical_loss": 3.590567555944183, + "tokens_seen": 1296957440 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004905757271815446, + "loss": 1.3827, + "theoretical_loss": 3.5905512403200266, + "tokens_seen": 1297022976 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004905596790371114, + "loss": 1.2773, + "theoretical_loss": 3.5905349257510624, + "tokens_seen": 1297088512 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490543630892678, + "loss": 1.3772, + "theoretical_loss": 3.5905186122371697, + "tokens_seen": 1297154048 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004905275827482448, + "loss": 1.3389, + "theoretical_loss": 3.5905022997782265, + "tokens_seen": 1297219584 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004905115346038114, + "loss": 1.3729, + "theoretical_loss": 3.5904859883741125, + "tokens_seen": 1297285120 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004904954864593782, + "loss": 1.3347, + "theoretical_loss": 3.5904696780247045, + "tokens_seen": 1297350656 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004904794383149448, + "loss": 1.3732, + "theoretical_loss": 3.5904533687298823, + "tokens_seen": 1297416192 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004904633901705116, + "loss": 1.3731, + "theoretical_loss": 3.5904370604895237, + "tokens_seen": 1297481728 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004904473420260782, + "loss": 1.3021, + "theoretical_loss": 3.590420753303508, + "tokens_seen": 1297547264 + }, + { + "epoch": 0.39, + "objective/train/docs_used": 482938, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.43950617313385, + "objective/train/theoretical_loss": 3.5904044471717134, + "objective/train/tokens_used": 1318072800, + "theoretical_loss": 3.5904044471717134, + "tokens_seen": 1297612800 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490431293881645, + "loss": 1.4395, + "theoretical_loss": 3.5904044471717134, + "tokens_seen": 1297612800 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004904152457372116, + "loss": 1.4711, + "theoretical_loss": 3.5903881420940182, + "tokens_seen": 1297678336 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004903991975927784, + "loss": 1.3343, + "theoretical_loss": 3.5903718380703014, + "tokens_seen": 1297743872 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490383149448345, + "loss": 1.3322, + "theoretical_loss": 3.5903555351004415, + "tokens_seen": 1297809408 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004903671013039118, + "loss": 1.3601, + "theoretical_loss": 3.5903392331843174, + "tokens_seen": 1297874944 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004903510531594784, + "loss": 1.36, + "theoretical_loss": 3.5903229323218078, + "tokens_seen": 1297940480 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004903350050150452, + "loss": 1.3443, + "theoretical_loss": 3.5903066325127906, + "tokens_seen": 1298006016 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004903189568706118, + "loss": 1.3674, + "theoretical_loss": 3.590290333757146, + "tokens_seen": 1298071552 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004903029087261786, + "loss": 1.4098, + "theoretical_loss": 3.590274036054751, + "tokens_seen": 1298137088 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004902868605817452, + "loss": 1.3149, + "theoretical_loss": 3.5902577394054864, + "tokens_seen": 1298202624 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490270812437312, + "loss": 1.3303, + "theoretical_loss": 3.590241443809229, + "tokens_seen": 1298268160 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004902547642928786, + "loss": 1.2568, + "theoretical_loss": 3.590225149265859, + "tokens_seen": 1298333696 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004902387161484454, + "loss": 1.3388, + "theoretical_loss": 3.5902088557752547, + "tokens_seen": 1298399232 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490222668004012, + "loss": 1.4095, + "theoretical_loss": 3.5901925633372946, + "tokens_seen": 1298464768 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004902066198595788, + "loss": 1.3515, + "theoretical_loss": 3.590176271951858, + "tokens_seen": 1298530304 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004901905717151454, + "loss": 1.4678, + "theoretical_loss": 3.5901599816188243, + "tokens_seen": 1298595840 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004901745235707122, + "loss": 1.36, + "theoretical_loss": 3.5901436923380716, + "tokens_seen": 1298661376 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004901584754262788, + "loss": 1.4082, + "theoretical_loss": 3.5901274041094786, + "tokens_seen": 1298726912 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004901424272818456, + "loss": 1.4081, + "theoretical_loss": 3.590111116932925, + "tokens_seen": 1298792448 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004901263791374122, + "loss": 1.3278, + "theoretical_loss": 3.5900948308082894, + "tokens_seen": 1298857984 + }, + { + "epoch": 0.39, + "learning_rate": 0.000490110330992979, + "loss": 1.4102, + "theoretical_loss": 3.5900785457354516, + "tokens_seen": 1298923520 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004900942828485456, + "loss": 1.3431, + "theoretical_loss": 3.590062261714289, + "tokens_seen": 1298989056 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004900782347041124, + "loss": 1.4741, + "theoretical_loss": 3.5900459787446817, + "tokens_seen": 1299054592 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004900621865596791, + "loss": 1.354, + "theoretical_loss": 3.590029696826509, + "tokens_seen": 1299120128 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004900461384152458, + "loss": 1.4014, + "theoretical_loss": 3.5900134159596493, + "tokens_seen": 1299185664 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004900300902708125, + "loss": 1.2951, + "theoretical_loss": 3.5899971361439817, + "tokens_seen": 1299251200 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004900140421263792, + "loss": 1.4475, + "theoretical_loss": 3.589980857379386, + "tokens_seen": 1299316736 + }, + { + "epoch": 0.39, + "learning_rate": 0.000489997993981946, + "loss": 1.3668, + "theoretical_loss": 3.5899645796657405, + "tokens_seen": 1299382272 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004899819458375126, + "loss": 1.3294, + "theoretical_loss": 3.5899483030029247, + "tokens_seen": 1299447808 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004899658976930794, + "loss": 1.4595, + "theoretical_loss": 3.5899320273908177, + "tokens_seen": 1299513344 + }, + { + "epoch": 0.39, + "learning_rate": 0.000489949849548646, + "loss": 1.357, + "theoretical_loss": 3.5899157528292993, + "tokens_seen": 1299578880 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004899338014042128, + "loss": 1.3995, + "theoretical_loss": 3.5898994793182477, + "tokens_seen": 1299644416 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004899177532597794, + "loss": 1.4376, + "theoretical_loss": 3.589883206857543, + "tokens_seen": 1299709952 + }, + { + "epoch": 0.39, + "learning_rate": 0.000489901705115346, + "loss": 1.2592, + "theoretical_loss": 3.5898669354470636, + "tokens_seen": 1299775488 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004898856569709128, + "loss": 1.2428, + "theoretical_loss": 3.58985066508669, + "tokens_seen": 1299841024 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004898696088264794, + "loss": 1.3755, + "theoretical_loss": 3.5898343957763004, + "tokens_seen": 1299906560 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004898535606820462, + "loss": 1.3065, + "theoretical_loss": 3.589818127515774, + "tokens_seen": 1299972096 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004898375125376128, + "loss": 1.3409, + "theoretical_loss": 3.5898018603049913, + "tokens_seen": 1300037632 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004898214643931796, + "loss": 1.3768, + "theoretical_loss": 3.58978559414383, + "tokens_seen": 1300103168 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004898054162487462, + "loss": 1.3357, + "theoretical_loss": 3.589769329032171, + "tokens_seen": 1300168704 + }, + { + "epoch": 0.39, + "learning_rate": 0.000489789368104313, + "loss": 1.3461, + "theoretical_loss": 3.5897530649698934, + "tokens_seen": 1300234240 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004897733199598797, + "loss": 1.3653, + "theoretical_loss": 3.5897368019568754, + "tokens_seen": 1300299776 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004897572718154463, + "loss": 1.3596, + "theoretical_loss": 3.589720539992998, + "tokens_seen": 1300365312 + }, + { + "epoch": 0.39, + "learning_rate": 0.000489741223671013, + "loss": 1.2832, + "theoretical_loss": 3.58970427907814, + "tokens_seen": 1300430848 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004897251755265797, + "loss": 1.3607, + "theoretical_loss": 3.58968801921218, + "tokens_seen": 1300496384 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004897091273821465, + "loss": 1.2992, + "theoretical_loss": 3.5896717603949995, + "tokens_seen": 1300561920 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004896930792377131, + "loss": 1.3228, + "theoretical_loss": 3.589655502626476, + "tokens_seen": 1300627456 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004896770310932799, + "loss": 1.3222, + "theoretical_loss": 3.5896392459064903, + "tokens_seen": 1300692992 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004896609829488465, + "loss": 1.3588, + "theoretical_loss": 3.5896229902349215, + "tokens_seen": 1300758528 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004896449348044133, + "loss": 1.3066, + "theoretical_loss": 3.5896067356116492, + "tokens_seen": 1300824064 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004896288866599799, + "loss": 1.3307, + "theoretical_loss": 3.5895904820365527, + "tokens_seen": 1300889600 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004896128385155467, + "loss": 1.2479, + "theoretical_loss": 3.589574229509512, + "tokens_seen": 1300955136 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004895967903711133, + "loss": 1.3547, + "theoretical_loss": 3.589557978030407, + "tokens_seen": 1301020672 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004895807422266801, + "loss": 1.5, + "theoretical_loss": 3.5895417275991166, + "tokens_seen": 1301086208 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004895646940822467, + "loss": 1.3405, + "theoretical_loss": 3.5895254782155206, + "tokens_seen": 1301151744 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004895486459378135, + "loss": 1.3085, + "theoretical_loss": 3.589509229879499, + "tokens_seen": 1301217280 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004895325977933801, + "loss": 1.42, + "theoretical_loss": 3.589492982590932, + "tokens_seen": 1301282816 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004895165496489469, + "loss": 1.4847, + "theoretical_loss": 3.5894767363496984, + "tokens_seen": 1301348352 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004895005015045135, + "loss": 1.3654, + "theoretical_loss": 3.5894604911556782, + "tokens_seen": 1301413888 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004894844533600803, + "loss": 1.3116, + "theoretical_loss": 3.5894442470087515, + "tokens_seen": 1301479424 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004894684052156469, + "loss": 1.2777, + "theoretical_loss": 3.5894280039087976, + "tokens_seen": 1301544960 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004894523570712137, + "loss": 1.3853, + "theoretical_loss": 3.589411761855697, + "tokens_seen": 1301610496 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004894363089267803, + "loss": 1.2608, + "theoretical_loss": 3.5893955208493287, + "tokens_seen": 1301676032 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004894202607823471, + "loss": 1.3417, + "theoretical_loss": 3.589379280889573, + "tokens_seen": 1301741568 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004894042126379137, + "loss": 1.3603, + "theoretical_loss": 3.589363041976309, + "tokens_seen": 1301807104 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004893881644934805, + "loss": 1.2634, + "theoretical_loss": 3.5893468041094185, + "tokens_seen": 1301872640 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004893721163490471, + "loss": 1.3342, + "theoretical_loss": 3.58933056728878, + "tokens_seen": 1301938176 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004893560682046139, + "loss": 1.2544, + "theoretical_loss": 3.589314331514273, + "tokens_seen": 1302003712 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004893400200601805, + "loss": 1.3007, + "theoretical_loss": 3.589298096785779, + "tokens_seen": 1302069248 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004893239719157473, + "loss": 1.318, + "theoretical_loss": 3.589281863103176, + "tokens_seen": 1302134784 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004893079237713139, + "loss": 1.3764, + "theoretical_loss": 3.5892656304663455, + "tokens_seen": 1302200320 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004892918756268807, + "loss": 1.3253, + "theoretical_loss": 3.5892493988751673, + "tokens_seen": 1302265856 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004892758274824473, + "loss": 1.548, + "theoretical_loss": 3.5892331683295207, + "tokens_seen": 1302331392 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004892597793380141, + "loss": 1.4027, + "theoretical_loss": 3.5892169388292867, + "tokens_seen": 1302396928 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004892437311935807, + "loss": 1.2989, + "theoretical_loss": 3.589200710374344, + "tokens_seen": 1302462464 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004892276830491475, + "loss": 1.4334, + "theoretical_loss": 3.5891844829645745, + "tokens_seen": 1302528000 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004892116349047141, + "loss": 1.3576, + "theoretical_loss": 3.589168256599857, + "tokens_seen": 1302593536 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004891955867602809, + "loss": 1.337, + "theoretical_loss": 3.5891520312800718, + "tokens_seen": 1302659072 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004891795386158475, + "loss": 1.4184, + "theoretical_loss": 3.5891358070051, + "tokens_seen": 1302724608 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004891634904714143, + "loss": 1.217, + "theoretical_loss": 3.5891195837748198, + "tokens_seen": 1302790144 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004891474423269809, + "loss": 1.333, + "theoretical_loss": 3.589103361589113, + "tokens_seen": 1302855680 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004891313941825477, + "loss": 1.3052, + "theoretical_loss": 3.5890871404478597, + "tokens_seen": 1302921216 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004891153460381143, + "loss": 1.3781, + "theoretical_loss": 3.58907092035094, + "tokens_seen": 1302986752 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004890992978936811, + "loss": 1.3047, + "theoretical_loss": 3.5890547012982337, + "tokens_seen": 1303052288 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004890832497492477, + "loss": 1.3681, + "theoretical_loss": 3.589038483289621, + "tokens_seen": 1303117824 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004890672016048145, + "loss": 1.2954, + "theoretical_loss": 3.5890222663249824, + "tokens_seen": 1303183360 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004890511534603811, + "loss": 1.3569, + "theoretical_loss": 3.589006050404199, + "tokens_seen": 1303248896 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004890351053159479, + "loss": 1.3495, + "theoretical_loss": 3.58898983552715, + "tokens_seen": 1303314432 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004890190571715145, + "loss": 1.3864, + "theoretical_loss": 3.5889736216937163, + "tokens_seen": 1303379968 + }, + { + "epoch": 0.39, + "learning_rate": 0.0004890030090270813, + "loss": 1.3695, + "theoretical_loss": 3.588957408903778, + "tokens_seen": 1303445504 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004889869608826479, + "loss": 1.339, + "theoretical_loss": 3.5889411971572156, + "tokens_seen": 1303511040 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004889709127382147, + "loss": 1.2319, + "theoretical_loss": 3.5889249864539097, + "tokens_seen": 1303576576 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004889548645937813, + "loss": 1.363, + "theoretical_loss": 3.5889087767937404, + "tokens_seen": 1303642112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004889388164493481, + "loss": 1.3232, + "theoretical_loss": 3.588892568176588, + "tokens_seen": 1303707648 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004889227683049147, + "loss": 1.3325, + "theoretical_loss": 3.5888763606023337, + "tokens_seen": 1303773184 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004889067201604815, + "loss": 1.3786, + "theoretical_loss": 3.588860154070857, + "tokens_seen": 1303838720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004888906720160481, + "loss": 1.3644, + "theoretical_loss": 3.5888439485820394, + "tokens_seen": 1303904256 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004888746238716149, + "loss": 1.3349, + "theoretical_loss": 3.5888277441357603, + "tokens_seen": 1303969792 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004888585757271816, + "loss": 1.3457, + "theoretical_loss": 3.5888115407319017, + "tokens_seen": 1304035328 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004888425275827483, + "loss": 1.2551, + "theoretical_loss": 3.5887953383703426, + "tokens_seen": 1304100864 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 485689, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3419523239135742, + "objective/train/theoretical_loss": 3.588779137050965, + "objective/train/tokens_used": 1324626400, + "theoretical_loss": 3.588779137050965, + "tokens_seen": 1304166400 + }, + { + "epoch": 0.4, + "learning_rate": 0.000488826479438315, + "loss": 1.342, + "theoretical_loss": 3.588779137050965, + "tokens_seen": 1304166400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004888104312938817, + "loss": 1.2728, + "theoretical_loss": 3.588762936773648, + "tokens_seen": 1304231936 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004887943831494484, + "loss": 1.4621, + "theoretical_loss": 3.5887467375382736, + "tokens_seen": 1304297472 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004887783350050151, + "loss": 1.2612, + "theoretical_loss": 3.5887305393447217, + "tokens_seen": 1304363008 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004887622868605818, + "loss": 1.2334, + "theoretical_loss": 3.5887143421928727, + "tokens_seen": 1304428544 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004887462387161485, + "loss": 1.3514, + "theoretical_loss": 3.588698146082608, + "tokens_seen": 1304494080 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004887301905717152, + "loss": 1.2871, + "theoretical_loss": 3.5886819510138084, + "tokens_seen": 1304559616 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004887141424272819, + "loss": 1.2728, + "theoretical_loss": 3.5886657569863543, + "tokens_seen": 1304625152 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004886980942828486, + "loss": 1.4655, + "theoretical_loss": 3.588649564000126, + "tokens_seen": 1304690688 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004886820461384153, + "loss": 1.3694, + "theoretical_loss": 3.5886333720550043, + "tokens_seen": 1304756224 + }, + { + "epoch": 0.4, + "learning_rate": 0.000488665997993982, + "loss": 1.3393, + "theoretical_loss": 3.5886171811508705, + "tokens_seen": 1304821760 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004886499498495487, + "loss": 1.3238, + "theoretical_loss": 3.588600991287605, + "tokens_seen": 1304887296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004886339017051155, + "loss": 1.3076, + "theoretical_loss": 3.588584802465089, + "tokens_seen": 1304952832 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004886178535606821, + "loss": 1.2777, + "theoretical_loss": 3.5885686146832025, + "tokens_seen": 1305018368 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004886018054162489, + "loss": 1.3379, + "theoretical_loss": 3.5885524279418277, + "tokens_seen": 1305083904 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004885857572718155, + "loss": 1.3575, + "theoretical_loss": 3.5885362422408447, + "tokens_seen": 1305149440 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004885697091273823, + "loss": 1.3288, + "theoretical_loss": 3.5885200575801344, + "tokens_seen": 1305214976 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004885536609829489, + "loss": 1.3714, + "theoretical_loss": 3.588503873959578, + "tokens_seen": 1305280512 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004885376128385157, + "loss": 1.2647, + "theoretical_loss": 3.588487691379055, + "tokens_seen": 1305346048 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004885215646940823, + "loss": 1.3034, + "theoretical_loss": 3.588471509838449, + "tokens_seen": 1305411584 + }, + { + "epoch": 0.4, + "learning_rate": 0.000488505516549649, + "loss": 1.3815, + "theoretical_loss": 3.5884553293376387, + "tokens_seen": 1305477120 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004884894684052157, + "loss": 1.3603, + "theoretical_loss": 3.588439149876506, + "tokens_seen": 1305542656 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004884734202607823, + "loss": 1.3132, + "theoretical_loss": 3.5884229714549316, + "tokens_seen": 1305608192 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004884573721163491, + "loss": 1.296, + "theoretical_loss": 3.588406794072797, + "tokens_seen": 1305673728 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004884413239719157, + "loss": 1.2768, + "theoretical_loss": 3.588390617729983, + "tokens_seen": 1305739264 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004884252758274825, + "loss": 1.3806, + "theoretical_loss": 3.5883744424263706, + "tokens_seen": 1305804800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004884092276830492, + "loss": 1.3761, + "theoretical_loss": 3.5883582681618407, + "tokens_seen": 1305870336 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004883931795386159, + "loss": 1.3657, + "theoretical_loss": 3.588342094936275, + "tokens_seen": 1305935872 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004883771313941826, + "loss": 1.4147, + "theoretical_loss": 3.5883259227495543, + "tokens_seen": 1306001408 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004883610832497492, + "loss": 1.3355, + "theoretical_loss": 3.5883097516015594, + "tokens_seen": 1306066944 + }, + { + "epoch": 0.4, + "learning_rate": 0.000488345035105316, + "loss": 1.3096, + "theoretical_loss": 3.588293581492172, + "tokens_seen": 1306132480 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004883289869608826, + "loss": 1.3311, + "theoretical_loss": 3.5882774124212733, + "tokens_seen": 1306198016 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004883129388164494, + "loss": 1.3806, + "theoretical_loss": 3.588261244388744, + "tokens_seen": 1306263552 + }, + { + "epoch": 0.4, + "learning_rate": 0.000488296890672016, + "loss": 1.3526, + "theoretical_loss": 3.5882450773944656, + "tokens_seen": 1306329088 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004882808425275828, + "loss": 1.369, + "theoretical_loss": 3.5882289114383195, + "tokens_seen": 1306394624 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048826479438314946, + "loss": 1.4674, + "theoretical_loss": 3.5882127465201865, + "tokens_seen": 1306460160 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004882487462387162, + "loss": 1.3043, + "theoretical_loss": 3.5881965826399487, + "tokens_seen": 1306525696 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048823269809428286, + "loss": 1.3011, + "theoretical_loss": 3.588180419797487, + "tokens_seen": 1306591232 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004882166499498496, + "loss": 1.3203, + "theoretical_loss": 3.5881642579926822, + "tokens_seen": 1306656768 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048820060180541626, + "loss": 1.3969, + "theoretical_loss": 3.588148097225416, + "tokens_seen": 1306722304 + }, + { + "epoch": 0.4, + "learning_rate": 0.000488184553660983, + "loss": 1.3625, + "theoretical_loss": 3.58813193749557, + "tokens_seen": 1306787840 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048816850551654966, + "loss": 1.4131, + "theoretical_loss": 3.5881157788030253, + "tokens_seen": 1306853376 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004881524573721164, + "loss": 1.4272, + "theoretical_loss": 3.5880996211476637, + "tokens_seen": 1306918912 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048813640922768306, + "loss": 1.4197, + "theoretical_loss": 3.588083464529366, + "tokens_seen": 1306984448 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048812036108324976, + "loss": 1.307, + "theoretical_loss": 3.5880673089480144, + "tokens_seen": 1307049984 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048810431293881646, + "loss": 1.2693, + "theoretical_loss": 3.5880511544034897, + "tokens_seen": 1307115520 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048808826479438316, + "loss": 1.3606, + "theoretical_loss": 3.5880350008956734, + "tokens_seen": 1307181056 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048807221664994986, + "loss": 1.2795, + "theoretical_loss": 3.588018848424448, + "tokens_seen": 1307246592 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048805616850551656, + "loss": 1.3469, + "theoretical_loss": 3.5880026969896934, + "tokens_seen": 1307312128 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048804012036108327, + "loss": 1.3535, + "theoretical_loss": 3.587986546591292, + "tokens_seen": 1307377664 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048802407221664997, + "loss": 1.3968, + "theoretical_loss": 3.587970397229126, + "tokens_seen": 1307443200 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048800802407221667, + "loss": 1.3537, + "theoretical_loss": 3.587954248903076, + "tokens_seen": 1307508736 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048799197592778337, + "loss": 1.3063, + "theoretical_loss": 3.5879381016130236, + "tokens_seen": 1307574272 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048797592778335, + "loss": 1.3529, + "theoretical_loss": 3.5879219553588513, + "tokens_seen": 1307639808 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048795987963891677, + "loss": 1.3823, + "theoretical_loss": 3.5879058101404397, + "tokens_seen": 1307705344 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004879438314944834, + "loss": 1.3933, + "theoretical_loss": 3.587889665957671, + "tokens_seen": 1307770880 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048792778335005017, + "loss": 1.3154, + "theoretical_loss": 3.587873522810427, + "tokens_seen": 1307836416 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004879117352056168, + "loss": 1.3412, + "theoretical_loss": 3.5878573806985887, + "tokens_seen": 1307901952 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048789568706118357, + "loss": 1.4919, + "theoretical_loss": 3.5878412396220387, + "tokens_seen": 1307967488 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004878796389167502, + "loss": 1.4946, + "theoretical_loss": 3.587825099580658, + "tokens_seen": 1308033024 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048786359077231697, + "loss": 1.31, + "theoretical_loss": 3.5878089605743293, + "tokens_seen": 1308098560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004878475426278836, + "loss": 1.2915, + "theoretical_loss": 3.5877928226029336, + "tokens_seen": 1308164096 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004878314944834504, + "loss": 1.3938, + "theoretical_loss": 3.5877766856663524, + "tokens_seen": 1308229632 + }, + { + "epoch": 0.4, + "learning_rate": 0.000487815446339017, + "loss": 1.3162, + "theoretical_loss": 3.587760549764468, + "tokens_seen": 1308295168 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004877993981945838, + "loss": 1.3693, + "theoretical_loss": 3.5877444148971627, + "tokens_seen": 1308360704 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004877833500501504, + "loss": 1.4153, + "theoretical_loss": 3.5877282810643174, + "tokens_seen": 1308426240 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004877673019057172, + "loss": 1.3724, + "theoretical_loss": 3.5877121482658145, + "tokens_seen": 1308491776 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004877512537612838, + "loss": 1.4093, + "theoretical_loss": 3.587696016501536, + "tokens_seen": 1308557312 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004877352056168506, + "loss": 1.3879, + "theoretical_loss": 3.5876798857713634, + "tokens_seen": 1308622848 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004877191574724172, + "loss": 1.3376, + "theoretical_loss": 3.5876637560751785, + "tokens_seen": 1308688384 + }, + { + "epoch": 0.4, + "learning_rate": 0.000487703109327984, + "loss": 1.4198, + "theoretical_loss": 3.5876476274128644, + "tokens_seen": 1308753920 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048768706118355073, + "loss": 1.2811, + "theoretical_loss": 3.587631499784301, + "tokens_seen": 1308819456 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004876710130391174, + "loss": 1.3598, + "theoretical_loss": 3.5876153731893727, + "tokens_seen": 1308884992 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048765496489468413, + "loss": 1.2942, + "theoretical_loss": 3.58759924762796, + "tokens_seen": 1308950528 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004876389167502508, + "loss": 1.3667, + "theoretical_loss": 3.587583123099945, + "tokens_seen": 1309016064 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048762286860581754, + "loss": 1.3412, + "theoretical_loss": 3.58756699960521, + "tokens_seen": 1309081600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004876068204613842, + "loss": 1.3063, + "theoretical_loss": 3.587550877143637, + "tokens_seen": 1309147136 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048759077231695094, + "loss": 1.2426, + "theoretical_loss": 3.5875347557151085, + "tokens_seen": 1309212672 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004875747241725176, + "loss": 1.4103, + "theoretical_loss": 3.5875186353195065, + "tokens_seen": 1309278208 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048755867602808434, + "loss": 1.4368, + "theoretical_loss": 3.5875025159567118, + "tokens_seen": 1309343744 + }, + { + "epoch": 0.4, + "learning_rate": 0.000487542627883651, + "loss": 1.3601, + "theoretical_loss": 3.5874863976266083, + "tokens_seen": 1309409280 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048752657973921774, + "loss": 1.3382, + "theoretical_loss": 3.5874702803290774, + "tokens_seen": 1309474816 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004875105315947844, + "loss": 1.3524, + "theoretical_loss": 3.5874541640640016, + "tokens_seen": 1309540352 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004874944834503511, + "loss": 1.3833, + "theoretical_loss": 3.5874380488312623, + "tokens_seen": 1309605888 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004874784353059178, + "loss": 1.448, + "theoretical_loss": 3.5874219346307425, + "tokens_seen": 1309671424 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004874623871614845, + "loss": 1.3587, + "theoretical_loss": 3.5874058214623243, + "tokens_seen": 1309736960 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004874463390170512, + "loss": 1.3468, + "theoretical_loss": 3.5873897093258895, + "tokens_seen": 1309802496 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004874302908726179, + "loss": 1.5457, + "theoretical_loss": 3.5873735982213213, + "tokens_seen": 1309868032 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004874142427281846, + "loss": 1.3924, + "theoretical_loss": 3.587357488148501, + "tokens_seen": 1309933568 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004873981945837513, + "loss": 1.3103, + "theoretical_loss": 3.587341379107311, + "tokens_seen": 1309999104 + }, + { + "epoch": 0.4, + "learning_rate": 0.000487382146439318, + "loss": 1.214, + "theoretical_loss": 3.587325271097635, + "tokens_seen": 1310064640 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004873660982948847, + "loss": 1.3035, + "theoretical_loss": 3.587309164119353, + "tokens_seen": 1310130176 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048735005015045134, + "loss": 1.2772, + "theoretical_loss": 3.5872930581723494, + "tokens_seen": 1310195712 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004873340020060181, + "loss": 1.2264, + "theoretical_loss": 3.587276953256506, + "tokens_seen": 1310261248 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048731795386158474, + "loss": 1.4391, + "theoretical_loss": 3.587260849371705, + "tokens_seen": 1310326784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004873019057171515, + "loss": 1.3222, + "theoretical_loss": 3.587244746517829, + "tokens_seen": 1310392320 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048728585757271814, + "loss": 1.2783, + "theoretical_loss": 3.5872286446947603, + "tokens_seen": 1310457856 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004872698094282849, + "loss": 1.3099, + "theoretical_loss": 3.587212543902381, + "tokens_seen": 1310523392 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048725376128385154, + "loss": 1.2667, + "theoretical_loss": 3.5871964441405746, + "tokens_seen": 1310588928 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004872377131394183, + "loss": 1.315, + "theoretical_loss": 3.587180345409222, + "tokens_seen": 1310654464 + }, + { + "debugging/Compilability": 1.0, + "debugging/distinct-1-grams": 0.7338505403722974, + "debugging/entropy-1-grams": 5.332375534521528, + "debugging/length": 494.75, + "debugging/num_segments": 20, + "debugging/score": 0.003694114610569988, + "debugging/score_std": 0.0028829754501364007, + "epoch": 0.4, + "objective/train/docs_used": 488027, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2713736295700073, + "objective/train/theoretical_loss": 3.587164247708208, + "objective/train/tokens_used": 1331180000, + "theoretical_loss": 3.587164247708208, + "tokens_seen": 1310720000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048722166499498494, + "loss": 1.2714, + "theoretical_loss": 3.587164247708208, + "tokens_seen": 1310720000 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004872056168505517, + "loss": 1.3085, + "theoretical_loss": 3.5871481510374137, + "tokens_seen": 1310785536 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048718956870611834, + "loss": 1.3033, + "theoretical_loss": 3.5871320553967214, + "tokens_seen": 1310851072 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004871735205616851, + "loss": 1.2543, + "theoretical_loss": 3.587115960786014, + "tokens_seen": 1310916608 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048715747241725175, + "loss": 1.27, + "theoretical_loss": 3.5870998672051746, + "tokens_seen": 1310982144 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004871414242728185, + "loss": 1.3743, + "theoretical_loss": 3.5870837746540856, + "tokens_seen": 1311047680 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048712537612838515, + "loss": 1.3394, + "theoretical_loss": 3.5870676831326294, + "tokens_seen": 1311113216 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004871093279839519, + "loss": 1.3747, + "theoretical_loss": 3.5870515926406887, + "tokens_seen": 1311178752 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048709327983951855, + "loss": 1.3786, + "theoretical_loss": 3.5870355031781465, + "tokens_seen": 1311244288 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004870772316950853, + "loss": 1.3659, + "theoretical_loss": 3.587019414744885, + "tokens_seen": 1311309824 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048706118355065195, + "loss": 1.3317, + "theoretical_loss": 3.587003327340787, + "tokens_seen": 1311375360 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004870451354062187, + "loss": 1.3903, + "theoretical_loss": 3.586987240965736, + "tokens_seen": 1311440896 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048702908726178535, + "loss": 1.3819, + "theoretical_loss": 3.5869711556196133, + "tokens_seen": 1311506432 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004870130391173521, + "loss": 1.1929, + "theoretical_loss": 3.5869550713023033, + "tokens_seen": 1311571968 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048699699097291875, + "loss": 1.4414, + "theoretical_loss": 3.5869389880136877, + "tokens_seen": 1311637504 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004869809428284855, + "loss": 1.3629, + "theoretical_loss": 3.5869229057536494, + "tokens_seen": 1311703040 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048696489468405215, + "loss": 1.2372, + "theoretical_loss": 3.586906824522072, + "tokens_seen": 1311768576 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004869488465396189, + "loss": 1.2518, + "theoretical_loss": 3.586890744318838, + "tokens_seen": 1311834112 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048693279839518555, + "loss": 1.3421, + "theoretical_loss": 3.5868746651438297, + "tokens_seen": 1311899648 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004869167502507523, + "loss": 1.3072, + "theoretical_loss": 3.5868585869969305, + "tokens_seen": 1311965184 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048690070210631896, + "loss": 1.353, + "theoretical_loss": 3.5868425098780228, + "tokens_seen": 1312030720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004868846539618857, + "loss": 1.2978, + "theoretical_loss": 3.5868264337869906, + "tokens_seen": 1312096256 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048686860581745236, + "loss": 1.38, + "theoretical_loss": 3.5868103587237155, + "tokens_seen": 1312161792 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004868525576730191, + "loss": 1.2725, + "theoretical_loss": 3.586794284688082, + "tokens_seen": 1312227328 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048683650952858576, + "loss": 1.2638, + "theoretical_loss": 3.586778211679972, + "tokens_seen": 1312292864 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004868204613841525, + "loss": 1.3201, + "theoretical_loss": 3.5867621396992684, + "tokens_seen": 1312358400 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048680441323971916, + "loss": 1.2499, + "theoretical_loss": 3.5867460687458546, + "tokens_seen": 1312423936 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004867883650952859, + "loss": 1.2213, + "theoretical_loss": 3.5867299988196137, + "tokens_seen": 1312489472 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048677231695085256, + "loss": 1.3925, + "theoretical_loss": 3.5867139299204287, + "tokens_seen": 1312555008 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004867562688064193, + "loss": 1.3085, + "theoretical_loss": 3.586697862048183, + "tokens_seen": 1312620544 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048674022066198596, + "loss": 1.3541, + "theoretical_loss": 3.586681795202759, + "tokens_seen": 1312686080 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048672417251755266, + "loss": 1.3701, + "theoretical_loss": 3.5866657293840403, + "tokens_seen": 1312751616 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048670812437311936, + "loss": 1.3521, + "theoretical_loss": 3.58664966459191, + "tokens_seen": 1312817152 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048669207622868606, + "loss": 1.3801, + "theoretical_loss": 3.586633600826251, + "tokens_seen": 1312882688 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048667602808425277, + "loss": 1.2963, + "theoretical_loss": 3.586617538086947, + "tokens_seen": 1312948224 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048665997993981947, + "loss": 1.3905, + "theoretical_loss": 3.5866014763738807, + "tokens_seen": 1313013760 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048664393179538617, + "loss": 1.3277, + "theoretical_loss": 3.5865854156869355, + "tokens_seen": 1313079296 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048662788365095287, + "loss": 1.3214, + "theoretical_loss": 3.5865693560259944, + "tokens_seen": 1313144832 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048661183550651957, + "loss": 1.3011, + "theoretical_loss": 3.586553297390941, + "tokens_seen": 1313210368 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048659578736208627, + "loss": 1.4338, + "theoretical_loss": 3.5865372397816584, + "tokens_seen": 1313275904 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004865797392176529, + "loss": 1.4519, + "theoretical_loss": 3.5865211831980304, + "tokens_seen": 1313341440 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048656369107321967, + "loss": 1.2878, + "theoretical_loss": 3.5865051276399393, + "tokens_seen": 1313406976 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004865476429287863, + "loss": 1.2516, + "theoretical_loss": 3.5864890731072694, + "tokens_seen": 1313472512 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048653159478435307, + "loss": 1.3272, + "theoretical_loss": 3.5864730195999037, + "tokens_seen": 1313538048 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004865155466399197, + "loss": 1.4128, + "theoretical_loss": 3.586456967117725, + "tokens_seen": 1313603584 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048649949849548647, + "loss": 1.3357, + "theoretical_loss": 3.5864409156606176, + "tokens_seen": 1313669120 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004864834503510532, + "loss": 1.2836, + "theoretical_loss": 3.5864248652284645, + "tokens_seen": 1313734656 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004864674022066199, + "loss": 1.3622, + "theoretical_loss": 3.586408815821149, + "tokens_seen": 1313800192 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048645135406218663, + "loss": 1.3898, + "theoretical_loss": 3.5863927674385545, + "tokens_seen": 1313865728 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004864353059177533, + "loss": 1.2999, + "theoretical_loss": 3.586376720080565, + "tokens_seen": 1313931264 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048641925777332003, + "loss": 1.3923, + "theoretical_loss": 3.5863606737470635, + "tokens_seen": 1313996800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004864032096288867, + "loss": 1.3222, + "theoretical_loss": 3.5863446284379337, + "tokens_seen": 1314062336 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048638716148445343, + "loss": 1.3079, + "theoretical_loss": 3.586328584153059, + "tokens_seen": 1314127872 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004863711133400201, + "loss": 1.3798, + "theoretical_loss": 3.586312540892323, + "tokens_seen": 1314193408 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048635506519558683, + "loss": 1.244, + "theoretical_loss": 3.586296498655609, + "tokens_seen": 1314258944 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004863390170511535, + "loss": 1.3498, + "theoretical_loss": 3.5862804574428013, + "tokens_seen": 1314324480 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048632296890672023, + "loss": 1.4809, + "theoretical_loss": 3.5862644172537825, + "tokens_seen": 1314390016 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004863069207622869, + "loss": 1.3219, + "theoretical_loss": 3.5862483780884373, + "tokens_seen": 1314455552 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048629087261785363, + "loss": 1.3029, + "theoretical_loss": 3.5862323399466485, + "tokens_seen": 1314521088 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004862748244734203, + "loss": 1.4382, + "theoretical_loss": 3.5862163028283005, + "tokens_seen": 1314586624 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048625877632898704, + "loss": 1.358, + "theoretical_loss": 3.586200266733276, + "tokens_seen": 1314652160 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004862427281845537, + "loss": 1.244, + "theoretical_loss": 3.586184231661459, + "tokens_seen": 1314717696 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048622668004012044, + "loss": 1.3769, + "theoretical_loss": 3.586168197612734, + "tokens_seen": 1314783232 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004862106318956871, + "loss": 1.3677, + "theoretical_loss": 3.5861521645869843, + "tokens_seen": 1314848768 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048619458375125384, + "loss": 1.3918, + "theoretical_loss": 3.586136132584093, + "tokens_seen": 1314914304 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004861785356068205, + "loss": 1.3105, + "theoretical_loss": 3.5861201016039446, + "tokens_seen": 1314979840 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048616248746238724, + "loss": 1.3203, + "theoretical_loss": 3.5861040716464228, + "tokens_seen": 1315045376 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004861464393179539, + "loss": 1.3251, + "theoretical_loss": 3.5860880427114106, + "tokens_seen": 1315110912 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048613039117352064, + "loss": 1.3659, + "theoretical_loss": 3.586072014798793, + "tokens_seen": 1315176448 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004861143430290873, + "loss": 1.4116, + "theoretical_loss": 3.5860559879084537, + "tokens_seen": 1315241984 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048609829488465404, + "loss": 1.4674, + "theoretical_loss": 3.5860399620402754, + "tokens_seen": 1315307520 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004860822467402207, + "loss": 1.3426, + "theoretical_loss": 3.5860239371941436, + "tokens_seen": 1315373056 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004860661985957874, + "loss": 1.3208, + "theoretical_loss": 3.586007913369941, + "tokens_seen": 1315438592 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004860501504513541, + "loss": 1.3324, + "theoretical_loss": 3.585991890567552, + "tokens_seen": 1315504128 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004860341023069208, + "loss": 1.4115, + "theoretical_loss": 3.5859758687868606, + "tokens_seen": 1315569664 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004860180541624875, + "loss": 1.3624, + "theoretical_loss": 3.58595984802775, + "tokens_seen": 1315635200 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004860020060180542, + "loss": 1.4268, + "theoretical_loss": 3.5859438282901057, + "tokens_seen": 1315700736 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004859859578736209, + "loss": 1.3786, + "theoretical_loss": 3.5859278095738105, + "tokens_seen": 1315766272 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004859699097291876, + "loss": 1.3274, + "theoretical_loss": 3.585911791878748, + "tokens_seen": 1315831808 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048595386158475424, + "loss": 1.3789, + "theoretical_loss": 3.5858957752048037, + "tokens_seen": 1315897344 + }, + { + "epoch": 0.4, + "learning_rate": 0.000485937813440321, + "loss": 1.4683, + "theoretical_loss": 3.585879759551861, + "tokens_seen": 1315962880 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048592176529588764, + "loss": 1.3758, + "theoretical_loss": 3.5858637449198034, + "tokens_seen": 1316028416 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004859057171514544, + "loss": 1.4362, + "theoretical_loss": 3.5858477313085158, + "tokens_seen": 1316093952 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048588966900702104, + "loss": 1.3339, + "theoretical_loss": 3.585831718717882, + "tokens_seen": 1316159488 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004858736208625878, + "loss": 1.3785, + "theoretical_loss": 3.585815707147786, + "tokens_seen": 1316225024 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048585757271815444, + "loss": 1.4597, + "theoretical_loss": 3.5857996965981123, + "tokens_seen": 1316290560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004858415245737212, + "loss": 1.3115, + "theoretical_loss": 3.5857836870687443, + "tokens_seen": 1316356096 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048582547642928784, + "loss": 1.3056, + "theoretical_loss": 3.585767678559567, + "tokens_seen": 1316421632 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004858094282848546, + "loss": 1.4212, + "theoretical_loss": 3.585751671070464, + "tokens_seen": 1316487168 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048579338014042125, + "loss": 1.4203, + "theoretical_loss": 3.5857356646013203, + "tokens_seen": 1316552704 + }, + { + "epoch": 0.4, + "learning_rate": 0.000485777331995988, + "loss": 1.3631, + "theoretical_loss": 3.5857196591520197, + "tokens_seen": 1316618240 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048576128385155465, + "loss": 1.3509, + "theoretical_loss": 3.585703654722446, + "tokens_seen": 1316683776 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004857452357071214, + "loss": 1.3571, + "theoretical_loss": 3.5856876513124845, + "tokens_seen": 1316749312 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048572918756268805, + "loss": 1.3515, + "theoretical_loss": 3.585671648922019, + "tokens_seen": 1316814848 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004857131394182548, + "loss": 1.3558, + "theoretical_loss": 3.585655647550933, + "tokens_seen": 1316880384 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048569709127382145, + "loss": 1.3738, + "theoretical_loss": 3.5856396471991117, + "tokens_seen": 1316945920 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004856810431293882, + "loss": 1.4116, + "theoretical_loss": 3.5856236478664396, + "tokens_seen": 1317011456 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048566499498495485, + "loss": 1.3294, + "theoretical_loss": 3.5856076495528013, + "tokens_seen": 1317076992 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004856489468405216, + "loss": 1.3236, + "theoretical_loss": 3.58559165225808, + "tokens_seen": 1317142528 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048563289869608825, + "loss": 1.3268, + "theoretical_loss": 3.5855756559821614, + "tokens_seen": 1317208064 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 490258, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4434093236923218, + "objective/train/theoretical_loss": 3.585559660724929, + "objective/train/tokens_used": 1337733600, + "theoretical_loss": 3.585559660724929, + "tokens_seen": 1317273600 + }, + { + "epoch": 0.4, + "learning_rate": 0.000485616850551655, + "loss": 1.4434, + "theoretical_loss": 3.585559660724929, + "tokens_seen": 1317273600 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048560080240722165, + "loss": 1.3381, + "theoretical_loss": 3.5855436664862674, + "tokens_seen": 1317339136 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004855847542627884, + "loss": 1.4074, + "theoretical_loss": 3.5855276732660615, + "tokens_seen": 1317404672 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048556870611835505, + "loss": 1.2577, + "theoretical_loss": 3.5855116810641956, + "tokens_seen": 1317470208 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004855526579739218, + "loss": 1.3753, + "theoretical_loss": 3.5854956898805543, + "tokens_seen": 1317535744 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048553660982948846, + "loss": 1.3051, + "theoretical_loss": 3.5854796997150222, + "tokens_seen": 1317601280 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004855205616850552, + "loss": 1.3406, + "theoretical_loss": 3.5854637105674834, + "tokens_seen": 1317666816 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048550451354062186, + "loss": 1.3688, + "theoretical_loss": 3.5854477224378227, + "tokens_seen": 1317732352 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004854884653961886, + "loss": 1.3471, + "theoretical_loss": 3.5854317353259244, + "tokens_seen": 1317797888 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048547241725175526, + "loss": 1.3389, + "theoretical_loss": 3.585415749231674, + "tokens_seen": 1317863424 + }, + { + "epoch": 0.4, + "learning_rate": 0.000485456369107322, + "loss": 1.4264, + "theoretical_loss": 3.585399764154955, + "tokens_seen": 1317928960 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048544032096288866, + "loss": 1.3513, + "theoretical_loss": 3.585383780095653, + "tokens_seen": 1317994496 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004854242728184554, + "loss": 1.3545, + "theoretical_loss": 3.585367797053652, + "tokens_seen": 1318060032 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048540822467402206, + "loss": 1.3604, + "theoretical_loss": 3.5853518150288366, + "tokens_seen": 1318125568 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004853921765295888, + "loss": 1.3615, + "theoretical_loss": 3.5853358340210923, + "tokens_seen": 1318191104 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048537612838515546, + "loss": 1.3818, + "theoretical_loss": 3.5853198540303035, + "tokens_seen": 1318256640 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004853600802407222, + "loss": 1.2952, + "theoretical_loss": 3.585303875056354, + "tokens_seen": 1318322176 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048534403209628886, + "loss": 1.3283, + "theoretical_loss": 3.5852878970991298, + "tokens_seen": 1318387712 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004853279839518556, + "loss": 1.4748, + "theoretical_loss": 3.5852719201585153, + "tokens_seen": 1318453248 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048531193580742226, + "loss": 1.3092, + "theoretical_loss": 3.5852559442343948, + "tokens_seen": 1318518784 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048529588766298897, + "loss": 1.3577, + "theoretical_loss": 3.5852399693266532, + "tokens_seen": 1318584320 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004852798395185557, + "loss": 1.3248, + "theoretical_loss": 3.585223995435176, + "tokens_seen": 1318649856 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048526379137412237, + "loss": 1.3277, + "theoretical_loss": 3.5852080225598475, + "tokens_seen": 1318715392 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004852477432296891, + "loss": 1.3976, + "theoretical_loss": 3.585192050700553, + "tokens_seen": 1318780928 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048523169508525577, + "loss": 1.2901, + "theoretical_loss": 3.5851760798571766, + "tokens_seen": 1318846464 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004852156469408225, + "loss": 1.3311, + "theoretical_loss": 3.585160110029604, + "tokens_seen": 1318912000 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048519959879638917, + "loss": 1.3312, + "theoretical_loss": 3.58514414121772, + "tokens_seen": 1318977536 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004851835506519559, + "loss": 1.3021, + "theoretical_loss": 3.585128173421409, + "tokens_seen": 1319043072 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048516750250752257, + "loss": 1.3134, + "theoretical_loss": 3.585112206640556, + "tokens_seen": 1319108608 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004851514543630893, + "loss": 1.3407, + "theoretical_loss": 3.585096240875047, + "tokens_seen": 1319174144 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048513540621865597, + "loss": 1.3388, + "theoretical_loss": 3.5850802761247658, + "tokens_seen": 1319239680 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004851193580742227, + "loss": 1.3474, + "theoretical_loss": 3.585064312389598, + "tokens_seen": 1319305216 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004851033099297894, + "loss": 1.3484, + "theoretical_loss": 3.585048349669429, + "tokens_seen": 1319370752 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048508726178535613, + "loss": 1.3791, + "theoretical_loss": 3.5850323879641426, + "tokens_seen": 1319436288 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004850712136409228, + "loss": 1.296, + "theoretical_loss": 3.585016427273625, + "tokens_seen": 1319501824 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048505516549648953, + "loss": 1.236, + "theoretical_loss": 3.5850004675977614, + "tokens_seen": 1319567360 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004850391173520562, + "loss": 1.3732, + "theoretical_loss": 3.5849845089364356, + "tokens_seen": 1319632896 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048502306920762293, + "loss": 1.2649, + "theoretical_loss": 3.5849685512895344, + "tokens_seen": 1319698432 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004850070210631896, + "loss": 1.3596, + "theoretical_loss": 3.584952594656942, + "tokens_seen": 1319763968 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048499097291875633, + "loss": 1.3363, + "theoretical_loss": 3.584936639038543, + "tokens_seen": 1319829504 + }, + { + "epoch": 0.4, + "learning_rate": 0.000484974924774323, + "loss": 1.406, + "theoretical_loss": 3.584920684434224, + "tokens_seen": 1319895040 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048495887662988973, + "loss": 1.3739, + "theoretical_loss": 3.584904730843869, + "tokens_seen": 1319960576 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004849428284854564, + "loss": 1.3652, + "theoretical_loss": 3.5848887782673637, + "tokens_seen": 1320026112 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048492678034102313, + "loss": 1.44, + "theoretical_loss": 3.5848728267045935, + "tokens_seen": 1320091648 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004849107321965898, + "loss": 1.2977, + "theoretical_loss": 3.5848568761554436, + "tokens_seen": 1320157184 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048489468405215654, + "loss": 1.3272, + "theoretical_loss": 3.5848409266197994, + "tokens_seen": 1320222720 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004848786359077232, + "loss": 1.3862, + "theoretical_loss": 3.5848249780975454, + "tokens_seen": 1320288256 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048486258776328994, + "loss": 1.3262, + "theoretical_loss": 3.584809030588568, + "tokens_seen": 1320353792 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004848465396188566, + "loss": 1.224, + "theoretical_loss": 3.5847930840927518, + "tokens_seen": 1320419328 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048483049147442334, + "loss": 1.3475, + "theoretical_loss": 3.584777138609982, + "tokens_seen": 1320484864 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048481444332999, + "loss": 1.3117, + "theoretical_loss": 3.584761194140145, + "tokens_seen": 1320550400 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048479839518555674, + "loss": 1.3439, + "theoretical_loss": 3.584745250683125, + "tokens_seen": 1320615936 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004847823470411234, + "loss": 1.2912, + "theoretical_loss": 3.584729308238808, + "tokens_seen": 1320681472 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048476629889669014, + "loss": 1.269, + "theoretical_loss": 3.5847133668070796, + "tokens_seen": 1320747008 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004847502507522568, + "loss": 1.3337, + "theoretical_loss": 3.584697426387825, + "tokens_seen": 1320812544 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048473420260782354, + "loss": 1.2758, + "theoretical_loss": 3.58468148698093, + "tokens_seen": 1320878080 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004847181544633902, + "loss": 1.3283, + "theoretical_loss": 3.5846655485862793, + "tokens_seen": 1320943616 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048470210631895694, + "loss": 1.2983, + "theoretical_loss": 3.5846496112037585, + "tokens_seen": 1321009152 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004846860581745236, + "loss": 1.3868, + "theoretical_loss": 3.5846336748332543, + "tokens_seen": 1321074688 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004846700100300903, + "loss": 1.3388, + "theoretical_loss": 3.584617739474651, + "tokens_seen": 1321140224 + }, + { + "epoch": 0.4, + "learning_rate": 0.000484653961885657, + "loss": 1.293, + "theoretical_loss": 3.5846018051278348, + "tokens_seen": 1321205760 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004846379137412237, + "loss": 1.4021, + "theoretical_loss": 3.58458587179269, + "tokens_seen": 1321271296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004846218655967904, + "loss": 1.2689, + "theoretical_loss": 3.5845699394691044, + "tokens_seen": 1321336832 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004846058174523571, + "loss": 1.2744, + "theoretical_loss": 3.584554008156962, + "tokens_seen": 1321402368 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004845897693079238, + "loss": 1.3116, + "theoretical_loss": 3.5845380778561493, + "tokens_seen": 1321467904 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004845737211634905, + "loss": 1.2575, + "theoretical_loss": 3.584522148566551, + "tokens_seen": 1321533440 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004845576730190572, + "loss": 1.4803, + "theoretical_loss": 3.5845062202880538, + "tokens_seen": 1321598976 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004845416248746239, + "loss": 1.287, + "theoretical_loss": 3.5844902930205422, + "tokens_seen": 1321664512 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048452557673019054, + "loss": 1.3212, + "theoretical_loss": 3.584474366763903, + "tokens_seen": 1321730048 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004845095285857573, + "loss": 1.4839, + "theoretical_loss": 3.584458441518022, + "tokens_seen": 1321795584 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048449348044132394, + "loss": 1.3754, + "theoretical_loss": 3.584442517282784, + "tokens_seen": 1321861120 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004844774322968907, + "loss": 1.3105, + "theoretical_loss": 3.584426594058075, + "tokens_seen": 1321926656 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048446138415245734, + "loss": 1.3541, + "theoretical_loss": 3.5844106718437807, + "tokens_seen": 1321992192 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004844453360080241, + "loss": 1.4122, + "theoretical_loss": 3.584394750639788, + "tokens_seen": 1322057728 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048442928786359075, + "loss": 1.5195, + "theoretical_loss": 3.5843788304459814, + "tokens_seen": 1322123264 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004844132397191575, + "loss": 1.3238, + "theoretical_loss": 3.5843629112622475, + "tokens_seen": 1322188800 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048439719157472415, + "loss": 1.3389, + "theoretical_loss": 3.584346993088472, + "tokens_seen": 1322254336 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004843811434302909, + "loss": 1.3776, + "theoretical_loss": 3.58433107592454, + "tokens_seen": 1322319872 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048436509528585755, + "loss": 1.3575, + "theoretical_loss": 3.5843151597703393, + "tokens_seen": 1322385408 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004843490471414243, + "loss": 1.5465, + "theoretical_loss": 3.5842992446257536, + "tokens_seen": 1322450944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048433299899699095, + "loss": 1.355, + "theoretical_loss": 3.58428333049067, + "tokens_seen": 1322516480 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004843169508525577, + "loss": 1.555, + "theoretical_loss": 3.5842674173649742, + "tokens_seen": 1322582016 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048430090270812435, + "loss": 1.428, + "theoretical_loss": 3.5842515052485524, + "tokens_seen": 1322647552 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004842848545636911, + "loss": 1.4352, + "theoretical_loss": 3.5842355941412904, + "tokens_seen": 1322713088 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048426880641925775, + "loss": 1.3766, + "theoretical_loss": 3.5842196840430742, + "tokens_seen": 1322778624 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004842527582748245, + "loss": 1.3745, + "theoretical_loss": 3.5842037749537896, + "tokens_seen": 1322844160 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048423671013039115, + "loss": 1.3196, + "theoretical_loss": 3.5841878668733234, + "tokens_seen": 1322909696 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004842206619859579, + "loss": 1.3457, + "theoretical_loss": 3.5841719598015604, + "tokens_seen": 1322975232 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048420461384152455, + "loss": 1.4035, + "theoretical_loss": 3.584156053738388, + "tokens_seen": 1323040768 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004841885656970913, + "loss": 1.3861, + "theoretical_loss": 3.5841401486836917, + "tokens_seen": 1323106304 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048417251755265796, + "loss": 1.3558, + "theoretical_loss": 3.5841242446373576, + "tokens_seen": 1323171840 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004841564694082247, + "loss": 1.4982, + "theoretical_loss": 3.584108341599271, + "tokens_seen": 1323237376 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048414042126379136, + "loss": 1.2147, + "theoretical_loss": 3.5840924395693197, + "tokens_seen": 1323302912 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004841243731193581, + "loss": 1.3806, + "theoretical_loss": 3.5840765385473894, + "tokens_seen": 1323368448 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048410832497492476, + "loss": 1.3838, + "theoretical_loss": 3.584060638533365, + "tokens_seen": 1323433984 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004840922768304915, + "loss": 1.3775, + "theoretical_loss": 3.584044739527134, + "tokens_seen": 1323499520 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048407622868605827, + "loss": 1.325, + "theoretical_loss": 3.584028841528583, + "tokens_seen": 1323565056 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004840601805416249, + "loss": 1.3866, + "theoretical_loss": 3.5840129445375966, + "tokens_seen": 1323630592 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004840441323971916, + "loss": 1.3693, + "theoretical_loss": 3.583997048554062, + "tokens_seen": 1323696128 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004840280842527583, + "loss": 1.512, + "theoretical_loss": 3.5839811535778656, + "tokens_seen": 1323761664 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 492513, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.5778007507324219, + "objective/train/theoretical_loss": 3.583965259608894, + "objective/train/tokens_used": 1344287200, + "theoretical_loss": 3.583965259608894, + "tokens_seen": 1323827200 + }, + { + "epoch": 0.4, + "learning_rate": 0.000484012036108325, + "loss": 1.5778, + "theoretical_loss": 3.583965259608894, + "tokens_seen": 1323827200 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004839959879638917, + "loss": 1.4359, + "theoretical_loss": 3.583949366647033, + "tokens_seen": 1323892736 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004839799398194584, + "loss": 1.4146, + "theoretical_loss": 3.5839334746921683, + "tokens_seen": 1323958272 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004839638916750251, + "loss": 1.2936, + "theoretical_loss": 3.5839175837441877, + "tokens_seen": 1324023808 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004839478435305918, + "loss": 1.3821, + "theoretical_loss": 3.583901693802977, + "tokens_seen": 1324089344 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004839317953861585, + "loss": 1.4843, + "theoretical_loss": 3.583885804868422, + "tokens_seen": 1324154880 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004839157472417252, + "loss": 1.3568, + "theoretical_loss": 3.5838699169404094, + "tokens_seen": 1324220416 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048389969909729187, + "loss": 1.3515, + "theoretical_loss": 3.583854030018826, + "tokens_seen": 1324285952 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004838836509528586, + "loss": 1.2285, + "theoretical_loss": 3.583838144103558, + "tokens_seen": 1324351488 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048386760280842527, + "loss": 1.5031, + "theoretical_loss": 3.583822259194492, + "tokens_seen": 1324417024 + }, + { + "epoch": 0.4, + "learning_rate": 0.000483851554663992, + "loss": 1.4862, + "theoretical_loss": 3.5838063752915144, + "tokens_seen": 1324482560 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048383550651955867, + "loss": 1.3821, + "theoretical_loss": 3.5837904923945114, + "tokens_seen": 1324548096 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004838194583751254, + "loss": 1.369, + "theoretical_loss": 3.5837746105033697, + "tokens_seen": 1324613632 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048380341023069207, + "loss": 1.4341, + "theoretical_loss": 3.5837587296179763, + "tokens_seen": 1324679168 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004837873620862588, + "loss": 1.4186, + "theoretical_loss": 3.5837428497382167, + "tokens_seen": 1324744704 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048377131394182547, + "loss": 1.2924, + "theoretical_loss": 3.583726970863979, + "tokens_seen": 1324810240 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004837552657973922, + "loss": 1.331, + "theoretical_loss": 3.5837110929951486, + "tokens_seen": 1324875776 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048373921765295887, + "loss": 1.4991, + "theoretical_loss": 3.5836952161316122, + "tokens_seen": 1324941312 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048372316950852563, + "loss": 1.3259, + "theoretical_loss": 3.5836793402732567, + "tokens_seen": 1325006848 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004837071213640923, + "loss": 1.2971, + "theoretical_loss": 3.5836634654199693, + "tokens_seen": 1325072384 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048369107321965903, + "loss": 1.36, + "theoretical_loss": 3.5836475915716357, + "tokens_seen": 1325137920 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004836750250752257, + "loss": 1.387, + "theoretical_loss": 3.5836317187281432, + "tokens_seen": 1325203456 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048365897693079243, + "loss": 1.3151, + "theoretical_loss": 3.583615846889378, + "tokens_seen": 1325268992 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004836429287863591, + "loss": 1.3827, + "theoretical_loss": 3.5835999760552273, + "tokens_seen": 1325334528 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048362688064192583, + "loss": 1.3891, + "theoretical_loss": 3.5835841062255773, + "tokens_seen": 1325400064 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004836108324974925, + "loss": 1.3612, + "theoretical_loss": 3.583568237400316, + "tokens_seen": 1325465600 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048359478435305923, + "loss": 1.2996, + "theoretical_loss": 3.583552369579328, + "tokens_seen": 1325531136 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004835787362086259, + "loss": 1.4834, + "theoretical_loss": 3.5835365027625024, + "tokens_seen": 1325596672 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048356268806419263, + "loss": 1.559, + "theoretical_loss": 3.583520636949725, + "tokens_seen": 1325662208 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004835466399197593, + "loss": 1.3595, + "theoretical_loss": 3.5835047721408824, + "tokens_seen": 1325727744 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048353059177532604, + "loss": 1.2683, + "theoretical_loss": 3.583488908335861, + "tokens_seen": 1325793280 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004835145436308927, + "loss": 1.4152, + "theoretical_loss": 3.583473045534549, + "tokens_seen": 1325858816 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048349849548645944, + "loss": 1.4743, + "theoretical_loss": 3.583457183736833, + "tokens_seen": 1325924352 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004834824473420261, + "loss": 1.3644, + "theoretical_loss": 3.583441322942599, + "tokens_seen": 1325989888 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048346639919759284, + "loss": 1.435, + "theoretical_loss": 3.583425463151735, + "tokens_seen": 1326055424 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004834503510531595, + "loss": 1.4551, + "theoretical_loss": 3.5834096043641264, + "tokens_seen": 1326120960 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048343430290872624, + "loss": 1.4503, + "theoretical_loss": 3.583393746579662, + "tokens_seen": 1326186496 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004834182547642929, + "loss": 1.3431, + "theoretical_loss": 3.5833778897982276, + "tokens_seen": 1326252032 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048340220661985964, + "loss": 1.498, + "theoretical_loss": 3.5833620340197108, + "tokens_seen": 1326317568 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004833861584754263, + "loss": 1.509, + "theoretical_loss": 3.5833461792439985, + "tokens_seen": 1326383104 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048337011033099304, + "loss": 1.3536, + "theoretical_loss": 3.5833303254709774, + "tokens_seen": 1326448640 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004833540621865597, + "loss": 1.3994, + "theoretical_loss": 3.5833144727005344, + "tokens_seen": 1326514176 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048333801404212644, + "loss": 1.3858, + "theoretical_loss": 3.583298620932557, + "tokens_seen": 1326579712 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004833219658976931, + "loss": 1.559, + "theoretical_loss": 3.583282770166933, + "tokens_seen": 1326645248 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048330591775325984, + "loss": 1.4668, + "theoretical_loss": 3.583266920403548, + "tokens_seen": 1326710784 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004832898696088265, + "loss": 1.3675, + "theoretical_loss": 3.58325107164229, + "tokens_seen": 1326776320 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004832738214643932, + "loss": 1.4063, + "theoretical_loss": 3.583235223883046, + "tokens_seen": 1326841856 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004832577733199599, + "loss": 1.3488, + "theoretical_loss": 3.5832193771257033, + "tokens_seen": 1326907392 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004832417251755266, + "loss": 1.4438, + "theoretical_loss": 3.583203531370149, + "tokens_seen": 1326972928 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004832256770310933, + "loss": 1.4793, + "theoretical_loss": 3.5831876866162693, + "tokens_seen": 1327038464 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048320962888666, + "loss": 1.4573, + "theoretical_loss": 3.5831718428639534, + "tokens_seen": 1327104000 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004831935807422267, + "loss": 1.2947, + "theoretical_loss": 3.583156000113087, + "tokens_seen": 1327169536 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004831775325977934, + "loss": 1.4691, + "theoretical_loss": 3.583140158363558, + "tokens_seen": 1327235072 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004831614844533601, + "loss": 1.4518, + "theoretical_loss": 3.5831243176152534, + "tokens_seen": 1327300608 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004831454363089268, + "loss": 1.3616, + "theoretical_loss": 3.583108477868061, + "tokens_seen": 1327366144 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048312938816449344, + "loss": 1.3246, + "theoretical_loss": 3.583092639121867, + "tokens_seen": 1327431680 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004831133400200602, + "loss": 1.4339, + "theoretical_loss": 3.58307680137656, + "tokens_seen": 1327497216 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048309729187562684, + "loss": 1.4837, + "theoretical_loss": 3.5830609646320264, + "tokens_seen": 1327562752 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004830812437311936, + "loss": 1.3413, + "theoretical_loss": 3.5830451288881537, + "tokens_seen": 1327628288 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048306519558676025, + "loss": 1.4533, + "theoretical_loss": 3.58302929414483, + "tokens_seen": 1327693824 + }, + { + "epoch": 0.4, + "learning_rate": 0.000483049147442327, + "loss": 1.3469, + "theoretical_loss": 3.583013460401942, + "tokens_seen": 1327759360 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048303309929789365, + "loss": 1.3675, + "theoretical_loss": 3.5829976276593776, + "tokens_seen": 1327824896 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004830170511534604, + "loss": 1.5206, + "theoretical_loss": 3.582981795917024, + "tokens_seen": 1327890432 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048300100300902705, + "loss": 1.3897, + "theoretical_loss": 3.5829659651747683, + "tokens_seen": 1327955968 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004829849548645938, + "loss": 1.3544, + "theoretical_loss": 3.5829501354324984, + "tokens_seen": 1328021504 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048296890672016045, + "loss": 1.4445, + "theoretical_loss": 3.5829343066901016, + "tokens_seen": 1328087040 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004829528585757272, + "loss": 1.3336, + "theoretical_loss": 3.582918478947466, + "tokens_seen": 1328152576 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048293681043129385, + "loss": 1.3904, + "theoretical_loss": 3.582902652204478, + "tokens_seen": 1328218112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004829207622868606, + "loss": 1.3598, + "theoretical_loss": 3.582886826461026, + "tokens_seen": 1328283648 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048290471414242725, + "loss": 1.3827, + "theoretical_loss": 3.582871001716997, + "tokens_seen": 1328349184 + }, + { + "epoch": 0.4, + "learning_rate": 0.000482888665997994, + "loss": 1.3759, + "theoretical_loss": 3.582855177972279, + "tokens_seen": 1328414720 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048287261785356076, + "loss": 1.2483, + "theoretical_loss": 3.58283935522676, + "tokens_seen": 1328480256 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004828565697091274, + "loss": 1.2129, + "theoretical_loss": 3.582823533480327, + "tokens_seen": 1328545792 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048284052156469416, + "loss": 1.3968, + "theoretical_loss": 3.5828077127328672, + "tokens_seen": 1328611328 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004828244734202608, + "loss": 1.2826, + "theoretical_loss": 3.5827918929842695, + "tokens_seen": 1328676864 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048280842527582756, + "loss": 1.3402, + "theoretical_loss": 3.5827760742344203, + "tokens_seen": 1328742400 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004827923771313942, + "loss": 1.3853, + "theoretical_loss": 3.5827602564832084, + "tokens_seen": 1328807936 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048277632898696097, + "loss": 1.3877, + "theoretical_loss": 3.582744439730521, + "tokens_seen": 1328873472 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004827602808425276, + "loss": 1.4173, + "theoretical_loss": 3.5827286239762453, + "tokens_seen": 1328939008 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048274423269809437, + "loss": 1.3291, + "theoretical_loss": 3.58271280922027, + "tokens_seen": 1329004544 + }, + { + "epoch": 0.4, + "learning_rate": 0.000482728184553661, + "loss": 1.3664, + "theoretical_loss": 3.582696995462482, + "tokens_seen": 1329070080 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048271213640922777, + "loss": 1.3755, + "theoretical_loss": 3.5826811827027703, + "tokens_seen": 1329135616 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004826960882647944, + "loss": 1.3161, + "theoretical_loss": 3.582665370941021, + "tokens_seen": 1329201152 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048268004012036117, + "loss": 1.4936, + "theoretical_loss": 3.5826495601771233, + "tokens_seen": 1329266688 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004826639919759278, + "loss": 1.2432, + "theoretical_loss": 3.582633750410964, + "tokens_seen": 1329332224 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004826479438314945, + "loss": 1.3436, + "theoretical_loss": 3.5826179416424324, + "tokens_seen": 1329397760 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004826318956870612, + "loss": 1.2663, + "theoretical_loss": 3.582602133871415, + "tokens_seen": 1329463296 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004826158475426279, + "loss": 1.383, + "theoretical_loss": 3.5825863270978, + "tokens_seen": 1329528832 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004825997993981946, + "loss": 1.4087, + "theoretical_loss": 3.582570521321476, + "tokens_seen": 1329594368 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004825837512537613, + "loss": 1.3944, + "theoretical_loss": 3.58255471654233, + "tokens_seen": 1329659904 + }, + { + "epoch": 0.4, + "learning_rate": 0.000482567703109328, + "loss": 1.4783, + "theoretical_loss": 3.58253891276025, + "tokens_seen": 1329725440 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004825516549648947, + "loss": 1.3198, + "theoretical_loss": 3.582523109975125, + "tokens_seen": 1329790976 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004825356068204614, + "loss": 1.3854, + "theoretical_loss": 3.582507308186842, + "tokens_seen": 1329856512 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004825195586760281, + "loss": 1.4281, + "theoretical_loss": 3.5824915073952894, + "tokens_seen": 1329922048 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048250351053159477, + "loss": 1.3726, + "theoretical_loss": 3.582475707600355, + "tokens_seen": 1329987584 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004824874623871615, + "loss": 1.3258, + "theoretical_loss": 3.582459908801927, + "tokens_seen": 1330053120 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048247141424272817, + "loss": 1.6153, + "theoretical_loss": 3.5824441109998935, + "tokens_seen": 1330118656 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004824553660982949, + "loss": 1.2354, + "theoretical_loss": 3.582428314194142, + "tokens_seen": 1330184192 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048243931795386157, + "loss": 1.376, + "theoretical_loss": 3.5824125183845617, + "tokens_seen": 1330249728 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004824232698094283, + "loss": 1.2546, + "theoretical_loss": 3.5823967235710397, + "tokens_seen": 1330315264 + }, + { + "epoch": 0.4, + "objective/train/docs_used": 494809, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3804559707641602, + "objective/train/theoretical_loss": 3.582380929753465, + "objective/train/tokens_used": 1350840800, + "theoretical_loss": 3.582380929753465, + "tokens_seen": 1330380800 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048240722166499497, + "loss": 1.3805, + "theoretical_loss": 3.582380929753465, + "tokens_seen": 1330380800 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004823911735205617, + "loss": 1.3298, + "theoretical_loss": 3.5823651369317244, + "tokens_seen": 1330446336 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048237512537612837, + "loss": 1.3782, + "theoretical_loss": 3.5823493451057074, + "tokens_seen": 1330511872 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048235907723169513, + "loss": 1.3961, + "theoretical_loss": 3.5823335542753014, + "tokens_seen": 1330577408 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004823430290872618, + "loss": 1.3952, + "theoretical_loss": 3.582317764440395, + "tokens_seen": 1330642944 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048232698094282853, + "loss": 1.4105, + "theoretical_loss": 3.5823019756008767, + "tokens_seen": 1330708480 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004823109327983952, + "loss": 1.2972, + "theoretical_loss": 3.582286187756634, + "tokens_seen": 1330774016 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048229488465396193, + "loss": 1.4388, + "theoretical_loss": 3.5822704009075554, + "tokens_seen": 1330839552 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004822788365095286, + "loss": 1.4809, + "theoretical_loss": 3.5822546150535293, + "tokens_seen": 1330905088 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048226278836509533, + "loss": 1.4376, + "theoretical_loss": 3.582238830194444, + "tokens_seen": 1330970624 + }, + { + "epoch": 0.4, + "learning_rate": 0.000482246740220662, + "loss": 1.3194, + "theoretical_loss": 3.5822230463301876, + "tokens_seen": 1331036160 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048223069207622873, + "loss": 1.4648, + "theoretical_loss": 3.5822072634606483, + "tokens_seen": 1331101696 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004822146439317954, + "loss": 1.3833, + "theoretical_loss": 3.582191481585715, + "tokens_seen": 1331167232 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048219859578736213, + "loss": 1.4194, + "theoretical_loss": 3.5821757007052764, + "tokens_seen": 1331232768 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004821825476429288, + "loss": 1.3617, + "theoretical_loss": 3.5821599208192194, + "tokens_seen": 1331298304 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048216649949849553, + "loss": 1.4178, + "theoretical_loss": 3.5821441419274334, + "tokens_seen": 1331363840 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004821504513540622, + "loss": 1.3283, + "theoretical_loss": 3.582128364029807, + "tokens_seen": 1331429376 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048213440320962894, + "loss": 1.3432, + "theoretical_loss": 3.582112587126228, + "tokens_seen": 1331494912 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004821183550651956, + "loss": 1.4373, + "theoretical_loss": 3.582096811216585, + "tokens_seen": 1331560448 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048210230692076234, + "loss": 1.293, + "theoretical_loss": 3.5820810363007665, + "tokens_seen": 1331625984 + }, + { + "epoch": 0.4, + "learning_rate": 0.000482086258776329, + "loss": 1.4073, + "theoretical_loss": 3.5820652623786615, + "tokens_seen": 1331691520 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048207021063189574, + "loss": 1.2961, + "theoretical_loss": 3.582049489450158, + "tokens_seen": 1331757056 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004820541624874624, + "loss": 1.3313, + "theoretical_loss": 3.582033717515144, + "tokens_seen": 1331822592 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048203811434302914, + "loss": 1.2824, + "theoretical_loss": 3.58201794657351, + "tokens_seen": 1331888128 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004820220661985958, + "loss": 1.2963, + "theoretical_loss": 3.582002176625142, + "tokens_seen": 1331953664 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048200601805416254, + "loss": 1.3334, + "theoretical_loss": 3.5819864076699304, + "tokens_seen": 1332019200 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004819899699097292, + "loss": 1.3703, + "theoretical_loss": 3.581970639707763, + "tokens_seen": 1332084736 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048197392176529594, + "loss": 1.4374, + "theoretical_loss": 3.5819548727385286, + "tokens_seen": 1332150272 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004819578736208626, + "loss": 1.391, + "theoretical_loss": 3.581939106762116, + "tokens_seen": 1332215808 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048194182547642934, + "loss": 1.3336, + "theoretical_loss": 3.5819233417784133, + "tokens_seen": 1332281344 + }, + { + "epoch": 0.4, + "learning_rate": 0.000481925777331996, + "loss": 1.3019, + "theoretical_loss": 3.5819075777873097, + "tokens_seen": 1332346880 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048190972918756275, + "loss": 1.3282, + "theoretical_loss": 3.581891814788694, + "tokens_seen": 1332412416 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004818936810431294, + "loss": 1.2818, + "theoretical_loss": 3.5818760527824542, + "tokens_seen": 1332477952 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004818776328986961, + "loss": 1.2704, + "theoretical_loss": 3.5818602917684794, + "tokens_seen": 1332543488 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004818615847542628, + "loss": 1.3217, + "theoretical_loss": 3.581844531746659, + "tokens_seen": 1332609024 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004818455366098295, + "loss": 1.2989, + "theoretical_loss": 3.5818287727168805, + "tokens_seen": 1332674560 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004818294884653962, + "loss": 1.4602, + "theoretical_loss": 3.5818130146790335, + "tokens_seen": 1332740096 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004818134403209629, + "loss": 1.4705, + "theoretical_loss": 3.581797257633007, + "tokens_seen": 1332805632 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004817973921765296, + "loss": 1.4806, + "theoretical_loss": 3.581781501578689, + "tokens_seen": 1332871168 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004817813440320963, + "loss": 1.3942, + "theoretical_loss": 3.581765746515969, + "tokens_seen": 1332936704 + }, + { + "epoch": 0.4, + "learning_rate": 0.000481765295887663, + "loss": 1.3049, + "theoretical_loss": 3.5817499924447356, + "tokens_seen": 1333002240 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004817492477432297, + "loss": 1.3911, + "theoretical_loss": 3.5817342393648772, + "tokens_seen": 1333067776 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048173319959879634, + "loss": 1.2953, + "theoretical_loss": 3.5817184872762833, + "tokens_seen": 1333133312 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004817171514543631, + "loss": 1.3615, + "theoretical_loss": 3.581702736178843, + "tokens_seen": 1333198848 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048170110330992974, + "loss": 1.3318, + "theoretical_loss": 3.5816869860724445, + "tokens_seen": 1333264384 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004816850551654965, + "loss": 1.3363, + "theoretical_loss": 3.581671236956977, + "tokens_seen": 1333329920 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048166900702106325, + "loss": 1.3148, + "theoretical_loss": 3.5816554888323298, + "tokens_seen": 1333395456 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004816529588766299, + "loss": 1.3972, + "theoretical_loss": 3.5816397416983916, + "tokens_seen": 1333460992 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048163691073219666, + "loss": 1.3458, + "theoretical_loss": 3.5816239955550513, + "tokens_seen": 1333526528 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004816208625877633, + "loss": 1.2335, + "theoretical_loss": 3.581608250402198, + "tokens_seen": 1333592064 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048160481444333006, + "loss": 1.2356, + "theoretical_loss": 3.581592506239721, + "tokens_seen": 1333657600 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004815887662988967, + "loss": 1.3857, + "theoretical_loss": 3.581576763067509, + "tokens_seen": 1333723136 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048157271815446346, + "loss": 1.3917, + "theoretical_loss": 3.581561020885451, + "tokens_seen": 1333788672 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004815566700100301, + "loss": 1.3809, + "theoretical_loss": 3.5815452796934357, + "tokens_seen": 1333854208 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048154062186559686, + "loss": 1.2747, + "theoretical_loss": 3.5815295394913536, + "tokens_seen": 1333919744 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004815245737211635, + "loss": 1.3342, + "theoretical_loss": 3.5815138002790925, + "tokens_seen": 1333985280 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048150852557673026, + "loss": 1.305, + "theoretical_loss": 3.581498062056542, + "tokens_seen": 1334050816 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004814924774322969, + "loss": 1.3556, + "theoretical_loss": 3.581482324823591, + "tokens_seen": 1334116352 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048147642928786366, + "loss": 1.4225, + "theoretical_loss": 3.5814665885801293, + "tokens_seen": 1334181888 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004814603811434303, + "loss": 1.3662, + "theoretical_loss": 3.581450853326045, + "tokens_seen": 1334247424 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048144433299899706, + "loss": 1.3646, + "theoretical_loss": 3.5814351190612284, + "tokens_seen": 1334312960 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004814282848545637, + "loss": 1.4115, + "theoretical_loss": 3.5814193857855683, + "tokens_seen": 1334378496 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048141223671013046, + "loss": 1.3019, + "theoretical_loss": 3.581403653498954, + "tokens_seen": 1334444032 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004813961885656971, + "loss": 1.3591, + "theoretical_loss": 3.5813879222012748, + "tokens_seen": 1334509568 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048138014042126387, + "loss": 1.3464, + "theoretical_loss": 3.5813721918924193, + "tokens_seen": 1334575104 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004813640922768305, + "loss": 1.2909, + "theoretical_loss": 3.581356462572278, + "tokens_seen": 1334640640 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048134804413239727, + "loss": 1.3067, + "theoretical_loss": 3.5813407342407384, + "tokens_seen": 1334706176 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004813319959879639, + "loss": 1.3416, + "theoretical_loss": 3.581325006897692, + "tokens_seen": 1334771712 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048131594784353067, + "loss": 1.4946, + "theoretical_loss": 3.581309280543027, + "tokens_seen": 1334837248 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004812998996990973, + "loss": 1.3132, + "theoretical_loss": 3.5812935551766327, + "tokens_seen": 1334902784 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048128385155466407, + "loss": 1.4133, + "theoretical_loss": 3.581277830798399, + "tokens_seen": 1334968320 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004812678034102307, + "loss": 1.5752, + "theoretical_loss": 3.5812621074082145, + "tokens_seen": 1335033856 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004812517552657974, + "loss": 1.3167, + "theoretical_loss": 3.5812463850059695, + "tokens_seen": 1335099392 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004812357071213641, + "loss": 1.4602, + "theoretical_loss": 3.5812306635915525, + "tokens_seen": 1335164928 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004812196589769308, + "loss": 1.3602, + "theoretical_loss": 3.581214943164854, + "tokens_seen": 1335230464 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004812036108324975, + "loss": 1.455, + "theoretical_loss": 3.581199223725762, + "tokens_seen": 1335296000 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004811875626880642, + "loss": 1.341, + "theoretical_loss": 3.5811835052741685, + "tokens_seen": 1335361536 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004811715145436309, + "loss": 1.3879, + "theoretical_loss": 3.58116778780996, + "tokens_seen": 1335427072 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004811554663991976, + "loss": 1.3624, + "theoretical_loss": 3.5811520713330283, + "tokens_seen": 1335492608 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004811394182547643, + "loss": 1.5269, + "theoretical_loss": 3.5811363558432623, + "tokens_seen": 1335558144 + }, + { + "epoch": 0.4, + "learning_rate": 0.000481123370110331, + "loss": 1.319, + "theoretical_loss": 3.5811206413405507, + "tokens_seen": 1335623680 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048110732196589767, + "loss": 1.2031, + "theoretical_loss": 3.581104927824784, + "tokens_seen": 1335689216 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004810912738214644, + "loss": 1.3426, + "theoretical_loss": 3.581089215295852, + "tokens_seen": 1335754752 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048107522567703107, + "loss": 1.2605, + "theoretical_loss": 3.581073503753643, + "tokens_seen": 1335820288 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004810591775325978, + "loss": 1.3419, + "theoretical_loss": 3.5810577931980476, + "tokens_seen": 1335885824 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048104312938816447, + "loss": 1.3017, + "theoretical_loss": 3.5810420836289554, + "tokens_seen": 1335951360 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004810270812437312, + "loss": 1.3258, + "theoretical_loss": 3.5810263750462568, + "tokens_seen": 1336016896 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048101103309929787, + "loss": 1.3061, + "theoretical_loss": 3.58101066744984, + "tokens_seen": 1336082432 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004809949849548646, + "loss": 1.2742, + "theoretical_loss": 3.580994960839595, + "tokens_seen": 1336147968 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004809789368104313, + "loss": 1.3636, + "theoretical_loss": 3.5809792552154125, + "tokens_seen": 1336213504 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048096288866599803, + "loss": 1.2836, + "theoretical_loss": 3.5809635505771817, + "tokens_seen": 1336279040 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004809468405215647, + "loss": 1.3422, + "theoretical_loss": 3.5809478469247917, + "tokens_seen": 1336344576 + }, + { + "epoch": 0.4, + "learning_rate": 0.00048093079237713143, + "loss": 1.2662, + "theoretical_loss": 3.580932144258133, + "tokens_seen": 1336410112 + }, + { + "epoch": 0.4, + "learning_rate": 0.0004809147442326981, + "loss": 1.3081, + "theoretical_loss": 3.580916442577096, + "tokens_seen": 1336475648 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048089869608826483, + "loss": 1.3957, + "theoretical_loss": 3.5809007418815693, + "tokens_seen": 1336541184 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004808826479438315, + "loss": 1.3491, + "theoretical_loss": 3.580885042171443, + "tokens_seen": 1336606720 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048086659979939823, + "loss": 1.3505, + "theoretical_loss": 3.5808693434466075, + "tokens_seen": 1336672256 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004808505516549649, + "loss": 1.3307, + "theoretical_loss": 3.580853645706952, + "tokens_seen": 1336737792 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048083450351053163, + "loss": 1.4147, + "theoretical_loss": 3.580837948952367, + "tokens_seen": 1336803328 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004808184553660983, + "loss": 1.2936, + "theoretical_loss": 3.580822253182742, + "tokens_seen": 1336868864 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 497137, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4211281538009644, + "objective/train/theoretical_loss": 3.5808065583979665, + "objective/train/tokens_used": 1357394400, + "theoretical_loss": 3.5808065583979665, + "tokens_seen": 1336934400 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048080240722166503, + "loss": 1.4211, + "theoretical_loss": 3.5808065583979665, + "tokens_seen": 1336934400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004807863590772317, + "loss": 1.4064, + "theoretical_loss": 3.5807908645979314, + "tokens_seen": 1336999936 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048077031093279844, + "loss": 1.4192, + "theoretical_loss": 3.580775171782526, + "tokens_seen": 1337065472 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004807542627883651, + "loss": 1.2654, + "theoretical_loss": 3.5807594799516402, + "tokens_seen": 1337131008 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048073821464393184, + "loss": 1.3964, + "theoretical_loss": 3.5807437891051643, + "tokens_seen": 1337196544 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004807221664994985, + "loss": 1.4137, + "theoretical_loss": 3.5807280992429886, + "tokens_seen": 1337262080 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048070611835506524, + "loss": 1.2533, + "theoretical_loss": 3.5807124103650025, + "tokens_seen": 1337327616 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004806900702106319, + "loss": 1.2643, + "theoretical_loss": 3.580696722471096, + "tokens_seen": 1337393152 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048067402206619864, + "loss": 1.3305, + "theoretical_loss": 3.58068103556116, + "tokens_seen": 1337458688 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004806579739217653, + "loss": 1.4488, + "theoretical_loss": 3.5806653496350838, + "tokens_seen": 1337524224 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048064192577733204, + "loss": 1.2965, + "theoretical_loss": 3.5806496646927575, + "tokens_seen": 1337589760 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004806258776328987, + "loss": 1.2746, + "theoretical_loss": 3.5806339807340715, + "tokens_seen": 1337655296 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048060982948846544, + "loss": 1.394, + "theoretical_loss": 3.5806182977589156, + "tokens_seen": 1337720832 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004805937813440321, + "loss": 1.3587, + "theoretical_loss": 3.580602615767181, + "tokens_seen": 1337786368 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048057773319959884, + "loss": 1.2241, + "theoretical_loss": 3.5805869347587564, + "tokens_seen": 1337851904 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004805616850551655, + "loss": 1.3497, + "theoretical_loss": 3.580571254733533, + "tokens_seen": 1337917440 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048054563691073224, + "loss": 1.3274, + "theoretical_loss": 3.5805555756914, + "tokens_seen": 1337982976 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004805295887662989, + "loss": 1.2813, + "theoretical_loss": 3.5805398976322484, + "tokens_seen": 1338048512 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048051354062186565, + "loss": 1.3641, + "theoretical_loss": 3.5805242205559686, + "tokens_seen": 1338114048 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004804974924774323, + "loss": 1.3494, + "theoretical_loss": 3.5805085444624503, + "tokens_seen": 1338179584 + }, + { + "epoch": 0.41, + "learning_rate": 0.000480481444332999, + "loss": 1.2709, + "theoretical_loss": 3.580492869351584, + "tokens_seen": 1338245120 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048046539618856575, + "loss": 1.3863, + "theoretical_loss": 3.58047719522326, + "tokens_seen": 1338310656 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004804493480441324, + "loss": 1.285, + "theoretical_loss": 3.5804615220773686, + "tokens_seen": 1338376192 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048043329989969915, + "loss": 1.3062, + "theoretical_loss": 3.5804458499138, + "tokens_seen": 1338441728 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004804172517552658, + "loss": 1.4543, + "theoretical_loss": 3.5804301787324446, + "tokens_seen": 1338507264 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048040120361083255, + "loss": 1.2713, + "theoretical_loss": 3.5804145085331927, + "tokens_seen": 1338572800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004803851554663992, + "loss": 1.3703, + "theoretical_loss": 3.580398839315935, + "tokens_seen": 1338638336 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048036910732196595, + "loss": 1.2884, + "theoretical_loss": 3.5803831710805616, + "tokens_seen": 1338703872 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004803530591775326, + "loss": 1.3762, + "theoretical_loss": 3.580367503826963, + "tokens_seen": 1338769408 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048033701103309935, + "loss": 1.2917, + "theoretical_loss": 3.5803518375550296, + "tokens_seen": 1338834944 + }, + { + "epoch": 0.41, + "learning_rate": 0.000480320962888666, + "loss": 1.4213, + "theoretical_loss": 3.580336172264652, + "tokens_seen": 1338900480 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048030491474423275, + "loss": 1.2936, + "theoretical_loss": 3.5803205079557197, + "tokens_seen": 1338966016 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004802888665997994, + "loss": 1.4326, + "theoretical_loss": 3.5803048446281247, + "tokens_seen": 1339031552 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048027281845536616, + "loss": 1.2648, + "theoretical_loss": 3.5802891822817564, + "tokens_seen": 1339097088 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004802567703109328, + "loss": 1.4112, + "theoretical_loss": 3.580273520916506, + "tokens_seen": 1339162624 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048024072216649956, + "loss": 1.3841, + "theoretical_loss": 3.5802578605322637, + "tokens_seen": 1339228160 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004802246740220662, + "loss": 1.2599, + "theoretical_loss": 3.58024220112892, + "tokens_seen": 1339293696 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048020862587763296, + "loss": 1.294, + "theoretical_loss": 3.580226542706365, + "tokens_seen": 1339359232 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004801925777331996, + "loss": 1.2737, + "theoretical_loss": 3.5802108852644903, + "tokens_seen": 1339424768 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048017652958876636, + "loss": 1.4205, + "theoretical_loss": 3.580195228803186, + "tokens_seen": 1339490304 + }, + { + "epoch": 0.41, + "learning_rate": 0.000480160481444333, + "loss": 1.3492, + "theoretical_loss": 3.5801795733223427, + "tokens_seen": 1339555840 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048014443329989976, + "loss": 1.4028, + "theoretical_loss": 3.5801639188218513, + "tokens_seen": 1339621376 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004801283851554664, + "loss": 1.2572, + "theoretical_loss": 3.5801482653016015, + "tokens_seen": 1339686912 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048011233701103316, + "loss": 1.2915, + "theoretical_loss": 3.5801326127614854, + "tokens_seen": 1339752448 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004800962888665998, + "loss": 1.3093, + "theoretical_loss": 3.5801169612013926, + "tokens_seen": 1339817984 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048008024072216656, + "loss": 1.4577, + "theoretical_loss": 3.580101310621214, + "tokens_seen": 1339883520 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004800641925777332, + "loss": 1.3208, + "theoretical_loss": 3.580085661020841, + "tokens_seen": 1339949056 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048004814443329996, + "loss": 1.2864, + "theoretical_loss": 3.5800700124001636, + "tokens_seen": 1340014592 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004800320962888666, + "loss": 1.3211, + "theoretical_loss": 3.5800543647590732, + "tokens_seen": 1340080128 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048001604814443337, + "loss": 1.3307, + "theoretical_loss": 3.5800387180974598, + "tokens_seen": 1340145664 + }, + { + "epoch": 0.41, + "learning_rate": 0.00048, + "loss": 1.3342, + "theoretical_loss": 3.580023072415215, + "tokens_seen": 1340211200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047998395185556677, + "loss": 1.3168, + "theoretical_loss": 3.5800074277122285, + "tokens_seen": 1340276736 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004799679037111334, + "loss": 1.304, + "theoretical_loss": 3.579991783988392, + "tokens_seen": 1340342272 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047995185556670017, + "loss": 1.279, + "theoretical_loss": 3.5799761412435966, + "tokens_seen": 1340407808 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004799358074222668, + "loss": 1.3357, + "theoretical_loss": 3.5799604994777328, + "tokens_seen": 1340473344 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047991975927783357, + "loss": 1.2842, + "theoretical_loss": 3.5799448586906912, + "tokens_seen": 1340538880 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004799037111334002, + "loss": 1.3776, + "theoretical_loss": 3.579929218882363, + "tokens_seen": 1340604416 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047988766298896697, + "loss": 1.1861, + "theoretical_loss": 3.5799135800526383, + "tokens_seen": 1340669952 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004798716148445336, + "loss": 1.3071, + "theoretical_loss": 3.57989794220141, + "tokens_seen": 1340735488 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047985556670010037, + "loss": 1.2771, + "theoretical_loss": 3.579882305328567, + "tokens_seen": 1340801024 + }, + { + "epoch": 0.41, + "learning_rate": 0.000479839518555667, + "loss": 1.2235, + "theoretical_loss": 3.5798666694340016, + "tokens_seen": 1340866560 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004798234704112337, + "loss": 1.3023, + "theoretical_loss": 3.5798510345176036, + "tokens_seen": 1340932096 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004798074222668004, + "loss": 1.283, + "theoretical_loss": 3.579835400579265, + "tokens_seen": 1340997632 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004797913741223671, + "loss": 1.3464, + "theoretical_loss": 3.5798197676188765, + "tokens_seen": 1341063168 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004797753259779338, + "loss": 1.3144, + "theoretical_loss": 3.57980413563633, + "tokens_seen": 1341128704 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004797592778335005, + "loss": 1.4342, + "theoretical_loss": 3.5797885046315145, + "tokens_seen": 1341194240 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004797432296890672, + "loss": 1.4295, + "theoretical_loss": 3.579772874604323, + "tokens_seen": 1341259776 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004797271815446339, + "loss": 1.3574, + "theoretical_loss": 3.5797572455546454, + "tokens_seen": 1341325312 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047971113340020057, + "loss": 1.3675, + "theoretical_loss": 3.579741617482374, + "tokens_seen": 1341390848 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004796950852557673, + "loss": 1.3323, + "theoretical_loss": 3.5797259903873986, + "tokens_seen": 1341456384 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047967903711133397, + "loss": 1.27, + "theoretical_loss": 3.579710364269611, + "tokens_seen": 1341521920 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004796629889669007, + "loss": 1.2401, + "theoretical_loss": 3.579694739128902, + "tokens_seen": 1341587456 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047964694082246737, + "loss": 1.3549, + "theoretical_loss": 3.5796791149651637, + "tokens_seen": 1341652992 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004796308926780341, + "loss": 1.2338, + "theoretical_loss": 3.5796634917782866, + "tokens_seen": 1341718528 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004796148445336008, + "loss": 1.1994, + "theoretical_loss": 3.579647869568162, + "tokens_seen": 1341784064 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047959879638916753, + "loss": 1.3127, + "theoretical_loss": 3.579632248334681, + "tokens_seen": 1341849600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004795827482447342, + "loss": 1.2336, + "theoretical_loss": 3.579616628077735, + "tokens_seen": 1341915136 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047956670010030093, + "loss": 1.4321, + "theoretical_loss": 3.579601008797215, + "tokens_seen": 1341980672 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004795506519558676, + "loss": 1.3906, + "theoretical_loss": 3.579585390493013, + "tokens_seen": 1342046208 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047953460381143433, + "loss": 1.2885, + "theoretical_loss": 3.5795697731650193, + "tokens_seen": 1342111744 + }, + { + "epoch": 0.41, + "learning_rate": 0.000479518555667001, + "loss": 1.2992, + "theoretical_loss": 3.5795541568131264, + "tokens_seen": 1342177280 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047950250752256773, + "loss": 1.3666, + "theoretical_loss": 3.5795385414372247, + "tokens_seen": 1342242816 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004794864593781344, + "loss": 1.3305, + "theoretical_loss": 3.579522927037206, + "tokens_seen": 1342308352 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047947041123370113, + "loss": 1.2088, + "theoretical_loss": 3.5795073136129614, + "tokens_seen": 1342373888 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004794543630892678, + "loss": 1.3668, + "theoretical_loss": 3.5794917011643825, + "tokens_seen": 1342439424 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047943831494483453, + "loss": 1.2808, + "theoretical_loss": 3.57947608969136, + "tokens_seen": 1342504960 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004794222668004012, + "loss": 1.3536, + "theoretical_loss": 3.579460479193787, + "tokens_seen": 1342570496 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047940621865596794, + "loss": 1.3445, + "theoretical_loss": 3.579444869671553, + "tokens_seen": 1342636032 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004793901705115346, + "loss": 1.3648, + "theoretical_loss": 3.5794292611245506, + "tokens_seen": 1342701568 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047937412236710134, + "loss": 1.5181, + "theoretical_loss": 3.5794136535526713, + "tokens_seen": 1342767104 + }, + { + "epoch": 0.41, + "learning_rate": 0.000479358074222668, + "loss": 1.4141, + "theoretical_loss": 3.5793980469558058, + "tokens_seen": 1342832640 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047934202607823474, + "loss": 1.3903, + "theoretical_loss": 3.5793824413338466, + "tokens_seen": 1342898176 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004793259779338014, + "loss": 1.4318, + "theoretical_loss": 3.579366836686684, + "tokens_seen": 1342963712 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047930992978936814, + "loss": 1.3041, + "theoretical_loss": 3.579351233014211, + "tokens_seen": 1343029248 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004792938816449348, + "loss": 1.338, + "theoretical_loss": 3.579335630316318, + "tokens_seen": 1343094784 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047927783350050154, + "loss": 1.2397, + "theoretical_loss": 3.5793200285928974, + "tokens_seen": 1343160320 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004792617853560683, + "loss": 1.2746, + "theoretical_loss": 3.57930442784384, + "tokens_seen": 1343225856 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047924573721163494, + "loss": 1.3885, + "theoretical_loss": 3.5792888280690383, + "tokens_seen": 1343291392 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004792296890672017, + "loss": 1.3411, + "theoretical_loss": 3.5792732292683835, + "tokens_seen": 1343356928 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047921364092276834, + "loss": 1.3771, + "theoretical_loss": 3.5792576314417666, + "tokens_seen": 1343422464 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 499917, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2781457901000977, + "objective/train/theoretical_loss": 3.5792420345890807, + "objective/train/tokens_used": 1363948000, + "theoretical_loss": 3.5792420345890807, + "tokens_seen": 1343488000 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047919759277833504, + "loss": 1.2781, + "theoretical_loss": 3.5792420345890807, + "tokens_seen": 1343488000 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047918154463390174, + "loss": 1.3075, + "theoretical_loss": 3.579226438710216, + "tokens_seen": 1343553536 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047916549648946845, + "loss": 1.306, + "theoretical_loss": 3.579210843805065, + "tokens_seen": 1343619072 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047914944834503515, + "loss": 1.4162, + "theoretical_loss": 3.5791952498735196, + "tokens_seen": 1343684608 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047913340020060185, + "loss": 1.2964, + "theoretical_loss": 3.579179656915471, + "tokens_seen": 1343750144 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047911735205616855, + "loss": 1.3694, + "theoretical_loss": 3.579164064930811, + "tokens_seen": 1343815680 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047910130391173525, + "loss": 1.2994, + "theoretical_loss": 3.5791484739194317, + "tokens_seen": 1343881216 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047908525576730195, + "loss": 1.2788, + "theoretical_loss": 3.5791328838812246, + "tokens_seen": 1343946752 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047906920762286865, + "loss": 1.4414, + "theoretical_loss": 3.579117294816082, + "tokens_seen": 1344012288 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004790531594784353, + "loss": 1.4117, + "theoretical_loss": 3.579101706723895, + "tokens_seen": 1344077824 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047903711133400205, + "loss": 1.3172, + "theoretical_loss": 3.579086119604556, + "tokens_seen": 1344143360 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004790210631895687, + "loss": 1.3232, + "theoretical_loss": 3.5790705334579567, + "tokens_seen": 1344208896 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047900501504513545, + "loss": 1.3176, + "theoretical_loss": 3.5790549482839884, + "tokens_seen": 1344274432 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004789889669007021, + "loss": 1.2187, + "theoretical_loss": 3.579039364082544, + "tokens_seen": 1344339968 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047897291875626885, + "loss": 1.1968, + "theoretical_loss": 3.5790237808535146, + "tokens_seen": 1344405504 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004789568706118355, + "loss": 1.3494, + "theoretical_loss": 3.5790081985967928, + "tokens_seen": 1344471040 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047894082246740225, + "loss": 1.307, + "theoretical_loss": 3.57899261731227, + "tokens_seen": 1344536576 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004789247743229689, + "loss": 1.245, + "theoretical_loss": 3.5789770369998384, + "tokens_seen": 1344602112 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047890872617853566, + "loss": 1.3467, + "theoretical_loss": 3.57896145765939, + "tokens_seen": 1344667648 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004788926780341023, + "loss": 1.3201, + "theoretical_loss": 3.5789458792908166, + "tokens_seen": 1344733184 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047887662988966906, + "loss": 1.3999, + "theoretical_loss": 3.5789303018940104, + "tokens_seen": 1344798720 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004788605817452357, + "loss": 1.321, + "theoretical_loss": 3.5789147254688634, + "tokens_seen": 1344864256 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047884453360080246, + "loss": 1.383, + "theoretical_loss": 3.5788991500152676, + "tokens_seen": 1344929792 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004788284854563691, + "loss": 1.284, + "theoretical_loss": 3.5788835755331148, + "tokens_seen": 1344995328 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047881243731193586, + "loss": 1.3318, + "theoretical_loss": 3.578868002022298, + "tokens_seen": 1345060864 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004787963891675025, + "loss": 1.2848, + "theoretical_loss": 3.578852429482708, + "tokens_seen": 1345126400 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047878034102306926, + "loss": 1.2566, + "theoretical_loss": 3.578836857914238, + "tokens_seen": 1345191936 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004787642928786359, + "loss": 1.4555, + "theoretical_loss": 3.5788212873167793, + "tokens_seen": 1345257472 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047874824473420266, + "loss": 1.3722, + "theoretical_loss": 3.5788057176902246, + "tokens_seen": 1345323008 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004787321965897693, + "loss": 1.3928, + "theoretical_loss": 3.5787901490344662, + "tokens_seen": 1345388544 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047871614844533606, + "loss": 1.2568, + "theoretical_loss": 3.5787745813493954, + "tokens_seen": 1345454080 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004787001003009027, + "loss": 1.3073, + "theoretical_loss": 3.5787590146349055, + "tokens_seen": 1345519616 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047868405215646946, + "loss": 1.3007, + "theoretical_loss": 3.5787434488908882, + "tokens_seen": 1345585152 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004786680040120361, + "loss": 1.3916, + "theoretical_loss": 3.578727884117236, + "tokens_seen": 1345650688 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047865195586760287, + "loss": 1.3254, + "theoretical_loss": 3.5787123203138402, + "tokens_seen": 1345716224 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004786359077231695, + "loss": 1.229, + "theoretical_loss": 3.5786967574805946, + "tokens_seen": 1345781760 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047861985957873627, + "loss": 1.2998, + "theoretical_loss": 3.57868119561739, + "tokens_seen": 1345847296 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004786038114343029, + "loss": 1.3216, + "theoretical_loss": 3.5786656347241195, + "tokens_seen": 1345912832 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047858776328986967, + "loss": 1.2245, + "theoretical_loss": 3.5786500748006755, + "tokens_seen": 1345978368 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004785717151454363, + "loss": 1.2944, + "theoretical_loss": 3.57863451584695, + "tokens_seen": 1346043904 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047855566700100307, + "loss": 1.3561, + "theoretical_loss": 3.578618957862836, + "tokens_seen": 1346109440 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004785396188565697, + "loss": 1.3347, + "theoretical_loss": 3.5786034008482246, + "tokens_seen": 1346174976 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047852357071213647, + "loss": 1.4416, + "theoretical_loss": 3.5785878448030095, + "tokens_seen": 1346240512 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004785075225677031, + "loss": 1.4023, + "theoretical_loss": 3.578572289727082, + "tokens_seen": 1346306048 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047849147442326987, + "loss": 1.2956, + "theoretical_loss": 3.5785567356203356, + "tokens_seen": 1346371584 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004784754262788365, + "loss": 1.3362, + "theoretical_loss": 3.5785411824826623, + "tokens_seen": 1346437120 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004784593781344033, + "loss": 1.3413, + "theoretical_loss": 3.578525630313954, + "tokens_seen": 1346502656 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004784433299899699, + "loss": 1.3491, + "theoretical_loss": 3.5785100791141042, + "tokens_seen": 1346568192 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004784272818455366, + "loss": 1.2966, + "theoretical_loss": 3.578494528883004, + "tokens_seen": 1346633728 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004784112337011033, + "loss": 1.4752, + "theoretical_loss": 3.578478979620548, + "tokens_seen": 1346699264 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047839518555667, + "loss": 1.362, + "theoretical_loss": 3.5784634313266266, + "tokens_seen": 1346764800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004783791374122367, + "loss": 1.3866, + "theoretical_loss": 3.578447884001134, + "tokens_seen": 1346830336 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004783630892678034, + "loss": 1.3264, + "theoretical_loss": 3.578432337643961, + "tokens_seen": 1346895872 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004783470411233701, + "loss": 1.3486, + "theoretical_loss": 3.578416792255002, + "tokens_seen": 1346961408 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004783309929789368, + "loss": 1.386, + "theoretical_loss": 3.5784012478341483, + "tokens_seen": 1347026944 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004783149448345035, + "loss": 1.4038, + "theoretical_loss": 3.5783857043812937, + "tokens_seen": 1347092480 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004782988966900702, + "loss": 1.3172, + "theoretical_loss": 3.5783701618963297, + "tokens_seen": 1347158016 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047828284854563687, + "loss": 1.2818, + "theoretical_loss": 3.578354620379149, + "tokens_seen": 1347223552 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004782668004012036, + "loss": 1.313, + "theoretical_loss": 3.578339079829645, + "tokens_seen": 1347289088 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004782507522567703, + "loss": 1.3035, + "theoretical_loss": 3.5783235402477103, + "tokens_seen": 1347354624 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047823470411233703, + "loss": 1.3606, + "theoretical_loss": 3.578308001633237, + "tokens_seen": 1347420160 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004782186559679037, + "loss": 1.2172, + "theoretical_loss": 3.578292463986118, + "tokens_seen": 1347485696 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047820260782347043, + "loss": 1.2926, + "theoretical_loss": 3.5782769273062467, + "tokens_seen": 1347551232 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004781865596790371, + "loss": 1.3069, + "theoretical_loss": 3.578261391593515, + "tokens_seen": 1347616768 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047817051153460383, + "loss": 1.3809, + "theoretical_loss": 3.5782458568478157, + "tokens_seen": 1347682304 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004781544633901705, + "loss": 1.257, + "theoretical_loss": 3.578230323069042, + "tokens_seen": 1347747840 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047813841524573723, + "loss": 1.3896, + "theoretical_loss": 3.5782147902570873, + "tokens_seen": 1347813376 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004781223671013039, + "loss": 1.3617, + "theoretical_loss": 3.578199258411843, + "tokens_seen": 1347878912 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047810631895687063, + "loss": 1.2893, + "theoretical_loss": 3.5781837275332027, + "tokens_seen": 1347944448 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004780902708124373, + "loss": 1.3173, + "theoretical_loss": 3.5781681976210598, + "tokens_seen": 1348009984 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047807422266800403, + "loss": 1.3151, + "theoretical_loss": 3.578152668675306, + "tokens_seen": 1348075520 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004780581745235708, + "loss": 1.362, + "theoretical_loss": 3.578137140695835, + "tokens_seen": 1348141056 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047804212637913744, + "loss": 1.3017, + "theoretical_loss": 3.578121613682539, + "tokens_seen": 1348206592 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004780260782347042, + "loss": 1.2552, + "theoretical_loss": 3.578106087635312, + "tokens_seen": 1348272128 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047801003009027084, + "loss": 1.3418, + "theoretical_loss": 3.578090562554046, + "tokens_seen": 1348337664 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004779939819458376, + "loss": 1.3723, + "theoretical_loss": 3.578075038438634, + "tokens_seen": 1348403200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047797793380140424, + "loss": 1.3997, + "theoretical_loss": 3.5780595152889694, + "tokens_seen": 1348468736 + }, + { + "epoch": 0.41, + "learning_rate": 0.000477961885656971, + "loss": 1.3579, + "theoretical_loss": 3.5780439931049455, + "tokens_seen": 1348534272 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047794583751253764, + "loss": 1.2925, + "theoretical_loss": 3.5780284718864546, + "tokens_seen": 1348599808 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004779297893681044, + "loss": 1.2944, + "theoretical_loss": 3.57801295163339, + "tokens_seen": 1348665344 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047791374122367104, + "loss": 1.3176, + "theoretical_loss": 3.5779974323456445, + "tokens_seen": 1348730880 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004778976930792378, + "loss": 1.3027, + "theoretical_loss": 3.5779819140231117, + "tokens_seen": 1348796416 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047788164493480444, + "loss": 1.3113, + "theoretical_loss": 3.577966396665684, + "tokens_seen": 1348861952 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004778655967903712, + "loss": 1.3593, + "theoretical_loss": 3.5779508802732547, + "tokens_seen": 1348927488 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047784954864593784, + "loss": 1.2706, + "theoretical_loss": 3.5779353648457173, + "tokens_seen": 1348993024 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004778335005015046, + "loss": 1.2685, + "theoretical_loss": 3.577919850382965, + "tokens_seen": 1349058560 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047781745235707124, + "loss": 1.3458, + "theoretical_loss": 3.57790433688489, + "tokens_seen": 1349124096 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047780140421263794, + "loss": 1.3477, + "theoretical_loss": 3.577888824351387, + "tokens_seen": 1349189632 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047778535606820465, + "loss": 1.3066, + "theoretical_loss": 3.577873312782347, + "tokens_seen": 1349255168 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047776930792377135, + "loss": 1.2661, + "theoretical_loss": 3.5778578021776655, + "tokens_seen": 1349320704 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047775325977933805, + "loss": 1.2772, + "theoretical_loss": 3.577842292537234, + "tokens_seen": 1349386240 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047773721163490475, + "loss": 1.2983, + "theoretical_loss": 3.5778267838609468, + "tokens_seen": 1349451776 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047772116349047145, + "loss": 1.348, + "theoretical_loss": 3.5778112761486964, + "tokens_seen": 1349517312 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047770511534603815, + "loss": 1.3265, + "theoretical_loss": 3.577795769400377, + "tokens_seen": 1349582848 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047768906720160485, + "loss": 1.4967, + "theoretical_loss": 3.577780263615881, + "tokens_seen": 1349648384 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047767301905717155, + "loss": 1.3812, + "theoretical_loss": 3.5777647587951016, + "tokens_seen": 1349713920 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004776569709127382, + "loss": 1.2698, + "theoretical_loss": 3.577749254937933, + "tokens_seen": 1349779456 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047764092276830495, + "loss": 1.3718, + "theoretical_loss": 3.577733752044267, + "tokens_seen": 1349844992 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004776248746238716, + "loss": 1.3164, + "theoretical_loss": 3.577718250113999, + "tokens_seen": 1349910528 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047760882647943835, + "loss": 1.4086, + "theoretical_loss": 3.5777027491470212, + "tokens_seen": 1349976064 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 502226, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3289717435836792, + "objective/train/theoretical_loss": 3.577687249143227, + "objective/train/tokens_used": 1370501600, + "theoretical_loss": 3.577687249143227, + "tokens_seen": 1350041600 + }, + { + "epoch": 0.41, + "learning_rate": 0.000477592778335005, + "loss": 1.329, + "theoretical_loss": 3.577687249143227, + "tokens_seen": 1350041600 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047757673019057175, + "loss": 1.254, + "theoretical_loss": 3.57767175010251, + "tokens_seen": 1350107136 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004775606820461384, + "loss": 1.2999, + "theoretical_loss": 3.577656252024763, + "tokens_seen": 1350172672 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047754463390170516, + "loss": 1.3069, + "theoretical_loss": 3.5776407549098805, + "tokens_seen": 1350238208 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004775285857572718, + "loss": 1.4239, + "theoretical_loss": 3.577625258757755, + "tokens_seen": 1350303744 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047751253761283856, + "loss": 1.3251, + "theoretical_loss": 3.5776097635682813, + "tokens_seen": 1350369280 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004774964894684052, + "loss": 1.4329, + "theoretical_loss": 3.577594269341351, + "tokens_seen": 1350434816 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047748044132397196, + "loss": 1.3824, + "theoretical_loss": 3.5775787760768587, + "tokens_seen": 1350500352 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004774643931795386, + "loss": 1.3294, + "theoretical_loss": 3.577563283774698, + "tokens_seen": 1350565888 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047744834503510536, + "loss": 1.3292, + "theoretical_loss": 3.5775477924347623, + "tokens_seen": 1350631424 + }, + { + "epoch": 0.41, + "learning_rate": 0.000477432296890672, + "loss": 1.2506, + "theoretical_loss": 3.5775323020569445, + "tokens_seen": 1350696960 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047741624874623876, + "loss": 1.3529, + "theoretical_loss": 3.5775168126411394, + "tokens_seen": 1350762496 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004774002006018054, + "loss": 1.3962, + "theoretical_loss": 3.5775013241872395, + "tokens_seen": 1350828032 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047738415245737216, + "loss": 1.3062, + "theoretical_loss": 3.577485836695139, + "tokens_seen": 1350893568 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004773681043129388, + "loss": 1.3871, + "theoretical_loss": 3.5774703501647314, + "tokens_seen": 1350959104 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047735205616850556, + "loss": 1.2407, + "theoretical_loss": 3.5774548645959103, + "tokens_seen": 1351024640 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004773360080240722, + "loss": 1.498, + "theoretical_loss": 3.5774393799885695, + "tokens_seen": 1351090176 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047731995987963896, + "loss": 1.2598, + "theoretical_loss": 3.577423896342602, + "tokens_seen": 1351155712 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004773039117352056, + "loss": 1.2646, + "theoretical_loss": 3.577408413657902, + "tokens_seen": 1351221248 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047728786359077237, + "loss": 1.3194, + "theoretical_loss": 3.577392931934363, + "tokens_seen": 1351286784 + }, + { + "epoch": 0.41, + "learning_rate": 0.000477271815446339, + "loss": 1.4036, + "theoretical_loss": 3.5773774511718797, + "tokens_seen": 1351352320 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047725576730190577, + "loss": 1.2369, + "theoretical_loss": 3.577361971370345, + "tokens_seen": 1351417856 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004772397191574724, + "loss": 1.2953, + "theoretical_loss": 3.577346492529652, + "tokens_seen": 1351483392 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047722367101303917, + "loss": 1.3781, + "theoretical_loss": 3.5773310146496953, + "tokens_seen": 1351548928 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004772076228686058, + "loss": 1.355, + "theoretical_loss": 3.5773155377303687, + "tokens_seen": 1351614464 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047719157472417257, + "loss": 1.3605, + "theoretical_loss": 3.577300061771566, + "tokens_seen": 1351680000 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004771755265797392, + "loss": 1.3204, + "theoretical_loss": 3.577284586773181, + "tokens_seen": 1351745536 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047715947843530597, + "loss": 1.3781, + "theoretical_loss": 3.577269112735107, + "tokens_seen": 1351811072 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004771434302908726, + "loss": 1.3341, + "theoretical_loss": 3.577253639657238, + "tokens_seen": 1351876608 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047712738214643937, + "loss": 1.3701, + "theoretical_loss": 3.577238167539469, + "tokens_seen": 1351942144 + }, + { + "epoch": 0.41, + "learning_rate": 0.000477111334002006, + "loss": 1.2415, + "theoretical_loss": 3.5772226963816927, + "tokens_seen": 1352007680 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047709528585757277, + "loss": 1.2885, + "theoretical_loss": 3.577207226183803, + "tokens_seen": 1352073216 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004770792377131394, + "loss": 1.259, + "theoretical_loss": 3.5771917569456946, + "tokens_seen": 1352138752 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004770631895687062, + "loss": 1.3357, + "theoretical_loss": 3.5771762886672605, + "tokens_seen": 1352204288 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004770471414242728, + "loss": 1.297, + "theoretical_loss": 3.5771608213483956, + "tokens_seen": 1352269824 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004770310932798395, + "loss": 1.3406, + "theoretical_loss": 3.5771453549889936, + "tokens_seen": 1352335360 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004770150451354062, + "loss": 1.2629, + "theoretical_loss": 3.577129889588948, + "tokens_seen": 1352400896 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004769989969909729, + "loss": 1.2293, + "theoretical_loss": 3.577114425148153, + "tokens_seen": 1352466432 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004769829488465396, + "loss": 1.291, + "theoretical_loss": 3.577098961666503, + "tokens_seen": 1352531968 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004769669007021063, + "loss": 1.3747, + "theoretical_loss": 3.577083499143892, + "tokens_seen": 1352597504 + }, + { + "epoch": 0.41, + "learning_rate": 0.000476950852557673, + "loss": 1.3149, + "theoretical_loss": 3.5770680375802137, + "tokens_seen": 1352663040 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004769348044132397, + "loss": 1.2075, + "theoretical_loss": 3.577052576975362, + "tokens_seen": 1352728576 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004769187562688064, + "loss": 1.3448, + "theoretical_loss": 3.5770371173292315, + "tokens_seen": 1352794112 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004769027081243731, + "loss": 1.2565, + "theoretical_loss": 3.5770216586417165, + "tokens_seen": 1352859648 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047688665997993977, + "loss": 1.2388, + "theoretical_loss": 3.5770062009127104, + "tokens_seen": 1352925184 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047687061183550653, + "loss": 1.2928, + "theoretical_loss": 3.576990744142108, + "tokens_seen": 1352990720 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004768545636910733, + "loss": 1.4184, + "theoretical_loss": 3.5769752883298027, + "tokens_seen": 1353056256 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047683851554663993, + "loss": 1.3356, + "theoretical_loss": 3.576959833475689, + "tokens_seen": 1353121792 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004768224674022067, + "loss": 1.4043, + "theoretical_loss": 3.576944379579662, + "tokens_seen": 1353187328 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047680641925777333, + "loss": 1.2567, + "theoretical_loss": 3.5769289266416147, + "tokens_seen": 1353252864 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004767903711133401, + "loss": 1.2528, + "theoretical_loss": 3.5769134746614415, + "tokens_seen": 1353318400 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047677432296890673, + "loss": 1.2634, + "theoretical_loss": 3.5768980236390373, + "tokens_seen": 1353383936 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004767582748244735, + "loss": 1.2012, + "theoretical_loss": 3.576882573574296, + "tokens_seen": 1353449472 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047674222668004013, + "loss": 1.3999, + "theoretical_loss": 3.5768671244671113, + "tokens_seen": 1353515008 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004767261785356069, + "loss": 1.3645, + "theoretical_loss": 3.5768516763173785, + "tokens_seen": 1353580544 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047671013039117353, + "loss": 1.2887, + "theoretical_loss": 3.5768362291249916, + "tokens_seen": 1353646080 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004766940822467403, + "loss": 1.2561, + "theoretical_loss": 3.576820782889844, + "tokens_seen": 1353711616 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047667803410230694, + "loss": 1.424, + "theoretical_loss": 3.5768053376118316, + "tokens_seen": 1353777152 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004766619859578737, + "loss": 1.4581, + "theoretical_loss": 3.576789893290848, + "tokens_seen": 1353842688 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047664593781344034, + "loss": 1.4128, + "theoretical_loss": 3.576774449926787, + "tokens_seen": 1353908224 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004766298896690071, + "loss": 1.3307, + "theoretical_loss": 3.576759007519544, + "tokens_seen": 1353973760 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047661384152457374, + "loss": 1.3269, + "theoretical_loss": 3.5767435660690126, + "tokens_seen": 1354039296 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004765977933801405, + "loss": 1.359, + "theoretical_loss": 3.5767281255750882, + "tokens_seen": 1354104832 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047658174523570714, + "loss": 1.4741, + "theoretical_loss": 3.5767126860376637, + "tokens_seen": 1354170368 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004765656970912739, + "loss": 1.2848, + "theoretical_loss": 3.576697247456635, + "tokens_seen": 1354235904 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047654964894684054, + "loss": 1.2783, + "theoretical_loss": 3.576681809831896, + "tokens_seen": 1354301440 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004765336008024073, + "loss": 1.2895, + "theoretical_loss": 3.5766663731633415, + "tokens_seen": 1354366976 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047651755265797394, + "loss": 1.3777, + "theoretical_loss": 3.5766509374508653, + "tokens_seen": 1354432512 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004765015045135407, + "loss": 1.3232, + "theoretical_loss": 3.5766355026943626, + "tokens_seen": 1354498048 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047648545636910734, + "loss": 1.3021, + "theoretical_loss": 3.576620068893728, + "tokens_seen": 1354563584 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004764694082246741, + "loss": 1.3806, + "theoretical_loss": 3.5766046360488555, + "tokens_seen": 1354629120 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047645336008024074, + "loss": 1.3729, + "theoretical_loss": 3.57658920415964, + "tokens_seen": 1354694656 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004764373119358075, + "loss": 1.2639, + "theoretical_loss": 3.5765737732259764, + "tokens_seen": 1354760192 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047642126379137415, + "loss": 1.2739, + "theoretical_loss": 3.5765583432477586, + "tokens_seen": 1354825728 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047640521564694085, + "loss": 1.3539, + "theoretical_loss": 3.5765429142248815, + "tokens_seen": 1354891264 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047638916750250755, + "loss": 1.3027, + "theoretical_loss": 3.5765274861572403, + "tokens_seen": 1354956800 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047637311935807425, + "loss": 1.3387, + "theoretical_loss": 3.5765120590447292, + "tokens_seen": 1355022336 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047635707121364095, + "loss": 1.2238, + "theoretical_loss": 3.5764966328872427, + "tokens_seen": 1355087872 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047634102306920765, + "loss": 1.4273, + "theoretical_loss": 3.576481207684676, + "tokens_seen": 1355153408 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047632497492477435, + "loss": 1.335, + "theoretical_loss": 3.576465783436923, + "tokens_seen": 1355218944 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047630892678034105, + "loss": 1.325, + "theoretical_loss": 3.576450360143879, + "tokens_seen": 1355284480 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047629287863590775, + "loss": 1.2816, + "theoretical_loss": 3.576434937805439, + "tokens_seen": 1355350016 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047627683049147445, + "loss": 1.3448, + "theoretical_loss": 3.576419516421497, + "tokens_seen": 1355415552 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004762607823470411, + "loss": 1.318, + "theoretical_loss": 3.5764040959919488, + "tokens_seen": 1355481088 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047624473420260785, + "loss": 1.2751, + "theoretical_loss": 3.5763886765166886, + "tokens_seen": 1355546624 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004762286860581745, + "loss": 1.4016, + "theoretical_loss": 3.576373257995611, + "tokens_seen": 1355612160 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047621263791374125, + "loss": 1.4011, + "theoretical_loss": 3.576357840428611, + "tokens_seen": 1355677696 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004761965897693079, + "loss": 1.313, + "theoretical_loss": 3.5763424238155834, + "tokens_seen": 1355743232 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047618054162487465, + "loss": 1.3232, + "theoretical_loss": 3.576327008156423, + "tokens_seen": 1355808768 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004761644934804413, + "loss": 1.2912, + "theoretical_loss": 3.5763115934510257, + "tokens_seen": 1355874304 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047614844533600806, + "loss": 1.3033, + "theoretical_loss": 3.576296179699285, + "tokens_seen": 1355939840 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004761323971915747, + "loss": 1.3214, + "theoretical_loss": 3.576280766901096, + "tokens_seen": 1356005376 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047611634904714146, + "loss": 1.2823, + "theoretical_loss": 3.5762653550563543, + "tokens_seen": 1356070912 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004761003009027081, + "loss": 1.3816, + "theoretical_loss": 3.576249944164955, + "tokens_seen": 1356136448 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047608425275827486, + "loss": 1.4176, + "theoretical_loss": 3.576234534226792, + "tokens_seen": 1356201984 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004760682046138415, + "loss": 1.3429, + "theoretical_loss": 3.576219125241761, + "tokens_seen": 1356267520 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047605215646940826, + "loss": 1.2261, + "theoretical_loss": 3.576203717209757, + "tokens_seen": 1356333056 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004760361083249749, + "loss": 1.3682, + "theoretical_loss": 3.576188310130674, + "tokens_seen": 1356398592 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047602006018054166, + "loss": 1.363, + "theoretical_loss": 3.576172904004409, + "tokens_seen": 1356464128 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004760040120361083, + "loss": 1.2552, + "theoretical_loss": 3.576157498830856, + "tokens_seen": 1356529664 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 504610, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3689444065093994, + "objective/train/theoretical_loss": 3.576142094609909, + "objective/train/tokens_used": 1377055200, + "theoretical_loss": 3.576142094609909, + "tokens_seen": 1356595200 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047598796389167506, + "loss": 1.3689, + "theoretical_loss": 3.576142094609909, + "tokens_seen": 1356595200 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004759719157472417, + "loss": 1.3924, + "theoretical_loss": 3.5761266913414653, + "tokens_seen": 1356660736 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047595586760280846, + "loss": 1.389, + "theoretical_loss": 3.576111289025418, + "tokens_seen": 1356726272 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004759398194583751, + "loss": 1.3605, + "theoretical_loss": 3.576095887661663, + "tokens_seen": 1356791808 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047592377131394186, + "loss": 1.413, + "theoretical_loss": 3.5760804872500955, + "tokens_seen": 1356857344 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004759077231695085, + "loss": 1.3602, + "theoretical_loss": 3.576065087790611, + "tokens_seen": 1356922880 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047589167502507527, + "loss": 1.2657, + "theoretical_loss": 3.5760496892831037, + "tokens_seen": 1356988416 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004758756268806419, + "loss": 1.3901, + "theoretical_loss": 3.5760342917274697, + "tokens_seen": 1357053952 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047585957873620867, + "loss": 1.2773, + "theoretical_loss": 3.576018895123603, + "tokens_seen": 1357119488 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004758435305917753, + "loss": 1.2303, + "theoretical_loss": 3.5760034994714003, + "tokens_seen": 1357185024 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047582748244734207, + "loss": 1.4073, + "theoretical_loss": 3.5759881047707562, + "tokens_seen": 1357250560 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004758114343029087, + "loss": 1.2457, + "theoretical_loss": 3.5759727110215658, + "tokens_seen": 1357316096 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047579538615847547, + "loss": 1.3617, + "theoretical_loss": 3.5759573182237245, + "tokens_seen": 1357381632 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004757793380140421, + "loss": 1.3999, + "theoretical_loss": 3.5759419263771273, + "tokens_seen": 1357447168 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047576328986960887, + "loss": 1.3262, + "theoretical_loss": 3.57592653548167, + "tokens_seen": 1357512704 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004757472417251755, + "loss": 1.3435, + "theoretical_loss": 3.575911145537247, + "tokens_seen": 1357578240 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047573119358074227, + "loss": 1.3865, + "theoretical_loss": 3.5758957565437552, + "tokens_seen": 1357643776 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004757151454363089, + "loss": 1.4123, + "theoretical_loss": 3.5758803685010885, + "tokens_seen": 1357709312 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004756990972918757, + "loss": 1.3022, + "theoretical_loss": 3.5758649814091426, + "tokens_seen": 1357774848 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004756830491474423, + "loss": 1.2743, + "theoretical_loss": 3.5758495952678135, + "tokens_seen": 1357840384 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004756670010030091, + "loss": 1.2625, + "theoretical_loss": 3.5758342100769958, + "tokens_seen": 1357905920 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004756509528585758, + "loss": 1.3185, + "theoretical_loss": 3.5758188258365853, + "tokens_seen": 1357971456 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004756349047141424, + "loss": 1.3254, + "theoretical_loss": 3.5758034425464773, + "tokens_seen": 1358036992 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004756188565697092, + "loss": 1.3401, + "theoretical_loss": 3.5757880602065675, + "tokens_seen": 1358102528 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004756028084252758, + "loss": 1.2874, + "theoretical_loss": 3.575772678816751, + "tokens_seen": 1358168064 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004755867602808426, + "loss": 1.3748, + "theoretical_loss": 3.5757572983769235, + "tokens_seen": 1358233600 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004755707121364092, + "loss": 1.3473, + "theoretical_loss": 3.5757419188869806, + "tokens_seen": 1358299136 + }, + { + "epoch": 0.41, + "learning_rate": 0.000475554663991976, + "loss": 1.2555, + "theoretical_loss": 3.5757265403468175, + "tokens_seen": 1358364672 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004755386158475426, + "loss": 1.3458, + "theoretical_loss": 3.5757111627563294, + "tokens_seen": 1358430208 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004755225677031094, + "loss": 1.2646, + "theoretical_loss": 3.575695786115413, + "tokens_seen": 1358495744 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047550651955867603, + "loss": 1.4207, + "theoretical_loss": 3.5756804104239626, + "tokens_seen": 1358561280 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004754904714142428, + "loss": 1.3334, + "theoretical_loss": 3.575665035681875, + "tokens_seen": 1358626816 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047547442326980943, + "loss": 1.2982, + "theoretical_loss": 3.575649661889045, + "tokens_seen": 1358692352 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004754583751253762, + "loss": 1.3559, + "theoretical_loss": 3.575634289045368, + "tokens_seen": 1358757888 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047544232698094283, + "loss": 1.2803, + "theoretical_loss": 3.5756189171507406, + "tokens_seen": 1358823424 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004754262788365096, + "loss": 1.3208, + "theoretical_loss": 3.5756035462050573, + "tokens_seen": 1358888960 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047541023069207623, + "loss": 1.405, + "theoretical_loss": 3.575588176208214, + "tokens_seen": 1358954496 + }, + { + "epoch": 0.41, + "learning_rate": 0.000475394182547643, + "loss": 1.3208, + "theoretical_loss": 3.5755728071601074, + "tokens_seen": 1359020032 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047537813440320963, + "loss": 1.369, + "theoretical_loss": 3.575557439060632, + "tokens_seen": 1359085568 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004753620862587764, + "loss": 1.4825, + "theoretical_loss": 3.575542071909684, + "tokens_seen": 1359151104 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047534603811434303, + "loss": 1.4192, + "theoretical_loss": 3.5755267057071594, + "tokens_seen": 1359216640 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004753299899699098, + "loss": 1.2503, + "theoretical_loss": 3.5755113404529535, + "tokens_seen": 1359282176 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047531394182547643, + "loss": 1.287, + "theoretical_loss": 3.5754959761469616, + "tokens_seen": 1359347712 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004752978936810432, + "loss": 1.3619, + "theoretical_loss": 3.5754806127890806, + "tokens_seen": 1359413248 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047528184553660984, + "loss": 1.2385, + "theoretical_loss": 3.5754652503792057, + "tokens_seen": 1359478784 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004752657973921766, + "loss": 1.383, + "theoretical_loss": 3.575449888917233, + "tokens_seen": 1359544320 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047524974924774324, + "loss": 1.3498, + "theoretical_loss": 3.5754345284030573, + "tokens_seen": 1359609856 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047523370110331, + "loss": 1.2882, + "theoretical_loss": 3.575419168836576, + "tokens_seen": 1359675392 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047521765295887664, + "loss": 1.3445, + "theoretical_loss": 3.5754038102176837, + "tokens_seen": 1359740928 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004752016048144434, + "loss": 1.2324, + "theoretical_loss": 3.575388452546277, + "tokens_seen": 1359806464 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047518555667001004, + "loss": 1.3445, + "theoretical_loss": 3.5753730958222514, + "tokens_seen": 1359872000 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004751695085255768, + "loss": 1.3003, + "theoretical_loss": 3.5753577400455026, + "tokens_seen": 1359937536 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047515346038114344, + "loss": 1.3646, + "theoretical_loss": 3.575342385215927, + "tokens_seen": 1360003072 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004751374122367102, + "loss": 1.2249, + "theoretical_loss": 3.575327031333421, + "tokens_seen": 1360068608 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047512136409227684, + "loss": 1.4736, + "theoretical_loss": 3.5753116783978793, + "tokens_seen": 1360134144 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004751053159478436, + "loss": 1.2682, + "theoretical_loss": 3.5752963264091986, + "tokens_seen": 1360199680 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047508926780341024, + "loss": 1.2617, + "theoretical_loss": 3.575280975367275, + "tokens_seen": 1360265216 + }, + { + "epoch": 0.41, + "learning_rate": 0.000475073219658977, + "loss": 1.4053, + "theoretical_loss": 3.5752656252720048, + "tokens_seen": 1360330752 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047505717151454364, + "loss": 1.4366, + "theoretical_loss": 3.5752502761232825, + "tokens_seen": 1360396288 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004750411233701104, + "loss": 1.3832, + "theoretical_loss": 3.575234927921006, + "tokens_seen": 1360461824 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047502507522567705, + "loss": 1.18, + "theoretical_loss": 3.57521958066507, + "tokens_seen": 1360527360 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047500902708124375, + "loss": 1.3697, + "theoretical_loss": 3.575204234355371, + "tokens_seen": 1360592896 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047499297893681045, + "loss": 1.1362, + "theoretical_loss": 3.5751888889918053, + "tokens_seen": 1360658432 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047497693079237715, + "loss": 1.2937, + "theoretical_loss": 3.575173544574269, + "tokens_seen": 1360723968 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047496088264794385, + "loss": 1.2482, + "theoretical_loss": 3.575158201102658, + "tokens_seen": 1360789504 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047494483450351055, + "loss": 1.3542, + "theoretical_loss": 3.5751428585768688, + "tokens_seen": 1360855040 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047492878635907725, + "loss": 1.4152, + "theoretical_loss": 3.5751275169967975, + "tokens_seen": 1360920576 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047491273821464395, + "loss": 1.3885, + "theoretical_loss": 3.5751121763623397, + "tokens_seen": 1360986112 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047489669007021065, + "loss": 1.3259, + "theoretical_loss": 3.5750968366733917, + "tokens_seen": 1361051648 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047488064192577735, + "loss": 1.312, + "theoretical_loss": 3.57508149792985, + "tokens_seen": 1361117184 + }, + { + "epoch": 0.41, + "learning_rate": 0.000474864593781344, + "loss": 1.312, + "theoretical_loss": 3.5750661601316116, + "tokens_seen": 1361182720 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047484854563691075, + "loss": 1.3383, + "theoretical_loss": 3.5750508232785707, + "tokens_seen": 1361248256 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004748324974924774, + "loss": 1.3856, + "theoretical_loss": 3.5750354873706254, + "tokens_seen": 1361313792 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047481644934804415, + "loss": 1.2762, + "theoretical_loss": 3.5750201524076712, + "tokens_seen": 1361379328 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004748004012036108, + "loss": 1.3203, + "theoretical_loss": 3.5750048183896044, + "tokens_seen": 1361444864 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047478435305917756, + "loss": 1.3774, + "theoretical_loss": 3.5749894853163218, + "tokens_seen": 1361510400 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004747683049147442, + "loss": 1.305, + "theoretical_loss": 3.5749741531877186, + "tokens_seen": 1361575936 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047475225677031096, + "loss": 1.3577, + "theoretical_loss": 3.5749588220036923, + "tokens_seen": 1361641472 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004747362086258776, + "loss": 1.2167, + "theoretical_loss": 3.574943491764139, + "tokens_seen": 1361707008 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047472016048144436, + "loss": 1.3095, + "theoretical_loss": 3.5749281624689546, + "tokens_seen": 1361772544 + }, + { + "epoch": 0.41, + "learning_rate": 0.000474704112337011, + "loss": 1.3813, + "theoretical_loss": 3.5749128341180354, + "tokens_seen": 1361838080 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047468806419257776, + "loss": 1.3288, + "theoretical_loss": 3.5748975067112783, + "tokens_seen": 1361903616 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004746720160481444, + "loss": 1.3904, + "theoretical_loss": 3.5748821802485797, + "tokens_seen": 1361969152 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047465596790371116, + "loss": 1.3665, + "theoretical_loss": 3.574866854729836, + "tokens_seen": 1362034688 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004746399197592778, + "loss": 1.4105, + "theoretical_loss": 3.5748515301549433, + "tokens_seen": 1362100224 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047462387161484456, + "loss": 1.3434, + "theoretical_loss": 3.5748362065237984, + "tokens_seen": 1362165760 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004746078234704112, + "loss": 1.3924, + "theoretical_loss": 3.5748208838362974, + "tokens_seen": 1362231296 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047459177532597796, + "loss": 1.3143, + "theoretical_loss": 3.574805562092337, + "tokens_seen": 1362296832 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004745757271815446, + "loss": 1.3317, + "theoretical_loss": 3.574790241291814, + "tokens_seen": 1362362368 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047455967903711136, + "loss": 1.3547, + "theoretical_loss": 3.574774921434625, + "tokens_seen": 1362427904 + }, + { + "epoch": 0.41, + "learning_rate": 0.000474543630892678, + "loss": 1.4376, + "theoretical_loss": 3.5747596025206656, + "tokens_seen": 1362493440 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047452758274824477, + "loss": 1.2675, + "theoretical_loss": 3.574744284549834, + "tokens_seen": 1362558976 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004745115346038114, + "loss": 1.4617, + "theoretical_loss": 3.5747289675220246, + "tokens_seen": 1362624512 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047449548645937817, + "loss": 1.2691, + "theoretical_loss": 3.5747136514371363, + "tokens_seen": 1362690048 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004744794383149448, + "loss": 1.3684, + "theoretical_loss": 3.5746983362950635, + "tokens_seen": 1362755584 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047446339017051157, + "loss": 1.3315, + "theoretical_loss": 3.5746830220957047, + "tokens_seen": 1362821120 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004744473420260783, + "loss": 1.3518, + "theoretical_loss": 3.5746677088389553, + "tokens_seen": 1362886656 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047443129388164497, + "loss": 1.2479, + "theoretical_loss": 3.574652396524713, + "tokens_seen": 1362952192 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004744152457372117, + "loss": 1.3815, + "theoretical_loss": 3.5746370851528733, + "tokens_seen": 1363017728 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047439919759277837, + "loss": 1.3417, + "theoretical_loss": 3.5746217747233335, + "tokens_seen": 1363083264 + }, + { + "epoch": 0.41, + "objective/train/docs_used": 506933, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2755460739135742, + "objective/train/theoretical_loss": 3.5746064652359903, + "objective/train/tokens_used": 1383608800, + "theoretical_loss": 3.5746064652359903, + "tokens_seen": 1363148800 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004743831494483451, + "loss": 1.2755, + "theoretical_loss": 3.5746064652359903, + "tokens_seen": 1363148800 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047436710130391177, + "loss": 1.3459, + "theoretical_loss": 3.574591156690741, + "tokens_seen": 1363214336 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004743510531594785, + "loss": 1.318, + "theoretical_loss": 3.574575849087481, + "tokens_seen": 1363279872 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004743350050150452, + "loss": 1.3719, + "theoretical_loss": 3.5745605424261084, + "tokens_seen": 1363345408 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004743189568706119, + "loss": 1.3463, + "theoretical_loss": 3.574545236706519, + "tokens_seen": 1363410944 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004743029087261786, + "loss": 1.3077, + "theoretical_loss": 3.57452993192861, + "tokens_seen": 1363476480 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004742868605817453, + "loss": 1.3513, + "theoretical_loss": 3.5745146280922784, + "tokens_seen": 1363542016 + }, + { + "epoch": 0.41, + "learning_rate": 0.000474270812437312, + "loss": 1.3576, + "theoretical_loss": 3.5744993251974204, + "tokens_seen": 1363607552 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004742547642928787, + "loss": 1.3845, + "theoretical_loss": 3.5744840232439334, + "tokens_seen": 1363673088 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004742387161484453, + "loss": 1.4352, + "theoretical_loss": 3.5744687222317144, + "tokens_seen": 1363738624 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004742226680040121, + "loss": 1.3912, + "theoretical_loss": 3.5744534221606594, + "tokens_seen": 1363804160 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004742066198595787, + "loss": 1.341, + "theoretical_loss": 3.5744381230306663, + "tokens_seen": 1363869696 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004741905717151455, + "loss": 1.4395, + "theoretical_loss": 3.5744228248416317, + "tokens_seen": 1363935232 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004741745235707121, + "loss": 1.2995, + "theoretical_loss": 3.5744075275934515, + "tokens_seen": 1364000768 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004741584754262789, + "loss": 1.3026, + "theoretical_loss": 3.5743922312860246, + "tokens_seen": 1364066304 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004741424272818455, + "loss": 1.2615, + "theoretical_loss": 3.574376935919246, + "tokens_seen": 1364131840 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004741263791374123, + "loss": 1.282, + "theoretical_loss": 3.574361641493014, + "tokens_seen": 1364197376 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047411033099297893, + "loss": 1.2689, + "theoretical_loss": 3.574346348007225, + "tokens_seen": 1364262912 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004740942828485457, + "loss": 1.2797, + "theoretical_loss": 3.574331055461776, + "tokens_seen": 1364328448 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047407823470411233, + "loss": 1.3042, + "theoretical_loss": 3.574315763856564, + "tokens_seen": 1364393984 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004740621865596791, + "loss": 1.3297, + "theoretical_loss": 3.5743004731914865, + "tokens_seen": 1364459520 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047404613841524573, + "loss": 1.3347, + "theoretical_loss": 3.57428518346644, + "tokens_seen": 1364525056 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004740300902708125, + "loss": 1.2712, + "theoretical_loss": 3.574269894681322, + "tokens_seen": 1364590592 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047401404212637913, + "loss": 1.3474, + "theoretical_loss": 3.5742546068360292, + "tokens_seen": 1364656128 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004739979939819459, + "loss": 1.4591, + "theoretical_loss": 3.574239319930459, + "tokens_seen": 1364721664 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047398194583751253, + "loss": 1.3154, + "theoretical_loss": 3.5742240339645077, + "tokens_seen": 1364787200 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004739658976930793, + "loss": 1.2665, + "theoretical_loss": 3.574208748938074, + "tokens_seen": 1364852736 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047394984954864593, + "loss": 1.2439, + "theoretical_loss": 3.574193464851054, + "tokens_seen": 1364918272 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004739338014042127, + "loss": 1.2604, + "theoretical_loss": 3.5741781817033447, + "tokens_seen": 1364983808 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047391775325977934, + "loss": 1.3372, + "theoretical_loss": 3.5741628994948433, + "tokens_seen": 1365049344 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004739017051153461, + "loss": 1.3382, + "theoretical_loss": 3.5741476182254477, + "tokens_seen": 1365114880 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047388565697091274, + "loss": 1.3015, + "theoretical_loss": 3.5741323378950542, + "tokens_seen": 1365180416 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004738696088264795, + "loss": 1.3576, + "theoretical_loss": 3.5741170585035613, + "tokens_seen": 1365245952 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047385356068204614, + "loss": 1.3596, + "theoretical_loss": 3.5741017800508645, + "tokens_seen": 1365311488 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004738375125376129, + "loss": 1.2894, + "theoretical_loss": 3.574086502536863, + "tokens_seen": 1365377024 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047382146439317954, + "loss": 1.2927, + "theoretical_loss": 3.574071225961452, + "tokens_seen": 1365442560 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004738054162487463, + "loss": 1.4027, + "theoretical_loss": 3.574055950324531, + "tokens_seen": 1365508096 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047378936810431294, + "loss": 1.3568, + "theoretical_loss": 3.574040675625996, + "tokens_seen": 1365573632 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004737733199598797, + "loss": 1.2129, + "theoretical_loss": 3.574025401865744, + "tokens_seen": 1365639168 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047375727181544634, + "loss": 1.2426, + "theoretical_loss": 3.574010129043673, + "tokens_seen": 1365704704 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004737412236710131, + "loss": 1.3304, + "theoretical_loss": 3.5739948571596805, + "tokens_seen": 1365770240 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047372517552657974, + "loss": 1.3808, + "theoretical_loss": 3.573979586213663, + "tokens_seen": 1365835776 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004737091273821465, + "loss": 1.4217, + "theoretical_loss": 3.573964316205519, + "tokens_seen": 1365901312 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047369307923771314, + "loss": 1.3107, + "theoretical_loss": 3.5739490471351454, + "tokens_seen": 1365966848 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004736770310932799, + "loss": 1.4238, + "theoretical_loss": 3.5739337790024397, + "tokens_seen": 1366032384 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047366098294884655, + "loss": 1.2907, + "theoretical_loss": 3.5739185118072987, + "tokens_seen": 1366097920 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004736449348044133, + "loss": 1.4382, + "theoretical_loss": 3.5739032455496207, + "tokens_seen": 1366163456 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047362888665997995, + "loss": 1.2872, + "theoretical_loss": 3.573887980229303, + "tokens_seen": 1366228992 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004736128385155467, + "loss": 1.3945, + "theoretical_loss": 3.5738727158462424, + "tokens_seen": 1366294528 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047359679037111335, + "loss": 1.3358, + "theoretical_loss": 3.5738574524003375, + "tokens_seen": 1366360064 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047358074222668005, + "loss": 1.4122, + "theoretical_loss": 3.573842189891485, + "tokens_seen": 1366425600 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047356469408224675, + "loss": 1.3658, + "theoretical_loss": 3.573826928319583, + "tokens_seen": 1366491136 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047354864593781345, + "loss": 1.361, + "theoretical_loss": 3.573811667684528, + "tokens_seen": 1366556672 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047353259779338015, + "loss": 1.3901, + "theoretical_loss": 3.5737964079862197, + "tokens_seen": 1366622208 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047351654964894685, + "loss": 1.2789, + "theoretical_loss": 3.573781149224553, + "tokens_seen": 1366687744 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047350050150451355, + "loss": 1.4016, + "theoretical_loss": 3.5737658913994275, + "tokens_seen": 1366753280 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047348445336008025, + "loss": 1.2951, + "theoretical_loss": 3.57375063451074, + "tokens_seen": 1366818816 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004734684052156469, + "loss": 1.2926, + "theoretical_loss": 3.573735378558388, + "tokens_seen": 1366884352 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047345235707121365, + "loss": 1.2819, + "theoretical_loss": 3.5737201235422695, + "tokens_seen": 1366949888 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004734363089267803, + "loss": 1.2934, + "theoretical_loss": 3.573704869462282, + "tokens_seen": 1367015424 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047342026078234706, + "loss": 1.3879, + "theoretical_loss": 3.5736896163183234, + "tokens_seen": 1367080960 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004734042126379137, + "loss": 1.3043, + "theoretical_loss": 3.573674364110291, + "tokens_seen": 1367146496 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047338816449348046, + "loss": 1.3697, + "theoretical_loss": 3.573659112838083, + "tokens_seen": 1367212032 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004733721163490471, + "loss": 1.2765, + "theoretical_loss": 3.573643862501597, + "tokens_seen": 1367277568 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047335606820461386, + "loss": 1.3316, + "theoretical_loss": 3.5736286131007304, + "tokens_seen": 1367343104 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004733400200601805, + "loss": 1.3278, + "theoretical_loss": 3.5736133646353805, + "tokens_seen": 1367408640 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047332397191574726, + "loss": 1.3413, + "theoretical_loss": 3.5735981171054467, + "tokens_seen": 1367474176 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004733079237713139, + "loss": 1.2912, + "theoretical_loss": 3.5735828705108252, + "tokens_seen": 1367539712 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047329187562688066, + "loss": 1.5102, + "theoretical_loss": 3.5735676248514148, + "tokens_seen": 1367605248 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004732758274824473, + "loss": 1.3138, + "theoretical_loss": 3.573552380127113, + "tokens_seen": 1367670784 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047325977933801406, + "loss": 1.2421, + "theoretical_loss": 3.5735371363378174, + "tokens_seen": 1367736320 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004732437311935808, + "loss": 1.3819, + "theoretical_loss": 3.5735218934834263, + "tokens_seen": 1367801856 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047322768304914746, + "loss": 1.3484, + "theoretical_loss": 3.573506651563837, + "tokens_seen": 1367867392 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004732116349047142, + "loss": 1.2567, + "theoretical_loss": 3.5734914105789484, + "tokens_seen": 1367932928 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047319558676028086, + "loss": 1.3272, + "theoretical_loss": 3.5734761705286573, + "tokens_seen": 1367998464 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004731795386158476, + "loss": 1.3198, + "theoretical_loss": 3.573460931412862, + "tokens_seen": 1368064000 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047316349047141427, + "loss": 1.2765, + "theoretical_loss": 3.57344569323146, + "tokens_seen": 1368129536 + }, + { + "epoch": 0.41, + "learning_rate": 0.000473147442326981, + "loss": 1.2478, + "theoretical_loss": 3.5734304559843504, + "tokens_seen": 1368195072 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047313139418254767, + "loss": 1.3179, + "theoretical_loss": 3.57341521967143, + "tokens_seen": 1368260608 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004731153460381144, + "loss": 1.2403, + "theoretical_loss": 3.573399984292598, + "tokens_seen": 1368326144 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047309929789368107, + "loss": 1.348, + "theoretical_loss": 3.5733847498477513, + "tokens_seen": 1368391680 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004730832497492478, + "loss": 1.4008, + "theoretical_loss": 3.5733695163367885, + "tokens_seen": 1368457216 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047306720160481447, + "loss": 1.3478, + "theoretical_loss": 3.573354283759607, + "tokens_seen": 1368522752 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004730511534603812, + "loss": 1.2918, + "theoretical_loss": 3.5733390521161055, + "tokens_seen": 1368588288 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047303510531594787, + "loss": 1.2553, + "theoretical_loss": 3.573323821406182, + "tokens_seen": 1368653824 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004730190571715146, + "loss": 1.3256, + "theoretical_loss": 3.5733085916297345, + "tokens_seen": 1368719360 + }, + { + "epoch": 0.41, + "learning_rate": 0.00047300300902708127, + "loss": 1.2417, + "theoretical_loss": 3.5732933627866608, + "tokens_seen": 1368784896 + }, + { + "epoch": 0.41, + "learning_rate": 0.000472986960882648, + "loss": 1.2841, + "theoretical_loss": 3.573278134876859, + "tokens_seen": 1368850432 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004729709127382147, + "loss": 1.2524, + "theoretical_loss": 3.5732629079002276, + "tokens_seen": 1368915968 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004729548645937814, + "loss": 1.2514, + "theoretical_loss": 3.5732476818566647, + "tokens_seen": 1368981504 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004729388164493481, + "loss": 1.3561, + "theoretical_loss": 3.5732324567460685, + "tokens_seen": 1369047040 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004729227683049148, + "loss": 1.2223, + "theoretical_loss": 3.573217232568337, + "tokens_seen": 1369112576 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004729067201604815, + "loss": 1.3839, + "theoretical_loss": 3.5732020093233685, + "tokens_seen": 1369178112 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004728906720160482, + "loss": 1.3273, + "theoretical_loss": 3.573186787011061, + "tokens_seen": 1369243648 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004728746238716149, + "loss": 1.3877, + "theoretical_loss": 3.5731715656313128, + "tokens_seen": 1369309184 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004728585757271816, + "loss": 1.3396, + "theoretical_loss": 3.5731563451840227, + "tokens_seen": 1369374720 + }, + { + "epoch": 0.41, + "learning_rate": 0.0004728425275827483, + "loss": 1.3665, + "theoretical_loss": 3.5731411256690877, + "tokens_seen": 1369440256 + }, + { + "epoch": 0.42, + "learning_rate": 0.000472826479438315, + "loss": 1.393, + "theoretical_loss": 3.573125907086407, + "tokens_seen": 1369505792 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004728104312938816, + "loss": 1.3231, + "theoretical_loss": 3.5731106894358793, + "tokens_seen": 1369571328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004727943831494484, + "loss": 1.2747, + "theoretical_loss": 3.5730954727174016, + "tokens_seen": 1369636864 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 509233, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3990371227264404, + "objective/train/theoretical_loss": 3.573080256930874, + "objective/train/tokens_used": 1390162400, + "theoretical_loss": 3.573080256930874, + "tokens_seen": 1369702400 + }, + { + "epoch": 0.42, + "learning_rate": 0.000472778335005015, + "loss": 1.399, + "theoretical_loss": 3.573080256930874, + "tokens_seen": 1369702400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004727622868605818, + "loss": 1.2675, + "theoretical_loss": 3.573065042076193, + "tokens_seen": 1369767936 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047274623871614843, + "loss": 1.2946, + "theoretical_loss": 3.5730498281532577, + "tokens_seen": 1369833472 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004727301905717152, + "loss": 1.3197, + "theoretical_loss": 3.5730346151619665, + "tokens_seen": 1369899008 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047271414242728183, + "loss": 1.3664, + "theoretical_loss": 3.5730194031022178, + "tokens_seen": 1369964544 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004726980942828486, + "loss": 1.2699, + "theoretical_loss": 3.57300419197391, + "tokens_seen": 1370030080 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047268204613841523, + "loss": 1.2982, + "theoretical_loss": 3.572988981776942, + "tokens_seen": 1370095616 + }, + { + "epoch": 0.42, + "learning_rate": 0.000472665997993982, + "loss": 1.2831, + "theoretical_loss": 3.5729737725112116, + "tokens_seen": 1370161152 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047264994984954863, + "loss": 1.3052, + "theoretical_loss": 3.572958564176617, + "tokens_seen": 1370226688 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004726339017051154, + "loss": 1.2842, + "theoretical_loss": 3.5729433567730573, + "tokens_seen": 1370292224 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047261785356068203, + "loss": 1.3242, + "theoretical_loss": 3.5729281503004304, + "tokens_seen": 1370357760 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004726018054162488, + "loss": 1.3144, + "theoretical_loss": 3.5729129447586354, + "tokens_seen": 1370423296 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047258575727181543, + "loss": 1.2223, + "theoretical_loss": 3.5728977401475706, + "tokens_seen": 1370488832 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004725697091273822, + "loss": 1.3098, + "theoretical_loss": 3.572882536467134, + "tokens_seen": 1370554368 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047255366098294884, + "loss": 1.3608, + "theoretical_loss": 3.572867333717225, + "tokens_seen": 1370619904 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004725376128385156, + "loss": 1.3118, + "theoretical_loss": 3.5728521318977418, + "tokens_seen": 1370685440 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047252156469408224, + "loss": 1.3579, + "theoretical_loss": 3.572836931008583, + "tokens_seen": 1370750976 + }, + { + "epoch": 0.42, + "learning_rate": 0.000472505516549649, + "loss": 1.2289, + "theoretical_loss": 3.572821731049647, + "tokens_seen": 1370816512 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047248946840521564, + "loss": 1.3623, + "theoretical_loss": 3.5728065320208326, + "tokens_seen": 1370882048 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004724734202607824, + "loss": 1.4139, + "theoretical_loss": 3.572791333922038, + "tokens_seen": 1370947584 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047245737211634904, + "loss": 1.3908, + "theoretical_loss": 3.5727761367531627, + "tokens_seen": 1371013120 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004724413239719158, + "loss": 1.3392, + "theoretical_loss": 3.5727609405141045, + "tokens_seen": 1371078656 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047242527582748244, + "loss": 1.3989, + "theoretical_loss": 3.572745745204763, + "tokens_seen": 1371144192 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004724092276830492, + "loss": 1.341, + "theoretical_loss": 3.5727305508250358, + "tokens_seen": 1371209728 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047239317953861584, + "loss": 1.3235, + "theoretical_loss": 3.572715357374822, + "tokens_seen": 1371275264 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004723771313941826, + "loss": 1.3842, + "theoretical_loss": 3.5727001648540204, + "tokens_seen": 1371340800 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047236108324974924, + "loss": 1.4012, + "theoretical_loss": 3.57268497326253, + "tokens_seen": 1371406336 + }, + { + "epoch": 0.42, + "learning_rate": 0.000472345035105316, + "loss": 1.403, + "theoretical_loss": 3.5726697826002494, + "tokens_seen": 1371471872 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047232898696088264, + "loss": 1.4424, + "theoretical_loss": 3.572654592867077, + "tokens_seen": 1371537408 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004723129388164494, + "loss": 1.3377, + "theoretical_loss": 3.572639404062912, + "tokens_seen": 1371602944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047229689067201605, + "loss": 1.3142, + "theoretical_loss": 3.572624216187653, + "tokens_seen": 1371668480 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004722808425275828, + "loss": 1.3724, + "theoretical_loss": 3.5726090292411996, + "tokens_seen": 1371734016 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047226479438314945, + "loss": 1.3937, + "theoretical_loss": 3.5725938432234488, + "tokens_seen": 1371799552 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004722487462387162, + "loss": 1.2729, + "theoretical_loss": 3.572578658134301, + "tokens_seen": 1371865088 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047223269809428285, + "loss": 1.3033, + "theoretical_loss": 3.5725634739736547, + "tokens_seen": 1371930624 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004722166499498496, + "loss": 1.3636, + "theoretical_loss": 3.572548290741408, + "tokens_seen": 1371996160 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047220060180541625, + "loss": 1.3229, + "theoretical_loss": 3.5725331084374616, + "tokens_seen": 1372061696 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047218455366098295, + "loss": 1.3014, + "theoretical_loss": 3.5725179270617127, + "tokens_seen": 1372127232 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047216850551654965, + "loss": 1.3252, + "theoretical_loss": 3.572502746614061, + "tokens_seen": 1372192768 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047215245737211635, + "loss": 1.2903, + "theoretical_loss": 3.572487567094405, + "tokens_seen": 1372258304 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047213640922768305, + "loss": 1.3226, + "theoretical_loss": 3.5724723885026437, + "tokens_seen": 1372323840 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047212036108324975, + "loss": 1.2974, + "theoretical_loss": 3.5724572108386767, + "tokens_seen": 1372389376 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047210431293881645, + "loss": 1.326, + "theoretical_loss": 3.5724420341024024, + "tokens_seen": 1372454912 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047208826479438315, + "loss": 1.3068, + "theoretical_loss": 3.5724268582937198, + "tokens_seen": 1372520448 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047207221664994985, + "loss": 1.3465, + "theoretical_loss": 3.572411683412528, + "tokens_seen": 1372585984 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047205616850551656, + "loss": 1.3779, + "theoretical_loss": 3.5723965094587267, + "tokens_seen": 1372651520 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004720401203610833, + "loss": 1.2395, + "theoretical_loss": 3.5723813364322137, + "tokens_seen": 1372717056 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047202407221664996, + "loss": 1.4497, + "theoretical_loss": 3.5723661643328892, + "tokens_seen": 1372782592 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004720080240722167, + "loss": 1.4084, + "theoretical_loss": 3.5723509931606516, + "tokens_seen": 1372848128 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047199197592778336, + "loss": 1.3181, + "theoretical_loss": 3.5723358229154005, + "tokens_seen": 1372913664 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004719759277833501, + "loss": 1.3805, + "theoretical_loss": 3.5723206535970347, + "tokens_seen": 1372979200 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047195987963891676, + "loss": 1.3241, + "theoretical_loss": 3.5723054852054528, + "tokens_seen": 1373044736 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004719438314944835, + "loss": 1.3877, + "theoretical_loss": 3.572290317740555, + "tokens_seen": 1373110272 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047192778335005016, + "loss": 1.3816, + "theoretical_loss": 3.57227515120224, + "tokens_seen": 1373175808 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004719117352056169, + "loss": 1.3376, + "theoretical_loss": 3.5722599855904065, + "tokens_seen": 1373241344 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047189568706118356, + "loss": 1.2687, + "theoretical_loss": 3.5722448209049547, + "tokens_seen": 1373306880 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004718796389167503, + "loss": 1.3956, + "theoretical_loss": 3.5722296571457828, + "tokens_seen": 1373372416 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047186359077231696, + "loss": 1.3031, + "theoretical_loss": 3.5722144943127905, + "tokens_seen": 1373437952 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004718475426278837, + "loss": 1.2999, + "theoretical_loss": 3.5721993324058774, + "tokens_seen": 1373503488 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047183149448345036, + "loss": 1.3025, + "theoretical_loss": 3.572184171424942, + "tokens_seen": 1373569024 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004718154463390171, + "loss": 1.3342, + "theoretical_loss": 3.572169011369884, + "tokens_seen": 1373634560 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047179939819458377, + "loss": 1.3576, + "theoretical_loss": 3.5721538522406027, + "tokens_seen": 1373700096 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004717833500501505, + "loss": 1.3475, + "theoretical_loss": 3.572138694036997, + "tokens_seen": 1373765632 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047176730190571717, + "loss": 1.2409, + "theoretical_loss": 3.572123536758967, + "tokens_seen": 1373831168 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004717512537612839, + "loss": 1.338, + "theoretical_loss": 3.5721083804064113, + "tokens_seen": 1373896704 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047173520561685057, + "loss": 1.3102, + "theoretical_loss": 3.5720932249792297, + "tokens_seen": 1373962240 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004717191574724173, + "loss": 1.3332, + "theoretical_loss": 3.572078070477321, + "tokens_seen": 1374027776 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047170310932798397, + "loss": 1.4378, + "theoretical_loss": 3.5720629169005855, + "tokens_seen": 1374093312 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004716870611835507, + "loss": 1.3552, + "theoretical_loss": 3.572047764248922, + "tokens_seen": 1374158848 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047167101303911737, + "loss": 1.2945, + "theoretical_loss": 3.5720326125222295, + "tokens_seen": 1374224384 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004716549648946841, + "loss": 1.3533, + "theoretical_loss": 3.5720174617204083, + "tokens_seen": 1374289920 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047163891675025077, + "loss": 1.2844, + "theoretical_loss": 3.5720023118433577, + "tokens_seen": 1374355456 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004716228686058175, + "loss": 1.3998, + "theoretical_loss": 3.5719871628909763, + "tokens_seen": 1374420992 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004716068204613842, + "loss": 1.3579, + "theoretical_loss": 3.5719720148631646, + "tokens_seen": 1374486528 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047159077231695093, + "loss": 1.3977, + "theoretical_loss": 3.5719568677598215, + "tokens_seen": 1374552064 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004715747241725176, + "loss": 1.3521, + "theoretical_loss": 3.5719417215808473, + "tokens_seen": 1374617600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004715586760280843, + "loss": 1.391, + "theoretical_loss": 3.57192657632614, + "tokens_seen": 1374683136 + }, + { + "epoch": 0.42, + "learning_rate": 0.000471542627883651, + "loss": 1.4113, + "theoretical_loss": 3.571911431995601, + "tokens_seen": 1374748672 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004715265797392177, + "loss": 1.2138, + "theoretical_loss": 3.5718962885891283, + "tokens_seen": 1374814208 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004715105315947844, + "loss": 1.4014, + "theoretical_loss": 3.571881146106622, + "tokens_seen": 1374879744 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004714944834503511, + "loss": 1.3126, + "theoretical_loss": 3.5718660045479824, + "tokens_seen": 1374945280 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004714784353059178, + "loss": 1.3772, + "theoretical_loss": 3.5718508639131077, + "tokens_seen": 1375010816 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004714623871614845, + "loss": 1.3009, + "theoretical_loss": 3.5718357242018985, + "tokens_seen": 1375076352 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004714463390170512, + "loss": 1.2095, + "theoretical_loss": 3.5718205854142546, + "tokens_seen": 1375141888 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004714302908726179, + "loss": 1.3845, + "theoretical_loss": 3.571805447550075, + "tokens_seen": 1375207424 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004714142427281845, + "loss": 1.1866, + "theoretical_loss": 3.57179031060926, + "tokens_seen": 1375272960 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004713981945837513, + "loss": 1.3587, + "theoretical_loss": 3.5717751745917083, + "tokens_seen": 1375338496 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047138214643931793, + "loss": 1.3074, + "theoretical_loss": 3.571760039497321, + "tokens_seen": 1375404032 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004713660982948847, + "loss": 1.3769, + "theoretical_loss": 3.571744905325996, + "tokens_seen": 1375469568 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047135005015045133, + "loss": 1.3107, + "theoretical_loss": 3.571729772077635, + "tokens_seen": 1375535104 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004713340020060181, + "loss": 1.2723, + "theoretical_loss": 3.571714639752136, + "tokens_seen": 1375600640 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047131795386158473, + "loss": 1.4162, + "theoretical_loss": 3.5716995083494005, + "tokens_seen": 1375666176 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004713019057171515, + "loss": 1.3573, + "theoretical_loss": 3.5716843778693264, + "tokens_seen": 1375731712 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047128585757271813, + "loss": 1.3128, + "theoretical_loss": 3.571669248311815, + "tokens_seen": 1375797248 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004712698094282849, + "loss": 1.3507, + "theoretical_loss": 3.571654119676765, + "tokens_seen": 1375862784 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047125376128385153, + "loss": 1.3592, + "theoretical_loss": 3.571638991964077, + "tokens_seen": 1375928320 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004712377131394183, + "loss": 1.3497, + "theoretical_loss": 3.571623865173651, + "tokens_seen": 1375993856 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047122166499498493, + "loss": 1.3595, + "theoretical_loss": 3.571608739305386, + "tokens_seen": 1376059392 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004712056168505517, + "loss": 1.3265, + "theoretical_loss": 3.5715936143591827, + "tokens_seen": 1376124928 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047118956870611834, + "loss": 1.3186, + "theoretical_loss": 3.5715784903349403, + "tokens_seen": 1376190464 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 511919, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3057072162628174, + "objective/train/theoretical_loss": 3.5715633672325593, + "objective/train/tokens_used": 1396716000, + "theoretical_loss": 3.5715633672325593, + "tokens_seen": 1376256000 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004711735205616851, + "loss": 1.3057, + "theoretical_loss": 3.5715633672325593, + "tokens_seen": 1376256000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047115747241725174, + "loss": 1.3116, + "theoretical_loss": 3.5715482450519387, + "tokens_seen": 1376321536 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004711414242728185, + "loss": 1.4218, + "theoretical_loss": 3.5715331237929795, + "tokens_seen": 1376387072 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047112537612838514, + "loss": 1.2671, + "theoretical_loss": 3.571518003455581, + "tokens_seen": 1376452608 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004711093279839519, + "loss": 1.2651, + "theoretical_loss": 3.5715028840396434, + "tokens_seen": 1376518144 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047109327983951854, + "loss": 1.4601, + "theoretical_loss": 3.5714877655450668, + "tokens_seen": 1376583680 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004710772316950853, + "loss": 1.3303, + "theoretical_loss": 3.571472647971751, + "tokens_seen": 1376649216 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047106118355065194, + "loss": 1.3254, + "theoretical_loss": 3.571457531319595, + "tokens_seen": 1376714752 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004710451354062187, + "loss": 1.2739, + "theoretical_loss": 3.571442415588501, + "tokens_seen": 1376780288 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047102908726178534, + "loss": 1.3225, + "theoretical_loss": 3.5714273007783675, + "tokens_seen": 1376845824 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004710130391173521, + "loss": 1.3325, + "theoretical_loss": 3.5714121868890953, + "tokens_seen": 1376911360 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047099699097291874, + "loss": 1.3653, + "theoretical_loss": 3.571397073920584, + "tokens_seen": 1376976896 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004709809428284855, + "loss": 1.2956, + "theoretical_loss": 3.5713819618727336, + "tokens_seen": 1377042432 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047096489468405214, + "loss": 1.3381, + "theoretical_loss": 3.571366850745444, + "tokens_seen": 1377107968 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004709488465396189, + "loss": 1.2431, + "theoretical_loss": 3.5713517405386166, + "tokens_seen": 1377173504 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047093279839518555, + "loss": 1.2974, + "theoretical_loss": 3.57133663125215, + "tokens_seen": 1377239040 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004709167502507523, + "loss": 1.3959, + "theoretical_loss": 3.5713215228859454, + "tokens_seen": 1377304576 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047090070210631895, + "loss": 1.3303, + "theoretical_loss": 3.571306415439903, + "tokens_seen": 1377370112 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004708846539618857, + "loss": 1.3909, + "theoretical_loss": 3.5712913089139215, + "tokens_seen": 1377435648 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004708686058174524, + "loss": 1.2787, + "theoretical_loss": 3.5712762033079026, + "tokens_seen": 1377501184 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004708525576730191, + "loss": 1.3855, + "theoretical_loss": 3.571261098621746, + "tokens_seen": 1377566720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004708365095285858, + "loss": 1.392, + "theoretical_loss": 3.571245994855352, + "tokens_seen": 1377632256 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004708204613841525, + "loss": 1.3189, + "theoretical_loss": 3.5712308920086215, + "tokens_seen": 1377697792 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004708044132397192, + "loss": 1.3859, + "theoretical_loss": 3.571215790081453, + "tokens_seen": 1377763328 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047078836509528585, + "loss": 1.3748, + "theoretical_loss": 3.5712006890737484, + "tokens_seen": 1377828864 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004707723169508526, + "loss": 1.4173, + "theoretical_loss": 3.571185588985408, + "tokens_seen": 1377894400 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047075626880641925, + "loss": 1.3699, + "theoretical_loss": 3.571170489816331, + "tokens_seen": 1377959936 + }, + { + "epoch": 0.42, + "learning_rate": 0.000470740220661986, + "loss": 1.4584, + "theoretical_loss": 3.571155391566418, + "tokens_seen": 1378025472 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047072417251755265, + "loss": 1.2975, + "theoretical_loss": 3.57114029423557, + "tokens_seen": 1378091008 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004707081243731194, + "loss": 1.2812, + "theoretical_loss": 3.5711251978236866, + "tokens_seen": 1378156544 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047069207622868605, + "loss": 1.3482, + "theoretical_loss": 3.5711101023306693, + "tokens_seen": 1378222080 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004706760280842528, + "loss": 1.3363, + "theoretical_loss": 3.5710950077564165, + "tokens_seen": 1378287616 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047065997993981946, + "loss": 1.375, + "theoretical_loss": 3.571079914100831, + "tokens_seen": 1378353152 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004706439317953862, + "loss": 1.3471, + "theoretical_loss": 3.5710648213638114, + "tokens_seen": 1378418688 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047062788365095286, + "loss": 1.3864, + "theoretical_loss": 3.5710497295452592, + "tokens_seen": 1378484224 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004706118355065196, + "loss": 1.4107, + "theoretical_loss": 3.5710346386450738, + "tokens_seen": 1378549760 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047059578736208626, + "loss": 1.3843, + "theoretical_loss": 3.571019548663157, + "tokens_seen": 1378615296 + }, + { + "epoch": 0.42, + "learning_rate": 0.000470579739217653, + "loss": 1.3285, + "theoretical_loss": 3.5710044595994077, + "tokens_seen": 1378680832 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047056369107321966, + "loss": 1.3296, + "theoretical_loss": 3.570989371453728, + "tokens_seen": 1378746368 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004705476429287864, + "loss": 1.2689, + "theoretical_loss": 3.5709742842260175, + "tokens_seen": 1378811904 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047053159478435306, + "loss": 1.4323, + "theoretical_loss": 3.570959197916176, + "tokens_seen": 1378877440 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004705155466399198, + "loss": 1.404, + "theoretical_loss": 3.570944112524106, + "tokens_seen": 1378942976 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047049949849548646, + "loss": 1.2897, + "theoretical_loss": 3.5709290280497066, + "tokens_seen": 1379008512 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004704834503510532, + "loss": 1.4583, + "theoretical_loss": 3.5709139444928786, + "tokens_seen": 1379074048 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047046740220661986, + "loss": 1.3838, + "theoretical_loss": 3.5708988618535225, + "tokens_seen": 1379139584 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004704513540621866, + "loss": 1.2354, + "theoretical_loss": 3.5708837801315396, + "tokens_seen": 1379205120 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047043530591775327, + "loss": 1.3134, + "theoretical_loss": 3.57086869932683, + "tokens_seen": 1379270656 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047041925777332, + "loss": 1.4117, + "theoretical_loss": 3.570853619439294, + "tokens_seen": 1379336192 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047040320962888667, + "loss": 1.3577, + "theoretical_loss": 3.5708385404688325, + "tokens_seen": 1379401728 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004703871614844534, + "loss": 1.3392, + "theoretical_loss": 3.5708234624153468, + "tokens_seen": 1379467264 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047037111334002007, + "loss": 1.3189, + "theoretical_loss": 3.5708083852787365, + "tokens_seen": 1379532800 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004703550651955868, + "loss": 1.3501, + "theoretical_loss": 3.570793309058903, + "tokens_seen": 1379598336 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047033901705115347, + "loss": 1.3168, + "theoretical_loss": 3.570778233755747, + "tokens_seen": 1379663872 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004703229689067202, + "loss": 1.3486, + "theoretical_loss": 3.5707631593691693, + "tokens_seen": 1379729408 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047030692076228687, + "loss": 1.3696, + "theoretical_loss": 3.57074808589907, + "tokens_seen": 1379794944 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004702908726178536, + "loss": 1.4136, + "theoretical_loss": 3.5707330133453503, + "tokens_seen": 1379860480 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047027482447342027, + "loss": 1.437, + "theoretical_loss": 3.570717941707911, + "tokens_seen": 1379926016 + }, + { + "epoch": 0.42, + "learning_rate": 0.000470258776328987, + "loss": 1.413, + "theoretical_loss": 3.570702870986653, + "tokens_seen": 1379991552 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047024272818455367, + "loss": 1.359, + "theoretical_loss": 3.5706878011814767, + "tokens_seen": 1380057088 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047022668004012043, + "loss": 1.3371, + "theoretical_loss": 3.570672732292283, + "tokens_seen": 1380122624 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004702106318956871, + "loss": 1.2711, + "theoretical_loss": 3.570657664318973, + "tokens_seen": 1380188160 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047019458375125383, + "loss": 1.2963, + "theoretical_loss": 3.5706425972614477, + "tokens_seen": 1380253696 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004701785356068205, + "loss": 1.3117, + "theoretical_loss": 3.570627531119607, + "tokens_seen": 1380319232 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004701624874623872, + "loss": 1.3786, + "theoretical_loss": 3.570612465893353, + "tokens_seen": 1380384768 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004701464393179539, + "loss": 1.3104, + "theoretical_loss": 3.570597401582586, + "tokens_seen": 1380450304 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004701303911735206, + "loss": 1.3505, + "theoretical_loss": 3.570582338187207, + "tokens_seen": 1380515840 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004701143430290873, + "loss": 1.3493, + "theoretical_loss": 3.5705672757071167, + "tokens_seen": 1380581376 + }, + { + "epoch": 0.42, + "learning_rate": 0.000470098294884654, + "loss": 1.3857, + "theoretical_loss": 3.570552214142216, + "tokens_seen": 1380646912 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004700822467402207, + "loss": 1.3383, + "theoretical_loss": 3.570537153492406, + "tokens_seen": 1380712448 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004700661985957874, + "loss": 1.4309, + "theoretical_loss": 3.5705220937575888, + "tokens_seen": 1380777984 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004700501504513541, + "loss": 1.3362, + "theoretical_loss": 3.5705070349376635, + "tokens_seen": 1380843520 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004700341023069208, + "loss": 1.3107, + "theoretical_loss": 3.570491977032532, + "tokens_seen": 1380909056 + }, + { + "epoch": 0.42, + "learning_rate": 0.00047001805416248743, + "loss": 1.3194, + "theoretical_loss": 3.570476920042095, + "tokens_seen": 1380974592 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004700020060180542, + "loss": 1.3611, + "theoretical_loss": 3.5704618639662544, + "tokens_seen": 1381040128 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046998595787362083, + "loss": 1.3969, + "theoretical_loss": 3.5704468088049106, + "tokens_seen": 1381105664 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004699699097291876, + "loss": 1.4348, + "theoretical_loss": 3.5704317545579647, + "tokens_seen": 1381171200 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046995386158475423, + "loss": 1.2334, + "theoretical_loss": 3.570416701225317, + "tokens_seen": 1381236736 + }, + { + "epoch": 0.42, + "learning_rate": 0.000469937813440321, + "loss": 1.3401, + "theoretical_loss": 3.57040164880687, + "tokens_seen": 1381302272 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046992176529588763, + "loss": 1.4052, + "theoretical_loss": 3.570386597302524, + "tokens_seen": 1381367808 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004699057171514544, + "loss": 1.3339, + "theoretical_loss": 3.57037154671218, + "tokens_seen": 1381433344 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046988966900702103, + "loss": 1.404, + "theoretical_loss": 3.57035649703574, + "tokens_seen": 1381498880 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004698736208625878, + "loss": 1.2547, + "theoretical_loss": 3.570341448273105, + "tokens_seen": 1381564416 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046985757271815443, + "loss": 1.3016, + "theoretical_loss": 3.570326400424175, + "tokens_seen": 1381629952 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004698415245737212, + "loss": 1.3483, + "theoretical_loss": 3.570311353488852, + "tokens_seen": 1381695488 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046982547642928784, + "loss": 1.2808, + "theoretical_loss": 3.5702963074670375, + "tokens_seen": 1381761024 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004698094282848546, + "loss": 1.2409, + "theoretical_loss": 3.570281262358632, + "tokens_seen": 1381826560 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046979338014042124, + "loss": 1.2707, + "theoretical_loss": 3.5702662181635376, + "tokens_seen": 1381892096 + }, + { + "epoch": 0.42, + "learning_rate": 0.000469777331995988, + "loss": 1.3852, + "theoretical_loss": 3.5702511748816548, + "tokens_seen": 1381957632 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046976128385155464, + "loss": 1.3146, + "theoretical_loss": 3.5702361325128846, + "tokens_seen": 1382023168 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004697452357071214, + "loss": 1.3751, + "theoretical_loss": 3.57022109105713, + "tokens_seen": 1382088704 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046972918756268804, + "loss": 1.2817, + "theoretical_loss": 3.5702060505142903, + "tokens_seen": 1382154240 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004697131394182548, + "loss": 1.2621, + "theoretical_loss": 3.570191010884267, + "tokens_seen": 1382219776 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046969709127382144, + "loss": 1.2952, + "theoretical_loss": 3.5701759721669633, + "tokens_seen": 1382285312 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004696810431293882, + "loss": 1.3364, + "theoretical_loss": 3.5701609343622787, + "tokens_seen": 1382350848 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046966499498495495, + "loss": 1.3896, + "theoretical_loss": 3.5701458974701152, + "tokens_seen": 1382416384 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004696489468405216, + "loss": 1.3594, + "theoretical_loss": 3.570130861490374, + "tokens_seen": 1382481920 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046963289869608835, + "loss": 1.3203, + "theoretical_loss": 3.570115826422956, + "tokens_seen": 1382547456 + }, + { + "epoch": 0.42, + "learning_rate": 0.000469616850551655, + "loss": 1.3741, + "theoretical_loss": 3.5701007922677643, + "tokens_seen": 1382612992 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046960080240722175, + "loss": 1.3478, + "theoretical_loss": 3.570085759024699, + "tokens_seen": 1382678528 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004695847542627884, + "loss": 1.3615, + "theoretical_loss": 3.570070726693661, + "tokens_seen": 1382744064 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 514348, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.307570219039917, + "objective/train/theoretical_loss": 3.570055695274553, + "objective/train/tokens_used": 1403269600, + "theoretical_loss": 3.570055695274553, + "tokens_seen": 1382809600 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046956870611835515, + "loss": 1.3076, + "theoretical_loss": 3.570055695274553, + "tokens_seen": 1382809600 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004695526579739218, + "loss": 1.3562, + "theoretical_loss": 3.570040664767276, + "tokens_seen": 1382875136 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004695366098294885, + "loss": 1.3211, + "theoretical_loss": 3.570025635171731, + "tokens_seen": 1382940672 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004695205616850552, + "loss": 1.3842, + "theoretical_loss": 3.5700106064878203, + "tokens_seen": 1383006208 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004695045135406219, + "loss": 1.3712, + "theoretical_loss": 3.569995578715445, + "tokens_seen": 1383071744 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004694884653961886, + "loss": 1.3276, + "theoretical_loss": 3.5699805518545062, + "tokens_seen": 1383137280 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004694724172517553, + "loss": 1.4608, + "theoretical_loss": 3.569965525904906, + "tokens_seen": 1383202816 + }, + { + "epoch": 0.42, + "learning_rate": 0.000469456369107322, + "loss": 1.4588, + "theoretical_loss": 3.569950500866547, + "tokens_seen": 1383268352 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004694403209628887, + "loss": 1.339, + "theoretical_loss": 3.569935476739328, + "tokens_seen": 1383333888 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004694242728184554, + "loss": 1.4518, + "theoretical_loss": 3.569920453523153, + "tokens_seen": 1383399424 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004694082246740221, + "loss": 1.3438, + "theoretical_loss": 3.569905431217923, + "tokens_seen": 1383464960 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046939217652958875, + "loss": 1.4339, + "theoretical_loss": 3.5698904098235387, + "tokens_seen": 1383530496 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004693761283851555, + "loss": 1.3643, + "theoretical_loss": 3.569875389339903, + "tokens_seen": 1383596032 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046936008024072215, + "loss": 1.3943, + "theoretical_loss": 3.569860369766917, + "tokens_seen": 1383661568 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004693440320962889, + "loss": 1.296, + "theoretical_loss": 3.5698453511044823, + "tokens_seen": 1383727104 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046932798395185555, + "loss": 1.3721, + "theoretical_loss": 3.5698303333525003, + "tokens_seen": 1383792640 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004693119358074223, + "loss": 1.2607, + "theoretical_loss": 3.5698153165108732, + "tokens_seen": 1383858176 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046929588766298896, + "loss": 1.3272, + "theoretical_loss": 3.5698003005795025, + "tokens_seen": 1383923712 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004692798395185557, + "loss": 1.2926, + "theoretical_loss": 3.5697852855582903, + "tokens_seen": 1383989248 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046926379137412236, + "loss": 1.3637, + "theoretical_loss": 3.5697702714471378, + "tokens_seen": 1384054784 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004692477432296891, + "loss": 1.3099, + "theoretical_loss": 3.5697552582459466, + "tokens_seen": 1384120320 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046923169508525576, + "loss": 1.3682, + "theoretical_loss": 3.569740245954619, + "tokens_seen": 1384185856 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004692156469408225, + "loss": 1.4145, + "theoretical_loss": 3.5697252345730566, + "tokens_seen": 1384251392 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046919959879638916, + "loss": 1.3158, + "theoretical_loss": 3.5697102241011613, + "tokens_seen": 1384316928 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004691835506519559, + "loss": 1.425, + "theoretical_loss": 3.569695214538835, + "tokens_seen": 1384382464 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046916750250752256, + "loss": 1.3216, + "theoretical_loss": 3.569680205885979, + "tokens_seen": 1384448000 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004691514543630893, + "loss": 1.29, + "theoretical_loss": 3.5696651981424954, + "tokens_seen": 1384513536 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046913540621865596, + "loss": 1.333, + "theoretical_loss": 3.5696501913082863, + "tokens_seen": 1384579072 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004691193580742227, + "loss": 1.2732, + "theoretical_loss": 3.5696351853832535, + "tokens_seen": 1384644608 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046910330992978936, + "loss": 1.4411, + "theoretical_loss": 3.5696201803672984, + "tokens_seen": 1384710144 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004690872617853561, + "loss": 1.3525, + "theoretical_loss": 3.569605176260324, + "tokens_seen": 1384775680 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046907121364092276, + "loss": 1.2996, + "theoretical_loss": 3.569590173062231, + "tokens_seen": 1384841216 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004690551654964895, + "loss": 1.4854, + "theoretical_loss": 3.569575170772922, + "tokens_seen": 1384906752 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046903911735205617, + "loss": 1.3901, + "theoretical_loss": 3.5695601693922985, + "tokens_seen": 1384972288 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004690230692076229, + "loss": 1.3324, + "theoretical_loss": 3.5695451689202633, + "tokens_seen": 1385037824 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046900702106318957, + "loss": 1.2868, + "theoretical_loss": 3.569530169356718, + "tokens_seen": 1385103360 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004689909729187563, + "loss": 1.3132, + "theoretical_loss": 3.569515170701564, + "tokens_seen": 1385168896 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046897492477432297, + "loss": 1.2657, + "theoretical_loss": 3.569500172954704, + "tokens_seen": 1385234432 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004689588766298897, + "loss": 1.3536, + "theoretical_loss": 3.5694851761160398, + "tokens_seen": 1385299968 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046894282848545637, + "loss": 1.4167, + "theoretical_loss": 3.5694701801854736, + "tokens_seen": 1385365504 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004689267803410231, + "loss": 1.3369, + "theoretical_loss": 3.569455185162907, + "tokens_seen": 1385431040 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046891073219658977, + "loss": 1.2751, + "theoretical_loss": 3.5694401910482423, + "tokens_seen": 1385496576 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004688946840521565, + "loss": 1.3334, + "theoretical_loss": 3.569425197841382, + "tokens_seen": 1385562112 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046887863590772317, + "loss": 1.3401, + "theoretical_loss": 3.5694102055422277, + "tokens_seen": 1385627648 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046886258776328993, + "loss": 1.3018, + "theoretical_loss": 3.5693952141506813, + "tokens_seen": 1385693184 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004688465396188566, + "loss": 1.3573, + "theoretical_loss": 3.569380223666646, + "tokens_seen": 1385758720 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046883049147442333, + "loss": 1.3583, + "theoretical_loss": 3.5693652340900233, + "tokens_seen": 1385824256 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046881444332999, + "loss": 1.3873, + "theoretical_loss": 3.569350245420715, + "tokens_seen": 1385889792 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046879839518555673, + "loss": 1.3982, + "theoretical_loss": 3.5693352576586235, + "tokens_seen": 1385955328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004687823470411234, + "loss": 1.3021, + "theoretical_loss": 3.5693202708036513, + "tokens_seen": 1386020864 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004687662988966901, + "loss": 1.2766, + "theoretical_loss": 3.5693052848557003, + "tokens_seen": 1386086400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004687502507522568, + "loss": 1.3087, + "theoretical_loss": 3.569290299814673, + "tokens_seen": 1386151936 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004687342026078235, + "loss": 1.329, + "theoretical_loss": 3.5692753156804713, + "tokens_seen": 1386217472 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004687181544633902, + "loss": 1.2732, + "theoretical_loss": 3.569260332452998, + "tokens_seen": 1386283008 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004687021063189569, + "loss": 1.4239, + "theoretical_loss": 3.569245350132155, + "tokens_seen": 1386348544 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004686860581745236, + "loss": 1.423, + "theoretical_loss": 3.569230368717844, + "tokens_seen": 1386414080 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004686700100300903, + "loss": 1.3174, + "theoretical_loss": 3.569215388209968, + "tokens_seen": 1386479616 + }, + { + "epoch": 0.42, + "learning_rate": 0.000468653961885657, + "loss": 1.3089, + "theoretical_loss": 3.56920040860843, + "tokens_seen": 1386545152 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004686379137412237, + "loss": 1.256, + "theoretical_loss": 3.569185429913131, + "tokens_seen": 1386610688 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046862186559679033, + "loss": 1.3413, + "theoretical_loss": 3.5691704521239735, + "tokens_seen": 1386676224 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004686058174523571, + "loss": 1.3289, + "theoretical_loss": 3.569155475240861, + "tokens_seen": 1386741760 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046858976930792373, + "loss": 1.3027, + "theoretical_loss": 3.5691404992636944, + "tokens_seen": 1386807296 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004685737211634905, + "loss": 1.3084, + "theoretical_loss": 3.569125524192377, + "tokens_seen": 1386872832 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046855767301905713, + "loss": 1.4122, + "theoretical_loss": 3.5691105500268114, + "tokens_seen": 1386938368 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004685416248746239, + "loss": 1.3388, + "theoretical_loss": 3.5690955767669, + "tokens_seen": 1387003904 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046852557673019053, + "loss": 1.3675, + "theoretical_loss": 3.569080604412544, + "tokens_seen": 1387069440 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004685095285857573, + "loss": 1.3449, + "theoretical_loss": 3.569065632963647, + "tokens_seen": 1387134976 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046849348044132393, + "loss": 1.3098, + "theoretical_loss": 3.5690506624201115, + "tokens_seen": 1387200512 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004684774322968907, + "loss": 1.3003, + "theoretical_loss": 3.5690356927818394, + "tokens_seen": 1387266048 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046846138415245744, + "loss": 1.3001, + "theoretical_loss": 3.5690207240487335, + "tokens_seen": 1387331584 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004684453360080241, + "loss": 1.4622, + "theoretical_loss": 3.5690057562206965, + "tokens_seen": 1387397120 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046842928786359084, + "loss": 1.412, + "theoretical_loss": 3.5689907892976303, + "tokens_seen": 1387462656 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004684132397191575, + "loss": 1.2579, + "theoretical_loss": 3.568975823279438, + "tokens_seen": 1387528192 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046839719157472425, + "loss": 1.4374, + "theoretical_loss": 3.5689608581660224, + "tokens_seen": 1387593728 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004683811434302909, + "loss": 1.2835, + "theoretical_loss": 3.5689458939572853, + "tokens_seen": 1387659264 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046836509528585765, + "loss": 1.2982, + "theoretical_loss": 3.5689309306531296, + "tokens_seen": 1387724800 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004683490471414243, + "loss": 1.1982, + "theoretical_loss": 3.5689159682534584, + "tokens_seen": 1387790336 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046833299899699105, + "loss": 1.3998, + "theoretical_loss": 3.568901006758174, + "tokens_seen": 1387855872 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004683169508525577, + "loss": 1.4569, + "theoretical_loss": 3.5688860461671785, + "tokens_seen": 1387921408 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046830090270812445, + "loss": 1.3002, + "theoretical_loss": 3.5688710864803745, + "tokens_seen": 1387986944 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004682848545636911, + "loss": 1.2934, + "theoretical_loss": 3.568856127697666, + "tokens_seen": 1388052480 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046826880641925785, + "loss": 1.3139, + "theoretical_loss": 3.5688411698189544, + "tokens_seen": 1388118016 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004682527582748245, + "loss": 1.3133, + "theoretical_loss": 3.5688262128441433, + "tokens_seen": 1388183552 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046823671013039125, + "loss": 1.2833, + "theoretical_loss": 3.5688112567731345, + "tokens_seen": 1388249088 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004682206619859579, + "loss": 1.4265, + "theoretical_loss": 3.5687963016058313, + "tokens_seen": 1388314624 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046820461384152465, + "loss": 1.3156, + "theoretical_loss": 3.568781347342136, + "tokens_seen": 1388380160 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004681885656970913, + "loss": 1.3489, + "theoretical_loss": 3.568766393981952, + "tokens_seen": 1388445696 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046817251755265805, + "loss": 1.1778, + "theoretical_loss": 3.5687514415251815, + "tokens_seen": 1388511232 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004681564694082247, + "loss": 1.3327, + "theoretical_loss": 3.5687364899717275, + "tokens_seen": 1388576768 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046814042126379146, + "loss": 1.484, + "theoretical_loss": 3.5687215393214933, + "tokens_seen": 1388642304 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004681243731193581, + "loss": 1.3473, + "theoretical_loss": 3.5687065895743806, + "tokens_seen": 1388707840 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004681083249749248, + "loss": 1.4164, + "theoretical_loss": 3.568691640730293, + "tokens_seen": 1388773376 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004680922768304915, + "loss": 1.3657, + "theoretical_loss": 3.5686766927891336, + "tokens_seen": 1388838912 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004680762286860582, + "loss": 1.4299, + "theoretical_loss": 3.568661745750804, + "tokens_seen": 1388904448 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004680601805416249, + "loss": 1.3546, + "theoretical_loss": 3.5686467996152085, + "tokens_seen": 1388969984 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004680441323971916, + "loss": 1.3886, + "theoretical_loss": 3.5686318543822493, + "tokens_seen": 1389035520 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004680280842527583, + "loss": 1.3335, + "theoretical_loss": 3.5686169100518295, + "tokens_seen": 1389101056 + }, + { + "epoch": 0.42, + "learning_rate": 0.000468012036108325, + "loss": 1.3099, + "theoretical_loss": 3.5686019666238513, + "tokens_seen": 1389166592 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046799598796389165, + "loss": 1.3742, + "theoretical_loss": 3.568587024098219, + "tokens_seen": 1389232128 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004679799398194584, + "loss": 1.2583, + "theoretical_loss": 3.5685720824748346, + "tokens_seen": 1389297664 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 516805, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2978594303131104, + "objective/train/theoretical_loss": 3.568557141753601, + "objective/train/tokens_used": 1409823200, + "theoretical_loss": 3.568557141753601, + "tokens_seen": 1389363200 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046796389167502505, + "loss": 1.2979, + "theoretical_loss": 3.568557141753601, + "tokens_seen": 1389363200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004679478435305918, + "loss": 1.4327, + "theoretical_loss": 3.568542201934422, + "tokens_seen": 1389428736 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046793179538615846, + "loss": 1.3143, + "theoretical_loss": 3.5685272630171996, + "tokens_seen": 1389494272 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004679157472417252, + "loss": 1.3657, + "theoretical_loss": 3.568512325001837, + "tokens_seen": 1389559808 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046789969909729186, + "loss": 1.3678, + "theoretical_loss": 3.5684973878882382, + "tokens_seen": 1389625344 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004678836509528586, + "loss": 1.4045, + "theoretical_loss": 3.568482451676305, + "tokens_seen": 1389690880 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046786760280842526, + "loss": 1.3125, + "theoretical_loss": 3.568467516365941, + "tokens_seen": 1389756416 + }, + { + "epoch": 0.42, + "learning_rate": 0.000467851554663992, + "loss": 1.4398, + "theoretical_loss": 3.5684525819570494, + "tokens_seen": 1389821952 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046783550651955866, + "loss": 1.41, + "theoretical_loss": 3.5684376484495335, + "tokens_seen": 1389887488 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004678194583751254, + "loss": 1.4037, + "theoretical_loss": 3.5684227158432953, + "tokens_seen": 1389953024 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046780341023069206, + "loss": 1.4708, + "theoretical_loss": 3.5684077841382393, + "tokens_seen": 1390018560 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004677873620862588, + "loss": 1.3492, + "theoretical_loss": 3.5683928533342675, + "tokens_seen": 1390084096 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046777131394182546, + "loss": 1.2384, + "theoretical_loss": 3.5683779234312833, + "tokens_seen": 1390149632 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004677552657973922, + "loss": 1.3256, + "theoretical_loss": 3.5683629944291906, + "tokens_seen": 1390215168 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046773921765295886, + "loss": 1.2677, + "theoretical_loss": 3.568348066327892, + "tokens_seen": 1390280704 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004677231695085256, + "loss": 1.2435, + "theoretical_loss": 3.56833313912729, + "tokens_seen": 1390346240 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046770712136409226, + "loss": 1.3228, + "theoretical_loss": 3.5683182128272897, + "tokens_seen": 1390411776 + }, + { + "epoch": 0.42, + "learning_rate": 0.000467691073219659, + "loss": 1.3258, + "theoretical_loss": 3.5683032874277925, + "tokens_seen": 1390477312 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046767502507522567, + "loss": 1.3505, + "theoretical_loss": 3.568288362928702, + "tokens_seen": 1390542848 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004676589769307924, + "loss": 1.4418, + "theoretical_loss": 3.5682734393299222, + "tokens_seen": 1390608384 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046764292878635907, + "loss": 1.4107, + "theoretical_loss": 3.568258516631356, + "tokens_seen": 1390673920 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004676268806419258, + "loss": 1.3546, + "theoretical_loss": 3.5682435948329063, + "tokens_seen": 1390739456 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046761083249749247, + "loss": 1.3394, + "theoretical_loss": 3.5682286739344766, + "tokens_seen": 1390804992 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004675947843530592, + "loss": 1.352, + "theoretical_loss": 3.5682137539359706, + "tokens_seen": 1390870528 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046757873620862587, + "loss": 1.305, + "theoretical_loss": 3.5681988348372915, + "tokens_seen": 1390936064 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004675626880641926, + "loss": 1.2855, + "theoretical_loss": 3.568183916638342, + "tokens_seen": 1391001600 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046754663991975927, + "loss": 1.3043, + "theoretical_loss": 3.5681689993390258, + "tokens_seen": 1391067136 + }, + { + "epoch": 0.42, + "learning_rate": 0.000467530591775326, + "loss": 1.3446, + "theoretical_loss": 3.568154082939247, + "tokens_seen": 1391132672 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046751454363089267, + "loss": 1.3726, + "theoretical_loss": 3.5681391674389076, + "tokens_seen": 1391198208 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004674984954864594, + "loss": 1.2586, + "theoretical_loss": 3.568124252837912, + "tokens_seen": 1391263744 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004674824473420261, + "loss": 1.2859, + "theoretical_loss": 3.5681093391361633, + "tokens_seen": 1391329280 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046746639919759283, + "loss": 1.4271, + "theoretical_loss": 3.568094426333565, + "tokens_seen": 1391394816 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004674503510531595, + "loss": 1.3049, + "theoretical_loss": 3.5680795144300212, + "tokens_seen": 1391460352 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046743430290872623, + "loss": 1.2888, + "theoretical_loss": 3.5680646034254337, + "tokens_seen": 1391525888 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004674182547642929, + "loss": 1.371, + "theoretical_loss": 3.5680496933197077, + "tokens_seen": 1391591424 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046740220661985963, + "loss": 1.3525, + "theoretical_loss": 3.5680347841127453, + "tokens_seen": 1391656960 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004673861584754263, + "loss": 1.344, + "theoretical_loss": 3.5680198758044512, + "tokens_seen": 1391722496 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046737011033099303, + "loss": 1.2947, + "theoretical_loss": 3.568004968394728, + "tokens_seen": 1391788032 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004673540621865597, + "loss": 1.3157, + "theoretical_loss": 3.56799006188348, + "tokens_seen": 1391853568 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004673380140421264, + "loss": 1.3583, + "theoretical_loss": 3.5679751562706103, + "tokens_seen": 1391919104 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004673219658976931, + "loss": 1.2443, + "theoretical_loss": 3.5679602515560225, + "tokens_seen": 1391984640 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004673059177532598, + "loss": 1.3716, + "theoretical_loss": 3.5679453477396197, + "tokens_seen": 1392050176 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004672898696088265, + "loss": 1.3156, + "theoretical_loss": 3.5679304448213065, + "tokens_seen": 1392115712 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004672738214643932, + "loss": 1.2818, + "theoretical_loss": 3.5679155428009857, + "tokens_seen": 1392181248 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046725777331995994, + "loss": 1.2897, + "theoretical_loss": 3.5679006416785617, + "tokens_seen": 1392246784 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004672417251755266, + "loss": 1.4612, + "theoretical_loss": 3.567885741453937, + "tokens_seen": 1392312320 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046722567703109334, + "loss": 1.3714, + "theoretical_loss": 3.567870842127016, + "tokens_seen": 1392377856 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046720962888666, + "loss": 1.3122, + "theoretical_loss": 3.567855943697703, + "tokens_seen": 1392443392 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046719358074222674, + "loss": 1.3, + "theoretical_loss": 3.5678410461659, + "tokens_seen": 1392508928 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004671775325977934, + "loss": 1.3352, + "theoretical_loss": 3.567826149531512, + "tokens_seen": 1392574464 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046716148445336014, + "loss": 1.2922, + "theoretical_loss": 3.567811253794442, + "tokens_seen": 1392640000 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004671454363089268, + "loss": 1.3598, + "theoretical_loss": 3.5677963589545953, + "tokens_seen": 1392705536 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046712938816449354, + "loss": 1.3301, + "theoretical_loss": 3.5677814650118735, + "tokens_seen": 1392771072 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004671133400200602, + "loss": 1.3728, + "theoretical_loss": 3.567766571966181, + "tokens_seen": 1392836608 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046709729187562694, + "loss": 1.4485, + "theoretical_loss": 3.5677516798174223, + "tokens_seen": 1392902144 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004670812437311936, + "loss": 1.4254, + "theoretical_loss": 3.567736788565501, + "tokens_seen": 1392967680 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046706519558676034, + "loss": 1.128, + "theoretical_loss": 3.56772189821032, + "tokens_seen": 1393033216 + }, + { + "epoch": 0.42, + "learning_rate": 0.000467049147442327, + "loss": 1.2006, + "theoretical_loss": 3.567707008751784, + "tokens_seen": 1393098752 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046703309929789375, + "loss": 1.4698, + "theoretical_loss": 3.567692120189797, + "tokens_seen": 1393164288 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004670170511534604, + "loss": 1.3123, + "theoretical_loss": 3.567677232524262, + "tokens_seen": 1393229824 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046700100300902715, + "loss": 1.3653, + "theoretical_loss": 3.567662345755083, + "tokens_seen": 1393295360 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004669849548645938, + "loss": 1.3056, + "theoretical_loss": 3.5676474598821644, + "tokens_seen": 1393360896 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046696890672016055, + "loss": 1.3164, + "theoretical_loss": 3.56763257490541, + "tokens_seen": 1393426432 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004669528585757272, + "loss": 1.2833, + "theoretical_loss": 3.567617690824724, + "tokens_seen": 1393491968 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046693681043129395, + "loss": 1.2698, + "theoretical_loss": 3.567602807640009, + "tokens_seen": 1393557504 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004669207622868606, + "loss": 1.3201, + "theoretical_loss": 3.5675879253511704, + "tokens_seen": 1393623040 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046690471414242735, + "loss": 1.3893, + "theoretical_loss": 3.5675730439581113, + "tokens_seen": 1393688576 + }, + { + "epoch": 0.42, + "learning_rate": 0.000466888665997994, + "loss": 1.321, + "theoretical_loss": 3.5675581634607356, + "tokens_seen": 1393754112 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046687261785356075, + "loss": 1.3051, + "theoretical_loss": 3.567543283858948, + "tokens_seen": 1393819648 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004668565697091274, + "loss": 1.3812, + "theoretical_loss": 3.567528405152652, + "tokens_seen": 1393885184 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046684052156469415, + "loss": 1.3736, + "theoretical_loss": 3.5675135273417515, + "tokens_seen": 1393950720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004668244734202608, + "loss": 1.338, + "theoretical_loss": 3.5674986504261508, + "tokens_seen": 1394016256 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046680842527582755, + "loss": 1.2458, + "theoretical_loss": 3.5674837744057544, + "tokens_seen": 1394081792 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004667923771313942, + "loss": 1.2936, + "theoretical_loss": 3.567468899280465, + "tokens_seen": 1394147328 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046677632898696096, + "loss": 1.3215, + "theoretical_loss": 3.5674540250501883, + "tokens_seen": 1394212864 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004667602808425276, + "loss": 1.3492, + "theoretical_loss": 3.5674391517148267, + "tokens_seen": 1394278400 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046674423269809436, + "loss": 1.3968, + "theoretical_loss": 3.567424279274286, + "tokens_seen": 1394343936 + }, + { + "epoch": 0.42, + "learning_rate": 0.000466728184553661, + "loss": 1.3651, + "theoretical_loss": 3.567409407728469, + "tokens_seen": 1394409472 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004667121364092277, + "loss": 1.281, + "theoretical_loss": 3.56739453707728, + "tokens_seen": 1394475008 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004666960882647944, + "loss": 1.2546, + "theoretical_loss": 3.567379667320624, + "tokens_seen": 1394540544 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004666800401203611, + "loss": 1.2913, + "theoretical_loss": 3.5673647984584043, + "tokens_seen": 1394606080 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004666639919759278, + "loss": 1.2969, + "theoretical_loss": 3.567349930490525, + "tokens_seen": 1394671616 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004666479438314945, + "loss": 1.3206, + "theoretical_loss": 3.567335063416891, + "tokens_seen": 1394737152 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004666318956870612, + "loss": 1.3181, + "theoretical_loss": 3.5673201972374065, + "tokens_seen": 1394802688 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004666158475426279, + "loss": 1.331, + "theoretical_loss": 3.567305331951975, + "tokens_seen": 1394868224 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004665997993981946, + "loss": 1.2935, + "theoretical_loss": 3.5672904675605013, + "tokens_seen": 1394933760 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004665837512537613, + "loss": 1.3487, + "theoretical_loss": 3.567275604062889, + "tokens_seen": 1394999296 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046656770310932796, + "loss": 1.4092, + "theoretical_loss": 3.5672607414590427, + "tokens_seen": 1395064832 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004665516549648947, + "loss": 1.323, + "theoretical_loss": 3.5672458797488673, + "tokens_seen": 1395130368 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046653560682046136, + "loss": 1.2688, + "theoretical_loss": 3.567231018932266, + "tokens_seen": 1395195904 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004665195586760281, + "loss": 1.2153, + "theoretical_loss": 3.5672161590091442, + "tokens_seen": 1395261440 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046650351053159476, + "loss": 1.3545, + "theoretical_loss": 3.5672012999794056, + "tokens_seen": 1395326976 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004664874623871615, + "loss": 1.2968, + "theoretical_loss": 3.5671864418429546, + "tokens_seen": 1395392512 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046647141424272816, + "loss": 1.2883, + "theoretical_loss": 3.567171584599695, + "tokens_seen": 1395458048 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004664553660982949, + "loss": 1.3741, + "theoretical_loss": 3.5671567282495324, + "tokens_seen": 1395523584 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046643931795386156, + "loss": 1.2901, + "theoretical_loss": 3.5671418727923703, + "tokens_seen": 1395589120 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004664232698094283, + "loss": 1.2847, + "theoretical_loss": 3.567127018228113, + "tokens_seen": 1395654656 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046640722166499496, + "loss": 1.2737, + "theoretical_loss": 3.5671121645566655, + "tokens_seen": 1395720192 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004663911735205617, + "loss": 1.2533, + "theoretical_loss": 3.567097311777932, + "tokens_seen": 1395785728 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046637512537612836, + "loss": 1.327, + "theoretical_loss": 3.5670824598918163, + "tokens_seen": 1395851264 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 519031, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3424731492996216, + "objective/train/theoretical_loss": 3.5670676088982236, + "objective/train/tokens_used": 1416376800, + "theoretical_loss": 3.5670676088982236, + "tokens_seen": 1395916800 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004663590772316951, + "loss": 1.3425, + "theoretical_loss": 3.5670676088982236, + "tokens_seen": 1395916800 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046634302908726176, + "loss": 1.314, + "theoretical_loss": 3.5670527587970584, + "tokens_seen": 1395982336 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004663269809428285, + "loss": 1.3074, + "theoretical_loss": 3.5670379095882248, + "tokens_seen": 1396047872 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046631093279839517, + "loss": 1.347, + "theoretical_loss": 3.5670230612716276, + "tokens_seen": 1396113408 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004662948846539619, + "loss": 1.3213, + "theoretical_loss": 3.5670082138471706, + "tokens_seen": 1396178944 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046627883650952857, + "loss": 1.2985, + "theoretical_loss": 3.5669933673147596, + "tokens_seen": 1396244480 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004662627883650953, + "loss": 1.2952, + "theoretical_loss": 3.566978521674298, + "tokens_seen": 1396310016 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046624674022066197, + "loss": 1.3722, + "theoretical_loss": 3.566963676925691, + "tokens_seen": 1396375552 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004662306920762287, + "loss": 1.3849, + "theoretical_loss": 3.5669488330688424, + "tokens_seen": 1396441088 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046621464393179537, + "loss": 1.2799, + "theoretical_loss": 3.566933990103658, + "tokens_seen": 1396506624 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004661985957873621, + "loss": 1.2835, + "theoretical_loss": 3.566919148030041, + "tokens_seen": 1396572160 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046618254764292877, + "loss": 1.2214, + "theoretical_loss": 3.5669043068478974, + "tokens_seen": 1396637696 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004661664994984955, + "loss": 1.3022, + "theoretical_loss": 3.5668894665571305, + "tokens_seen": 1396703232 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046615045135406217, + "loss": 1.3323, + "theoretical_loss": 3.566874627157646, + "tokens_seen": 1396768768 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004661344032096289, + "loss": 1.3246, + "theoretical_loss": 3.5668597886493485, + "tokens_seen": 1396834304 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004661183550651956, + "loss": 1.3851, + "theoretical_loss": 3.5668449510321416, + "tokens_seen": 1396899840 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046610230692076233, + "loss": 1.3983, + "theoretical_loss": 3.5668301143059313, + "tokens_seen": 1396965376 + }, + { + "epoch": 0.42, + "learning_rate": 0.000466086258776329, + "loss": 1.3395, + "theoretical_loss": 3.5668152784706213, + "tokens_seen": 1397030912 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046607021063189573, + "loss": 1.3437, + "theoretical_loss": 3.566800443526117, + "tokens_seen": 1397096448 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046605416248746243, + "loss": 1.4844, + "theoretical_loss": 3.5667856094723227, + "tokens_seen": 1397161984 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046603811434302913, + "loss": 1.2337, + "theoretical_loss": 3.566770776309143, + "tokens_seen": 1397227520 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046602206619859583, + "loss": 1.3691, + "theoretical_loss": 3.5667559440364833, + "tokens_seen": 1397293056 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046600601805416253, + "loss": 1.2595, + "theoretical_loss": 3.566741112654248, + "tokens_seen": 1397358592 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046598996990972923, + "loss": 1.3505, + "theoretical_loss": 3.566726282162342, + "tokens_seen": 1397424128 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046597392176529593, + "loss": 1.2619, + "theoretical_loss": 3.5667114525606696, + "tokens_seen": 1397489664 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046595787362086263, + "loss": 1.3575, + "theoretical_loss": 3.566696623849136, + "tokens_seen": 1397555200 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004659418254764293, + "loss": 1.24, + "theoretical_loss": 3.5666817960276465, + "tokens_seen": 1397620736 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046592577733199603, + "loss": 1.3804, + "theoretical_loss": 3.5666669690961057, + "tokens_seen": 1397686272 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004659097291875627, + "loss": 1.2465, + "theoretical_loss": 3.566652143054418, + "tokens_seen": 1397751808 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046589368104312944, + "loss": 1.3509, + "theoretical_loss": 3.5666373179024884, + "tokens_seen": 1397817344 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004658776328986961, + "loss": 1.3506, + "theoretical_loss": 3.566622493640222, + "tokens_seen": 1397882880 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046586158475426284, + "loss": 1.4594, + "theoretical_loss": 3.5666076702675236, + "tokens_seen": 1397948416 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004658455366098295, + "loss": 1.2582, + "theoretical_loss": 3.5665928477842987, + "tokens_seen": 1398013952 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046582948846539624, + "loss": 1.2375, + "theoretical_loss": 3.566578026190451, + "tokens_seen": 1398079488 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004658134403209629, + "loss": 1.3442, + "theoretical_loss": 3.5665632054858865, + "tokens_seen": 1398145024 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046579739217652964, + "loss": 1.3795, + "theoretical_loss": 3.56654838567051, + "tokens_seen": 1398210560 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004657813440320963, + "loss": 1.3074, + "theoretical_loss": 3.5665335667442255, + "tokens_seen": 1398276096 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046576529588766304, + "loss": 1.229, + "theoretical_loss": 3.5665187487069394, + "tokens_seen": 1398341632 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004657492477432297, + "loss": 1.3413, + "theoretical_loss": 3.5665039315585556, + "tokens_seen": 1398407168 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046573319959879644, + "loss": 1.4075, + "theoretical_loss": 3.56648911529898, + "tokens_seen": 1398472704 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004657171514543631, + "loss": 1.4026, + "theoretical_loss": 3.5664742999281174, + "tokens_seen": 1398538240 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046570110330992984, + "loss": 1.308, + "theoretical_loss": 3.566459485445873, + "tokens_seen": 1398603776 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004656850551654965, + "loss": 1.2977, + "theoretical_loss": 3.5664446718521505, + "tokens_seen": 1398669312 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046566900702106325, + "loss": 1.4099, + "theoretical_loss": 3.566429859146857, + "tokens_seen": 1398734848 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004656529588766299, + "loss": 1.3368, + "theoretical_loss": 3.566415047329896, + "tokens_seen": 1398800384 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046563691073219665, + "loss": 1.2476, + "theoretical_loss": 3.566400236401174, + "tokens_seen": 1398865920 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004656208625877633, + "loss": 1.4526, + "theoretical_loss": 3.5663854263605943, + "tokens_seen": 1398931456 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046560481444333005, + "loss": 1.3831, + "theoretical_loss": 3.5663706172080643, + "tokens_seen": 1398996992 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004655887662988967, + "loss": 1.3055, + "theoretical_loss": 3.566355808943487, + "tokens_seen": 1399062528 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046557271815446345, + "loss": 1.2731, + "theoretical_loss": 3.5663410015667694, + "tokens_seen": 1399128064 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004655566700100301, + "loss": 1.2838, + "theoretical_loss": 3.5663261950778153, + "tokens_seen": 1399193600 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046554062186559685, + "loss": 1.1931, + "theoretical_loss": 3.56631138947653, + "tokens_seen": 1399259136 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004655245737211635, + "loss": 1.323, + "theoretical_loss": 3.56629658476282, + "tokens_seen": 1399324672 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046550852557673025, + "loss": 1.3853, + "theoretical_loss": 3.566281780936589, + "tokens_seen": 1399390208 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004654924774322969, + "loss": 1.3408, + "theoretical_loss": 3.566266977997743, + "tokens_seen": 1399455744 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046547642928786365, + "loss": 1.3012, + "theoretical_loss": 3.5662521759461874, + "tokens_seen": 1399521280 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004654603811434303, + "loss": 1.2703, + "theoretical_loss": 3.566237374781827, + "tokens_seen": 1399586816 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046544433299899705, + "loss": 1.2862, + "theoretical_loss": 3.5662225745045677, + "tokens_seen": 1399652352 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004654282848545637, + "loss": 1.2961, + "theoretical_loss": 3.566207775114314, + "tokens_seen": 1399717888 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046541223671013046, + "loss": 1.3395, + "theoretical_loss": 3.5661929766109717, + "tokens_seen": 1399783424 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004653961885656971, + "loss": 1.3395, + "theoretical_loss": 3.5661781789944462, + "tokens_seen": 1399848960 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046538014042126386, + "loss": 1.2486, + "theoretical_loss": 3.5661633822646426, + "tokens_seen": 1399914496 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004653640922768305, + "loss": 1.3583, + "theoretical_loss": 3.566148586421466, + "tokens_seen": 1399980032 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046534804413239726, + "loss": 1.2781, + "theoretical_loss": 3.5661337914648223, + "tokens_seen": 1400045568 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004653319959879639, + "loss": 1.3071, + "theoretical_loss": 3.566118997394617, + "tokens_seen": 1400111104 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004653159478435306, + "loss": 1.418, + "theoretical_loss": 3.566104204210755, + "tokens_seen": 1400176640 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004652998996990973, + "loss": 1.2718, + "theoretical_loss": 3.566089411913142, + "tokens_seen": 1400242176 + }, + { + "epoch": 0.42, + "learning_rate": 0.000465283851554664, + "loss": 1.2509, + "theoretical_loss": 3.5660746205016833, + "tokens_seen": 1400307712 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004652678034102307, + "loss": 1.3938, + "theoretical_loss": 3.5660598299762847, + "tokens_seen": 1400373248 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004652517552657974, + "loss": 1.3815, + "theoretical_loss": 3.566045040336851, + "tokens_seen": 1400438784 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004652357071213641, + "loss": 1.3006, + "theoretical_loss": 3.566030251583288, + "tokens_seen": 1400504320 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004652196589769308, + "loss": 1.3865, + "theoretical_loss": 3.5660154637155013, + "tokens_seen": 1400569856 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004652036108324975, + "loss": 1.2916, + "theoretical_loss": 3.5660006767333963, + "tokens_seen": 1400635392 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004651875626880642, + "loss": 1.3443, + "theoretical_loss": 3.5659858906368784, + "tokens_seen": 1400700928 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046517151454363086, + "loss": 1.3464, + "theoretical_loss": 3.5659711054258536, + "tokens_seen": 1400766464 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004651554663991976, + "loss": 1.4045, + "theoretical_loss": 3.5659563211002268, + "tokens_seen": 1400832000 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046513941825476426, + "loss": 1.3088, + "theoretical_loss": 3.5659415376599037, + "tokens_seen": 1400897536 + }, + { + "epoch": 0.42, + "learning_rate": 0.000465123370110331, + "loss": 1.3167, + "theoretical_loss": 3.5659267551047904, + "tokens_seen": 1400963072 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046510732196589766, + "loss": 1.3169, + "theoretical_loss": 3.565911973434792, + "tokens_seen": 1401028608 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004650912738214644, + "loss": 1.337, + "theoretical_loss": 3.565897192649815, + "tokens_seen": 1401094144 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046507522567703106, + "loss": 1.33, + "theoretical_loss": 3.565882412749763, + "tokens_seen": 1401159680 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004650591775325978, + "loss": 1.2848, + "theoretical_loss": 3.5658676337345434, + "tokens_seen": 1401225216 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046504312938816446, + "loss": 1.4035, + "theoretical_loss": 3.5658528556040614, + "tokens_seen": 1401290752 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004650270812437312, + "loss": 1.2589, + "theoretical_loss": 3.5658380783582224, + "tokens_seen": 1401356288 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046501103309929786, + "loss": 1.2923, + "theoretical_loss": 3.5658233019969328, + "tokens_seen": 1401421824 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004649949849548646, + "loss": 1.3336, + "theoretical_loss": 3.5658085265200974, + "tokens_seen": 1401487360 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046497893681043126, + "loss": 1.3291, + "theoretical_loss": 3.5657937519276226, + "tokens_seen": 1401552896 + }, + { + "epoch": 0.42, + "learning_rate": 0.000464962888665998, + "loss": 1.3308, + "theoretical_loss": 3.565778978219413, + "tokens_seen": 1401618432 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046494684052156467, + "loss": 1.3245, + "theoretical_loss": 3.565764205395376, + "tokens_seen": 1401683968 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004649307923771314, + "loss": 1.3335, + "theoretical_loss": 3.565749433455416, + "tokens_seen": 1401749504 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046491474423269807, + "loss": 1.3348, + "theoretical_loss": 3.5657346623994393, + "tokens_seen": 1401815040 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004648986960882648, + "loss": 1.323, + "theoretical_loss": 3.565719892227351, + "tokens_seen": 1401880576 + }, + { + "epoch": 0.42, + "learning_rate": 0.00046488264794383147, + "loss": 1.265, + "theoretical_loss": 3.5657051229390584, + "tokens_seen": 1401946112 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004648665997993982, + "loss": 1.3571, + "theoretical_loss": 3.565690354534466, + "tokens_seen": 1402011648 + }, + { + "epoch": 0.42, + "learning_rate": 0.000464850551654965, + "loss": 1.3185, + "theoretical_loss": 3.56567558701348, + "tokens_seen": 1402077184 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004648345035105316, + "loss": 1.2633, + "theoretical_loss": 3.5656608203760065, + "tokens_seen": 1402142720 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004648184553660984, + "loss": 1.2693, + "theoretical_loss": 3.5656460546219515, + "tokens_seen": 1402208256 + }, + { + "epoch": 0.42, + "learning_rate": 0.000464802407221665, + "loss": 1.367, + "theoretical_loss": 3.5656312897512197, + "tokens_seen": 1402273792 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004647863590772318, + "loss": 1.3532, + "theoretical_loss": 3.5656165257637182, + "tokens_seen": 1402339328 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004647703109327984, + "loss": 1.3515, + "theoretical_loss": 3.5656017626593526, + "tokens_seen": 1402404864 + }, + { + "epoch": 0.42, + "objective/train/docs_used": 521467, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3597064018249512, + "objective/train/theoretical_loss": 3.565587000438028, + "objective/train/tokens_used": 1422930400, + "theoretical_loss": 3.565587000438028, + "tokens_seen": 1402470400 + }, + { + "epoch": 0.42, + "learning_rate": 0.0004647542627883652, + "loss": 1.3597, + "theoretical_loss": 3.565587000438028, + "tokens_seen": 1402470400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046473821464393183, + "loss": 1.3772, + "theoretical_loss": 3.5655722390996516, + "tokens_seen": 1402535936 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004647221664994986, + "loss": 1.3595, + "theoretical_loss": 3.5655574786441284, + "tokens_seen": 1402601472 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046470611835506523, + "loss": 1.3776, + "theoretical_loss": 3.565542719071365, + "tokens_seen": 1402667008 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046469007021063193, + "loss": 1.3746, + "theoretical_loss": 3.5655279603812664, + "tokens_seen": 1402732544 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046467402206619863, + "loss": 1.2982, + "theoretical_loss": 3.5655132025737393, + "tokens_seen": 1402798080 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046465797392176533, + "loss": 1.4482, + "theoretical_loss": 3.5654984456486902, + "tokens_seen": 1402863616 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046464192577733203, + "loss": 1.478, + "theoretical_loss": 3.5654836896060242, + "tokens_seen": 1402929152 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046462587763289873, + "loss": 1.3265, + "theoretical_loss": 3.5654689344456476, + "tokens_seen": 1402994688 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046460982948846543, + "loss": 1.2369, + "theoretical_loss": 3.5654541801674666, + "tokens_seen": 1403060224 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046459378134403213, + "loss": 1.3278, + "theoretical_loss": 3.5654394267713867, + "tokens_seen": 1403125760 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046457773319959883, + "loss": 1.2742, + "theoretical_loss": 3.565424674257315, + "tokens_seen": 1403191296 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046456168505516553, + "loss": 1.4877, + "theoretical_loss": 3.5654099226251565, + "tokens_seen": 1403256832 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004645456369107322, + "loss": 1.2952, + "theoretical_loss": 3.565395171874818, + "tokens_seen": 1403322368 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046452958876629894, + "loss": 1.3011, + "theoretical_loss": 3.5653804220062053, + "tokens_seen": 1403387904 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004645135406218656, + "loss": 1.3814, + "theoretical_loss": 3.565365673019225, + "tokens_seen": 1403453440 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046449749247743234, + "loss": 1.289, + "theoretical_loss": 3.5653509249137825, + "tokens_seen": 1403518976 + }, + { + "epoch": 0.43, + "learning_rate": 0.000464481444332999, + "loss": 1.4867, + "theoretical_loss": 3.565336177689784, + "tokens_seen": 1403584512 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046446539618856574, + "loss": 1.3922, + "theoretical_loss": 3.5653214313471366, + "tokens_seen": 1403650048 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004644493480441324, + "loss": 1.3381, + "theoretical_loss": 3.565306685885745, + "tokens_seen": 1403715584 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046443329989969914, + "loss": 1.2971, + "theoretical_loss": 3.565291941305517, + "tokens_seen": 1403781120 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004644172517552658, + "loss": 1.3881, + "theoretical_loss": 3.5652771976063575, + "tokens_seen": 1403846656 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046440120361083254, + "loss": 1.3929, + "theoretical_loss": 3.5652624547881735, + "tokens_seen": 1403912192 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004643851554663992, + "loss": 1.2763, + "theoretical_loss": 3.565247712850871, + "tokens_seen": 1403977728 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046436910732196594, + "loss": 1.3859, + "theoretical_loss": 3.565232971794356, + "tokens_seen": 1404043264 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004643530591775326, + "loss": 1.2934, + "theoretical_loss": 3.5652182316185357, + "tokens_seen": 1404108800 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046433701103309934, + "loss": 1.3191, + "theoretical_loss": 3.565203492323315, + "tokens_seen": 1404174336 + }, + { + "epoch": 0.43, + "learning_rate": 0.000464320962888666, + "loss": 1.3099, + "theoretical_loss": 3.5651887539086013, + "tokens_seen": 1404239872 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046430491474423274, + "loss": 1.3448, + "theoretical_loss": 3.5651740163743004, + "tokens_seen": 1404305408 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004642888665997994, + "loss": 1.3937, + "theoretical_loss": 3.5651592797203184, + "tokens_seen": 1404370944 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046427281845536615, + "loss": 1.3051, + "theoretical_loss": 3.5651445439465625, + "tokens_seen": 1404436480 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004642567703109328, + "loss": 1.3497, + "theoretical_loss": 3.565129809052938, + "tokens_seen": 1404502016 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046424072216649955, + "loss": 1.3409, + "theoretical_loss": 3.565115075039352, + "tokens_seen": 1404567552 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004642246740220662, + "loss": 1.3614, + "theoretical_loss": 3.565100341905711, + "tokens_seen": 1404633088 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046420862587763295, + "loss": 1.3202, + "theoretical_loss": 3.565085609651921, + "tokens_seen": 1404698624 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004641925777331996, + "loss": 1.2957, + "theoretical_loss": 3.565070878277888, + "tokens_seen": 1404764160 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046417652958876635, + "loss": 1.3277, + "theoretical_loss": 3.5650561477835194, + "tokens_seen": 1404829696 + }, + { + "epoch": 0.43, + "learning_rate": 0.000464160481444333, + "loss": 1.4252, + "theoretical_loss": 3.565041418168721, + "tokens_seen": 1404895232 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046414443329989975, + "loss": 1.3377, + "theoretical_loss": 3.5650266894333993, + "tokens_seen": 1404960768 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004641283851554664, + "loss": 1.5115, + "theoretical_loss": 3.565011961577461, + "tokens_seen": 1405026304 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046411233701103315, + "loss": 1.3088, + "theoretical_loss": 3.5649972346008125, + "tokens_seen": 1405091840 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004640962888665998, + "loss": 1.2481, + "theoretical_loss": 3.56498250850336, + "tokens_seen": 1405157376 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046408024072216655, + "loss": 1.3651, + "theoretical_loss": 3.5649677832850104, + "tokens_seen": 1405222912 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004640641925777332, + "loss": 1.341, + "theoretical_loss": 3.5649530589456697, + "tokens_seen": 1405288448 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046404814443329996, + "loss": 1.2964, + "theoretical_loss": 3.5649383354852455, + "tokens_seen": 1405353984 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004640320962888666, + "loss": 1.2671, + "theoretical_loss": 3.564923612903643, + "tokens_seen": 1405419520 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046401604814443336, + "loss": 1.3461, + "theoretical_loss": 3.56490889120077, + "tokens_seen": 1405485056 + }, + { + "epoch": 0.43, + "learning_rate": 0.000464, + "loss": 1.3043, + "theoretical_loss": 3.5648941703765322, + "tokens_seen": 1405550592 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046398395185556676, + "loss": 1.4168, + "theoretical_loss": 3.5648794504308365, + "tokens_seen": 1405616128 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004639679037111334, + "loss": 1.3962, + "theoretical_loss": 3.5648647313635893, + "tokens_seen": 1405681664 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046395185556670016, + "loss": 1.434, + "theoretical_loss": 3.5648500131746976, + "tokens_seen": 1405747200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004639358074222668, + "loss": 1.2422, + "theoretical_loss": 3.5648352958640688, + "tokens_seen": 1405812736 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004639197592778335, + "loss": 1.3435, + "theoretical_loss": 3.5648205794316077, + "tokens_seen": 1405878272 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004639037111334002, + "loss": 1.3265, + "theoretical_loss": 3.564805863877222, + "tokens_seen": 1405943808 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004638876629889669, + "loss": 1.3414, + "theoretical_loss": 3.5647911492008184, + "tokens_seen": 1406009344 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004638716148445336, + "loss": 1.3311, + "theoretical_loss": 3.5647764354023037, + "tokens_seen": 1406074880 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004638555667001003, + "loss": 1.3968, + "theoretical_loss": 3.5647617224815837, + "tokens_seen": 1406140416 + }, + { + "epoch": 0.43, + "learning_rate": 0.000463839518555667, + "loss": 1.3256, + "theoretical_loss": 3.5647470104385666, + "tokens_seen": 1406205952 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004638234704112337, + "loss": 1.3141, + "theoretical_loss": 3.564732299273158, + "tokens_seen": 1406271488 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004638074222668004, + "loss": 1.3753, + "theoretical_loss": 3.564717588985265, + "tokens_seen": 1406337024 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004637913741223671, + "loss": 1.3386, + "theoretical_loss": 3.564702879574795, + "tokens_seen": 1406402560 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046377532597793376, + "loss": 1.3091, + "theoretical_loss": 3.564688171041653, + "tokens_seen": 1406468096 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004637592778335005, + "loss": 1.2074, + "theoretical_loss": 3.5646734633857475, + "tokens_seen": 1406533632 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046374322968906716, + "loss": 1.3846, + "theoretical_loss": 3.5646587566069856, + "tokens_seen": 1406599168 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004637271815446339, + "loss": 1.3695, + "theoretical_loss": 3.5646440507052723, + "tokens_seen": 1406664704 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046371113340020056, + "loss": 1.2668, + "theoretical_loss": 3.5646293456805154, + "tokens_seen": 1406730240 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004636950852557673, + "loss": 1.3399, + "theoretical_loss": 3.5646146415326223, + "tokens_seen": 1406795776 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046367903711133396, + "loss": 1.3307, + "theoretical_loss": 3.564599938261499, + "tokens_seen": 1406861312 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004636629889669007, + "loss": 1.3605, + "theoretical_loss": 3.564585235867053, + "tokens_seen": 1406926848 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046364694082246747, + "loss": 1.3565, + "theoretical_loss": 3.5645705343491914, + "tokens_seen": 1406992384 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004636308926780341, + "loss": 1.3105, + "theoretical_loss": 3.5645558337078196, + "tokens_seen": 1407057920 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046361484453360087, + "loss": 1.3242, + "theoretical_loss": 3.5645411339428463, + "tokens_seen": 1407123456 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004635987963891675, + "loss": 1.5686, + "theoretical_loss": 3.5645264350541774, + "tokens_seen": 1407188992 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004635827482447343, + "loss": 1.4037, + "theoretical_loss": 3.56451173704172, + "tokens_seen": 1407254528 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004635667001003009, + "loss": 1.3177, + "theoretical_loss": 3.5644970399053815, + "tokens_seen": 1407320064 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004635506519558677, + "loss": 1.405, + "theoretical_loss": 3.5644823436450688, + "tokens_seen": 1407385600 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004635346038114343, + "loss": 1.3285, + "theoretical_loss": 3.5644676482606883, + "tokens_seen": 1407451136 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004635185556670011, + "loss": 1.4036, + "theoretical_loss": 3.5644529537521477, + "tokens_seen": 1407516672 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004635025075225677, + "loss": 1.2636, + "theoretical_loss": 3.5644382601193536, + "tokens_seen": 1407582208 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004634864593781345, + "loss": 1.3997, + "theoretical_loss": 3.564423567362213, + "tokens_seen": 1407647744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004634704112337011, + "loss": 1.3453, + "theoretical_loss": 3.564408875480633, + "tokens_seen": 1407713280 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004634543630892679, + "loss": 1.2563, + "theoretical_loss": 3.564394184474521, + "tokens_seen": 1407778816 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004634383149448345, + "loss": 1.3264, + "theoretical_loss": 3.5643794943437843, + "tokens_seen": 1407844352 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004634222668004013, + "loss": 1.3332, + "theoretical_loss": 3.5643648050883288, + "tokens_seen": 1407909888 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004634062186559679, + "loss": 1.4523, + "theoretical_loss": 3.564350116708063, + "tokens_seen": 1407975424 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004633901705115347, + "loss": 1.3395, + "theoretical_loss": 3.5643354292028935, + "tokens_seen": 1408040960 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046337412236710133, + "loss": 1.3483, + "theoretical_loss": 3.564320742572727, + "tokens_seen": 1408106496 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004633580742226681, + "loss": 1.3061, + "theoretical_loss": 3.564306056817471, + "tokens_seen": 1408172032 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046334202607823473, + "loss": 1.3773, + "theoretical_loss": 3.564291371937032, + "tokens_seen": 1408237568 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004633259779338015, + "loss": 1.295, + "theoretical_loss": 3.5642766879313186, + "tokens_seen": 1408303104 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046330992978936813, + "loss": 1.3849, + "theoretical_loss": 3.5642620048002374, + "tokens_seen": 1408368640 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046329388164493483, + "loss": 1.3827, + "theoretical_loss": 3.5642473225436953, + "tokens_seen": 1408434176 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046327783350050153, + "loss": 1.3322, + "theoretical_loss": 3.5642326411615994, + "tokens_seen": 1408499712 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046326178535606823, + "loss": 1.2662, + "theoretical_loss": 3.564217960653857, + "tokens_seen": 1408565248 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046324573721163493, + "loss": 1.3079, + "theoretical_loss": 3.564203281020376, + "tokens_seen": 1408630784 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046322968906720163, + "loss": 1.3051, + "theoretical_loss": 3.564188602261063, + "tokens_seen": 1408696320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046321364092276833, + "loss": 1.2846, + "theoretical_loss": 3.564173924375826, + "tokens_seen": 1408761856 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046319759277833503, + "loss": 1.3351, + "theoretical_loss": 3.5641592473645707, + "tokens_seen": 1408827392 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046318154463390174, + "loss": 1.2895, + "theoretical_loss": 3.564144571227206, + "tokens_seen": 1408892928 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046316549648946844, + "loss": 1.3542, + "theoretical_loss": 3.5641298959636387, + "tokens_seen": 1408958464 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 523754, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.329208493232727, + "objective/train/theoretical_loss": 3.5641152215737764, + "objective/train/tokens_used": 1429484000, + "theoretical_loss": 3.5641152215737764, + "tokens_seen": 1409024000 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004631494483450351, + "loss": 1.3292, + "theoretical_loss": 3.5641152215737764, + "tokens_seen": 1409024000 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046313340020060184, + "loss": 1.3223, + "theoretical_loss": 3.5641005480575263, + "tokens_seen": 1409089536 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004631173520561685, + "loss": 1.3911, + "theoretical_loss": 3.5640858754147953, + "tokens_seen": 1409155072 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046310130391173524, + "loss": 1.3743, + "theoretical_loss": 3.564071203645491, + "tokens_seen": 1409220608 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004630852557673019, + "loss": 1.198, + "theoretical_loss": 3.564056532749521, + "tokens_seen": 1409286144 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046306920762286864, + "loss": 1.3251, + "theoretical_loss": 3.5640418627267927, + "tokens_seen": 1409351680 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004630531594784353, + "loss": 1.2791, + "theoretical_loss": 3.5640271935772136, + "tokens_seen": 1409417216 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046303711133400204, + "loss": 1.336, + "theoretical_loss": 3.564012525300691, + "tokens_seen": 1409482752 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004630210631895687, + "loss": 1.4047, + "theoretical_loss": 3.563997857897132, + "tokens_seen": 1409548288 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046300501504513544, + "loss": 1.3, + "theoretical_loss": 3.563983191366445, + "tokens_seen": 1409613824 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004629889669007021, + "loss": 1.2639, + "theoretical_loss": 3.5639685257085363, + "tokens_seen": 1409679360 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046297291875626884, + "loss": 1.3074, + "theoretical_loss": 3.563953860923314, + "tokens_seen": 1409744896 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004629568706118355, + "loss": 1.3739, + "theoretical_loss": 3.5639391970106855, + "tokens_seen": 1409810432 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046294082246740224, + "loss": 1.3583, + "theoretical_loss": 3.5639245339705585, + "tokens_seen": 1409875968 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004629247743229689, + "loss": 1.361, + "theoretical_loss": 3.5639098718028404, + "tokens_seen": 1409941504 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046290872617853565, + "loss": 1.3626, + "theoretical_loss": 3.563895210507439, + "tokens_seen": 1410007040 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004628926780341023, + "loss": 1.332, + "theoretical_loss": 3.563880550084262, + "tokens_seen": 1410072576 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046287662988966905, + "loss": 1.2947, + "theoretical_loss": 3.563865890533216, + "tokens_seen": 1410138112 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004628605817452357, + "loss": 1.3904, + "theoretical_loss": 3.5638512318542093, + "tokens_seen": 1410203648 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046284453360080245, + "loss": 1.444, + "theoretical_loss": 3.563836574047149, + "tokens_seen": 1410269184 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004628284854563691, + "loss": 1.3319, + "theoretical_loss": 3.5638219171119436, + "tokens_seen": 1410334720 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046281243731193585, + "loss": 1.3275, + "theoretical_loss": 3.5638072610485, + "tokens_seen": 1410400256 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004627963891675025, + "loss": 1.3176, + "theoretical_loss": 3.563792605856726, + "tokens_seen": 1410465792 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046278034102306925, + "loss": 1.4438, + "theoretical_loss": 3.5637779515365295, + "tokens_seen": 1410531328 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004627642928786359, + "loss": 1.4999, + "theoretical_loss": 3.563763298087818, + "tokens_seen": 1410596864 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046274824473420265, + "loss": 1.2916, + "theoretical_loss": 3.5637486455104996, + "tokens_seen": 1410662400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004627321965897693, + "loss": 1.3592, + "theoretical_loss": 3.5637339938044814, + "tokens_seen": 1410727936 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046271614844533605, + "loss": 1.2847, + "theoretical_loss": 3.563719342969671, + "tokens_seen": 1410793472 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004627001003009027, + "loss": 1.3549, + "theoretical_loss": 3.5637046930059766, + "tokens_seen": 1410859008 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046268405215646945, + "loss": 1.3477, + "theoretical_loss": 3.563690043913306, + "tokens_seen": 1410924544 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004626680040120361, + "loss": 1.379, + "theoretical_loss": 3.5636753956915665, + "tokens_seen": 1410990080 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046265195586760286, + "loss": 1.4431, + "theoretical_loss": 3.5636607483406664, + "tokens_seen": 1411055616 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004626359077231695, + "loss": 1.3838, + "theoretical_loss": 3.5636461018605132, + "tokens_seen": 1411121152 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046261985957873626, + "loss": 1.3918, + "theoretical_loss": 3.5636314562510147, + "tokens_seen": 1411186688 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004626038114343029, + "loss": 1.2429, + "theoretical_loss": 3.5636168115120785, + "tokens_seen": 1411252224 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046258776328986966, + "loss": 1.2648, + "theoretical_loss": 3.5636021676436123, + "tokens_seen": 1411317760 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004625717151454363, + "loss": 1.244, + "theoretical_loss": 3.5635875246455253, + "tokens_seen": 1411383296 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046255566700100306, + "loss": 1.3634, + "theoretical_loss": 3.5635728825177235, + "tokens_seen": 1411448832 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004625396188565697, + "loss": 1.3342, + "theoretical_loss": 3.5635582412601163, + "tokens_seen": 1411514368 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004625235707121364, + "loss": 1.2653, + "theoretical_loss": 3.5635436008726105, + "tokens_seen": 1411579904 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004625075225677031, + "loss": 1.3149, + "theoretical_loss": 3.5635289613551144, + "tokens_seen": 1411645440 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004624914744232698, + "loss": 1.2444, + "theoretical_loss": 3.563514322707536, + "tokens_seen": 1411710976 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004624754262788365, + "loss": 1.428, + "theoretical_loss": 3.5634996849297833, + "tokens_seen": 1411776512 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004624593781344032, + "loss": 1.3087, + "theoretical_loss": 3.563485048021764, + "tokens_seen": 1411842048 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046244332998996996, + "loss": 1.4097, + "theoretical_loss": 3.5634704119833858, + "tokens_seen": 1411907584 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004624272818455366, + "loss": 1.3603, + "theoretical_loss": 3.563455776814558, + "tokens_seen": 1411973120 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046241123370110337, + "loss": 1.3136, + "theoretical_loss": 3.5634411425151864, + "tokens_seen": 1412038656 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046239518555667, + "loss": 1.3462, + "theoretical_loss": 3.563426509085181, + "tokens_seen": 1412104192 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046237913741223677, + "loss": 1.3554, + "theoretical_loss": 3.5634118765244485, + "tokens_seen": 1412169728 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004623630892678034, + "loss": 1.3869, + "theoretical_loss": 3.5633972448328977, + "tokens_seen": 1412235264 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046234704112337017, + "loss": 1.3766, + "theoretical_loss": 3.563382614010436, + "tokens_seen": 1412300800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004623309929789368, + "loss": 1.4448, + "theoretical_loss": 3.5633679840569723, + "tokens_seen": 1412366336 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046231494483450357, + "loss": 1.3589, + "theoretical_loss": 3.5633533549724143, + "tokens_seen": 1412431872 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004622988966900702, + "loss": 1.4197, + "theoretical_loss": 3.56333872675667, + "tokens_seen": 1412497408 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046228284854563697, + "loss": 1.2912, + "theoretical_loss": 3.563324099409647, + "tokens_seen": 1412562944 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004622668004012036, + "loss": 1.2566, + "theoretical_loss": 3.5633094729312536, + "tokens_seen": 1412628480 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046225075225677037, + "loss": 1.3669, + "theoretical_loss": 3.563294847321399, + "tokens_seen": 1412694016 + }, + { + "epoch": 0.43, + "learning_rate": 0.000462234704112337, + "loss": 1.3736, + "theoretical_loss": 3.5632802225799898, + "tokens_seen": 1412759552 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004622186559679038, + "loss": 1.2574, + "theoretical_loss": 3.563265598706936, + "tokens_seen": 1412825088 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004622026078234704, + "loss": 1.267, + "theoretical_loss": 3.563250975702143, + "tokens_seen": 1412890624 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004621865596790372, + "loss": 1.3351, + "theoretical_loss": 3.5632363535655216, + "tokens_seen": 1412956160 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004621705115346038, + "loss": 1.3539, + "theoretical_loss": 3.563221732296979, + "tokens_seen": 1413021696 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004621544633901706, + "loss": 1.3627, + "theoretical_loss": 3.563207111896424, + "tokens_seen": 1413087232 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004621384152457372, + "loss": 1.4642, + "theoretical_loss": 3.5631924923637635, + "tokens_seen": 1413152768 + }, + { + "epoch": 0.43, + "learning_rate": 0.000462122367101304, + "loss": 1.352, + "theoretical_loss": 3.5631778736989066, + "tokens_seen": 1413218304 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004621063189568706, + "loss": 1.2819, + "theoretical_loss": 3.5631632559017614, + "tokens_seen": 1413283840 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004620902708124374, + "loss": 1.2275, + "theoretical_loss": 3.563148638972236, + "tokens_seen": 1413349376 + }, + { + "epoch": 0.43, + "learning_rate": 0.000462074222668004, + "loss": 1.317, + "theoretical_loss": 3.563134022910239, + "tokens_seen": 1413414912 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004620581745235708, + "loss": 1.3092, + "theoretical_loss": 3.5631194077156785, + "tokens_seen": 1413480448 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004620421263791374, + "loss": 1.2779, + "theoretical_loss": 3.5631047933884634, + "tokens_seen": 1413545984 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004620260782347042, + "loss": 1.3045, + "theoretical_loss": 3.563090179928501, + "tokens_seen": 1413611520 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046201003009027083, + "loss": 1.3505, + "theoretical_loss": 3.5630755673357, + "tokens_seen": 1413677056 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004619939819458376, + "loss": 1.362, + "theoretical_loss": 3.563060955609969, + "tokens_seen": 1413742592 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046197793380140423, + "loss": 1.3617, + "theoretical_loss": 3.5630463447512164, + "tokens_seen": 1413808128 + }, + { + "epoch": 0.43, + "learning_rate": 0.000461961885656971, + "loss": 1.352, + "theoretical_loss": 3.5630317347593503, + "tokens_seen": 1413873664 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046194583751253763, + "loss": 1.2051, + "theoretical_loss": 3.5630171256342793, + "tokens_seen": 1413939200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004619297893681044, + "loss": 1.3972, + "theoretical_loss": 3.563002517375911, + "tokens_seen": 1414004736 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046191374122367103, + "loss": 1.3614, + "theoretical_loss": 3.5629879099841553, + "tokens_seen": 1414070272 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004618976930792378, + "loss": 1.2716, + "theoretical_loss": 3.5629733034589197, + "tokens_seen": 1414135808 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046188164493480443, + "loss": 1.2924, + "theoretical_loss": 3.5629586978001124, + "tokens_seen": 1414201344 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046186559679037113, + "loss": 1.231, + "theoretical_loss": 3.5629440930076424, + "tokens_seen": 1414266880 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046184954864593783, + "loss": 1.3701, + "theoretical_loss": 3.562929489081418, + "tokens_seen": 1414332416 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046183350050150453, + "loss": 1.3187, + "theoretical_loss": 3.562914886021348, + "tokens_seen": 1414397952 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046181745235707123, + "loss": 1.2471, + "theoretical_loss": 3.5629002838273403, + "tokens_seen": 1414463488 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046180140421263794, + "loss": 1.271, + "theoretical_loss": 3.5628856824993034, + "tokens_seen": 1414529024 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046178535606820464, + "loss": 1.3971, + "theoretical_loss": 3.5628710820371463, + "tokens_seen": 1414594560 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046176930792377134, + "loss": 1.274, + "theoretical_loss": 3.5628564824407776, + "tokens_seen": 1414660096 + }, + { + "epoch": 0.43, + "learning_rate": 0.000461753259779338, + "loss": 1.3325, + "theoretical_loss": 3.5628418837101057, + "tokens_seen": 1414725632 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046173721163490474, + "loss": 1.3369, + "theoretical_loss": 3.5628272858450387, + "tokens_seen": 1414791168 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004617211634904714, + "loss": 1.291, + "theoretical_loss": 3.5628126888454856, + "tokens_seen": 1414856704 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046170511534603814, + "loss": 1.3054, + "theoretical_loss": 3.562798092711355, + "tokens_seen": 1414922240 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004616890672016048, + "loss": 1.3955, + "theoretical_loss": 3.5627834974425556, + "tokens_seen": 1414987776 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046167301905717154, + "loss": 1.3381, + "theoretical_loss": 3.5627689030389957, + "tokens_seen": 1415053312 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004616569709127382, + "loss": 1.3516, + "theoretical_loss": 3.5627543095005842, + "tokens_seen": 1415118848 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046164092276830494, + "loss": 1.4086, + "theoretical_loss": 3.56273971682723, + "tokens_seen": 1415184384 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004616248746238716, + "loss": 1.3441, + "theoretical_loss": 3.562725125018841, + "tokens_seen": 1415249920 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046160882647943834, + "loss": 1.3016, + "theoretical_loss": 3.5627105340753267, + "tokens_seen": 1415315456 + }, + { + "epoch": 0.43, + "learning_rate": 0.000461592778335005, + "loss": 1.3541, + "theoretical_loss": 3.5626959439965953, + "tokens_seen": 1415380992 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046157673019057174, + "loss": 1.2824, + "theoretical_loss": 3.5626813547825558, + "tokens_seen": 1415446528 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004615606820461384, + "loss": 1.3103, + "theoretical_loss": 3.562666766433116, + "tokens_seen": 1415512064 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 526162, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3163809776306152, + "objective/train/theoretical_loss": 3.5626521789481864, + "objective/train/tokens_used": 1436037600, + "theoretical_loss": 3.5626521789481864, + "tokens_seen": 1415577600 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046154463390170515, + "loss": 1.3164, + "theoretical_loss": 3.5626521789481864, + "tokens_seen": 1415577600 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004615285857572718, + "loss": 1.4117, + "theoretical_loss": 3.5626375923276745, + "tokens_seen": 1415643136 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046151253761283855, + "loss": 1.3143, + "theoretical_loss": 3.562623006571489, + "tokens_seen": 1415708672 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004614964894684052, + "loss": 1.2076, + "theoretical_loss": 3.5626084216795393, + "tokens_seen": 1415774208 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046148044132397195, + "loss": 1.2925, + "theoretical_loss": 3.5625938376517334, + "tokens_seen": 1415839744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004614643931795386, + "loss": 1.271, + "theoretical_loss": 3.562579254487981, + "tokens_seen": 1415905280 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046144834503510535, + "loss": 1.2652, + "theoretical_loss": 3.5625646721881905, + "tokens_seen": 1415970816 + }, + { + "epoch": 0.43, + "learning_rate": 0.000461432296890672, + "loss": 1.387, + "theoretical_loss": 3.56255009075227, + "tokens_seen": 1416036352 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046141624874623875, + "loss": 1.2576, + "theoretical_loss": 3.5625355101801297, + "tokens_seen": 1416101888 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004614002006018054, + "loss": 1.302, + "theoretical_loss": 3.5625209304716776, + "tokens_seen": 1416167424 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046138415245737215, + "loss": 1.3985, + "theoretical_loss": 3.562506351626823, + "tokens_seen": 1416232960 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004613681043129388, + "loss": 1.268, + "theoretical_loss": 3.5624917736454744, + "tokens_seen": 1416298496 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046135205616850555, + "loss": 1.3214, + "theoretical_loss": 3.5624771965275412, + "tokens_seen": 1416364032 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004613360080240722, + "loss": 1.4271, + "theoretical_loss": 3.5624626202729317, + "tokens_seen": 1416429568 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046131995987963895, + "loss": 1.2925, + "theoretical_loss": 3.5624480448815556, + "tokens_seen": 1416495104 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004613039117352056, + "loss": 1.3611, + "theoretical_loss": 3.562433470353321, + "tokens_seen": 1416560640 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046128786359077236, + "loss": 1.3475, + "theoretical_loss": 3.562418896688137, + "tokens_seen": 1416626176 + }, + { + "epoch": 0.43, + "learning_rate": 0.000461271815446339, + "loss": 1.3112, + "theoretical_loss": 3.5624043238859135, + "tokens_seen": 1416691712 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046125576730190576, + "loss": 1.2868, + "theoretical_loss": 3.5623897519465584, + "tokens_seen": 1416757248 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046123971915747246, + "loss": 1.2953, + "theoretical_loss": 3.5623751808699806, + "tokens_seen": 1416822784 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046122367101303916, + "loss": 1.2282, + "theoretical_loss": 3.5623606106560906, + "tokens_seen": 1416888320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046120762286860586, + "loss": 1.3042, + "theoretical_loss": 3.562346041304796, + "tokens_seen": 1416953856 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046119157472417256, + "loss": 1.3712, + "theoretical_loss": 3.562331472816006, + "tokens_seen": 1417019392 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046117552657973926, + "loss": 1.304, + "theoretical_loss": 3.56231690518963, + "tokens_seen": 1417084928 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046115947843530596, + "loss": 1.3641, + "theoretical_loss": 3.562302338425577, + "tokens_seen": 1417150464 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046114343029087266, + "loss": 1.2728, + "theoretical_loss": 3.5622877725237565, + "tokens_seen": 1417216000 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046112738214643936, + "loss": 1.3634, + "theoretical_loss": 3.5622732074840764, + "tokens_seen": 1417281536 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046111133400200606, + "loss": 1.3645, + "theoretical_loss": 3.562258643306447, + "tokens_seen": 1417347072 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004610952858575727, + "loss": 1.2573, + "theoretical_loss": 3.562244079990777, + "tokens_seen": 1417412608 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046107923771313946, + "loss": 1.3143, + "theoretical_loss": 3.5622295175369754, + "tokens_seen": 1417478144 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004610631895687061, + "loss": 1.2769, + "theoretical_loss": 3.5622149559449516, + "tokens_seen": 1417543680 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046104714142427287, + "loss": 1.4289, + "theoretical_loss": 3.5622003952146146, + "tokens_seen": 1417609216 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004610310932798395, + "loss": 1.3622, + "theoretical_loss": 3.5621858353458737, + "tokens_seen": 1417674752 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046101504513540627, + "loss": 1.2252, + "theoretical_loss": 3.5621712763386375, + "tokens_seen": 1417740288 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004609989969909729, + "loss": 1.3212, + "theoretical_loss": 3.562156718192816, + "tokens_seen": 1417805824 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046098294884653967, + "loss": 1.2315, + "theoretical_loss": 3.5621421609083184, + "tokens_seen": 1417871360 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004609669007021063, + "loss": 1.4518, + "theoretical_loss": 3.5621276044850534, + "tokens_seen": 1417936896 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046095085255767307, + "loss": 1.3207, + "theoretical_loss": 3.5621130489229307, + "tokens_seen": 1418002432 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004609348044132397, + "loss": 1.3645, + "theoretical_loss": 3.5620984942218596, + "tokens_seen": 1418067968 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046091875626880647, + "loss": 1.4116, + "theoretical_loss": 3.562083940381749, + "tokens_seen": 1418133504 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004609027081243731, + "loss": 1.389, + "theoretical_loss": 3.5620693874025076, + "tokens_seen": 1418199040 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046088665997993987, + "loss": 1.3467, + "theoretical_loss": 3.562054835284046, + "tokens_seen": 1418264576 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004608706118355065, + "loss": 1.3373, + "theoretical_loss": 3.5620402840262733, + "tokens_seen": 1418330112 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046085456369107327, + "loss": 1.3271, + "theoretical_loss": 3.5620257336290977, + "tokens_seen": 1418395648 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004608385155466399, + "loss": 1.3998, + "theoretical_loss": 3.5620111840924302, + "tokens_seen": 1418461184 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004608224674022067, + "loss": 1.3629, + "theoretical_loss": 3.5619966354161785, + "tokens_seen": 1418526720 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004608064192577733, + "loss": 1.3916, + "theoretical_loss": 3.561982087600253, + "tokens_seen": 1418592256 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004607903711133401, + "loss": 1.3054, + "theoretical_loss": 3.561967540644563, + "tokens_seen": 1418657792 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004607743229689067, + "loss": 1.2783, + "theoretical_loss": 3.561952994549017, + "tokens_seen": 1418723328 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004607582748244735, + "loss": 1.3661, + "theoretical_loss": 3.5619384493135264, + "tokens_seen": 1418788864 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004607422266800401, + "loss": 1.2818, + "theoretical_loss": 3.5619239049379985, + "tokens_seen": 1418854400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004607261785356069, + "loss": 1.3706, + "theoretical_loss": 3.5619093614223436, + "tokens_seen": 1418919936 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004607101303911735, + "loss": 1.356, + "theoretical_loss": 3.561894818766471, + "tokens_seen": 1418985472 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004606940822467403, + "loss": 1.2135, + "theoretical_loss": 3.5618802769702906, + "tokens_seen": 1419051008 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004606780341023069, + "loss": 1.3391, + "theoretical_loss": 3.561865736033712, + "tokens_seen": 1419116544 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004606619859578737, + "loss": 1.3297, + "theoretical_loss": 3.5618511959566437, + "tokens_seen": 1419182080 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004606459378134403, + "loss": 1.3832, + "theoretical_loss": 3.561836656738996, + "tokens_seen": 1419247616 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004606298896690071, + "loss": 1.3287, + "theoretical_loss": 3.5618221183806784, + "tokens_seen": 1419313152 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046061384152457373, + "loss": 1.221, + "theoretical_loss": 3.5618075808815997, + "tokens_seen": 1419378688 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004605977933801405, + "loss": 1.3223, + "theoretical_loss": 3.5617930442416705, + "tokens_seen": 1419444224 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046058174523570713, + "loss": 1.2964, + "theoretical_loss": 3.5617785084608, + "tokens_seen": 1419509760 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004605656970912739, + "loss": 1.3209, + "theoretical_loss": 3.5617639735388975, + "tokens_seen": 1419575296 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046054964894684053, + "loss": 1.4327, + "theoretical_loss": 3.5617494394758724, + "tokens_seen": 1419640832 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004605336008024073, + "loss": 1.362, + "theoretical_loss": 3.561734906271635, + "tokens_seen": 1419706368 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046051755265797393, + "loss": 1.1641, + "theoretical_loss": 3.561720373926094, + "tokens_seen": 1419771904 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004605015045135407, + "loss": 1.3673, + "theoretical_loss": 3.56170584243916, + "tokens_seen": 1419837440 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046048545636910733, + "loss": 1.3514, + "theoretical_loss": 3.5616913118107423, + "tokens_seen": 1419902976 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046046940822467403, + "loss": 1.3695, + "theoretical_loss": 3.56167678204075, + "tokens_seen": 1419968512 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046045336008024073, + "loss": 1.2296, + "theoretical_loss": 3.561662253129094, + "tokens_seen": 1420034048 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046043731193580744, + "loss": 1.3885, + "theoretical_loss": 3.5616477250756824, + "tokens_seen": 1420099584 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046042126379137414, + "loss": 1.3921, + "theoretical_loss": 3.5616331978804263, + "tokens_seen": 1420165120 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046040521564694084, + "loss": 1.3946, + "theoretical_loss": 3.5616186715432345, + "tokens_seen": 1420230656 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046038916750250754, + "loss": 1.2273, + "theoretical_loss": 3.561604146064017, + "tokens_seen": 1420296192 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046037311935807424, + "loss": 1.2473, + "theoretical_loss": 3.5615896214426837, + "tokens_seen": 1420361728 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046035707121364094, + "loss": 1.2973, + "theoretical_loss": 3.5615750976791447, + "tokens_seen": 1420427264 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046034102306920764, + "loss": 1.3304, + "theoretical_loss": 3.561560574773309, + "tokens_seen": 1420492800 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004603249749247743, + "loss": 1.3584, + "theoretical_loss": 3.5615460527250864, + "tokens_seen": 1420558336 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046030892678034104, + "loss": 1.2116, + "theoretical_loss": 3.5615315315343876, + "tokens_seen": 1420623872 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004602928786359077, + "loss": 1.3706, + "theoretical_loss": 3.561517011201121, + "tokens_seen": 1420689408 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046027683049147444, + "loss": 1.3671, + "theoretical_loss": 3.5615024917251983, + "tokens_seen": 1420754944 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004602607823470411, + "loss": 1.3208, + "theoretical_loss": 3.5614879731065274, + "tokens_seen": 1420820480 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046024473420260784, + "loss": 1.3073, + "theoretical_loss": 3.56147345534502, + "tokens_seen": 1420886016 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004602286860581745, + "loss": 1.3606, + "theoretical_loss": 3.561458938440584, + "tokens_seen": 1420951552 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046021263791374124, + "loss": 1.4037, + "theoretical_loss": 3.5614444223931305, + "tokens_seen": 1421017088 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004601965897693079, + "loss": 1.3027, + "theoretical_loss": 3.561429907202569, + "tokens_seen": 1421082624 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046018054162487465, + "loss": 1.4339, + "theoretical_loss": 3.5614153928688097, + "tokens_seen": 1421148160 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004601644934804413, + "loss": 1.3034, + "theoretical_loss": 3.561400879391763, + "tokens_seen": 1421213696 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046014844533600805, + "loss": 1.291, + "theoretical_loss": 3.5613863667713375, + "tokens_seen": 1421279232 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004601323971915747, + "loss": 1.3695, + "theoretical_loss": 3.561371855007444, + "tokens_seen": 1421344768 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046011634904714145, + "loss": 1.4652, + "theoretical_loss": 3.561357344099992, + "tokens_seen": 1421410304 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004601003009027081, + "loss": 1.3962, + "theoretical_loss": 3.561342834048892, + "tokens_seen": 1421475840 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046008425275827485, + "loss": 1.2901, + "theoretical_loss": 3.5613283248540535, + "tokens_seen": 1421541376 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004600682046138415, + "loss": 1.3191, + "theoretical_loss": 3.561313816515387, + "tokens_seen": 1421606912 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046005215646940825, + "loss": 1.3844, + "theoretical_loss": 3.5612993090328025, + "tokens_seen": 1421672448 + }, + { + "epoch": 0.43, + "learning_rate": 0.000460036108324975, + "loss": 1.3615, + "theoretical_loss": 3.5612848024062096, + "tokens_seen": 1421737984 + }, + { + "epoch": 0.43, + "learning_rate": 0.00046002006018054165, + "loss": 1.3195, + "theoretical_loss": 3.5612702966355183, + "tokens_seen": 1421803520 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004600040120361084, + "loss": 1.2588, + "theoretical_loss": 3.5612557917206393, + "tokens_seen": 1421869056 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045998796389167505, + "loss": 1.2691, + "theoretical_loss": 3.561241287661482, + "tokens_seen": 1421934592 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004599719157472418, + "loss": 1.1993, + "theoretical_loss": 3.561226784457957, + "tokens_seen": 1422000128 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045995586760280845, + "loss": 1.4098, + "theoretical_loss": 3.561212282109974, + "tokens_seen": 1422065664 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 528901, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2613064050674438, + "objective/train/theoretical_loss": 3.5611977806174426, + "objective/train/tokens_used": 1442591200, + "theoretical_loss": 3.5611977806174426, + "tokens_seen": 1422131200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004599398194583752, + "loss": 1.2613, + "theoretical_loss": 3.5611977806174426, + "tokens_seen": 1422131200 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045992377131394186, + "loss": 1.4614, + "theoretical_loss": 3.5611832799802743, + "tokens_seen": 1422196736 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004599077231695086, + "loss": 1.3841, + "theoretical_loss": 3.5611687801983782, + "tokens_seen": 1422262272 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045989167502507526, + "loss": 1.2296, + "theoretical_loss": 3.561154281271665, + "tokens_seen": 1422327808 + }, + { + "epoch": 0.43, + "learning_rate": 0.000459875626880642, + "loss": 1.2395, + "theoretical_loss": 3.561139783200044, + "tokens_seen": 1422393344 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045985957873620866, + "loss": 1.2897, + "theoretical_loss": 3.5611252859834264, + "tokens_seen": 1422458880 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045984353059177536, + "loss": 1.2427, + "theoretical_loss": 3.561110789621722, + "tokens_seen": 1422524416 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045982748244734206, + "loss": 1.221, + "theoretical_loss": 3.561096294114841, + "tokens_seen": 1422589952 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045981143430290876, + "loss": 1.3949, + "theoretical_loss": 3.561081799462693, + "tokens_seen": 1422655488 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045979538615847546, + "loss": 1.2337, + "theoretical_loss": 3.5610673056651896, + "tokens_seen": 1422721024 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045977933801404216, + "loss": 1.3527, + "theoretical_loss": 3.56105281272224, + "tokens_seen": 1422786560 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045976328986960886, + "loss": 1.2724, + "theoretical_loss": 3.561038320633754, + "tokens_seen": 1422852096 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045974724172517556, + "loss": 1.3664, + "theoretical_loss": 3.5610238293996437, + "tokens_seen": 1422917632 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045973119358074226, + "loss": 1.3171, + "theoretical_loss": 3.561009339019818, + "tokens_seen": 1422983168 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045971514543630896, + "loss": 1.3557, + "theoretical_loss": 3.560994849494187, + "tokens_seen": 1423048704 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004596990972918756, + "loss": 1.3409, + "theoretical_loss": 3.5609803608226613, + "tokens_seen": 1423114240 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045968304914744236, + "loss": 1.3353, + "theoretical_loss": 3.560965873005152, + "tokens_seen": 1423179776 + }, + { + "epoch": 0.43, + "learning_rate": 0.000459667001003009, + "loss": 1.364, + "theoretical_loss": 3.5609513860415687, + "tokens_seen": 1423245312 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045965095285857577, + "loss": 1.3614, + "theoretical_loss": 3.5609368999318214, + "tokens_seen": 1423310848 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004596349047141424, + "loss": 1.3888, + "theoretical_loss": 3.5609224146758214, + "tokens_seen": 1423376384 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045961885656970917, + "loss": 1.4738, + "theoretical_loss": 3.560907930273479, + "tokens_seen": 1423441920 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004596028084252758, + "loss": 1.2748, + "theoretical_loss": 3.5608934467247035, + "tokens_seen": 1423507456 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045958676028084257, + "loss": 1.3029, + "theoretical_loss": 3.5608789640294063, + "tokens_seen": 1423572992 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004595707121364092, + "loss": 1.135, + "theoretical_loss": 3.5608644821874975, + "tokens_seen": 1423638528 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045955466399197597, + "loss": 1.2904, + "theoretical_loss": 3.5608500011988875, + "tokens_seen": 1423704064 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004595386158475426, + "loss": 1.2781, + "theoretical_loss": 3.5608355210634874, + "tokens_seen": 1423769600 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045952256770310937, + "loss": 1.4084, + "theoretical_loss": 3.5608210417812067, + "tokens_seen": 1423835136 + }, + { + "epoch": 0.43, + "learning_rate": 0.000459506519558676, + "loss": 1.253, + "theoretical_loss": 3.560806563351956, + "tokens_seen": 1423900672 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045949047141424277, + "loss": 1.27, + "theoretical_loss": 3.5607920857756463, + "tokens_seen": 1423966208 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004594744232698094, + "loss": 1.3286, + "theoretical_loss": 3.5607776090521877, + "tokens_seen": 1424031744 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004594583751253762, + "loss": 1.4164, + "theoretical_loss": 3.5607631331814913, + "tokens_seen": 1424097280 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004594423269809428, + "loss": 1.3805, + "theoretical_loss": 3.5607486581634666, + "tokens_seen": 1424162816 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004594262788365096, + "loss": 1.3715, + "theoretical_loss": 3.560734183998025, + "tokens_seen": 1424228352 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004594102306920762, + "loss": 1.2957, + "theoretical_loss": 3.5607197106850768, + "tokens_seen": 1424293888 + }, + { + "epoch": 0.43, + "learning_rate": 0.000459394182547643, + "loss": 1.3302, + "theoretical_loss": 3.560705238224532, + "tokens_seen": 1424359424 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004593781344032096, + "loss": 1.3183, + "theoretical_loss": 3.560690766616303, + "tokens_seen": 1424424960 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004593620862587764, + "loss": 1.2646, + "theoretical_loss": 3.560676295860298, + "tokens_seen": 1424490496 + }, + { + "epoch": 0.43, + "learning_rate": 0.000459346038114343, + "loss": 1.4003, + "theoretical_loss": 3.560661825956429, + "tokens_seen": 1424556032 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004593299899699098, + "loss": 1.3829, + "theoretical_loss": 3.5606473569046067, + "tokens_seen": 1424621568 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004593139418254764, + "loss": 1.4049, + "theoretical_loss": 3.560632888704741, + "tokens_seen": 1424687104 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004592978936810432, + "loss": 1.236, + "theoretical_loss": 3.560618421356743, + "tokens_seen": 1424752640 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004592818455366098, + "loss": 1.358, + "theoretical_loss": 3.5606039548605235, + "tokens_seen": 1424818176 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004592657973921766, + "loss": 1.4482, + "theoretical_loss": 3.560589489215993, + "tokens_seen": 1424883712 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045924974924774323, + "loss": 1.3307, + "theoretical_loss": 3.560575024423062, + "tokens_seen": 1424949248 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045923370110331, + "loss": 1.3417, + "theoretical_loss": 3.5605605604816413, + "tokens_seen": 1425014784 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045921765295887663, + "loss": 1.3152, + "theoretical_loss": 3.560546097391642, + "tokens_seen": 1425080320 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004592016048144434, + "loss": 1.3194, + "theoretical_loss": 3.560531635152974, + "tokens_seen": 1425145856 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045918555667001003, + "loss": 1.2848, + "theoretical_loss": 3.5605171737655485, + "tokens_seen": 1425211392 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004591695085255768, + "loss": 1.2504, + "theoretical_loss": 3.560502713229277, + "tokens_seen": 1425276928 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045915346038114343, + "loss": 1.2013, + "theoretical_loss": 3.5604882535440696, + "tokens_seen": 1425342464 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004591374122367102, + "loss": 1.2834, + "theoretical_loss": 3.5604737947098366, + "tokens_seen": 1425408000 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045912136409227683, + "loss": 1.3597, + "theoretical_loss": 3.560459336726489, + "tokens_seen": 1425473536 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004591053159478436, + "loss": 1.346, + "theoretical_loss": 3.5604448795939385, + "tokens_seen": 1425539072 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045908926780341023, + "loss": 1.3731, + "theoretical_loss": 3.560430423312095, + "tokens_seen": 1425604608 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045907321965897693, + "loss": 1.2119, + "theoretical_loss": 3.56041596788087, + "tokens_seen": 1425670144 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045905717151454364, + "loss": 1.3375, + "theoretical_loss": 3.5604015133001736, + "tokens_seen": 1425735680 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045904112337011034, + "loss": 1.3126, + "theoretical_loss": 3.5603870595699174, + "tokens_seen": 1425801216 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045902507522567704, + "loss": 1.4053, + "theoretical_loss": 3.560372606690011, + "tokens_seen": 1425866752 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045900902708124374, + "loss": 1.3572, + "theoretical_loss": 3.560358154660367, + "tokens_seen": 1425932288 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045899297893681044, + "loss": 1.3627, + "theoretical_loss": 3.560343703480896, + "tokens_seen": 1425997824 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045897693079237714, + "loss": 1.2872, + "theoretical_loss": 3.560329253151508, + "tokens_seen": 1426063360 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045896088264794384, + "loss": 1.3704, + "theoretical_loss": 3.560314803672114, + "tokens_seen": 1426128896 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045894483450351054, + "loss": 1.3746, + "theoretical_loss": 3.5603003550426258, + "tokens_seen": 1426194432 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004589287863590772, + "loss": 1.4251, + "theoretical_loss": 3.560285907262953, + "tokens_seen": 1426259968 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045891273821464394, + "loss": 1.3028, + "theoretical_loss": 3.5602714603330083, + "tokens_seen": 1426325504 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004588966900702106, + "loss": 1.3018, + "theoretical_loss": 3.560257014252702, + "tokens_seen": 1426391040 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045888064192577734, + "loss": 1.2822, + "theoretical_loss": 3.560242569021945, + "tokens_seen": 1426456576 + }, + { + "epoch": 0.43, + "learning_rate": 0.000458864593781344, + "loss": 1.3106, + "theoretical_loss": 3.5602281246406475, + "tokens_seen": 1426522112 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045884854563691074, + "loss": 1.4405, + "theoretical_loss": 3.5602136811087215, + "tokens_seen": 1426587648 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004588324974924775, + "loss": 1.388, + "theoretical_loss": 3.560199238426078, + "tokens_seen": 1426653184 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045881644934804415, + "loss": 1.2535, + "theoretical_loss": 3.560184796592628, + "tokens_seen": 1426718720 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004588004012036109, + "loss": 1.3652, + "theoretical_loss": 3.5601703556082818, + "tokens_seen": 1426784256 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045878435305917755, + "loss": 1.2633, + "theoretical_loss": 3.5601559154729516, + "tokens_seen": 1426849792 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004587683049147443, + "loss": 1.149, + "theoretical_loss": 3.5601414761865477, + "tokens_seen": 1426915328 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045875225677031095, + "loss": 1.2628, + "theoretical_loss": 3.560127037748982, + "tokens_seen": 1426980864 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004587362086258777, + "loss": 1.3181, + "theoretical_loss": 3.560112600160165, + "tokens_seen": 1427046400 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045872016048144435, + "loss": 1.3595, + "theoretical_loss": 3.5600981634200073, + "tokens_seen": 1427111936 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004587041123370111, + "loss": 1.2916, + "theoretical_loss": 3.5600837275284216, + "tokens_seen": 1427177472 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045868806419257775, + "loss": 1.2679, + "theoretical_loss": 3.5600692924853177, + "tokens_seen": 1427243008 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004586720160481445, + "loss": 1.399, + "theoretical_loss": 3.5600548582906075, + "tokens_seen": 1427308544 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045865596790371115, + "loss": 1.3741, + "theoretical_loss": 3.5600404249442015, + "tokens_seen": 1427374080 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004586399197592779, + "loss": 1.3628, + "theoretical_loss": 3.5600259924460116, + "tokens_seen": 1427439616 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045862387161484455, + "loss": 1.4589, + "theoretical_loss": 3.5600115607959486, + "tokens_seen": 1427505152 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004586078234704113, + "loss": 1.4418, + "theoretical_loss": 3.559997129993924, + "tokens_seen": 1427570688 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045859177532597795, + "loss": 1.3325, + "theoretical_loss": 3.559982700039849, + "tokens_seen": 1427636224 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004585757271815447, + "loss": 1.2705, + "theoretical_loss": 3.559968270933634, + "tokens_seen": 1427701760 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045855967903711136, + "loss": 1.3812, + "theoretical_loss": 3.559953842675192, + "tokens_seen": 1427767296 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004585436308926781, + "loss": 1.3691, + "theoretical_loss": 3.5599394152644326, + "tokens_seen": 1427832832 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045852758274824476, + "loss": 1.317, + "theoretical_loss": 3.559924988701268, + "tokens_seen": 1427898368 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004585115346038115, + "loss": 1.1933, + "theoretical_loss": 3.5599105629856096, + "tokens_seen": 1427963904 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045849548645937816, + "loss": 1.428, + "theoretical_loss": 3.5598961381173684, + "tokens_seen": 1428029440 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004584794383149449, + "loss": 1.2735, + "theoretical_loss": 3.5598817140964556, + "tokens_seen": 1428094976 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045846339017051156, + "loss": 1.2389, + "theoretical_loss": 3.5598672909227824, + "tokens_seen": 1428160512 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045844734202607826, + "loss": 1.325, + "theoretical_loss": 3.559852868596261, + "tokens_seen": 1428226048 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045843129388164496, + "loss": 1.254, + "theoretical_loss": 3.5598384471168014, + "tokens_seen": 1428291584 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045841524573721166, + "loss": 1.3172, + "theoretical_loss": 3.559824026484317, + "tokens_seen": 1428357120 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045839919759277836, + "loss": 1.4431, + "theoretical_loss": 3.5598096066987166, + "tokens_seen": 1428422656 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045838314944834506, + "loss": 1.3593, + "theoretical_loss": 3.5597951877599137, + "tokens_seen": 1428488192 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045836710130391176, + "loss": 1.3407, + "theoretical_loss": 3.559780769667819, + "tokens_seen": 1428553728 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045835105315947846, + "loss": 1.3539, + "theoretical_loss": 3.5597663524223444, + "tokens_seen": 1428619264 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 531074, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3276088237762451, + "objective/train/theoretical_loss": 3.559751936023401, + "objective/train/tokens_used": 1449144800, + "theoretical_loss": 3.559751936023401, + "tokens_seen": 1428684800 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045833500501504516, + "loss": 1.3276, + "theoretical_loss": 3.559751936023401, + "tokens_seen": 1428684800 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045831895687061186, + "loss": 1.3279, + "theoretical_loss": 3.5597375204709, + "tokens_seen": 1428750336 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004583029087261785, + "loss": 1.3713, + "theoretical_loss": 3.5597231057647525, + "tokens_seen": 1428815872 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045828686058174527, + "loss": 1.2637, + "theoretical_loss": 3.5597086919048713, + "tokens_seen": 1428881408 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004582708124373119, + "loss": 1.3384, + "theoretical_loss": 3.5596942788911665, + "tokens_seen": 1428946944 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045825476429287867, + "loss": 1.4591, + "theoretical_loss": 3.5596798667235507, + "tokens_seen": 1429012480 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004582387161484453, + "loss": 1.3764, + "theoretical_loss": 3.5596654554019356, + "tokens_seen": 1429078016 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045822266800401207, + "loss": 1.2585, + "theoretical_loss": 3.5596510449262313, + "tokens_seen": 1429143552 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004582066198595787, + "loss": 1.3328, + "theoretical_loss": 3.559636635296351, + "tokens_seen": 1429209088 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045819057171514547, + "loss": 1.2915, + "theoretical_loss": 3.559622226512205, + "tokens_seen": 1429274624 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004581745235707121, + "loss": 1.3806, + "theoretical_loss": 3.5596078185737055, + "tokens_seen": 1429340160 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045815847542627887, + "loss": 1.332, + "theoretical_loss": 3.559593411480764, + "tokens_seen": 1429405696 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004581424272818455, + "loss": 1.2909, + "theoretical_loss": 3.5595790052332923, + "tokens_seen": 1429471232 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045812637913741227, + "loss": 1.4022, + "theoretical_loss": 3.5595645998312015, + "tokens_seen": 1429536768 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004581103309929789, + "loss": 1.4123, + "theoretical_loss": 3.559550195274404, + "tokens_seen": 1429602304 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004580942828485457, + "loss": 1.2355, + "theoretical_loss": 3.5595357915628107, + "tokens_seen": 1429667840 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004580782347041123, + "loss": 1.3472, + "theoretical_loss": 3.559521388696334, + "tokens_seen": 1429733376 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004580621865596791, + "loss": 1.4524, + "theoretical_loss": 3.5595069866748847, + "tokens_seen": 1429798912 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004580461384152457, + "loss": 1.2845, + "theoretical_loss": 3.559492585498375, + "tokens_seen": 1429864448 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004580300902708125, + "loss": 1.2663, + "theoretical_loss": 3.559478185166717, + "tokens_seen": 1429929984 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004580140421263791, + "loss": 1.2899, + "theoretical_loss": 3.5594637856798217, + "tokens_seen": 1429995520 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004579979939819459, + "loss": 1.314, + "theoretical_loss": 3.5594493870376014, + "tokens_seen": 1430061056 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004579819458375125, + "loss": 1.2356, + "theoretical_loss": 3.5594349892399673, + "tokens_seen": 1430126592 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004579658976930793, + "loss": 1.2578, + "theoretical_loss": 3.5594205922868314, + "tokens_seen": 1430192128 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004579498495486459, + "loss": 1.3013, + "theoretical_loss": 3.559406196178106, + "tokens_seen": 1430257664 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004579338014042127, + "loss": 1.3137, + "theoretical_loss": 3.5593918009137018, + "tokens_seen": 1430323200 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004579177532597793, + "loss": 1.4152, + "theoretical_loss": 3.5593774064935317, + "tokens_seen": 1430388736 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004579017051153461, + "loss": 1.2833, + "theoretical_loss": 3.5593630129175065, + "tokens_seen": 1430454272 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045788565697091273, + "loss": 1.3981, + "theoretical_loss": 3.559348620185539, + "tokens_seen": 1430519808 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004578696088264795, + "loss": 1.331, + "theoretical_loss": 3.55933422829754, + "tokens_seen": 1430585344 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045785356068204613, + "loss": 1.4249, + "theoretical_loss": 3.5593198372534225, + "tokens_seen": 1430650880 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004578375125376129, + "loss": 1.3437, + "theoretical_loss": 3.5593054470530974, + "tokens_seen": 1430716416 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045782146439317953, + "loss": 1.3074, + "theoretical_loss": 3.559291057696477, + "tokens_seen": 1430781952 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004578054162487463, + "loss": 1.2422, + "theoretical_loss": 3.559276669183473, + "tokens_seen": 1430847488 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045778936810431293, + "loss": 1.4556, + "theoretical_loss": 3.559262281513998, + "tokens_seen": 1430913024 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004577733199598797, + "loss": 1.4517, + "theoretical_loss": 3.559247894687963, + "tokens_seen": 1430978560 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045775727181544633, + "loss": 1.29, + "theoretical_loss": 3.5592335087052804, + "tokens_seen": 1431044096 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004577412236710131, + "loss": 1.364, + "theoretical_loss": 3.559219123565862, + "tokens_seen": 1431109632 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045772517552657973, + "loss": 1.3541, + "theoretical_loss": 3.55920473926962, + "tokens_seen": 1431175168 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004577091273821465, + "loss": 1.3718, + "theoretical_loss": 3.559190355816466, + "tokens_seen": 1431240704 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045769307923771314, + "loss": 1.3834, + "theoretical_loss": 3.5591759732063117, + "tokens_seen": 1431306240 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045767703109327984, + "loss": 1.2698, + "theoretical_loss": 3.5591615914390706, + "tokens_seen": 1431371776 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045766098294884654, + "loss": 1.2868, + "theoretical_loss": 3.559147210514653, + "tokens_seen": 1431437312 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045764493480441324, + "loss": 1.2721, + "theoretical_loss": 3.5591328304329712, + "tokens_seen": 1431502848 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045762888665998, + "loss": 1.34, + "theoretical_loss": 3.559118451193938, + "tokens_seen": 1431568384 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045761283851554664, + "loss": 1.2657, + "theoretical_loss": 3.5591040727974654, + "tokens_seen": 1431633920 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004575967903711134, + "loss": 1.3244, + "theoretical_loss": 3.559089695243465, + "tokens_seen": 1431699456 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045758074222668004, + "loss": 1.3241, + "theoretical_loss": 3.559075318531849, + "tokens_seen": 1431764992 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004575646940822468, + "loss": 1.3756, + "theoretical_loss": 3.5590609426625295, + "tokens_seen": 1431830528 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045754864593781344, + "loss": 1.4187, + "theoretical_loss": 3.5590465676354186, + "tokens_seen": 1431896064 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004575325977933802, + "loss": 1.3138, + "theoretical_loss": 3.5590321934504283, + "tokens_seen": 1431961600 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045751654964894684, + "loss": 1.2411, + "theoretical_loss": 3.559017820107471, + "tokens_seen": 1432027136 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004575005015045136, + "loss": 1.3924, + "theoretical_loss": 3.5590034476064587, + "tokens_seen": 1432092672 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045748445336008024, + "loss": 1.303, + "theoretical_loss": 3.558989075947303, + "tokens_seen": 1432158208 + }, + { + "epoch": 0.43, + "learning_rate": 0.000457468405215647, + "loss": 1.3163, + "theoretical_loss": 3.558974705129917, + "tokens_seen": 1432223744 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045745235707121364, + "loss": 1.2993, + "theoretical_loss": 3.558960335154213, + "tokens_seen": 1432289280 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004574363089267804, + "loss": 1.262, + "theoretical_loss": 3.5589459660201026, + "tokens_seen": 1432354816 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045742026078234705, + "loss": 1.3465, + "theoretical_loss": 3.558931597727498, + "tokens_seen": 1432420352 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004574042126379138, + "loss": 1.262, + "theoretical_loss": 3.558917230276311, + "tokens_seen": 1432485888 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045738816449348045, + "loss": 1.2791, + "theoretical_loss": 3.5589028636664546, + "tokens_seen": 1432551424 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004573721163490472, + "loss": 1.3544, + "theoretical_loss": 3.558888497897841, + "tokens_seen": 1432616960 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045735606820461385, + "loss": 1.3066, + "theoretical_loss": 3.5588741329703826, + "tokens_seen": 1432682496 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004573400200601806, + "loss": 1.3741, + "theoretical_loss": 3.558859768883991, + "tokens_seen": 1432748032 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045732397191574725, + "loss": 1.3572, + "theoretical_loss": 3.5588454056385785, + "tokens_seen": 1432813568 + }, + { + "epoch": 0.43, + "learning_rate": 0.000457307923771314, + "loss": 1.3157, + "theoretical_loss": 3.5588310432340577, + "tokens_seen": 1432879104 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045729187562688065, + "loss": 1.3275, + "theoretical_loss": 3.5588166816703417, + "tokens_seen": 1432944640 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004572758274824474, + "loss": 1.2588, + "theoretical_loss": 3.5588023209473416, + "tokens_seen": 1433010176 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045725977933801405, + "loss": 1.314, + "theoretical_loss": 3.5587879610649704, + "tokens_seen": 1433075712 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004572437311935808, + "loss": 1.3565, + "theoretical_loss": 3.55877360202314, + "tokens_seen": 1433141248 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045722768304914745, + "loss": 1.3947, + "theoretical_loss": 3.558759243821763, + "tokens_seen": 1433206784 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004572116349047142, + "loss": 1.3499, + "theoretical_loss": 3.558744886460752, + "tokens_seen": 1433272320 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045719558676028085, + "loss": 1.3926, + "theoretical_loss": 3.558730529940019, + "tokens_seen": 1433337856 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004571795386158476, + "loss": 1.2199, + "theoretical_loss": 3.5587161742594766, + "tokens_seen": 1433403392 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045716349047141426, + "loss": 1.3009, + "theoretical_loss": 3.5587018194190376, + "tokens_seen": 1433468928 + }, + { + "epoch": 0.43, + "learning_rate": 0.000457147442326981, + "loss": 1.3989, + "theoretical_loss": 3.558687465418614, + "tokens_seen": 1433534464 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045713139418254766, + "loss": 1.3379, + "theoretical_loss": 3.558673112258118, + "tokens_seen": 1433600000 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004571153460381144, + "loss": 1.2549, + "theoretical_loss": 3.5586587599374626, + "tokens_seen": 1433665536 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045709929789368106, + "loss": 1.3359, + "theoretical_loss": 3.5586444084565603, + "tokens_seen": 1433731072 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004570832497492478, + "loss": 1.3149, + "theoretical_loss": 3.5586300578153227, + "tokens_seen": 1433796608 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045706720160481446, + "loss": 1.3254, + "theoretical_loss": 3.5586157080136633, + "tokens_seen": 1433862144 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045705115346038116, + "loss": 1.1916, + "theoretical_loss": 3.558601359051494, + "tokens_seen": 1433927680 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045703510531594786, + "loss": 1.3266, + "theoretical_loss": 3.5585870109287274, + "tokens_seen": 1433993216 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045701905717151456, + "loss": 1.4213, + "theoretical_loss": 3.558572663645277, + "tokens_seen": 1434058752 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045700300902708126, + "loss": 1.4105, + "theoretical_loss": 3.5585583172010535, + "tokens_seen": 1434124288 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045698696088264796, + "loss": 1.3423, + "theoretical_loss": 3.5585439715959715, + "tokens_seen": 1434189824 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045697091273821466, + "loss": 1.3644, + "theoretical_loss": 3.558529626829942, + "tokens_seen": 1434255360 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045695486459378136, + "loss": 1.4234, + "theoretical_loss": 3.558515282902878, + "tokens_seen": 1434320896 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045693881644934807, + "loss": 1.2907, + "theoretical_loss": 3.5585009398146927, + "tokens_seen": 1434386432 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045692276830491477, + "loss": 1.3925, + "theoretical_loss": 3.558486597565298, + "tokens_seen": 1434451968 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004569067201604814, + "loss": 1.4285, + "theoretical_loss": 3.558472256154607, + "tokens_seen": 1434517504 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045689067201604817, + "loss": 1.4192, + "theoretical_loss": 3.558457915582532, + "tokens_seen": 1434583040 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004568746238716148, + "loss": 1.2009, + "theoretical_loss": 3.558443575848986, + "tokens_seen": 1434648576 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045685857572718157, + "loss": 1.3405, + "theoretical_loss": 3.558429236953881, + "tokens_seen": 1434714112 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004568425275827482, + "loss": 1.3271, + "theoretical_loss": 3.558414898897131, + "tokens_seen": 1434779648 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045682647943831497, + "loss": 1.2574, + "theoretical_loss": 3.558400561678647, + "tokens_seen": 1434845184 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004568104312938816, + "loss": 1.2725, + "theoretical_loss": 3.558386225298343, + "tokens_seen": 1434910720 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045679438314944837, + "loss": 1.2698, + "theoretical_loss": 3.558371889756131, + "tokens_seen": 1434976256 + }, + { + "epoch": 0.43, + "learning_rate": 0.000456778335005015, + "loss": 1.3206, + "theoretical_loss": 3.5583575550519244, + "tokens_seen": 1435041792 + }, + { + "epoch": 0.43, + "learning_rate": 0.00045676228686058177, + "loss": 1.267, + "theoretical_loss": 3.558343221185635, + "tokens_seen": 1435107328 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004567462387161484, + "loss": 1.3424, + "theoretical_loss": 3.5583288881571766, + "tokens_seen": 1435172864 + }, + { + "epoch": 0.43, + "objective/train/docs_used": 533435, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3692102432250977, + "objective/train/theoretical_loss": 3.5583145559664615, + "objective/train/tokens_used": 1455698400, + "theoretical_loss": 3.5583145559664615, + "tokens_seen": 1435238400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004567301905717152, + "loss": 1.3692, + "theoretical_loss": 3.5583145559664615, + "tokens_seen": 1435238400 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004567141424272818, + "loss": 1.3471, + "theoretical_loss": 3.5583002246134026, + "tokens_seen": 1435303936 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004566980942828486, + "loss": 1.3702, + "theoretical_loss": 3.558285894097912, + "tokens_seen": 1435369472 + }, + { + "epoch": 0.43, + "learning_rate": 0.0004566820461384152, + "loss": 1.3615, + "theoretical_loss": 3.558271564419903, + "tokens_seen": 1435435008 + }, + { + "epoch": 0.44, + "learning_rate": 0.000456665997993982, + "loss": 1.2935, + "theoretical_loss": 3.558257235579289, + "tokens_seen": 1435500544 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004566499498495486, + "loss": 1.4826, + "theoretical_loss": 3.5582429075759823, + "tokens_seen": 1435566080 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004566339017051154, + "loss": 1.3581, + "theoretical_loss": 3.558228580409896, + "tokens_seen": 1435631616 + }, + { + "epoch": 0.44, + "learning_rate": 0.000456617853560682, + "loss": 1.4226, + "theoretical_loss": 3.5582142540809425, + "tokens_seen": 1435697152 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004566018054162488, + "loss": 1.3595, + "theoretical_loss": 3.558199928589035, + "tokens_seen": 1435762688 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004565857572718154, + "loss": 1.3396, + "theoretical_loss": 3.5581856039340862, + "tokens_seen": 1435828224 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004565697091273822, + "loss": 1.3781, + "theoretical_loss": 3.5581712801160092, + "tokens_seen": 1435893760 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004565536609829488, + "loss": 1.3728, + "theoretical_loss": 3.5581569571347167, + "tokens_seen": 1435959296 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004565376128385156, + "loss": 1.4916, + "theoretical_loss": 3.558142634990122, + "tokens_seen": 1436024832 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045652156469408223, + "loss": 1.3632, + "theoretical_loss": 3.558128313682138, + "tokens_seen": 1436090368 + }, + { + "epoch": 0.44, + "learning_rate": 0.000456505516549649, + "loss": 1.3157, + "theoretical_loss": 3.558113993210677, + "tokens_seen": 1436155904 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045648946840521563, + "loss": 1.4538, + "theoretical_loss": 3.5580996735756525, + "tokens_seen": 1436221440 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004564734202607824, + "loss": 1.3215, + "theoretical_loss": 3.5580853547769777, + "tokens_seen": 1436286976 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045645737211634903, + "loss": 1.3317, + "theoretical_loss": 3.558071036814565, + "tokens_seen": 1436352512 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004564413239719158, + "loss": 1.3665, + "theoretical_loss": 3.558056719688328, + "tokens_seen": 1436418048 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004564252758274825, + "loss": 1.3107, + "theoretical_loss": 3.5580424033981792, + "tokens_seen": 1436483584 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004564092276830492, + "loss": 1.4248, + "theoretical_loss": 3.5580280879440322, + "tokens_seen": 1436549120 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004563931795386159, + "loss": 1.4037, + "theoretical_loss": 3.5580137733257997, + "tokens_seen": 1436614656 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004563771313941826, + "loss": 1.3088, + "theoretical_loss": 3.557999459543394, + "tokens_seen": 1436680192 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004563610832497493, + "loss": 1.3124, + "theoretical_loss": 3.55798514659673, + "tokens_seen": 1436745728 + }, + { + "epoch": 0.44, + "learning_rate": 0.000456345035105316, + "loss": 1.2549, + "theoretical_loss": 3.5579708344857197, + "tokens_seen": 1436811264 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004563289869608827, + "loss": 1.2851, + "theoretical_loss": 3.557956523210276, + "tokens_seen": 1436876800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004563129388164494, + "loss": 1.3351, + "theoretical_loss": 3.5579422127703113, + "tokens_seen": 1436942336 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004562968906720161, + "loss": 1.2891, + "theoretical_loss": 3.5579279031657407, + "tokens_seen": 1437007872 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045628084252758274, + "loss": 1.3256, + "theoretical_loss": 3.557913594396476, + "tokens_seen": 1437073408 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004562647943831495, + "loss": 1.3936, + "theoretical_loss": 3.5578992864624306, + "tokens_seen": 1437138944 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045624874623871614, + "loss": 1.3844, + "theoretical_loss": 3.557884979363518, + "tokens_seen": 1437204480 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004562326980942829, + "loss": 1.289, + "theoretical_loss": 3.557870673099651, + "tokens_seen": 1437270016 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045621664994984954, + "loss": 1.4092, + "theoretical_loss": 3.557856367670743, + "tokens_seen": 1437335552 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004562006018054163, + "loss": 1.2889, + "theoretical_loss": 3.557842063076707, + "tokens_seen": 1437401088 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045618455366098294, + "loss": 1.369, + "theoretical_loss": 3.5578277593174565, + "tokens_seen": 1437466624 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004561685055165497, + "loss": 1.3379, + "theoretical_loss": 3.5578134563929042, + "tokens_seen": 1437532160 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045615245737211634, + "loss": 1.2725, + "theoretical_loss": 3.5577991543029635, + "tokens_seen": 1437597696 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004561364092276831, + "loss": 1.3003, + "theoretical_loss": 3.557784853047548, + "tokens_seen": 1437663232 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045612036108324974, + "loss": 1.3481, + "theoretical_loss": 3.557770552626571, + "tokens_seen": 1437728768 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004561043129388165, + "loss": 1.2668, + "theoretical_loss": 3.557756253039946, + "tokens_seen": 1437794304 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045608826479438314, + "loss": 1.3916, + "theoretical_loss": 3.557741954287585, + "tokens_seen": 1437859840 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004560722166499499, + "loss": 1.4302, + "theoretical_loss": 3.5577276563694022, + "tokens_seen": 1437925376 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045605616850551655, + "loss": 1.3087, + "theoretical_loss": 3.5577133592853114, + "tokens_seen": 1437990912 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004560401203610833, + "loss": 1.3387, + "theoretical_loss": 3.557699063035225, + "tokens_seen": 1438056448 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045602407221664995, + "loss": 1.4207, + "theoretical_loss": 3.5576847676190573, + "tokens_seen": 1438121984 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004560080240722167, + "loss": 1.3344, + "theoretical_loss": 3.557670473036721, + "tokens_seen": 1438187520 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045599197592778335, + "loss": 1.2936, + "theoretical_loss": 3.557656179288129, + "tokens_seen": 1438253056 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004559759277833501, + "loss": 1.3795, + "theoretical_loss": 3.5576418863731956, + "tokens_seen": 1438318592 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045595987963891675, + "loss": 1.267, + "theoretical_loss": 3.5576275942918345, + "tokens_seen": 1438384128 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004559438314944835, + "loss": 1.3215, + "theoretical_loss": 3.557613303043958, + "tokens_seen": 1438449664 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045592778335005015, + "loss": 1.3551, + "theoretical_loss": 3.5575990126294794, + "tokens_seen": 1438515200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004559117352056169, + "loss": 1.3238, + "theoretical_loss": 3.5575847230483135, + "tokens_seen": 1438580736 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045589568706118355, + "loss": 1.4035, + "theoretical_loss": 3.5575704343003727, + "tokens_seen": 1438646272 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004558796389167503, + "loss": 1.4171, + "theoretical_loss": 3.557556146385571, + "tokens_seen": 1438711808 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045586359077231695, + "loss": 1.3099, + "theoretical_loss": 3.5575418593038215, + "tokens_seen": 1438777344 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004558475426278837, + "loss": 1.4322, + "theoretical_loss": 3.557527573055038, + "tokens_seen": 1438842880 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045583149448345035, + "loss": 1.3596, + "theoretical_loss": 3.5575132876391335, + "tokens_seen": 1438908416 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004558154463390171, + "loss": 1.3034, + "theoretical_loss": 3.557499003056022, + "tokens_seen": 1438973952 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045579939819458376, + "loss": 1.3159, + "theoretical_loss": 3.557484719305617, + "tokens_seen": 1439039488 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004557833500501505, + "loss": 1.307, + "theoretical_loss": 3.5574704363878316, + "tokens_seen": 1439105024 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045576730190571716, + "loss": 1.3131, + "theoretical_loss": 3.5574561543025798, + "tokens_seen": 1439170560 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004557512537612839, + "loss": 1.3627, + "theoretical_loss": 3.5574418730497745, + "tokens_seen": 1439236096 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045573520561685056, + "loss": 1.2798, + "theoretical_loss": 3.5574275926293306, + "tokens_seen": 1439301632 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004557191574724173, + "loss": 1.2942, + "theoretical_loss": 3.5574133130411605, + "tokens_seen": 1439367168 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045570310932798396, + "loss": 1.2875, + "theoretical_loss": 3.5573990342851785, + "tokens_seen": 1439432704 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004556870611835507, + "loss": 1.4283, + "theoretical_loss": 3.5573847563612975, + "tokens_seen": 1439498240 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045567101303911736, + "loss": 1.401, + "theoretical_loss": 3.557370479269432, + "tokens_seen": 1439563776 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004556549648946841, + "loss": 1.2632, + "theoretical_loss": 3.5573562030094945, + "tokens_seen": 1439629312 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045563891675025076, + "loss": 1.3499, + "theoretical_loss": 3.5573419275813993, + "tokens_seen": 1439694848 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045562286860581746, + "loss": 1.3635, + "theoretical_loss": 3.5573276529850606, + "tokens_seen": 1439760384 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045560682046138416, + "loss": 1.2856, + "theoretical_loss": 3.5573133792203917, + "tokens_seen": 1439825920 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045559077231695086, + "loss": 1.2752, + "theoretical_loss": 3.5572991062873056, + "tokens_seen": 1439891456 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045557472417251756, + "loss": 1.3685, + "theoretical_loss": 3.557284834185717, + "tokens_seen": 1439956992 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045555867602808427, + "loss": 1.3393, + "theoretical_loss": 3.557270562915539, + "tokens_seen": 1440022528 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045554262788365097, + "loss": 1.3935, + "theoretical_loss": 3.5572562924766853, + "tokens_seen": 1440088064 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045552657973921767, + "loss": 1.2651, + "theoretical_loss": 3.5572420228690707, + "tokens_seen": 1440153600 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004555105315947843, + "loss": 1.2417, + "theoretical_loss": 3.557227754092607, + "tokens_seen": 1440219136 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045549448345035107, + "loss": 1.3448, + "theoretical_loss": 3.55721348614721, + "tokens_seen": 1440284672 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004554784353059177, + "loss": 1.3011, + "theoretical_loss": 3.5571992190327926, + "tokens_seen": 1440350208 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045546238716148447, + "loss": 1.4258, + "theoretical_loss": 3.557184952749268, + "tokens_seen": 1440415744 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004554463390170511, + "loss": 1.194, + "theoretical_loss": 3.5571706872965505, + "tokens_seen": 1440481280 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045543029087261787, + "loss": 1.3559, + "theoretical_loss": 3.5571564226745545, + "tokens_seen": 1440546816 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004554142427281845, + "loss": 1.3863, + "theoretical_loss": 3.5571421588831935, + "tokens_seen": 1440612352 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045539819458375127, + "loss": 1.3189, + "theoretical_loss": 3.5571278959223807, + "tokens_seen": 1440677888 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004553821464393179, + "loss": 1.3154, + "theoretical_loss": 3.5571136337920306, + "tokens_seen": 1440743424 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004553660982948847, + "loss": 1.3774, + "theoretical_loss": 3.5570993724920568, + "tokens_seen": 1440808960 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004553500501504513, + "loss": 1.3831, + "theoretical_loss": 3.5570851120223734, + "tokens_seen": 1440874496 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004553340020060181, + "loss": 1.3932, + "theoretical_loss": 3.5570708523828944, + "tokens_seen": 1440940032 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004553179538615847, + "loss": 1.4187, + "theoretical_loss": 3.5570565935735337, + "tokens_seen": 1441005568 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004553019057171515, + "loss": 1.2635, + "theoretical_loss": 3.5570423355942045, + "tokens_seen": 1441071104 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004552858575727181, + "loss": 1.3086, + "theoretical_loss": 3.5570280784448216, + "tokens_seen": 1441136640 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004552698094282849, + "loss": 1.2543, + "theoretical_loss": 3.557013822125299, + "tokens_seen": 1441202176 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045525376128385163, + "loss": 1.4075, + "theoretical_loss": 3.5569995666355494, + "tokens_seen": 1441267712 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004552377131394183, + "loss": 1.3038, + "theoretical_loss": 3.556985311975488, + "tokens_seen": 1441333248 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045522166499498503, + "loss": 1.2999, + "theoretical_loss": 3.556971058145029, + "tokens_seen": 1441398784 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004552056168505517, + "loss": 1.4349, + "theoretical_loss": 3.556956805144085, + "tokens_seen": 1441464320 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045518956870611843, + "loss": 1.3576, + "theoretical_loss": 3.556942552972572, + "tokens_seen": 1441529856 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004551735205616851, + "loss": 1.3534, + "theoretical_loss": 3.556928301630402, + "tokens_seen": 1441595392 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045515747241725184, + "loss": 1.3412, + "theoretical_loss": 3.5569140511174906, + "tokens_seen": 1441660928 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004551414242728185, + "loss": 1.4579, + "theoretical_loss": 3.556899801433751, + "tokens_seen": 1441726464 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 535911, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2928510904312134, + "objective/train/theoretical_loss": 3.5568855525790974, + "objective/train/tokens_used": 1462252000, + "theoretical_loss": 3.5568855525790974, + "tokens_seen": 1441792000 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045512537612838524, + "loss": 1.2929, + "theoretical_loss": 3.5568855525790974, + "tokens_seen": 1441792000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004551093279839519, + "loss": 1.408, + "theoretical_loss": 3.556871304553444, + "tokens_seen": 1441857536 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045509327983951864, + "loss": 1.4566, + "theoretical_loss": 3.556857057356704, + "tokens_seen": 1441923072 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004550772316950853, + "loss": 1.4678, + "theoretical_loss": 3.556842810988794, + "tokens_seen": 1441988608 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045506118355065204, + "loss": 1.3322, + "theoretical_loss": 3.5568285654496252, + "tokens_seen": 1442054144 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004550451354062187, + "loss": 1.2664, + "theoretical_loss": 3.5568143207391136, + "tokens_seen": 1442119680 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045502908726178544, + "loss": 1.3498, + "theoretical_loss": 3.556800076857172, + "tokens_seen": 1442185216 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004550130391173521, + "loss": 1.3468, + "theoretical_loss": 3.5567858338037164, + "tokens_seen": 1442250752 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004549969909729188, + "loss": 1.3124, + "theoretical_loss": 3.5567715915786593, + "tokens_seen": 1442316288 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004549809428284855, + "loss": 1.3954, + "theoretical_loss": 3.556757350181915, + "tokens_seen": 1442381824 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004549648946840522, + "loss": 1.361, + "theoretical_loss": 3.556743109613399, + "tokens_seen": 1442447360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004549488465396189, + "loss": 1.4257, + "theoretical_loss": 3.556728869873024, + "tokens_seen": 1442512896 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004549327983951856, + "loss": 1.3287, + "theoretical_loss": 3.5567146309607054, + "tokens_seen": 1442578432 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004549167502507523, + "loss": 1.3907, + "theoretical_loss": 3.5567003928763565, + "tokens_seen": 1442643968 + }, + { + "epoch": 0.44, + "learning_rate": 0.000454900702106319, + "loss": 1.4321, + "theoretical_loss": 3.556686155619892, + "tokens_seen": 1442709504 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004548846539618857, + "loss": 1.193, + "theoretical_loss": 3.556671919191226, + "tokens_seen": 1442775040 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004548686058174524, + "loss": 1.2839, + "theoretical_loss": 3.556657683590273, + "tokens_seen": 1442840576 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045485255767301904, + "loss": 1.3445, + "theoretical_loss": 3.5566434488169474, + "tokens_seen": 1442906112 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004548365095285858, + "loss": 1.4217, + "theoretical_loss": 3.5566292148711627, + "tokens_seen": 1442971648 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045482046138415244, + "loss": 1.3689, + "theoretical_loss": 3.5566149817528343, + "tokens_seen": 1443037184 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004548044132397192, + "loss": 1.2921, + "theoretical_loss": 3.556600749461876, + "tokens_seen": 1443102720 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045478836509528584, + "loss": 1.2597, + "theoretical_loss": 3.556586517998202, + "tokens_seen": 1443168256 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004547723169508526, + "loss": 1.3729, + "theoretical_loss": 3.5565722873617265, + "tokens_seen": 1443233792 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045475626880641924, + "loss": 1.3851, + "theoretical_loss": 3.5565580575523645, + "tokens_seen": 1443299328 + }, + { + "epoch": 0.44, + "learning_rate": 0.000454740220661986, + "loss": 1.23, + "theoretical_loss": 3.55654382857003, + "tokens_seen": 1443364864 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045472417251755264, + "loss": 1.3095, + "theoretical_loss": 3.5565296004146365, + "tokens_seen": 1443430400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004547081243731194, + "loss": 1.3131, + "theoretical_loss": 3.5565153730861, + "tokens_seen": 1443495936 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045469207622868605, + "loss": 1.411, + "theoretical_loss": 3.5565011465843344, + "tokens_seen": 1443561472 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004546760280842528, + "loss": 1.3933, + "theoretical_loss": 3.556486920909254, + "tokens_seen": 1443627008 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045465997993981945, + "loss": 1.3136, + "theoretical_loss": 3.556472696060773, + "tokens_seen": 1443692544 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004546439317953862, + "loss": 1.3121, + "theoretical_loss": 3.5564584720388055, + "tokens_seen": 1443758080 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045462788365095285, + "loss": 1.3035, + "theoretical_loss": 3.556444248843267, + "tokens_seen": 1443823616 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004546118355065196, + "loss": 1.4264, + "theoretical_loss": 3.556430026474071, + "tokens_seen": 1443889152 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045459578736208625, + "loss": 1.3422, + "theoretical_loss": 3.5564158049311327, + "tokens_seen": 1443954688 + }, + { + "epoch": 0.44, + "learning_rate": 0.000454579739217653, + "loss": 1.2931, + "theoretical_loss": 3.556401584214367, + "tokens_seen": 1444020224 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045456369107321965, + "loss": 1.4171, + "theoretical_loss": 3.5563873643236867, + "tokens_seen": 1444085760 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004545476429287864, + "loss": 1.2191, + "theoretical_loss": 3.5563731452590077, + "tokens_seen": 1444151296 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045453159478435305, + "loss": 1.4141, + "theoretical_loss": 3.5563589270202445, + "tokens_seen": 1444216832 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004545155466399198, + "loss": 1.4151, + "theoretical_loss": 3.556344709607311, + "tokens_seen": 1444282368 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045449949849548645, + "loss": 1.3289, + "theoretical_loss": 3.5563304930201225, + "tokens_seen": 1444347904 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004544834503510532, + "loss": 1.2924, + "theoretical_loss": 3.556316277258593, + "tokens_seen": 1444413440 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045446740220661985, + "loss": 1.2285, + "theoretical_loss": 3.5563020623226373, + "tokens_seen": 1444478976 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004544513540621866, + "loss": 1.3004, + "theoretical_loss": 3.55628784821217, + "tokens_seen": 1444544512 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045443530591775326, + "loss": 1.4087, + "theoretical_loss": 3.5562736349271056, + "tokens_seen": 1444610048 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045441925777332, + "loss": 1.3973, + "theoretical_loss": 3.5562594224673587, + "tokens_seen": 1444675584 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045440320962888666, + "loss": 1.3356, + "theoretical_loss": 3.556245210832844, + "tokens_seen": 1444741120 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004543871614844534, + "loss": 1.3578, + "theoretical_loss": 3.556231000023477, + "tokens_seen": 1444806656 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045437111334002006, + "loss": 1.2681, + "theoretical_loss": 3.556216790039171, + "tokens_seen": 1444872192 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004543550651955868, + "loss": 1.4043, + "theoretical_loss": 3.5562025808798414, + "tokens_seen": 1444937728 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045433901705115346, + "loss": 1.3562, + "theoretical_loss": 3.556188372545403, + "tokens_seen": 1445003264 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004543229689067202, + "loss": 1.2564, + "theoretical_loss": 3.55617416503577, + "tokens_seen": 1445068800 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045430692076228686, + "loss": 1.2696, + "theoretical_loss": 3.5561599583508574, + "tokens_seen": 1445134336 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004542908726178536, + "loss": 1.3141, + "theoretical_loss": 3.55614575249058, + "tokens_seen": 1445199872 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045427482447342026, + "loss": 1.3014, + "theoretical_loss": 3.5561315474548527, + "tokens_seen": 1445265408 + }, + { + "epoch": 0.44, + "learning_rate": 0.000454258776328987, + "loss": 1.3159, + "theoretical_loss": 3.5561173432435895, + "tokens_seen": 1445330944 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045424272818455366, + "loss": 1.258, + "theoretical_loss": 3.556103139856706, + "tokens_seen": 1445396480 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045422668004012036, + "loss": 1.4078, + "theoretical_loss": 3.5560889372941165, + "tokens_seen": 1445462016 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045421063189568706, + "loss": 1.4696, + "theoretical_loss": 3.556074735555736, + "tokens_seen": 1445527552 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045419458375125377, + "loss": 1.2662, + "theoretical_loss": 3.556060534641479, + "tokens_seen": 1445593088 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045417853560682047, + "loss": 1.3786, + "theoretical_loss": 3.5560463345512607, + "tokens_seen": 1445658624 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045416248746238717, + "loss": 1.2777, + "theoretical_loss": 3.556032135284996, + "tokens_seen": 1445724160 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045414643931795387, + "loss": 1.2618, + "theoretical_loss": 3.5560179368425997, + "tokens_seen": 1445789696 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045413039117352057, + "loss": 1.3122, + "theoretical_loss": 3.5560037392239865, + "tokens_seen": 1445855232 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045411434302908727, + "loss": 1.2853, + "theoretical_loss": 3.555989542429071, + "tokens_seen": 1445920768 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045409829488465397, + "loss": 1.3723, + "theoretical_loss": 3.5559753464577684, + "tokens_seen": 1445986304 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004540822467402206, + "loss": 1.2437, + "theoretical_loss": 3.5559611513099934, + "tokens_seen": 1446051840 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045406619859578737, + "loss": 1.3741, + "theoretical_loss": 3.555946956985661, + "tokens_seen": 1446117376 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004540501504513541, + "loss": 1.3037, + "theoretical_loss": 3.5559327634846865, + "tokens_seen": 1446182912 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045403410230692077, + "loss": 1.2779, + "theoretical_loss": 3.5559185708069845, + "tokens_seen": 1446248448 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004540180541624875, + "loss": 1.387, + "theoretical_loss": 3.55590437895247, + "tokens_seen": 1446313984 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045400200601805417, + "loss": 1.224, + "theoretical_loss": 3.5558901879210576, + "tokens_seen": 1446379520 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045398595787362093, + "loss": 1.2929, + "theoretical_loss": 3.555875997712663, + "tokens_seen": 1446445056 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004539699097291876, + "loss": 1.2029, + "theoretical_loss": 3.5558618083272, + "tokens_seen": 1446510592 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045395386158475433, + "loss": 1.299, + "theoretical_loss": 3.5558476197645854, + "tokens_seen": 1446576128 + }, + { + "epoch": 0.44, + "learning_rate": 0.000453937813440321, + "loss": 1.3594, + "theoretical_loss": 3.5558334320247322, + "tokens_seen": 1446641664 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045392176529588773, + "loss": 1.3491, + "theoretical_loss": 3.555819245107557, + "tokens_seen": 1446707200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004539057171514544, + "loss": 1.3139, + "theoretical_loss": 3.555805059012974, + "tokens_seen": 1446772736 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045388966900702113, + "loss": 1.3071, + "theoretical_loss": 3.5557908737408983, + "tokens_seen": 1446838272 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004538736208625878, + "loss": 1.3492, + "theoretical_loss": 3.5557766892912452, + "tokens_seen": 1446903808 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045385757271815453, + "loss": 1.345, + "theoretical_loss": 3.55576250566393, + "tokens_seen": 1446969344 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004538415245737212, + "loss": 1.2677, + "theoretical_loss": 3.555748322858867, + "tokens_seen": 1447034880 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045382547642928793, + "loss": 1.333, + "theoretical_loss": 3.555734140875972, + "tokens_seen": 1447100416 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004538094282848546, + "loss": 1.3795, + "theoretical_loss": 3.5557199597151596, + "tokens_seen": 1447165952 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045379338014042134, + "loss": 1.2754, + "theoretical_loss": 3.555705779376346, + "tokens_seen": 1447231488 + }, + { + "epoch": 0.44, + "learning_rate": 0.000453777331995988, + "loss": 1.372, + "theoretical_loss": 3.5556915998594447, + "tokens_seen": 1447297024 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045376128385155474, + "loss": 1.3436, + "theoretical_loss": 3.555677421164372, + "tokens_seen": 1447362560 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004537452357071214, + "loss": 1.4133, + "theoretical_loss": 3.555663243291043, + "tokens_seen": 1447428096 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004537452357071214, + "loss": 1.4891, + "theoretical_loss": 3.555649066239372, + "tokens_seen": 1447493632 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045372918756268814, + "loss": 1.4083, + "theoretical_loss": 3.555634890009275, + "tokens_seen": 1447559168 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004537131394182548, + "loss": 1.3796, + "theoretical_loss": 3.5556207146006673, + "tokens_seen": 1447624704 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045369709127382154, + "loss": 1.3138, + "theoretical_loss": 3.5556065400134633, + "tokens_seen": 1447690240 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004536810431293882, + "loss": 1.2175, + "theoretical_loss": 3.5555923662475792, + "tokens_seen": 1447755776 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045366499498495494, + "loss": 1.3603, + "theoretical_loss": 3.55557819330293, + "tokens_seen": 1447821312 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004536489468405216, + "loss": 1.3076, + "theoretical_loss": 3.55556402117943, + "tokens_seen": 1447886848 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045363289869608834, + "loss": 1.4207, + "theoretical_loss": 3.5555498498769955, + "tokens_seen": 1447952384 + }, + { + "epoch": 0.44, + "learning_rate": 0.000453616850551655, + "loss": 1.3915, + "theoretical_loss": 3.555535679395541, + "tokens_seen": 1448017920 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004536008024072217, + "loss": 1.4125, + "theoretical_loss": 3.555521509734983, + "tokens_seen": 1448083456 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004535847542627884, + "loss": 1.3213, + "theoretical_loss": 3.555507340895235, + "tokens_seen": 1448148992 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004535687061183551, + "loss": 1.3486, + "theoretical_loss": 3.555493172876214, + "tokens_seen": 1448214528 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004535526579739218, + "loss": 1.4529, + "theoretical_loss": 3.555479005677835, + "tokens_seen": 1448280064 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 538246, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3327152729034424, + "objective/train/theoretical_loss": 3.5554648393000123, + "objective/train/tokens_used": 1468805600, + "theoretical_loss": 3.5554648393000123, + "tokens_seen": 1448345600 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004535366098294885, + "loss": 1.3327, + "theoretical_loss": 3.5554648393000123, + "tokens_seen": 1448345600 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004535205616850552, + "loss": 1.4212, + "theoretical_loss": 3.555450673742662, + "tokens_seen": 1448411136 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004535045135406219, + "loss": 1.3435, + "theoretical_loss": 3.5554365090057, + "tokens_seen": 1448476672 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004534884653961886, + "loss": 1.3577, + "theoretical_loss": 3.5554223450890405, + "tokens_seen": 1448542208 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004534724172517553, + "loss": 1.3275, + "theoretical_loss": 3.5554081819925996, + "tokens_seen": 1448607744 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045345636910732194, + "loss": 1.4685, + "theoretical_loss": 3.5553940197162923, + "tokens_seen": 1448673280 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004534403209628887, + "loss": 1.3989, + "theoretical_loss": 3.555379858260035, + "tokens_seen": 1448738816 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045342427281845534, + "loss": 1.4547, + "theoretical_loss": 3.5553656976237415, + "tokens_seen": 1448804352 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004534082246740221, + "loss": 1.4254, + "theoretical_loss": 3.5553515378073284, + "tokens_seen": 1448869888 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045339217652958874, + "loss": 1.3661, + "theoretical_loss": 3.5553373788107114, + "tokens_seen": 1448935424 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004533761283851555, + "loss": 1.4051, + "theoretical_loss": 3.555323220633805, + "tokens_seen": 1449000960 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045336008024072214, + "loss": 1.2453, + "theoretical_loss": 3.5553090632765247, + "tokens_seen": 1449066496 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004533440320962889, + "loss": 1.3799, + "theoretical_loss": 3.555294906738787, + "tokens_seen": 1449132032 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045332798395185555, + "loss": 1.4178, + "theoretical_loss": 3.5552807510205064, + "tokens_seen": 1449197568 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004533119358074223, + "loss": 1.3242, + "theoretical_loss": 3.555266596121599, + "tokens_seen": 1449263104 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045329588766298895, + "loss": 1.385, + "theoretical_loss": 3.55525244204198, + "tokens_seen": 1449328640 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004532798395185557, + "loss": 1.3647, + "theoretical_loss": 3.555238288781565, + "tokens_seen": 1449394176 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045326379137412235, + "loss": 1.3873, + "theoretical_loss": 3.5552241363402697, + "tokens_seen": 1449459712 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004532477432296891, + "loss": 1.3857, + "theoretical_loss": 3.55520998471801, + "tokens_seen": 1449525248 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045323169508525575, + "loss": 1.5158, + "theoretical_loss": 3.5551958339147003, + "tokens_seen": 1449590784 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004532156469408225, + "loss": 1.3591, + "theoretical_loss": 3.5551816839302575, + "tokens_seen": 1449656320 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045319959879638915, + "loss": 1.4488, + "theoretical_loss": 3.555167534764596, + "tokens_seen": 1449721856 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004531835506519559, + "loss": 1.4039, + "theoretical_loss": 3.555153386417633, + "tokens_seen": 1449787392 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045316750250752255, + "loss": 1.3059, + "theoretical_loss": 3.555139238889282, + "tokens_seen": 1449852928 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004531514543630893, + "loss": 1.3061, + "theoretical_loss": 3.5551250921794604, + "tokens_seen": 1449918464 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045313540621865595, + "loss": 1.3401, + "theoretical_loss": 3.5551109462880834, + "tokens_seen": 1449984000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004531193580742227, + "loss": 1.3197, + "theoretical_loss": 3.555096801215066, + "tokens_seen": 1450049536 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045310330992978935, + "loss": 1.4023, + "theoretical_loss": 3.555082656960325, + "tokens_seen": 1450115072 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004530872617853561, + "loss": 1.3615, + "theoretical_loss": 3.555068513523775, + "tokens_seen": 1450180608 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045307121364092276, + "loss": 1.3943, + "theoretical_loss": 3.555054370905332, + "tokens_seen": 1450246144 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004530551654964895, + "loss": 1.4088, + "theoretical_loss": 3.5550402291049124, + "tokens_seen": 1450311680 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045303911735205616, + "loss": 1.4791, + "theoretical_loss": 3.555026088122431, + "tokens_seen": 1450377216 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004530230692076229, + "loss": 1.3622, + "theoretical_loss": 3.555011947957804, + "tokens_seen": 1450442752 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045300702106318956, + "loss": 1.3869, + "theoretical_loss": 3.5549978086109473, + "tokens_seen": 1450508288 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004529909729187563, + "loss": 1.3472, + "theoretical_loss": 3.554983670081776, + "tokens_seen": 1450573824 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045297492477432296, + "loss": 1.4246, + "theoretical_loss": 3.554969532370207, + "tokens_seen": 1450639360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004529588766298897, + "loss": 1.3327, + "theoretical_loss": 3.5549553954761546, + "tokens_seen": 1450704896 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045294282848545636, + "loss": 1.3242, + "theoretical_loss": 3.554941259399536, + "tokens_seen": 1450770432 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004529267803410231, + "loss": 1.448, + "theoretical_loss": 3.5549271241402662, + "tokens_seen": 1450835968 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045291073219658976, + "loss": 1.2895, + "theoretical_loss": 3.554912989698261, + "tokens_seen": 1450901504 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004528946840521565, + "loss": 1.4287, + "theoretical_loss": 3.5548988560734367, + "tokens_seen": 1450967040 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045287863590772316, + "loss": 1.3027, + "theoretical_loss": 3.554884723265709, + "tokens_seen": 1451032576 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004528625877632899, + "loss": 1.3193, + "theoretical_loss": 3.5548705912749936, + "tokens_seen": 1451098112 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004528465396188566, + "loss": 1.3122, + "theoretical_loss": 3.554856460101206, + "tokens_seen": 1451163648 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045283049147442326, + "loss": 1.3542, + "theoretical_loss": 3.5548423297442633, + "tokens_seen": 1451229184 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045281444332999, + "loss": 1.3963, + "theoretical_loss": 3.5548282002040805, + "tokens_seen": 1451294720 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045279839518555667, + "loss": 1.2942, + "theoretical_loss": 3.554814071480573, + "tokens_seen": 1451360256 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004527823470411234, + "loss": 1.3055, + "theoretical_loss": 3.554799943573658, + "tokens_seen": 1451425792 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045276629889669007, + "loss": 1.5028, + "theoretical_loss": 3.5547858164832506, + "tokens_seen": 1451491328 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004527502507522568, + "loss": 1.3667, + "theoretical_loss": 3.554771690209267, + "tokens_seen": 1451556864 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045273420260782347, + "loss": 1.5709, + "theoretical_loss": 3.5547575647516227, + "tokens_seen": 1451622400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004527181544633902, + "loss": 1.3754, + "theoretical_loss": 3.5547434401102347, + "tokens_seen": 1451687936 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045270210631895687, + "loss": 1.3754, + "theoretical_loss": 3.554729316285018, + "tokens_seen": 1451753472 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004526860581745236, + "loss": 1.408, + "theoretical_loss": 3.5547151932758894, + "tokens_seen": 1451819008 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045267001003009027, + "loss": 1.3672, + "theoretical_loss": 3.554701071082764, + "tokens_seen": 1451884544 + }, + { + "epoch": 0.44, + "learning_rate": 0.000452653961885657, + "loss": 1.358, + "theoretical_loss": 3.5546869497055584, + "tokens_seen": 1451950080 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045263791374122367, + "loss": 1.465, + "theoretical_loss": 3.554672829144189, + "tokens_seen": 1452015616 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045262186559679043, + "loss": 1.3531, + "theoretical_loss": 3.5546587093985704, + "tokens_seen": 1452081152 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004526058174523571, + "loss": 1.4062, + "theoretical_loss": 3.5546445904686204, + "tokens_seen": 1452146688 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045258976930792383, + "loss": 1.3824, + "theoretical_loss": 3.554630472354254, + "tokens_seen": 1452212224 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004525737211634905, + "loss": 1.4698, + "theoretical_loss": 3.554616355055388, + "tokens_seen": 1452277760 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045255767301905723, + "loss": 1.269, + "theoretical_loss": 3.5546022385719374, + "tokens_seen": 1452343296 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004525416248746239, + "loss": 1.3237, + "theoretical_loss": 3.5545881229038194, + "tokens_seen": 1452408832 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045252557673019063, + "loss": 1.4595, + "theoretical_loss": 3.55457400805095, + "tokens_seen": 1452474368 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004525095285857573, + "loss": 1.3554, + "theoretical_loss": 3.554559894013245, + "tokens_seen": 1452539904 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045249348044132403, + "loss": 1.3506, + "theoretical_loss": 3.5545457807906202, + "tokens_seen": 1452605440 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004524774322968907, + "loss": 1.3866, + "theoretical_loss": 3.5545316683829924, + "tokens_seen": 1452670976 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045246138415245743, + "loss": 1.2921, + "theoretical_loss": 3.5545175567902776, + "tokens_seen": 1452736512 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004524453360080241, + "loss": 1.3639, + "theoretical_loss": 3.554503446012392, + "tokens_seen": 1452802048 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045242928786359083, + "loss": 1.2515, + "theoretical_loss": 3.5544893360492513, + "tokens_seen": 1452867584 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004524132397191575, + "loss": 1.381, + "theoretical_loss": 3.5544752269007724, + "tokens_seen": 1452933120 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045239719157472424, + "loss": 1.3186, + "theoretical_loss": 3.5544611185668713, + "tokens_seen": 1452998656 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004523811434302909, + "loss": 1.3532, + "theoretical_loss": 3.554447011047464, + "tokens_seen": 1453064192 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045236509528585764, + "loss": 1.2844, + "theoretical_loss": 3.554432904342467, + "tokens_seen": 1453129728 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004523490471414243, + "loss": 1.2265, + "theoretical_loss": 3.5544187984517968, + "tokens_seen": 1453195264 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045233299899699104, + "loss": 1.3127, + "theoretical_loss": 3.5544046933753695, + "tokens_seen": 1453260800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004523169508525577, + "loss": 1.2784, + "theoretical_loss": 3.5543905891131007, + "tokens_seen": 1453326336 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045230090270812444, + "loss": 1.3679, + "theoretical_loss": 3.554376485664907, + "tokens_seen": 1453391872 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004522848545636911, + "loss": 1.3276, + "theoretical_loss": 3.5543623830307056, + "tokens_seen": 1453457408 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045226880641925784, + "loss": 1.4009, + "theoretical_loss": 3.554348281210412, + "tokens_seen": 1453522944 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004522527582748245, + "loss": 1.2623, + "theoretical_loss": 3.5543341802039423, + "tokens_seen": 1453588480 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045223671013039124, + "loss": 1.1943, + "theoretical_loss": 3.5543200800112134, + "tokens_seen": 1453654016 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004522206619859579, + "loss": 1.4026, + "theoretical_loss": 3.554305980632142, + "tokens_seen": 1453719552 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004522046138415246, + "loss": 1.3789, + "theoretical_loss": 3.554291882066644, + "tokens_seen": 1453785088 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004521885656970913, + "loss": 1.327, + "theoretical_loss": 3.554277784314635, + "tokens_seen": 1453850624 + }, + { + "epoch": 0.44, + "learning_rate": 0.000452172517552658, + "loss": 1.3673, + "theoretical_loss": 3.5542636873760323, + "tokens_seen": 1453916160 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004521564694082247, + "loss": 1.4177, + "theoretical_loss": 3.554249591250753, + "tokens_seen": 1453981696 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004521404212637914, + "loss": 1.3999, + "theoretical_loss": 3.5542354959387117, + "tokens_seen": 1454047232 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004521243731193581, + "loss": 1.3112, + "theoretical_loss": 3.5542214014398263, + "tokens_seen": 1454112768 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004521083249749248, + "loss": 1.2614, + "theoretical_loss": 3.5542073077540124, + "tokens_seen": 1454178304 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004520922768304915, + "loss": 1.4152, + "theoretical_loss": 3.5541932148811872, + "tokens_seen": 1454243840 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004520762286860582, + "loss": 1.3382, + "theoretical_loss": 3.5541791228212665, + "tokens_seen": 1454309376 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045206018054162484, + "loss": 1.4543, + "theoretical_loss": 3.554165031574167, + "tokens_seen": 1454374912 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004520441323971916, + "loss": 1.2839, + "theoretical_loss": 3.5541509411398056, + "tokens_seen": 1454440448 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045202808425275824, + "loss": 1.3382, + "theoretical_loss": 3.5541368515180984, + "tokens_seen": 1454505984 + }, + { + "epoch": 0.44, + "learning_rate": 0.000452012036108325, + "loss": 1.2054, + "theoretical_loss": 3.5541227627089613, + "tokens_seen": 1454571520 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045199598796389164, + "loss": 1.3678, + "theoretical_loss": 3.554108674712312, + "tokens_seen": 1454637056 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004519799398194584, + "loss": 1.3412, + "theoretical_loss": 3.554094587528067, + "tokens_seen": 1454702592 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045196389167502504, + "loss": 1.295, + "theoretical_loss": 3.5540805011561423, + "tokens_seen": 1454768128 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004519478435305918, + "loss": 1.1925, + "theoretical_loss": 3.554066415596454, + "tokens_seen": 1454833664 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 540953, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3491237163543701, + "objective/train/theoretical_loss": 3.5540523308489194, + "objective/train/tokens_used": 1475359200, + "theoretical_loss": 3.5540523308489194, + "tokens_seen": 1454899200 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045193179538615845, + "loss": 1.3491, + "theoretical_loss": 3.5540523308489194, + "tokens_seen": 1454899200 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004519157472417252, + "loss": 1.3184, + "theoretical_loss": 3.5540382469134553, + "tokens_seen": 1454964736 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045189969909729185, + "loss": 1.3915, + "theoretical_loss": 3.5540241637899777, + "tokens_seen": 1455030272 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004518836509528586, + "loss": 1.2953, + "theoretical_loss": 3.5540100814784035, + "tokens_seen": 1455095808 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045186760280842525, + "loss": 1.4281, + "theoretical_loss": 3.553995999978649, + "tokens_seen": 1455161344 + }, + { + "epoch": 0.44, + "learning_rate": 0.000451851554663992, + "loss": 1.4111, + "theoretical_loss": 3.5539819192906315, + "tokens_seen": 1455226880 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045183550651955865, + "loss": 1.3857, + "theoretical_loss": 3.553967839414267, + "tokens_seen": 1455292416 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004518194583751254, + "loss": 1.3978, + "theoretical_loss": 3.5539537603494726, + "tokens_seen": 1455357952 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045180341023069205, + "loss": 1.3267, + "theoretical_loss": 3.553939682096165, + "tokens_seen": 1455423488 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004517873620862588, + "loss": 1.3251, + "theoretical_loss": 3.5539256046542604, + "tokens_seen": 1455489024 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045177131394182545, + "loss": 1.2794, + "theoretical_loss": 3.553911528023676, + "tokens_seen": 1455554560 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004517552657973922, + "loss": 1.3207, + "theoretical_loss": 3.5538974522043283, + "tokens_seen": 1455620096 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045173921765295885, + "loss": 1.2892, + "theoretical_loss": 3.553883377196134, + "tokens_seen": 1455685632 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004517231695085256, + "loss": 1.3564, + "theoretical_loss": 3.55386930299901, + "tokens_seen": 1455751168 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045170712136409226, + "loss": 1.2633, + "theoretical_loss": 3.553855229612873, + "tokens_seen": 1455816704 + }, + { + "epoch": 0.44, + "learning_rate": 0.000451691073219659, + "loss": 1.292, + "theoretical_loss": 3.55384115703764, + "tokens_seen": 1455882240 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045167502507522566, + "loss": 1.2397, + "theoretical_loss": 3.5538270852732268, + "tokens_seen": 1455947776 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004516589769307924, + "loss": 1.4024, + "theoretical_loss": 3.553813014319551, + "tokens_seen": 1456013312 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045164292878635917, + "loss": 1.3445, + "theoretical_loss": 3.55379894417653, + "tokens_seen": 1456078848 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004516268806419258, + "loss": 1.248, + "theoretical_loss": 3.5537848748440792, + "tokens_seen": 1456144384 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045161083249749257, + "loss": 1.2811, + "theoretical_loss": 3.5537708063221167, + "tokens_seen": 1456209920 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004515947843530592, + "loss": 1.3467, + "theoretical_loss": 3.5537567386105584, + "tokens_seen": 1456275456 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004515787362086259, + "loss": 1.3297, + "theoretical_loss": 3.5537426717093217, + "tokens_seen": 1456340992 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004515626880641926, + "loss": 1.3836, + "theoretical_loss": 3.553728605618323, + "tokens_seen": 1456406528 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004515466399197593, + "loss": 1.394, + "theoretical_loss": 3.5537145403374795, + "tokens_seen": 1456472064 + }, + { + "epoch": 0.44, + "learning_rate": 0.000451530591775326, + "loss": 1.3812, + "theoretical_loss": 3.553700475866708, + "tokens_seen": 1456537600 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004515145436308927, + "loss": 1.3164, + "theoretical_loss": 3.5536864122059257, + "tokens_seen": 1456603136 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004514984954864594, + "loss": 1.387, + "theoretical_loss": 3.5536723493550495, + "tokens_seen": 1456668672 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004514824473420261, + "loss": 1.305, + "theoretical_loss": 3.5536582873139952, + "tokens_seen": 1456734208 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004514663991975928, + "loss": 1.2573, + "theoretical_loss": 3.5536442260826813, + "tokens_seen": 1456799744 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004514503510531595, + "loss": 1.3157, + "theoretical_loss": 3.5536301656610236, + "tokens_seen": 1456865280 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045143430290872617, + "loss": 1.2292, + "theoretical_loss": 3.55361610604894, + "tokens_seen": 1456930816 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004514182547642929, + "loss": 1.3317, + "theoretical_loss": 3.553602047246347, + "tokens_seen": 1456996352 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045140220661985957, + "loss": 1.2486, + "theoretical_loss": 3.5535879892531614, + "tokens_seen": 1457061888 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004513861584754263, + "loss": 1.3215, + "theoretical_loss": 3.5535739320693, + "tokens_seen": 1457127424 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045137011033099297, + "loss": 1.3199, + "theoretical_loss": 3.5535598756946807, + "tokens_seen": 1457192960 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004513540621865597, + "loss": 1.349, + "theoretical_loss": 3.55354582012922, + "tokens_seen": 1457258496 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045133801404212637, + "loss": 1.3929, + "theoretical_loss": 3.553531765372835, + "tokens_seen": 1457324032 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004513219658976931, + "loss": 1.2761, + "theoretical_loss": 3.5535177114254424, + "tokens_seen": 1457389568 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045130591775325977, + "loss": 1.3573, + "theoretical_loss": 3.5535036582869592, + "tokens_seen": 1457455104 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004512898696088265, + "loss": 1.4112, + "theoretical_loss": 3.5534896059573033, + "tokens_seen": 1457520640 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045127382146439317, + "loss": 1.3185, + "theoretical_loss": 3.5534755544363916, + "tokens_seen": 1457586176 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004512577733199599, + "loss": 1.2044, + "theoretical_loss": 3.55346150372414, + "tokens_seen": 1457651712 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004512417251755266, + "loss": 1.2837, + "theoretical_loss": 3.5534474538204672, + "tokens_seen": 1457717248 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045122567703109333, + "loss": 1.2609, + "theoretical_loss": 3.55343340472529, + "tokens_seen": 1457782784 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045120962888666, + "loss": 1.3989, + "theoretical_loss": 3.553419356438524, + "tokens_seen": 1457848320 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045119358074222673, + "loss": 1.4084, + "theoretical_loss": 3.5534053089600883, + "tokens_seen": 1457913856 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004511775325977934, + "loss": 1.3779, + "theoretical_loss": 3.553391262289899, + "tokens_seen": 1457979392 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045116148445336013, + "loss": 1.4213, + "theoretical_loss": 3.5533772164278736, + "tokens_seen": 1458044928 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004511454363089268, + "loss": 1.2903, + "theoretical_loss": 3.5533631713739293, + "tokens_seen": 1458110464 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045112938816449353, + "loss": 1.2597, + "theoretical_loss": 3.553349127127983, + "tokens_seen": 1458176000 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004511133400200602, + "loss": 1.1899, + "theoretical_loss": 3.5533350836899524, + "tokens_seen": 1458241536 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045109729187562693, + "loss": 1.3491, + "theoretical_loss": 3.553321041059754, + "tokens_seen": 1458307072 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004510812437311936, + "loss": 1.2639, + "theoretical_loss": 3.553306999237306, + "tokens_seen": 1458372608 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045106519558676033, + "loss": 1.3105, + "theoretical_loss": 3.5532929582225243, + "tokens_seen": 1458438144 + }, + { + "epoch": 0.44, + "learning_rate": 0.000451049147442327, + "loss": 1.44, + "theoretical_loss": 3.5532789180153275, + "tokens_seen": 1458503680 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045103309929789374, + "loss": 1.2769, + "theoretical_loss": 3.5532648786156322, + "tokens_seen": 1458569216 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004510170511534604, + "loss": 1.3575, + "theoretical_loss": 3.553250840023356, + "tokens_seen": 1458634752 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045100100300902714, + "loss": 1.2588, + "theoretical_loss": 3.553236802238416, + "tokens_seen": 1458700288 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004509849548645938, + "loss": 1.252, + "theoretical_loss": 3.5532227652607293, + "tokens_seen": 1458765824 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045096890672016054, + "loss": 1.3698, + "theoretical_loss": 3.5532087290902132, + "tokens_seen": 1458831360 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004509528585757272, + "loss": 1.4818, + "theoretical_loss": 3.5531946937267858, + "tokens_seen": 1458896896 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045093681043129394, + "loss": 1.3153, + "theoretical_loss": 3.553180659170363, + "tokens_seen": 1458962432 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004509207622868606, + "loss": 1.305, + "theoretical_loss": 3.5531666254208636, + "tokens_seen": 1459027968 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045090471414242734, + "loss": 1.366, + "theoretical_loss": 3.5531525924782046, + "tokens_seen": 1459093504 + }, + { + "epoch": 0.44, + "learning_rate": 0.000450888665997994, + "loss": 1.2617, + "theoretical_loss": 3.553138560342303, + "tokens_seen": 1459159040 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045087261785356074, + "loss": 1.4243, + "theoretical_loss": 3.553124529013076, + "tokens_seen": 1459224576 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004508565697091274, + "loss": 1.2434, + "theoretical_loss": 3.5531104984904416, + "tokens_seen": 1459290112 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045084052156469414, + "loss": 1.3792, + "theoretical_loss": 3.5530964687743167, + "tokens_seen": 1459355648 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004508244734202608, + "loss": 1.4054, + "theoretical_loss": 3.5530824398646192, + "tokens_seen": 1459421184 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004508084252758275, + "loss": 1.4008, + "theoretical_loss": 3.5530684117612665, + "tokens_seen": 1459486720 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004507923771313942, + "loss": 1.217, + "theoretical_loss": 3.5530543844641755, + "tokens_seen": 1459552256 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004507763289869609, + "loss": 1.1986, + "theoretical_loss": 3.5530403579732646, + "tokens_seen": 1459617792 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004507602808425276, + "loss": 1.3397, + "theoretical_loss": 3.55302633228845, + "tokens_seen": 1459683328 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004507442326980943, + "loss": 1.3343, + "theoretical_loss": 3.5530123074096505, + "tokens_seen": 1459748864 + }, + { + "epoch": 0.44, + "learning_rate": 0.000450728184553661, + "loss": 1.4429, + "theoretical_loss": 3.552998283336782, + "tokens_seen": 1459814400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004507121364092277, + "loss": 1.3009, + "theoretical_loss": 3.5529842600697643, + "tokens_seen": 1459879936 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004506960882647944, + "loss": 1.3921, + "theoretical_loss": 3.5529702376085126, + "tokens_seen": 1459945472 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004506800401203611, + "loss": 1.2916, + "theoretical_loss": 3.552956215952946, + "tokens_seen": 1460011008 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045066399197592774, + "loss": 1.402, + "theoretical_loss": 3.5529421951029816, + "tokens_seen": 1460076544 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004506479438314945, + "loss": 1.4513, + "theoretical_loss": 3.5529281750585366, + "tokens_seen": 1460142080 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045063189568706114, + "loss": 1.3962, + "theoretical_loss": 3.5529141558195287, + "tokens_seen": 1460207616 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004506158475426279, + "loss": 1.3605, + "theoretical_loss": 3.552900137385876, + "tokens_seen": 1460273152 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045059979939819454, + "loss": 1.4149, + "theoretical_loss": 3.5528861197574955, + "tokens_seen": 1460338688 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004505837512537613, + "loss": 1.2739, + "theoretical_loss": 3.552872102934305, + "tokens_seen": 1460404224 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045056770310932795, + "loss": 1.2752, + "theoretical_loss": 3.552858086916222, + "tokens_seen": 1460469760 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004505516549648947, + "loss": 1.4318, + "theoretical_loss": 3.552844071703164, + "tokens_seen": 1460535296 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045053560682046135, + "loss": 1.2587, + "theoretical_loss": 3.5528300572950497, + "tokens_seen": 1460600832 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004505195586760281, + "loss": 1.3297, + "theoretical_loss": 3.5528160436917955, + "tokens_seen": 1460666368 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045050351053159475, + "loss": 1.3227, + "theoretical_loss": 3.5528020308933197, + "tokens_seen": 1460731904 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004504874623871615, + "loss": 1.2963, + "theoretical_loss": 3.5527880188995393, + "tokens_seen": 1460797440 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045047141424272815, + "loss": 1.3906, + "theoretical_loss": 3.552774007710373, + "tokens_seen": 1460862976 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004504553660982949, + "loss": 1.26, + "theoretical_loss": 3.5527599973257376, + "tokens_seen": 1460928512 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045043931795386166, + "loss": 1.4034, + "theoretical_loss": 3.552745987745552, + "tokens_seen": 1460994048 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004504232698094283, + "loss": 1.3606, + "theoretical_loss": 3.5527319789697325, + "tokens_seen": 1461059584 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045040722166499506, + "loss": 1.3851, + "theoretical_loss": 3.5527179709981973, + "tokens_seen": 1461125120 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004503911735205617, + "loss": 1.3705, + "theoretical_loss": 3.5527039638308646, + "tokens_seen": 1461190656 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045037512537612846, + "loss": 1.3151, + "theoretical_loss": 3.552689957467652, + "tokens_seen": 1461256192 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004503590772316951, + "loss": 1.4017, + "theoretical_loss": 3.552675951908477, + "tokens_seen": 1461321728 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045034302908726186, + "loss": 1.3891, + "theoretical_loss": 3.5526619471532577, + "tokens_seen": 1461387264 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 543334, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.363938570022583, + "objective/train/theoretical_loss": 3.552647943201912, + "objective/train/tokens_used": 1481912800, + "theoretical_loss": 3.552647943201912, + "tokens_seen": 1461452800 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004503269809428285, + "loss": 1.3639, + "theoretical_loss": 3.552647943201912, + "tokens_seen": 1461452800 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045031093279839526, + "loss": 1.3725, + "theoretical_loss": 3.552633940054357, + "tokens_seen": 1461518336 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004502948846539619, + "loss": 1.3412, + "theoretical_loss": 3.5526199377105114, + "tokens_seen": 1461583872 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045027883650952867, + "loss": 1.2974, + "theoretical_loss": 3.552605936170292, + "tokens_seen": 1461649408 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004502627883650953, + "loss": 1.3317, + "theoretical_loss": 3.552591935433618, + "tokens_seen": 1461714944 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045024674022066207, + "loss": 1.4466, + "theoretical_loss": 3.5525779355004063, + "tokens_seen": 1461780480 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004502306920762287, + "loss": 1.2818, + "theoretical_loss": 3.552563936370575, + "tokens_seen": 1461846016 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045021464393179547, + "loss": 1.286, + "theoretical_loss": 3.5525499380440424, + "tokens_seen": 1461911552 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004501985957873621, + "loss": 1.2014, + "theoretical_loss": 3.5525359405207255, + "tokens_seen": 1461977088 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004501825476429288, + "loss": 1.2755, + "theoretical_loss": 3.552521943800543, + "tokens_seen": 1462042624 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004501664994984955, + "loss": 1.3259, + "theoretical_loss": 3.552507947883412, + "tokens_seen": 1462108160 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004501504513540622, + "loss": 1.435, + "theoretical_loss": 3.552493952769252, + "tokens_seen": 1462173696 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004501344032096289, + "loss": 1.3121, + "theoretical_loss": 3.552479958457979, + "tokens_seen": 1462239232 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004501183550651956, + "loss": 1.3485, + "theoretical_loss": 3.552465964949512, + "tokens_seen": 1462304768 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004501023069207623, + "loss": 1.2446, + "theoretical_loss": 3.5524519722437695, + "tokens_seen": 1462370304 + }, + { + "epoch": 0.44, + "learning_rate": 0.000450086258776329, + "loss": 1.2835, + "theoretical_loss": 3.5524379803406685, + "tokens_seen": 1462435840 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004500702106318957, + "loss": 1.248, + "theoretical_loss": 3.552423989240127, + "tokens_seen": 1462501376 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004500541624874624, + "loss": 1.2556, + "theoretical_loss": 3.552409998942063, + "tokens_seen": 1462566912 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045003811434302907, + "loss": 1.3219, + "theoretical_loss": 3.5523960094463956, + "tokens_seen": 1462632448 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004500220661985958, + "loss": 1.3279, + "theoretical_loss": 3.5523820207530417, + "tokens_seen": 1462697984 + }, + { + "epoch": 0.44, + "learning_rate": 0.00045000601805416247, + "loss": 1.292, + "theoretical_loss": 3.55236803286192, + "tokens_seen": 1462763520 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004499899699097292, + "loss": 1.3865, + "theoretical_loss": 3.552354045772948, + "tokens_seen": 1462829056 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044997392176529587, + "loss": 1.3358, + "theoretical_loss": 3.5523400594860437, + "tokens_seen": 1462894592 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004499578736208626, + "loss": 1.3409, + "theoretical_loss": 3.552326074001126, + "tokens_seen": 1462960128 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044994182547642927, + "loss": 1.2615, + "theoretical_loss": 3.5523120893181126, + "tokens_seen": 1463025664 + }, + { + "epoch": 0.44, + "learning_rate": 0.000449925777331996, + "loss": 1.3804, + "theoretical_loss": 3.5522981054369214, + "tokens_seen": 1463091200 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044990972918756267, + "loss": 1.3009, + "theoretical_loss": 3.5522841223574706, + "tokens_seen": 1463156736 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004498936810431294, + "loss": 1.31, + "theoretical_loss": 3.552270140079678, + "tokens_seen": 1463222272 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004498776328986961, + "loss": 1.2851, + "theoretical_loss": 3.5522561586034627, + "tokens_seen": 1463287808 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044986158475426283, + "loss": 1.3573, + "theoretical_loss": 3.5522421779287416, + "tokens_seen": 1463353344 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004498455366098295, + "loss": 1.3158, + "theoretical_loss": 3.5522281980554338, + "tokens_seen": 1463418880 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044982948846539623, + "loss": 1.3696, + "theoretical_loss": 3.5522142189834574, + "tokens_seen": 1463484416 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004498134403209629, + "loss": 1.3347, + "theoretical_loss": 3.5522002407127298, + "tokens_seen": 1463549952 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044979739217652963, + "loss": 1.3611, + "theoretical_loss": 3.55218626324317, + "tokens_seen": 1463615488 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004497813440320963, + "loss": 1.3655, + "theoretical_loss": 3.552172286574696, + "tokens_seen": 1463681024 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044976529588766303, + "loss": 1.2967, + "theoretical_loss": 3.5521583107072265, + "tokens_seen": 1463746560 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004497492477432297, + "loss": 1.2901, + "theoretical_loss": 3.5521443356406786, + "tokens_seen": 1463812096 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044973319959879643, + "loss": 1.2573, + "theoretical_loss": 3.552130361374971, + "tokens_seen": 1463877632 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004497171514543631, + "loss": 1.2834, + "theoretical_loss": 3.5521163879100226, + "tokens_seen": 1463943168 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044970110330992983, + "loss": 1.3041, + "theoretical_loss": 3.552102415245751, + "tokens_seen": 1464008704 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004496850551654965, + "loss": 1.3352, + "theoretical_loss": 3.5520884433820745, + "tokens_seen": 1464074240 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044966900702106324, + "loss": 1.3587, + "theoretical_loss": 3.552074472318912, + "tokens_seen": 1464139776 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004496529588766299, + "loss": 1.3471, + "theoretical_loss": 3.552060502056181, + "tokens_seen": 1464205312 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044963691073219664, + "loss": 1.2522, + "theoretical_loss": 3.5520465325938004, + "tokens_seen": 1464270848 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004496208625877633, + "loss": 1.4048, + "theoretical_loss": 3.5520325639316885, + "tokens_seen": 1464336384 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044960481444333004, + "loss": 1.3991, + "theoretical_loss": 3.552018596069763, + "tokens_seen": 1464401920 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004495887662988967, + "loss": 1.2989, + "theoretical_loss": 3.552004629007943, + "tokens_seen": 1464467456 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044957271815446344, + "loss": 1.4051, + "theoretical_loss": 3.551990662746147, + "tokens_seen": 1464532992 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004495566700100301, + "loss": 1.3656, + "theoretical_loss": 3.5519766972842923, + "tokens_seen": 1464598528 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044954062186559684, + "loss": 1.2831, + "theoretical_loss": 3.5519627326222976, + "tokens_seen": 1464664064 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004495245737211635, + "loss": 1.3304, + "theoretical_loss": 3.5519487687600826, + "tokens_seen": 1464729600 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044950852557673024, + "loss": 1.1919, + "theoretical_loss": 3.5519348056975644, + "tokens_seen": 1464795136 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004494924774322969, + "loss": 1.2845, + "theoretical_loss": 3.5519208434346616, + "tokens_seen": 1464860672 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044947642928786364, + "loss": 1.1987, + "theoretical_loss": 3.551906881971293, + "tokens_seen": 1464926208 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004494603811434303, + "loss": 1.4716, + "theoretical_loss": 3.551892921307377, + "tokens_seen": 1464991744 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044944433299899704, + "loss": 1.4319, + "theoretical_loss": 3.551878961442832, + "tokens_seen": 1465057280 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004494282848545637, + "loss": 1.3272, + "theoretical_loss": 3.5518650023775766, + "tokens_seen": 1465122816 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004494122367101304, + "loss": 1.298, + "theoretical_loss": 3.5518510441115287, + "tokens_seen": 1465188352 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004493961885656971, + "loss": 1.3329, + "theoretical_loss": 3.551837086644607, + "tokens_seen": 1465253888 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004493801404212638, + "loss": 1.2959, + "theoretical_loss": 3.5518231299767304, + "tokens_seen": 1465319424 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004493640922768305, + "loss": 1.3355, + "theoretical_loss": 3.5518091741078175, + "tokens_seen": 1465384960 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004493480441323972, + "loss": 1.2865, + "theoretical_loss": 3.5517952190377864, + "tokens_seen": 1465450496 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004493319959879639, + "loss": 1.3304, + "theoretical_loss": 3.5517812647665563, + "tokens_seen": 1465516032 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004493159478435306, + "loss": 1.2787, + "theoretical_loss": 3.5517673112940447, + "tokens_seen": 1465581568 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004492998996990973, + "loss": 1.2391, + "theoretical_loss": 3.5517533586201706, + "tokens_seen": 1465647104 + }, + { + "epoch": 0.44, + "learning_rate": 0.000449283851554664, + "loss": 1.3796, + "theoretical_loss": 3.5517394067448533, + "tokens_seen": 1465712640 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044926780341023064, + "loss": 1.2804, + "theoretical_loss": 3.5517254556680102, + "tokens_seen": 1465778176 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004492517552657974, + "loss": 1.2625, + "theoretical_loss": 3.5517115053895614, + "tokens_seen": 1465843712 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044923570712136415, + "loss": 1.4461, + "theoretical_loss": 3.5516975559094237, + "tokens_seen": 1465909248 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004492196589769308, + "loss": 1.3299, + "theoretical_loss": 3.551683607227517, + "tokens_seen": 1465974784 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044920361083249755, + "loss": 1.3704, + "theoretical_loss": 3.55166965934376, + "tokens_seen": 1466040320 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004491875626880642, + "loss": 1.3283, + "theoretical_loss": 3.5516557122580705, + "tokens_seen": 1466105856 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044917151454363096, + "loss": 1.2555, + "theoretical_loss": 3.551641765970368, + "tokens_seen": 1466171392 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004491554663991976, + "loss": 1.4086, + "theoretical_loss": 3.5516278204805705, + "tokens_seen": 1466236928 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044913941825476436, + "loss": 1.283, + "theoretical_loss": 3.551613875788597, + "tokens_seen": 1466302464 + }, + { + "epoch": 0.44, + "learning_rate": 0.000449123370110331, + "loss": 1.2665, + "theoretical_loss": 3.5515999318943665, + "tokens_seen": 1466368000 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044910732196589776, + "loss": 1.3185, + "theoretical_loss": 3.551585988797797, + "tokens_seen": 1466433536 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004490912738214644, + "loss": 1.3226, + "theoretical_loss": 3.5515720464988076, + "tokens_seen": 1466499072 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044907522567703116, + "loss": 1.3038, + "theoretical_loss": 3.551558104997318, + "tokens_seen": 1466564608 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004490591775325978, + "loss": 1.2753, + "theoretical_loss": 3.551544164293245, + "tokens_seen": 1466630144 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044904312938816456, + "loss": 1.2469, + "theoretical_loss": 3.551530224386509, + "tokens_seen": 1466695680 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004490270812437312, + "loss": 1.3693, + "theoretical_loss": 3.551516285277028, + "tokens_seen": 1466761216 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044901103309929796, + "loss": 1.4347, + "theoretical_loss": 3.5515023469647207, + "tokens_seen": 1466826752 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004489949849548646, + "loss": 1.2642, + "theoretical_loss": 3.5514884094495063, + "tokens_seen": 1466892288 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044897893681043136, + "loss": 1.416, + "theoretical_loss": 3.5514744727313032, + "tokens_seen": 1466957824 + }, + { + "epoch": 0.44, + "learning_rate": 0.000448962888665998, + "loss": 1.3319, + "theoretical_loss": 3.5514605368100307, + "tokens_seen": 1467023360 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044894684052156476, + "loss": 1.2981, + "theoretical_loss": 3.5514466016856074, + "tokens_seen": 1467088896 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004489307923771314, + "loss": 1.3548, + "theoretical_loss": 3.551432667357952, + "tokens_seen": 1467154432 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044891474423269817, + "loss": 1.3337, + "theoretical_loss": 3.551418733826984, + "tokens_seen": 1467219968 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004488986960882648, + "loss": 1.3969, + "theoretical_loss": 3.5514048010926214, + "tokens_seen": 1467285504 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044888264794383157, + "loss": 1.2897, + "theoretical_loss": 3.5513908691547833, + "tokens_seen": 1467351040 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004488665997993982, + "loss": 1.3637, + "theoretical_loss": 3.551376938013389, + "tokens_seen": 1467416576 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044885055165496497, + "loss": 1.2558, + "theoretical_loss": 3.551363007668357, + "tokens_seen": 1467482112 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004488345035105316, + "loss": 1.3688, + "theoretical_loss": 3.551349078119606, + "tokens_seen": 1467547648 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044881845536609837, + "loss": 1.2696, + "theoretical_loss": 3.5513351493670555, + "tokens_seen": 1467613184 + }, + { + "epoch": 0.44, + "learning_rate": 0.000448802407221665, + "loss": 1.3323, + "theoretical_loss": 3.5513212214106247, + "tokens_seen": 1467678720 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044878635907723177, + "loss": 1.1915, + "theoretical_loss": 3.5513072942502317, + "tokens_seen": 1467744256 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004487703109327984, + "loss": 1.2583, + "theoretical_loss": 3.551293367885796, + "tokens_seen": 1467809792 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004487542627883651, + "loss": 1.2452, + "theoretical_loss": 3.5512794423172362, + "tokens_seen": 1467875328 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004487382146439318, + "loss": 1.321, + "theoretical_loss": 3.551265517544471, + "tokens_seen": 1467940864 + }, + { + "epoch": 0.44, + "objective/train/docs_used": 545648, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.40727961063385, + "objective/train/theoretical_loss": 3.5512515935674207, + "objective/train/tokens_used": 1488466400, + "theoretical_loss": 3.5512515935674207, + "tokens_seen": 1468006400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004487221664994985, + "loss": 1.4073, + "theoretical_loss": 3.5512515935674207, + "tokens_seen": 1468006400 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004487061183550652, + "loss": 1.3977, + "theoretical_loss": 3.551237670386003, + "tokens_seen": 1468071936 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004486900702106319, + "loss": 1.3296, + "theoretical_loss": 3.5512237480001376, + "tokens_seen": 1468137472 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004486740220661986, + "loss": 1.2911, + "theoretical_loss": 3.551209826409744, + "tokens_seen": 1468203008 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004486579739217653, + "loss": 1.2927, + "theoretical_loss": 3.5511959056147395, + "tokens_seen": 1468268544 + }, + { + "epoch": 0.44, + "learning_rate": 0.000448641925777332, + "loss": 1.3705, + "theoretical_loss": 3.5511819856150453, + "tokens_seen": 1468334080 + }, + { + "epoch": 0.44, + "learning_rate": 0.0004486258776328987, + "loss": 1.3253, + "theoretical_loss": 3.5511680664105785, + "tokens_seen": 1468399616 + }, + { + "epoch": 0.44, + "learning_rate": 0.00044860982948846537, + "loss": 1.3514, + "theoretical_loss": 3.5511541480012596, + "tokens_seen": 1468465152 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004485937813440321, + "loss": 1.2592, + "theoretical_loss": 3.551140230387007, + "tokens_seen": 1468530688 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044857773319959877, + "loss": 1.2514, + "theoretical_loss": 3.55112631356774, + "tokens_seen": 1468596224 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004485616850551655, + "loss": 1.3596, + "theoretical_loss": 3.551112397543378, + "tokens_seen": 1468661760 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044854563691073217, + "loss": 1.2942, + "theoretical_loss": 3.5510984823138396, + "tokens_seen": 1468727296 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004485295887662989, + "loss": 1.4758, + "theoretical_loss": 3.5510845678790446, + "tokens_seen": 1468792832 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004485135406218656, + "loss": 1.311, + "theoretical_loss": 3.551070654238911, + "tokens_seen": 1468858368 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044849749247743233, + "loss": 1.4027, + "theoretical_loss": 3.551056741393359, + "tokens_seen": 1468923904 + }, + { + "epoch": 0.45, + "learning_rate": 0.000448481444332999, + "loss": 1.3155, + "theoretical_loss": 3.551042829342308, + "tokens_seen": 1468989440 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044846539618856573, + "loss": 1.3208, + "theoretical_loss": 3.5510289180856764, + "tokens_seen": 1469054976 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004484493480441324, + "loss": 1.3199, + "theoretical_loss": 3.5510150076233833, + "tokens_seen": 1469120512 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044843329989969913, + "loss": 1.3402, + "theoretical_loss": 3.551001097955348, + "tokens_seen": 1469186048 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004484172517552658, + "loss": 1.2363, + "theoretical_loss": 3.550987189081491, + "tokens_seen": 1469251584 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044840120361083253, + "loss": 1.2616, + "theoretical_loss": 3.5509732810017303, + "tokens_seen": 1469317120 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004483851554663992, + "loss": 1.3094, + "theoretical_loss": 3.550959373715985, + "tokens_seen": 1469382656 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044836910732196593, + "loss": 1.3571, + "theoretical_loss": 3.5509454672241745, + "tokens_seen": 1469448192 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004483530591775326, + "loss": 1.2945, + "theoretical_loss": 3.5509315615262187, + "tokens_seen": 1469513728 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044833701103309933, + "loss": 1.3095, + "theoretical_loss": 3.550917656622037, + "tokens_seen": 1469579264 + }, + { + "epoch": 0.45, + "learning_rate": 0.000448320962888666, + "loss": 1.3214, + "theoretical_loss": 3.5509037525115477, + "tokens_seen": 1469644800 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044830491474423274, + "loss": 1.2397, + "theoretical_loss": 3.5508898491946703, + "tokens_seen": 1469710336 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004482888665997994, + "loss": 1.369, + "theoretical_loss": 3.550875946671325, + "tokens_seen": 1469775872 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044827281845536614, + "loss": 1.3089, + "theoretical_loss": 3.55086204494143, + "tokens_seen": 1469841408 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004482567703109328, + "loss": 1.3342, + "theoretical_loss": 3.5508481440049056, + "tokens_seen": 1469906944 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044824072216649954, + "loss": 1.3311, + "theoretical_loss": 3.5508342438616705, + "tokens_seen": 1469972480 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004482246740220662, + "loss": 1.3048, + "theoretical_loss": 3.5508203445116444, + "tokens_seen": 1470038016 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044820862587763294, + "loss": 1.2818, + "theoretical_loss": 3.5508064459547466, + "tokens_seen": 1470103552 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004481925777331996, + "loss": 1.2266, + "theoretical_loss": 3.5507925481908966, + "tokens_seen": 1470169088 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044817652958876634, + "loss": 1.2647, + "theoretical_loss": 3.5507786512200132, + "tokens_seen": 1470234624 + }, + { + "epoch": 0.45, + "learning_rate": 0.000448160481444333, + "loss": 1.2953, + "theoretical_loss": 3.5507647550420165, + "tokens_seen": 1470300160 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044814443329989974, + "loss": 1.2267, + "theoretical_loss": 3.550750859656826, + "tokens_seen": 1470365696 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004481283851554664, + "loss": 1.2999, + "theoretical_loss": 3.5507369650643605, + "tokens_seen": 1470431232 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044811233701103314, + "loss": 1.4075, + "theoretical_loss": 3.5507230712645397, + "tokens_seen": 1470496768 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004480962888665998, + "loss": 1.326, + "theoretical_loss": 3.550709178257284, + "tokens_seen": 1470562304 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044808024072216654, + "loss": 1.3229, + "theoretical_loss": 3.5506952860425107, + "tokens_seen": 1470627840 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004480641925777332, + "loss": 1.3001, + "theoretical_loss": 3.5506813946201414, + "tokens_seen": 1470693376 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044804814443329995, + "loss": 1.3099, + "theoretical_loss": 3.550667503990095, + "tokens_seen": 1470758912 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044803209628886665, + "loss": 1.3093, + "theoretical_loss": 3.5506536141522904, + "tokens_seen": 1470824448 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044801604814443335, + "loss": 1.2373, + "theoretical_loss": 3.5506397251066475, + "tokens_seen": 1470889984 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044800000000000005, + "loss": 1.3416, + "theoretical_loss": 3.5506258368530856, + "tokens_seen": 1470955520 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004479839518555667, + "loss": 1.2922, + "theoretical_loss": 3.5506119493915254, + "tokens_seen": 1471021056 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044796790371113345, + "loss": 1.3968, + "theoretical_loss": 3.5505980627218845, + "tokens_seen": 1471086592 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004479518555667001, + "loss": 1.2312, + "theoretical_loss": 3.550584176844084, + "tokens_seen": 1471152128 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044793580742226685, + "loss": 1.2555, + "theoretical_loss": 3.550570291758043, + "tokens_seen": 1471217664 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004479197592778335, + "loss": 1.2992, + "theoretical_loss": 3.550556407463681, + "tokens_seen": 1471283200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044790371113340025, + "loss": 1.364, + "theoretical_loss": 3.5505425239609174, + "tokens_seen": 1471348736 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004478876629889669, + "loss": 1.3824, + "theoretical_loss": 3.550528641249673, + "tokens_seen": 1471414272 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044787161484453365, + "loss": 1.2453, + "theoretical_loss": 3.5505147593298654, + "tokens_seen": 1471479808 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004478555667001003, + "loss": 1.3726, + "theoretical_loss": 3.550500878201416, + "tokens_seen": 1471545344 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044783951855566705, + "loss": 1.3333, + "theoretical_loss": 3.5504869978642435, + "tokens_seen": 1471610880 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004478234704112337, + "loss": 1.2799, + "theoretical_loss": 3.550473118318268, + "tokens_seen": 1471676416 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044780742226680046, + "loss": 1.3966, + "theoretical_loss": 3.550459239563409, + "tokens_seen": 1471741952 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004477913741223671, + "loss": 1.3386, + "theoretical_loss": 3.550445361599586, + "tokens_seen": 1471807488 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044777532597793386, + "loss": 1.3826, + "theoretical_loss": 3.550431484426719, + "tokens_seen": 1471873024 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004477592778335005, + "loss": 1.2897, + "theoretical_loss": 3.550417608044728, + "tokens_seen": 1471938560 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044774322968906726, + "loss": 1.2881, + "theoretical_loss": 3.550403732453532, + "tokens_seen": 1472004096 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004477271815446339, + "loss": 1.2842, + "theoretical_loss": 3.550389857653051, + "tokens_seen": 1472069632 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044771113340020066, + "loss": 1.3002, + "theoretical_loss": 3.550375983643205, + "tokens_seen": 1472135168 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004476950852557673, + "loss": 1.448, + "theoretical_loss": 3.5503621104239134, + "tokens_seen": 1472200704 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044767903711133406, + "loss": 1.3339, + "theoretical_loss": 3.5503482379950957, + "tokens_seen": 1472266240 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004476629889669007, + "loss": 1.3728, + "theoretical_loss": 3.5503343663566724, + "tokens_seen": 1472331776 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044764694082246746, + "loss": 1.2878, + "theoretical_loss": 3.550320495508563, + "tokens_seen": 1472397312 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004476308926780341, + "loss": 1.3691, + "theoretical_loss": 3.5503066254506876, + "tokens_seen": 1472462848 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044761484453360086, + "loss": 1.3045, + "theoretical_loss": 3.550292756182965, + "tokens_seen": 1472528384 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004475987963891675, + "loss": 1.4318, + "theoretical_loss": 3.550278887705316, + "tokens_seen": 1472593920 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044758274824473426, + "loss": 1.2801, + "theoretical_loss": 3.5502650200176604, + "tokens_seen": 1472659456 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004475667001003009, + "loss": 1.352, + "theoretical_loss": 3.5502511531199175, + "tokens_seen": 1472724992 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044755065195586767, + "loss": 1.2347, + "theoretical_loss": 3.5502372870120076, + "tokens_seen": 1472790528 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004475346038114343, + "loss": 1.3027, + "theoretical_loss": 3.5502234216938504, + "tokens_seen": 1472856064 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044751855566700107, + "loss": 1.3529, + "theoretical_loss": 3.550209557165366, + "tokens_seen": 1472921600 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004475025075225677, + "loss": 1.2382, + "theoretical_loss": 3.550195693426473, + "tokens_seen": 1472987136 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044748645937813447, + "loss": 1.4437, + "theoretical_loss": 3.5501818304770936, + "tokens_seen": 1473052672 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004474704112337011, + "loss": 1.2699, + "theoretical_loss": 3.550167968317146, + "tokens_seen": 1473118208 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044745436308926787, + "loss": 1.3806, + "theoretical_loss": 3.550154106946551, + "tokens_seen": 1473183744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004474383149448345, + "loss": 1.3676, + "theoretical_loss": 3.5501402463652276, + "tokens_seen": 1473249280 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044742226680040127, + "loss": 1.3897, + "theoretical_loss": 3.5501263865730968, + "tokens_seen": 1473314816 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004474062186559679, + "loss": 1.3625, + "theoretical_loss": 3.5501125275700782, + "tokens_seen": 1473380352 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044739017051153467, + "loss": 1.309, + "theoretical_loss": 3.5500986693560916, + "tokens_seen": 1473445888 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004473741223671013, + "loss": 1.3084, + "theoretical_loss": 3.5500848119310566, + "tokens_seen": 1473511424 + }, + { + "epoch": 0.45, + "learning_rate": 0.000447358074222668, + "loss": 1.4253, + "theoretical_loss": 3.5500709552948937, + "tokens_seen": 1473576960 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004473420260782347, + "loss": 1.3787, + "theoretical_loss": 3.550057099447523, + "tokens_seen": 1473642496 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004473259779338014, + "loss": 1.2574, + "theoretical_loss": 3.5500432443888643, + "tokens_seen": 1473708032 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004473099297893681, + "loss": 1.4116, + "theoretical_loss": 3.550029390118838, + "tokens_seen": 1473773568 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004472938816449348, + "loss": 1.3511, + "theoretical_loss": 3.550015536637364, + "tokens_seen": 1473839104 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004472778335005015, + "loss": 1.3906, + "theoretical_loss": 3.550001683944362, + "tokens_seen": 1473904640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004472617853560682, + "loss": 1.2916, + "theoretical_loss": 3.5499878320397524, + "tokens_seen": 1473970176 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004472457372116349, + "loss": 1.3938, + "theoretical_loss": 3.549973980923455, + "tokens_seen": 1474035712 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004472296890672016, + "loss": 1.3066, + "theoretical_loss": 3.54996013059539, + "tokens_seen": 1474101248 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044721364092276827, + "loss": 1.3018, + "theoretical_loss": 3.549946281055478, + "tokens_seen": 1474166784 + }, + { + "epoch": 0.45, + "learning_rate": 0.000447197592778335, + "loss": 1.2734, + "theoretical_loss": 3.5499324323036374, + "tokens_seen": 1474232320 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044718154463390167, + "loss": 1.3708, + "theoretical_loss": 3.5499185843397907, + "tokens_seen": 1474297856 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004471654964894684, + "loss": 1.3048, + "theoretical_loss": 3.549904737163857, + "tokens_seen": 1474363392 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044714944834503507, + "loss": 1.299, + "theoretical_loss": 3.549890890775756, + "tokens_seen": 1474428928 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044713340020060183, + "loss": 1.3026, + "theoretical_loss": 3.5498770451754083, + "tokens_seen": 1474494464 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 547951, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3319976329803467, + "objective/train/theoretical_loss": 3.549863200362734, + "objective/train/tokens_used": 1495020000, + "theoretical_loss": 3.549863200362734, + "tokens_seen": 1474560000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004471173520561685, + "loss": 1.332, + "theoretical_loss": 3.549863200362734, + "tokens_seen": 1474560000 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044710130391173523, + "loss": 1.3134, + "theoretical_loss": 3.5498493563376536, + "tokens_seen": 1474625536 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004470852557673019, + "loss": 1.4386, + "theoretical_loss": 3.5498355131000867, + "tokens_seen": 1474691072 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044706920762286863, + "loss": 1.359, + "theoretical_loss": 3.549821670649954, + "tokens_seen": 1474756608 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004470531594784353, + "loss": 1.2789, + "theoretical_loss": 3.5498078289871753, + "tokens_seen": 1474822144 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044703711133400203, + "loss": 1.2954, + "theoretical_loss": 3.549793988111671, + "tokens_seen": 1474887680 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004470210631895687, + "loss": 1.2997, + "theoretical_loss": 3.5497801480233617, + "tokens_seen": 1474953216 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044700501504513543, + "loss": 1.2418, + "theoretical_loss": 3.5497663087221674, + "tokens_seen": 1475018752 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004469889669007021, + "loss": 1.3878, + "theoretical_loss": 3.5497524702080083, + "tokens_seen": 1475084288 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044697291875626883, + "loss": 1.342, + "theoretical_loss": 3.5497386324808042, + "tokens_seen": 1475149824 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004469568706118355, + "loss": 1.2959, + "theoretical_loss": 3.5497247955404765, + "tokens_seen": 1475215360 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044694082246740224, + "loss": 1.3584, + "theoretical_loss": 3.5497109593869447, + "tokens_seen": 1475280896 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004469247743229689, + "loss": 1.3185, + "theoretical_loss": 3.5496971240201294, + "tokens_seen": 1475346432 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044690872617853564, + "loss": 1.3648, + "theoretical_loss": 3.5496832894399506, + "tokens_seen": 1475411968 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004468926780341023, + "loss": 1.3327, + "theoretical_loss": 3.5496694556463293, + "tokens_seen": 1475477504 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044687662988966904, + "loss": 1.2851, + "theoretical_loss": 3.549655622639185, + "tokens_seen": 1475543040 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004468605817452357, + "loss": 1.2983, + "theoretical_loss": 3.5496417904184385, + "tokens_seen": 1475608576 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044684453360080244, + "loss": 1.3841, + "theoretical_loss": 3.54962795898401, + "tokens_seen": 1475674112 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004468284854563692, + "loss": 1.3175, + "theoretical_loss": 3.5496141283358202, + "tokens_seen": 1475739648 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044681243731193584, + "loss": 1.4327, + "theoretical_loss": 3.5496002984737896, + "tokens_seen": 1475805184 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004467963891675026, + "loss": 1.3732, + "theoretical_loss": 3.5495864693978376, + "tokens_seen": 1475870720 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044678034102306924, + "loss": 1.2741, + "theoretical_loss": 3.5495726411078863, + "tokens_seen": 1475936256 + }, + { + "epoch": 0.45, + "learning_rate": 0.000446764292878636, + "loss": 1.3538, + "theoretical_loss": 3.5495588136038547, + "tokens_seen": 1476001792 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044674824473420264, + "loss": 1.3423, + "theoretical_loss": 3.549544986885664, + "tokens_seen": 1476067328 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044673219658976934, + "loss": 1.405, + "theoretical_loss": 3.549531160953234, + "tokens_seen": 1476132864 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044671614844533604, + "loss": 1.3027, + "theoretical_loss": 3.5495173358064855, + "tokens_seen": 1476198400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044670010030090274, + "loss": 1.4164, + "theoretical_loss": 3.549503511445339, + "tokens_seen": 1476263936 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044668405215646945, + "loss": 1.3146, + "theoretical_loss": 3.5494896878697153, + "tokens_seen": 1476329472 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044666800401203615, + "loss": 1.3058, + "theoretical_loss": 3.549475865079534, + "tokens_seen": 1476395008 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044665195586760285, + "loss": 1.2721, + "theoretical_loss": 3.5494620430747164, + "tokens_seen": 1476460544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044663590772316955, + "loss": 1.266, + "theoretical_loss": 3.549448221855183, + "tokens_seen": 1476526080 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044661985957873625, + "loss": 1.3751, + "theoretical_loss": 3.549434401420854, + "tokens_seen": 1476591616 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044660381143430295, + "loss": 1.3199, + "theoretical_loss": 3.5494205817716504, + "tokens_seen": 1476657152 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004465877632898696, + "loss": 1.3725, + "theoretical_loss": 3.549406762907492, + "tokens_seen": 1476722688 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044657171514543635, + "loss": 1.3905, + "theoretical_loss": 3.5493929448283, + "tokens_seen": 1476788224 + }, + { + "epoch": 0.45, + "learning_rate": 0.000446555667001003, + "loss": 1.4814, + "theoretical_loss": 3.549379127533995, + "tokens_seen": 1476853760 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044653961885656975, + "loss": 1.3502, + "theoretical_loss": 3.5493653110244976, + "tokens_seen": 1476919296 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004465235707121364, + "loss": 1.3787, + "theoretical_loss": 3.549351495299728, + "tokens_seen": 1476984832 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044650752256770315, + "loss": 1.3734, + "theoretical_loss": 3.5493376803596064, + "tokens_seen": 1477050368 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004464914744232698, + "loss": 1.3891, + "theoretical_loss": 3.5493238662040545, + "tokens_seen": 1477115904 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044647542627883655, + "loss": 1.2902, + "theoretical_loss": 3.5493100528329924, + "tokens_seen": 1477181440 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004464593781344032, + "loss": 1.2597, + "theoretical_loss": 3.549296240246341, + "tokens_seen": 1477246976 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044644332998996995, + "loss": 1.2842, + "theoretical_loss": 3.5492824284440205, + "tokens_seen": 1477312512 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004464272818455366, + "loss": 1.3007, + "theoretical_loss": 3.5492686174259522, + "tokens_seen": 1477378048 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044641123370110336, + "loss": 1.2914, + "theoretical_loss": 3.549254807192056, + "tokens_seen": 1477443584 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044639518555667, + "loss": 1.4232, + "theoretical_loss": 3.5492409977422534, + "tokens_seen": 1477509120 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044637913741223676, + "loss": 1.3141, + "theoretical_loss": 3.549227189076465, + "tokens_seen": 1477574656 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004463630892678034, + "loss": 1.2546, + "theoretical_loss": 3.54921338119461, + "tokens_seen": 1477640192 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044634704112337016, + "loss": 1.3413, + "theoretical_loss": 3.549199574096612, + "tokens_seen": 1477705728 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004463309929789368, + "loss": 1.3096, + "theoretical_loss": 3.549185767782389, + "tokens_seen": 1477771264 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044631494483450356, + "loss": 1.3083, + "theoretical_loss": 3.549171962251863, + "tokens_seen": 1477836800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004462988966900702, + "loss": 1.3701, + "theoretical_loss": 3.5491581575049547, + "tokens_seen": 1477902336 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044628284854563696, + "loss": 1.3422, + "theoretical_loss": 3.549144353541585, + "tokens_seen": 1477967872 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004462668004012036, + "loss": 1.3588, + "theoretical_loss": 3.5491305503616744, + "tokens_seen": 1478033408 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044625075225677036, + "loss": 1.3721, + "theoretical_loss": 3.5491167479651438, + "tokens_seen": 1478098944 + }, + { + "epoch": 0.45, + "learning_rate": 0.000446234704112337, + "loss": 1.4232, + "theoretical_loss": 3.5491029463519137, + "tokens_seen": 1478164480 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044621865596790376, + "loss": 1.2567, + "theoretical_loss": 3.5490891455219056, + "tokens_seen": 1478230016 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004462026078234704, + "loss": 1.3554, + "theoretical_loss": 3.54907534547504, + "tokens_seen": 1478295552 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044618655967903716, + "loss": 1.2065, + "theoretical_loss": 3.549061546211237, + "tokens_seen": 1478361088 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004461705115346038, + "loss": 1.3051, + "theoretical_loss": 3.5490477477304188, + "tokens_seen": 1478426624 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044615446339017057, + "loss": 1.3211, + "theoretical_loss": 3.5490339500325057, + "tokens_seen": 1478492160 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004461384152457372, + "loss": 1.2433, + "theoretical_loss": 3.5490201531174184, + "tokens_seen": 1478557696 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044612236710130397, + "loss": 1.37, + "theoretical_loss": 3.549006356985078, + "tokens_seen": 1478623232 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004461063189568706, + "loss": 1.2017, + "theoretical_loss": 3.548992561635405, + "tokens_seen": 1478688768 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044609027081243737, + "loss": 1.3446, + "theoretical_loss": 3.548978767068321, + "tokens_seen": 1478754304 + }, + { + "epoch": 0.45, + "learning_rate": 0.000446074222668004, + "loss": 1.2797, + "theoretical_loss": 3.548964973283746, + "tokens_seen": 1478819840 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044605817452357077, + "loss": 1.3944, + "theoretical_loss": 3.548951180281602, + "tokens_seen": 1478885376 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004460421263791374, + "loss": 1.3488, + "theoretical_loss": 3.548937388061809, + "tokens_seen": 1478950912 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044602607823470417, + "loss": 1.3356, + "theoretical_loss": 3.548923596624288, + "tokens_seen": 1479016448 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004460100300902708, + "loss": 1.2518, + "theoretical_loss": 3.548909805968961, + "tokens_seen": 1479081984 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044599398194583757, + "loss": 1.2746, + "theoretical_loss": 3.548896016095748, + "tokens_seen": 1479147520 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004459779338014042, + "loss": 1.3226, + "theoretical_loss": 3.5488822270045706, + "tokens_seen": 1479213056 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004459618856569709, + "loss": 1.4181, + "theoretical_loss": 3.5488684386953495, + "tokens_seen": 1479278592 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004459458375125376, + "loss": 1.3291, + "theoretical_loss": 3.5488546511680057, + "tokens_seen": 1479344128 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004459297893681043, + "loss": 1.3443, + "theoretical_loss": 3.54884086442246, + "tokens_seen": 1479409664 + }, + { + "epoch": 0.45, + "learning_rate": 0.000445913741223671, + "loss": 1.3269, + "theoretical_loss": 3.548827078458634, + "tokens_seen": 1479475200 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004458976930792377, + "loss": 1.2115, + "theoretical_loss": 3.5488132932764485, + "tokens_seen": 1479540736 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004458816449348044, + "loss": 1.3137, + "theoretical_loss": 3.5487995088758244, + "tokens_seen": 1479606272 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004458655967903711, + "loss": 1.4427, + "theoretical_loss": 3.548785725256683, + "tokens_seen": 1479671808 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004458495486459378, + "loss": 1.2377, + "theoretical_loss": 3.548771942418945, + "tokens_seen": 1479737344 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004458335005015045, + "loss": 1.4055, + "theoretical_loss": 3.5487581603625324, + "tokens_seen": 1479802880 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044581745235707117, + "loss": 1.3115, + "theoretical_loss": 3.548744379087365, + "tokens_seen": 1479868416 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004458014042126379, + "loss": 1.3672, + "theoretical_loss": 3.548730598593365, + "tokens_seen": 1479933952 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044578535606820457, + "loss": 1.4162, + "theoretical_loss": 3.548716818880453, + "tokens_seen": 1479999488 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044576930792377133, + "loss": 1.3405, + "theoretical_loss": 3.5487030399485504, + "tokens_seen": 1480065024 + }, + { + "epoch": 0.45, + "learning_rate": 0.000445753259779338, + "loss": 1.331, + "theoretical_loss": 3.5486892617975783, + "tokens_seen": 1480130560 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044573721163490473, + "loss": 1.3802, + "theoretical_loss": 3.5486754844274575, + "tokens_seen": 1480196096 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004457211634904714, + "loss": 1.3043, + "theoretical_loss": 3.54866170783811, + "tokens_seen": 1480261632 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044570511534603813, + "loss": 1.3275, + "theoretical_loss": 3.548647932029456, + "tokens_seen": 1480327168 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004456890672016048, + "loss": 1.3577, + "theoretical_loss": 3.548634157001417, + "tokens_seen": 1480392704 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044567301905717153, + "loss": 1.198, + "theoretical_loss": 3.548620382753915, + "tokens_seen": 1480458240 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004456569709127382, + "loss": 1.2379, + "theoretical_loss": 3.54860660928687, + "tokens_seen": 1480523776 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044564092276830493, + "loss": 1.3608, + "theoretical_loss": 3.5485928366002044, + "tokens_seen": 1480589312 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004456248746238717, + "loss": 1.3115, + "theoretical_loss": 3.5485790646938384, + "tokens_seen": 1480654848 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044560882647943833, + "loss": 1.1759, + "theoretical_loss": 3.548565293567694, + "tokens_seen": 1480720384 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004455927783350051, + "loss": 1.3103, + "theoretical_loss": 3.5485515232216924, + "tokens_seen": 1480785920 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044557673019057173, + "loss": 1.2354, + "theoretical_loss": 3.5485377536557543, + "tokens_seen": 1480851456 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004455606820461385, + "loss": 1.2697, + "theoretical_loss": 3.5485239848698016, + "tokens_seen": 1480916992 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044554463390170514, + "loss": 1.3431, + "theoretical_loss": 3.5485102168637557, + "tokens_seen": 1480982528 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004455285857572719, + "loss": 1.358, + "theoretical_loss": 3.548496449637537, + "tokens_seen": 1481048064 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 550093, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.346829891204834, + "objective/train/theoretical_loss": 3.548482683191067, + "objective/train/tokens_used": 1501573600, + "theoretical_loss": 3.548482683191067, + "tokens_seen": 1481113600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044551253761283854, + "loss": 1.3468, + "theoretical_loss": 3.548482683191067, + "tokens_seen": 1481113600 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004454964894684053, + "loss": 1.3067, + "theoretical_loss": 3.5484689175242687, + "tokens_seen": 1481179136 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044548044132397194, + "loss": 1.3328, + "theoretical_loss": 3.548455152637062, + "tokens_seen": 1481244672 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004454643931795387, + "loss": 1.3505, + "theoretical_loss": 3.5484413885293673, + "tokens_seen": 1481310208 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044544834503510534, + "loss": 1.3124, + "theoretical_loss": 3.548427625201108, + "tokens_seen": 1481375744 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004454322968906721, + "loss": 1.3195, + "theoretical_loss": 3.548413862652205, + "tokens_seen": 1481441280 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044541624874623874, + "loss": 1.3799, + "theoretical_loss": 3.5484001008825787, + "tokens_seen": 1481506816 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004454002006018055, + "loss": 1.3305, + "theoretical_loss": 3.5483863398921516, + "tokens_seen": 1481572352 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044538415245737214, + "loss": 1.3817, + "theoretical_loss": 3.5483725796808443, + "tokens_seen": 1481637888 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004453681043129389, + "loss": 1.3974, + "theoretical_loss": 3.5483588202485787, + "tokens_seen": 1481703424 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044535205616850554, + "loss": 1.3505, + "theoretical_loss": 3.548345061595276, + "tokens_seen": 1481768960 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044533600802407224, + "loss": 1.2846, + "theoretical_loss": 3.5483313037208575, + "tokens_seen": 1481834496 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044531995987963895, + "loss": 1.2619, + "theoretical_loss": 3.5483175466252455, + "tokens_seen": 1481900032 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044530391173520565, + "loss": 1.2989, + "theoretical_loss": 3.5483037903083607, + "tokens_seen": 1481965568 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044528786359077235, + "loss": 1.4002, + "theoretical_loss": 3.5482900347701247, + "tokens_seen": 1482031104 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044527181544633905, + "loss": 1.264, + "theoretical_loss": 3.5482762800104592, + "tokens_seen": 1482096640 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044525576730190575, + "loss": 1.2798, + "theoretical_loss": 3.5482625260292853, + "tokens_seen": 1482162176 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044523971915747245, + "loss": 1.3202, + "theoretical_loss": 3.5482487728265255, + "tokens_seen": 1482227712 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044522367101303915, + "loss": 1.3835, + "theoretical_loss": 3.5482350204020996, + "tokens_seen": 1482293248 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044520762286860585, + "loss": 1.351, + "theoretical_loss": 3.548221268755931, + "tokens_seen": 1482358784 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004451915747241725, + "loss": 1.2828, + "theoretical_loss": 3.54820751788794, + "tokens_seen": 1482424320 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044517552657973925, + "loss": 1.274, + "theoretical_loss": 3.548193767798049, + "tokens_seen": 1482489856 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004451594784353059, + "loss": 1.3303, + "theoretical_loss": 3.548180018486179, + "tokens_seen": 1482555392 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044514343029087265, + "loss": 1.2672, + "theoretical_loss": 3.548166269952252, + "tokens_seen": 1482620928 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004451273821464393, + "loss": 1.2873, + "theoretical_loss": 3.548152522196189, + "tokens_seen": 1482686464 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044511133400200605, + "loss": 1.3307, + "theoretical_loss": 3.548138775217912, + "tokens_seen": 1482752000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004450952858575727, + "loss": 1.362, + "theoretical_loss": 3.548125029017343, + "tokens_seen": 1482817536 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044507923771313945, + "loss": 1.2649, + "theoretical_loss": 3.5481112835944026, + "tokens_seen": 1482883072 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004450631895687061, + "loss": 1.3459, + "theoretical_loss": 3.5480975389490137, + "tokens_seen": 1482948608 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044504714142427286, + "loss": 1.3717, + "theoretical_loss": 3.548083795081097, + "tokens_seen": 1483014144 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004450310932798395, + "loss": 1.2262, + "theoretical_loss": 3.5480700519905746, + "tokens_seen": 1483079680 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044501504513540626, + "loss": 1.2428, + "theoretical_loss": 3.548056309677368, + "tokens_seen": 1483145216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004449989969909729, + "loss": 1.317, + "theoretical_loss": 3.548042568141399, + "tokens_seen": 1483210752 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044498294884653966, + "loss": 1.3148, + "theoretical_loss": 3.5480288273825895, + "tokens_seen": 1483276288 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004449669007021063, + "loss": 1.3163, + "theoretical_loss": 3.5480150874008602, + "tokens_seen": 1483341824 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044495085255767306, + "loss": 1.2372, + "theoretical_loss": 3.5480013481961343, + "tokens_seen": 1483407360 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004449348044132397, + "loss": 1.3642, + "theoretical_loss": 3.5479876097683327, + "tokens_seen": 1483472896 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044491875626880646, + "loss": 1.2956, + "theoretical_loss": 3.5479738721173772, + "tokens_seen": 1483538432 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004449027081243731, + "loss": 1.2574, + "theoretical_loss": 3.5479601352431898, + "tokens_seen": 1483603968 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044488665997993986, + "loss": 1.3651, + "theoretical_loss": 3.547946399145692, + "tokens_seen": 1483669504 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004448706118355065, + "loss": 1.3506, + "theoretical_loss": 3.547932663824806, + "tokens_seen": 1483735040 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044485456369107326, + "loss": 1.3806, + "theoretical_loss": 3.547918929280453, + "tokens_seen": 1483800576 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004448385155466399, + "loss": 1.3091, + "theoretical_loss": 3.5479051955125556, + "tokens_seen": 1483866112 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044482246740220666, + "loss": 1.3187, + "theoretical_loss": 3.5478914625210343, + "tokens_seen": 1483931648 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004448064192577733, + "loss": 1.2769, + "theoretical_loss": 3.547877730305812, + "tokens_seen": 1483997184 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044479037111334007, + "loss": 1.2529, + "theoretical_loss": 3.547863998866811, + "tokens_seen": 1484062720 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004447743229689067, + "loss": 1.3039, + "theoretical_loss": 3.5478502682039514, + "tokens_seen": 1484128256 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044475827482447347, + "loss": 1.2772, + "theoretical_loss": 3.5478365383171564, + "tokens_seen": 1484193792 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004447422266800401, + "loss": 1.3102, + "theoretical_loss": 3.547822809206348, + "tokens_seen": 1484259328 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044472617853560687, + "loss": 1.3263, + "theoretical_loss": 3.5478090808714473, + "tokens_seen": 1484324864 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004447101303911735, + "loss": 1.2701, + "theoretical_loss": 3.5477953533123765, + "tokens_seen": 1484390400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044469408224674027, + "loss": 1.2452, + "theoretical_loss": 3.5477816265290576, + "tokens_seen": 1484455936 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004446780341023069, + "loss": 1.3363, + "theoretical_loss": 3.5477679005214124, + "tokens_seen": 1484521472 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044466198595787367, + "loss": 1.3845, + "theoretical_loss": 3.547754175289363, + "tokens_seen": 1484587008 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004446459378134403, + "loss": 1.3068, + "theoretical_loss": 3.5477404508328307, + "tokens_seen": 1484652544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044462988966900707, + "loss": 1.3454, + "theoretical_loss": 3.5477267271517388, + "tokens_seen": 1484718080 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004446138415245737, + "loss": 1.327, + "theoretical_loss": 3.547713004246008, + "tokens_seen": 1484783616 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004445977933801405, + "loss": 1.2674, + "theoretical_loss": 3.547699282115561, + "tokens_seen": 1484849152 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004445817452357071, + "loss": 1.2927, + "theoretical_loss": 3.547685560760319, + "tokens_seen": 1484914688 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004445656970912738, + "loss": 1.2465, + "theoretical_loss": 3.5476718401802048, + "tokens_seen": 1484980224 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004445496489468405, + "loss": 1.3779, + "theoretical_loss": 3.54765812037514, + "tokens_seen": 1485045760 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004445336008024072, + "loss": 1.3258, + "theoretical_loss": 3.547644401345047, + "tokens_seen": 1485111296 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004445175526579739, + "loss": 1.2795, + "theoretical_loss": 3.5476306830898467, + "tokens_seen": 1485176832 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004445015045135406, + "loss": 1.4357, + "theoretical_loss": 3.547616965609463, + "tokens_seen": 1485242368 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004444854563691073, + "loss": 1.2517, + "theoretical_loss": 3.5476032489038163, + "tokens_seen": 1485307904 + }, + { + "epoch": 0.45, + "learning_rate": 0.000444469408224674, + "loss": 1.379, + "theoretical_loss": 3.5475895329728298, + "tokens_seen": 1485373440 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004444533600802407, + "loss": 1.2067, + "theoretical_loss": 3.5475758178164245, + "tokens_seen": 1485438976 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004444373119358074, + "loss": 1.3601, + "theoretical_loss": 3.5475621034345233, + "tokens_seen": 1485504512 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004444212637913742, + "loss": 1.3116, + "theoretical_loss": 3.547548389827048, + "tokens_seen": 1485570048 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004444052156469408, + "loss": 1.3158, + "theoretical_loss": 3.547534676993921, + "tokens_seen": 1485635584 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004443891675025076, + "loss": 1.3546, + "theoretical_loss": 3.547520964935064, + "tokens_seen": 1485701120 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044437311935807423, + "loss": 1.3404, + "theoretical_loss": 3.5475072536503998, + "tokens_seen": 1485766656 + }, + { + "epoch": 0.45, + "learning_rate": 0.000444357071213641, + "loss": 1.3451, + "theoretical_loss": 3.547493543139849, + "tokens_seen": 1485832192 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044434102306920763, + "loss": 1.2684, + "theoretical_loss": 3.5474798334033357, + "tokens_seen": 1485897728 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004443249749247744, + "loss": 1.2992, + "theoretical_loss": 3.547466124440781, + "tokens_seen": 1485963264 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044430892678034103, + "loss": 1.3418, + "theoretical_loss": 3.5474524162521073, + "tokens_seen": 1486028800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004442928786359078, + "loss": 1.3284, + "theoretical_loss": 3.5474387088372366, + "tokens_seen": 1486094336 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044427683049147443, + "loss": 1.3231, + "theoretical_loss": 3.547425002196091, + "tokens_seen": 1486159872 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004442607823470412, + "loss": 1.3145, + "theoretical_loss": 3.547411296328593, + "tokens_seen": 1486225408 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044424473420260783, + "loss": 1.372, + "theoretical_loss": 3.5473975912346654, + "tokens_seen": 1486290944 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004442286860581746, + "loss": 1.4514, + "theoretical_loss": 3.547383886914229, + "tokens_seen": 1486356480 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044421263791374123, + "loss": 1.3054, + "theoretical_loss": 3.5473701833672076, + "tokens_seen": 1486422016 + }, + { + "epoch": 0.45, + "learning_rate": 0.000444196589769308, + "loss": 1.337, + "theoretical_loss": 3.547356480593522, + "tokens_seen": 1486487552 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044418054162487464, + "loss": 1.2573, + "theoretical_loss": 3.5473427785930958, + "tokens_seen": 1486553088 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004441644934804414, + "loss": 1.365, + "theoretical_loss": 3.5473290773658506, + "tokens_seen": 1486618624 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044414844533600804, + "loss": 1.3582, + "theoretical_loss": 3.5473153769117083, + "tokens_seen": 1486684160 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004441323971915748, + "loss": 1.2215, + "theoretical_loss": 3.5473016772305916, + "tokens_seen": 1486749696 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044411634904714144, + "loss": 1.3768, + "theoretical_loss": 3.5472879783224234, + "tokens_seen": 1486815232 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004441003009027082, + "loss": 1.258, + "theoretical_loss": 3.547274280187125, + "tokens_seen": 1486880768 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044408425275827484, + "loss": 1.3907, + "theoretical_loss": 3.5472605828246193, + "tokens_seen": 1486946304 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004440682046138416, + "loss": 1.3206, + "theoretical_loss": 3.547246886234829, + "tokens_seen": 1487011840 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044405215646940824, + "loss": 1.4084, + "theoretical_loss": 3.5472331904176757, + "tokens_seen": 1487077376 + }, + { + "epoch": 0.45, + "learning_rate": 0.000444036108324975, + "loss": 1.325, + "theoretical_loss": 3.547219495373082, + "tokens_seen": 1487142912 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044402006018054164, + "loss": 1.2505, + "theoretical_loss": 3.5472058011009704, + "tokens_seen": 1487208448 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004440040120361084, + "loss": 1.3017, + "theoretical_loss": 3.547192107601263, + "tokens_seen": 1487273984 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044398796389167504, + "loss": 1.2906, + "theoretical_loss": 3.5471784148738825, + "tokens_seen": 1487339520 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004439719157472418, + "loss": 1.2879, + "theoretical_loss": 3.5471647229187515, + "tokens_seen": 1487405056 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044395586760280844, + "loss": 1.2747, + "theoretical_loss": 3.547151031735792, + "tokens_seen": 1487470592 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044393981945837515, + "loss": 1.4101, + "theoretical_loss": 3.5471373413249268, + "tokens_seen": 1487536128 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044392377131394185, + "loss": 1.3327, + "theoretical_loss": 3.547123651686078, + "tokens_seen": 1487601664 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 552839, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2520471811294556, + "objective/train/theoretical_loss": 3.5471099628191682, + "objective/train/tokens_used": 1508127200, + "theoretical_loss": 3.5471099628191682, + "tokens_seen": 1487667200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044390772316950855, + "loss": 1.252, + "theoretical_loss": 3.5471099628191682, + "tokens_seen": 1487667200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044389167502507525, + "loss": 1.3614, + "theoretical_loss": 3.5470962747241197, + "tokens_seen": 1487732736 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044387562688064195, + "loss": 1.3273, + "theoretical_loss": 3.5470825874008556, + "tokens_seen": 1487798272 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044385957873620865, + "loss": 1.2692, + "theoretical_loss": 3.5470689008492977, + "tokens_seen": 1487863808 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044384353059177535, + "loss": 1.2675, + "theoretical_loss": 3.547055215069369, + "tokens_seen": 1487929344 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044382748244734205, + "loss": 1.3648, + "theoretical_loss": 3.5470415300609917, + "tokens_seen": 1487994880 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044381143430290875, + "loss": 1.4491, + "theoretical_loss": 3.547027845824088, + "tokens_seen": 1488060416 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004437953861584754, + "loss": 1.3927, + "theoretical_loss": 3.5470141623585807, + "tokens_seen": 1488125952 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044377933801404215, + "loss": 1.2622, + "theoretical_loss": 3.5470004796643932, + "tokens_seen": 1488191488 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004437632898696088, + "loss": 1.3168, + "theoretical_loss": 3.5469867977414467, + "tokens_seen": 1488257024 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044374724172517555, + "loss": 1.2918, + "theoretical_loss": 3.546973116589665, + "tokens_seen": 1488322560 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004437311935807422, + "loss": 1.3062, + "theoretical_loss": 3.5469594362089696, + "tokens_seen": 1488388096 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044371514543630895, + "loss": 1.3345, + "theoretical_loss": 3.5469457565992837, + "tokens_seen": 1488453632 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004436990972918756, + "loss": 1.3075, + "theoretical_loss": 3.54693207776053, + "tokens_seen": 1488519168 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044368304914744236, + "loss": 1.4096, + "theoretical_loss": 3.546918399692631, + "tokens_seen": 1488584704 + }, + { + "epoch": 0.45, + "learning_rate": 0.000443667001003009, + "loss": 1.3527, + "theoretical_loss": 3.5469047223955084, + "tokens_seen": 1488650240 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044365095285857576, + "loss": 1.4191, + "theoretical_loss": 3.5468910458690863, + "tokens_seen": 1488715776 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004436349047141424, + "loss": 1.3616, + "theoretical_loss": 3.5468773701132865, + "tokens_seen": 1488781312 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044361885656970916, + "loss": 1.4021, + "theoretical_loss": 3.5468636951280317, + "tokens_seen": 1488846848 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004436028084252758, + "loss": 1.34, + "theoretical_loss": 3.5468500209132454, + "tokens_seen": 1488912384 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044358676028084256, + "loss": 1.2889, + "theoretical_loss": 3.546836347468849, + "tokens_seen": 1488977920 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004435707121364092, + "loss": 1.3153, + "theoretical_loss": 3.546822674794766, + "tokens_seen": 1489043456 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044355466399197596, + "loss": 1.4289, + "theoretical_loss": 3.5468090028909187, + "tokens_seen": 1489108992 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004435386158475426, + "loss": 1.2667, + "theoretical_loss": 3.54679533175723, + "tokens_seen": 1489174528 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044352256770310936, + "loss": 1.22, + "theoretical_loss": 3.5467816613936227, + "tokens_seen": 1489240064 + }, + { + "epoch": 0.45, + "learning_rate": 0.000443506519558676, + "loss": 1.2542, + "theoretical_loss": 3.54676799180002, + "tokens_seen": 1489305600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044349047141424276, + "loss": 1.2101, + "theoretical_loss": 3.5467543229763434, + "tokens_seen": 1489371136 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004434744232698094, + "loss": 1.2924, + "theoretical_loss": 3.546740654922517, + "tokens_seen": 1489436672 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044345837512537616, + "loss": 1.3755, + "theoretical_loss": 3.546726987638462, + "tokens_seen": 1489502208 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004434423269809428, + "loss": 1.2557, + "theoretical_loss": 3.5467133211241033, + "tokens_seen": 1489567744 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044342627883650957, + "loss": 1.3304, + "theoretical_loss": 3.5466996553793617, + "tokens_seen": 1489633280 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004434102306920762, + "loss": 1.3482, + "theoretical_loss": 3.5466859904041614, + "tokens_seen": 1489698816 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044339418254764297, + "loss": 1.3395, + "theoretical_loss": 3.5466723261984243, + "tokens_seen": 1489764352 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004433781344032096, + "loss": 1.3074, + "theoretical_loss": 3.5466586627620735, + "tokens_seen": 1489829888 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044336208625877637, + "loss": 1.2583, + "theoretical_loss": 3.5466450000950323, + "tokens_seen": 1489895424 + }, + { + "epoch": 0.45, + "learning_rate": 0.000443346038114343, + "loss": 1.3397, + "theoretical_loss": 3.5466313381972228, + "tokens_seen": 1489960960 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044332998996990977, + "loss": 1.2548, + "theoretical_loss": 3.5466176770685682, + "tokens_seen": 1490026496 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004433139418254764, + "loss": 1.3125, + "theoretical_loss": 3.546604016708992, + "tokens_seen": 1490092032 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044329789368104317, + "loss": 1.2394, + "theoretical_loss": 3.546590357118416, + "tokens_seen": 1490157568 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004432818455366098, + "loss": 1.2813, + "theoretical_loss": 3.546576698296763, + "tokens_seen": 1490223104 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044326579739217657, + "loss": 1.2812, + "theoretical_loss": 3.546563040243957, + "tokens_seen": 1490288640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004432497492477432, + "loss": 1.3855, + "theoretical_loss": 3.5465493829599204, + "tokens_seen": 1490354176 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044323370110331, + "loss": 1.4125, + "theoretical_loss": 3.546535726444576, + "tokens_seen": 1490419712 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004432176529588767, + "loss": 1.2767, + "theoretical_loss": 3.546522070697847, + "tokens_seen": 1490485248 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004432016048144434, + "loss": 1.2617, + "theoretical_loss": 3.546508415719656, + "tokens_seen": 1490550784 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004431855566700101, + "loss": 1.3866, + "theoretical_loss": 3.5464947615099263, + "tokens_seen": 1490616320 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004431695085255767, + "loss": 1.2394, + "theoretical_loss": 3.5464811080685807, + "tokens_seen": 1490681856 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004431534603811435, + "loss": 1.3111, + "theoretical_loss": 3.5464674553955415, + "tokens_seen": 1490747392 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004431374122367101, + "loss": 1.3298, + "theoretical_loss": 3.546453803490733, + "tokens_seen": 1490812928 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004431213640922769, + "loss": 1.3221, + "theoretical_loss": 3.5464401523540783, + "tokens_seen": 1490878464 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004431053159478435, + "loss": 1.46, + "theoretical_loss": 3.546426501985499, + "tokens_seen": 1490944000 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004430892678034103, + "loss": 1.3196, + "theoretical_loss": 3.5464128523849183, + "tokens_seen": 1491009536 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004430732196589769, + "loss": 1.3927, + "theoretical_loss": 3.5463992035522605, + "tokens_seen": 1491075072 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004430571715145437, + "loss": 1.2611, + "theoretical_loss": 3.546385555487448, + "tokens_seen": 1491140608 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004430411233701103, + "loss": 1.3853, + "theoretical_loss": 3.5463719081904035, + "tokens_seen": 1491206144 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004430250752256771, + "loss": 1.3382, + "theoretical_loss": 3.54635826166105, + "tokens_seen": 1491271680 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044300902708124373, + "loss": 1.3294, + "theoretical_loss": 3.5463446158993115, + "tokens_seen": 1491337216 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004429929789368105, + "loss": 1.4029, + "theoretical_loss": 3.5463309709051103, + "tokens_seen": 1491402752 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044297693079237713, + "loss": 1.354, + "theoretical_loss": 3.54631732667837, + "tokens_seen": 1491468288 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004429608826479439, + "loss": 1.2805, + "theoretical_loss": 3.546303683219013, + "tokens_seen": 1491533824 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044294483450351053, + "loss": 1.3653, + "theoretical_loss": 3.546290040526963, + "tokens_seen": 1491599360 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004429287863590773, + "loss": 1.3708, + "theoretical_loss": 3.546276398602143, + "tokens_seen": 1491664896 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044291273821464393, + "loss": 1.4117, + "theoretical_loss": 3.546262757444476, + "tokens_seen": 1491730432 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004428966900702107, + "loss": 1.3608, + "theoretical_loss": 3.5462491170538852, + "tokens_seen": 1491795968 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044288064192577733, + "loss": 1.3849, + "theoretical_loss": 3.5462354774302947, + "tokens_seen": 1491861504 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004428645937813441, + "loss": 1.3164, + "theoretical_loss": 3.546221838573626, + "tokens_seen": 1491927040 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044284854563691073, + "loss": 1.3567, + "theoretical_loss": 3.546208200483804, + "tokens_seen": 1491992576 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004428324974924775, + "loss": 1.3146, + "theoretical_loss": 3.54619456316075, + "tokens_seen": 1492058112 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044281644934804414, + "loss": 1.3546, + "theoretical_loss": 3.546180926604389, + "tokens_seen": 1492123648 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004428004012036109, + "loss": 1.3633, + "theoretical_loss": 3.546167290814643, + "tokens_seen": 1492189184 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044278435305917754, + "loss": 1.3144, + "theoretical_loss": 3.5461536557914357, + "tokens_seen": 1492254720 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004427683049147443, + "loss": 1.375, + "theoretical_loss": 3.546140021534691, + "tokens_seen": 1492320256 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044275225677031094, + "loss": 1.2074, + "theoretical_loss": 3.546126388044331, + "tokens_seen": 1492385792 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004427362086258777, + "loss": 1.4078, + "theoretical_loss": 3.54611275532028, + "tokens_seen": 1492451328 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044272016048144434, + "loss": 1.3007, + "theoretical_loss": 3.5460991233624606, + "tokens_seen": 1492516864 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004427041123370111, + "loss": 1.4347, + "theoretical_loss": 3.5460854921707963, + "tokens_seen": 1492582400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044268806419257774, + "loss": 1.3536, + "theoretical_loss": 3.54607186174521, + "tokens_seen": 1492647936 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004426720160481445, + "loss": 1.3678, + "theoretical_loss": 3.5460582320856258, + "tokens_seen": 1492713472 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044265596790371114, + "loss": 1.3904, + "theoretical_loss": 3.5460446031919663, + "tokens_seen": 1492779008 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004426399197592779, + "loss": 1.3478, + "theoretical_loss": 3.546030975064155, + "tokens_seen": 1492844544 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044262387161484454, + "loss": 1.334, + "theoretical_loss": 3.546017347702116, + "tokens_seen": 1492910080 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004426078234704113, + "loss": 1.2745, + "theoretical_loss": 3.5460037211057718, + "tokens_seen": 1492975616 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044259177532597794, + "loss": 1.3961, + "theoretical_loss": 3.545990095275046, + "tokens_seen": 1493041152 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004425757271815447, + "loss": 1.2414, + "theoretical_loss": 3.5459764702098617, + "tokens_seen": 1493106688 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044255967903711135, + "loss": 1.3756, + "theoretical_loss": 3.545962845910143, + "tokens_seen": 1493172224 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004425436308926781, + "loss": 1.2753, + "theoretical_loss": 3.5459492223758127, + "tokens_seen": 1493237760 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044252758274824475, + "loss": 1.376, + "theoretical_loss": 3.5459355996067945, + "tokens_seen": 1493303296 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044251153460381145, + "loss": 1.3671, + "theoretical_loss": 3.5459219776030118, + "tokens_seen": 1493368832 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044249548645937815, + "loss": 1.234, + "theoretical_loss": 3.5459083563643876, + "tokens_seen": 1493434368 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044247943831494485, + "loss": 1.4601, + "theoretical_loss": 3.5458947358908466, + "tokens_seen": 1493499904 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044246339017051155, + "loss": 1.2758, + "theoretical_loss": 3.5458811161823105, + "tokens_seen": 1493565440 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044244734202607825, + "loss": 1.2842, + "theoretical_loss": 3.545867497238704, + "tokens_seen": 1493630976 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044243129388164495, + "loss": 1.4104, + "theoretical_loss": 3.5458538790599503, + "tokens_seen": 1493696512 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044241524573721165, + "loss": 1.3347, + "theoretical_loss": 3.5458402616459725, + "tokens_seen": 1493762048 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004423991975927783, + "loss": 1.2422, + "theoretical_loss": 3.545826644996694, + "tokens_seen": 1493827584 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044238314944834505, + "loss": 1.264, + "theoretical_loss": 3.5458130291120398, + "tokens_seen": 1493893120 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004423671013039117, + "loss": 1.335, + "theoretical_loss": 3.545799413991932, + "tokens_seen": 1493958656 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044235105315947845, + "loss": 1.2383, + "theoretical_loss": 3.545785799636294, + "tokens_seen": 1494024192 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004423350050150451, + "loss": 1.3448, + "theoretical_loss": 3.54577218604505, + "tokens_seen": 1494089728 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044231895687061186, + "loss": 1.2843, + "theoretical_loss": 3.545758573218124, + "tokens_seen": 1494155264 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 555072, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2695960998535156, + "objective/train/theoretical_loss": 3.545744961155438, + "objective/train/tokens_used": 1514680800, + "theoretical_loss": 3.545744961155438, + "tokens_seen": 1494220800 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004423029087261785, + "loss": 1.2696, + "theoretical_loss": 3.545744961155438, + "tokens_seen": 1494220800 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044228686058174526, + "loss": 1.395, + "theoretical_loss": 3.545731349856917, + "tokens_seen": 1494286336 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004422708124373119, + "loss": 1.2225, + "theoretical_loss": 3.545717739322484, + "tokens_seen": 1494351872 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044225476429287866, + "loss": 1.2914, + "theoretical_loss": 3.545704129552063, + "tokens_seen": 1494417408 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004422387161484453, + "loss": 1.3028, + "theoretical_loss": 3.545690520545577, + "tokens_seen": 1494482944 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044222266800401206, + "loss": 1.2555, + "theoretical_loss": 3.5456769123029503, + "tokens_seen": 1494548480 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004422066198595787, + "loss": 1.2978, + "theoretical_loss": 3.5456633048241057, + "tokens_seen": 1494614016 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044219057171514546, + "loss": 1.3748, + "theoretical_loss": 3.545649698108968, + "tokens_seen": 1494679552 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004421745235707121, + "loss": 1.3337, + "theoretical_loss": 3.5456360921574595, + "tokens_seen": 1494745088 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044215847542627886, + "loss": 1.2587, + "theoretical_loss": 3.5456224869695045, + "tokens_seen": 1494810624 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004421424272818455, + "loss": 1.3179, + "theoretical_loss": 3.5456088825450274, + "tokens_seen": 1494876160 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044212637913741226, + "loss": 1.3238, + "theoretical_loss": 3.5455952788839507, + "tokens_seen": 1494941696 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004421103309929789, + "loss": 1.2663, + "theoretical_loss": 3.545581675986199, + "tokens_seen": 1495007232 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044209428284854566, + "loss": 1.405, + "theoretical_loss": 3.5455680738516953, + "tokens_seen": 1495072768 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004420782347041123, + "loss": 1.3562, + "theoretical_loss": 3.545554472480364, + "tokens_seen": 1495138304 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044206218655967907, + "loss": 1.4667, + "theoretical_loss": 3.545540871872128, + "tokens_seen": 1495203840 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004420461384152457, + "loss": 1.2213, + "theoretical_loss": 3.5455272720269115, + "tokens_seen": 1495269376 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044203009027081247, + "loss": 1.2025, + "theoretical_loss": 3.5455136729446384, + "tokens_seen": 1495334912 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004420140421263792, + "loss": 1.3512, + "theoretical_loss": 3.5455000746252328, + "tokens_seen": 1495400448 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044199799398194587, + "loss": 1.2653, + "theoretical_loss": 3.5454864770686174, + "tokens_seen": 1495465984 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004419819458375126, + "loss": 1.3609, + "theoretical_loss": 3.5454728802747164, + "tokens_seen": 1495531520 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044196589769307927, + "loss": 1.3301, + "theoretical_loss": 3.5454592842434547, + "tokens_seen": 1495597056 + }, + { + "epoch": 0.45, + "learning_rate": 0.000441949849548646, + "loss": 1.4047, + "theoretical_loss": 3.5454456889747545, + "tokens_seen": 1495662592 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044193380140421267, + "loss": 1.3312, + "theoretical_loss": 3.5454320944685405, + "tokens_seen": 1495728128 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004419177532597794, + "loss": 1.3437, + "theoretical_loss": 3.545418500724736, + "tokens_seen": 1495793664 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044190170511534607, + "loss": 1.3012, + "theoretical_loss": 3.545404907743266, + "tokens_seen": 1495859200 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044188565697091277, + "loss": 1.2737, + "theoretical_loss": 3.545391315524053, + "tokens_seen": 1495924736 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004418696088264795, + "loss": 1.3555, + "theoretical_loss": 3.5453777240670217, + "tokens_seen": 1495990272 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004418535606820462, + "loss": 1.2596, + "theoretical_loss": 3.5453641333720953, + "tokens_seen": 1496055808 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004418375125376129, + "loss": 1.2644, + "theoretical_loss": 3.5453505434391985, + "tokens_seen": 1496121344 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004418214643931796, + "loss": 1.2849, + "theoretical_loss": 3.5453369542682545, + "tokens_seen": 1496186880 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004418054162487463, + "loss": 1.3075, + "theoretical_loss": 3.545323365859187, + "tokens_seen": 1496252416 + }, + { + "epoch": 0.45, + "learning_rate": 0.000441789368104313, + "loss": 1.3142, + "theoretical_loss": 3.545309778211921, + "tokens_seen": 1496317952 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004417733199598797, + "loss": 1.2922, + "theoretical_loss": 3.54529619132638, + "tokens_seen": 1496383488 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004417572718154464, + "loss": 1.3789, + "theoretical_loss": 3.5452826052024875, + "tokens_seen": 1496449024 + }, + { + "epoch": 0.45, + "learning_rate": 0.000441741223671013, + "loss": 1.416, + "theoretical_loss": 3.545269019840167, + "tokens_seen": 1496514560 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004417251755265798, + "loss": 1.3475, + "theoretical_loss": 3.545255435239344, + "tokens_seen": 1496580096 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004417091273821464, + "loss": 1.2382, + "theoretical_loss": 3.5452418513999415, + "tokens_seen": 1496645632 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004416930792377132, + "loss": 1.2691, + "theoretical_loss": 3.5452282683218836, + "tokens_seen": 1496711168 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004416770310932798, + "loss": 1.2882, + "theoretical_loss": 3.5452146860050946, + "tokens_seen": 1496776704 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004416609829488466, + "loss": 1.2575, + "theoretical_loss": 3.5452011044494975, + "tokens_seen": 1496842240 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044164493480441323, + "loss": 1.3278, + "theoretical_loss": 3.5451875236550174, + "tokens_seen": 1496907776 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044162888665998, + "loss": 1.2949, + "theoretical_loss": 3.5451739436215783, + "tokens_seen": 1496973312 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044161283851554663, + "loss": 1.2646, + "theoretical_loss": 3.545160364349104, + "tokens_seen": 1497038848 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004415967903711134, + "loss": 1.2665, + "theoretical_loss": 3.5451467858375176, + "tokens_seen": 1497104384 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044158074222668003, + "loss": 1.3272, + "theoretical_loss": 3.5451332080867446, + "tokens_seen": 1497169920 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004415646940822468, + "loss": 1.3308, + "theoretical_loss": 3.5451196310967084, + "tokens_seen": 1497235456 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044154864593781343, + "loss": 1.2986, + "theoretical_loss": 3.5451060548673334, + "tokens_seen": 1497300992 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004415325977933802, + "loss": 1.1769, + "theoretical_loss": 3.545092479398543, + "tokens_seen": 1497366528 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044151654964894683, + "loss": 1.4046, + "theoretical_loss": 3.545078904690262, + "tokens_seen": 1497432064 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004415005015045136, + "loss": 1.4659, + "theoretical_loss": 3.545065330742414, + "tokens_seen": 1497497600 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044148445336008023, + "loss": 1.3918, + "theoretical_loss": 3.545051757554924, + "tokens_seen": 1497563136 + }, + { + "epoch": 0.45, + "learning_rate": 0.000441468405215647, + "loss": 1.3874, + "theoretical_loss": 3.545038185127715, + "tokens_seen": 1497628672 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044145235707121364, + "loss": 1.2795, + "theoretical_loss": 3.545024613460712, + "tokens_seen": 1497694208 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004414363089267804, + "loss": 1.2219, + "theoretical_loss": 3.545011042553839, + "tokens_seen": 1497759744 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044142026078234704, + "loss": 1.3235, + "theoretical_loss": 3.5449974724070197, + "tokens_seen": 1497825280 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004414042126379138, + "loss": 1.2953, + "theoretical_loss": 3.544983903020178, + "tokens_seen": 1497890816 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044138816449348044, + "loss": 1.2982, + "theoretical_loss": 3.5449703343932395, + "tokens_seen": 1497956352 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004413721163490472, + "loss": 1.3224, + "theoretical_loss": 3.5449567665261275, + "tokens_seen": 1498021888 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044135606820461384, + "loss": 1.3612, + "theoretical_loss": 3.544943199418766, + "tokens_seen": 1498087424 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004413400200601806, + "loss": 1.2426, + "theoretical_loss": 3.5449296330710798, + "tokens_seen": 1498152960 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044132397191574724, + "loss": 1.2339, + "theoretical_loss": 3.5449160674829927, + "tokens_seen": 1498218496 + }, + { + "epoch": 0.45, + "learning_rate": 0.000441307923771314, + "loss": 1.3427, + "theoretical_loss": 3.544902502654429, + "tokens_seen": 1498284032 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044129187562688064, + "loss": 1.2567, + "theoretical_loss": 3.5448889385853137, + "tokens_seen": 1498349568 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004412758274824474, + "loss": 1.3323, + "theoretical_loss": 3.5448753752755695, + "tokens_seen": 1498415104 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044125977933801404, + "loss": 1.3567, + "theoretical_loss": 3.544861812725122, + "tokens_seen": 1498480640 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004412437311935808, + "loss": 1.2061, + "theoretical_loss": 3.544848250933895, + "tokens_seen": 1498546176 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044122768304914744, + "loss": 1.3519, + "theoretical_loss": 3.5448346899018133, + "tokens_seen": 1498611712 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004412116349047142, + "loss": 1.2727, + "theoretical_loss": 3.5448211296288004, + "tokens_seen": 1498677248 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044119558676028085, + "loss": 1.3435, + "theoretical_loss": 3.5448075701147808, + "tokens_seen": 1498742784 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004411795386158476, + "loss": 1.2795, + "theoretical_loss": 3.54479401135968, + "tokens_seen": 1498808320 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044116349047141425, + "loss": 1.2347, + "theoretical_loss": 3.5447804533634204, + "tokens_seen": 1498873856 + }, + { + "epoch": 0.45, + "learning_rate": 0.000441147442326981, + "loss": 1.2796, + "theoretical_loss": 3.5447668961259278, + "tokens_seen": 1498939392 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044113139418254765, + "loss": 1.3538, + "theoretical_loss": 3.5447533396471256, + "tokens_seen": 1499004928 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044111534603811435, + "loss": 1.2659, + "theoretical_loss": 3.5447397839269392, + "tokens_seen": 1499070464 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044109929789368105, + "loss": 1.2887, + "theoretical_loss": 3.5447262289652928, + "tokens_seen": 1499136000 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044108324974924775, + "loss": 1.2366, + "theoretical_loss": 3.54471267476211, + "tokens_seen": 1499201536 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044106720160481445, + "loss": 1.314, + "theoretical_loss": 3.5446991213173153, + "tokens_seen": 1499267072 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044105115346038115, + "loss": 1.2411, + "theoretical_loss": 3.544685568630834, + "tokens_seen": 1499332608 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044103510531594785, + "loss": 1.3048, + "theoretical_loss": 3.54467201670259, + "tokens_seen": 1499398144 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044101905717151455, + "loss": 1.4471, + "theoretical_loss": 3.5446584655325073, + "tokens_seen": 1499463680 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044100300902708125, + "loss": 1.3201, + "theoretical_loss": 3.5446449151205113, + "tokens_seen": 1499529216 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044098696088264795, + "loss": 1.2974, + "theoretical_loss": 3.544631365466526, + "tokens_seen": 1499594752 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004409709127382146, + "loss": 1.3607, + "theoretical_loss": 3.5446178165704754, + "tokens_seen": 1499660288 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044095486459378136, + "loss": 1.3531, + "theoretical_loss": 3.544604268432285, + "tokens_seen": 1499725824 + }, + { + "epoch": 0.45, + "learning_rate": 0.000440938816449348, + "loss": 1.3561, + "theoretical_loss": 3.544590721051878, + "tokens_seen": 1499791360 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044092276830491476, + "loss": 1.2993, + "theoretical_loss": 3.5445771744291803, + "tokens_seen": 1499856896 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004409067201604814, + "loss": 1.2991, + "theoretical_loss": 3.5445636285641156, + "tokens_seen": 1499922432 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044089067201604816, + "loss": 1.3714, + "theoretical_loss": 3.544550083456608, + "tokens_seen": 1499987968 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004408746238716148, + "loss": 1.4121, + "theoretical_loss": 3.5445365391065833, + "tokens_seen": 1500053504 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044085857572718156, + "loss": 1.2091, + "theoretical_loss": 3.544522995513965, + "tokens_seen": 1500119040 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004408425275827482, + "loss": 1.3053, + "theoretical_loss": 3.5445094526786782, + "tokens_seen": 1500184576 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044082647943831496, + "loss": 1.3489, + "theoretical_loss": 3.5444959106006473, + "tokens_seen": 1500250112 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004408104312938817, + "loss": 1.2063, + "theoretical_loss": 3.5444823692797973, + "tokens_seen": 1500315648 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044079438314944836, + "loss": 1.3623, + "theoretical_loss": 3.5444688287160515, + "tokens_seen": 1500381184 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004407783350050151, + "loss": 1.3298, + "theoretical_loss": 3.5444552889093357, + "tokens_seen": 1500446720 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044076228686058176, + "loss": 1.2896, + "theoretical_loss": 3.5444417498595744, + "tokens_seen": 1500512256 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004407462387161485, + "loss": 1.2691, + "theoretical_loss": 3.5444282115666916, + "tokens_seen": 1500577792 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044073019057171516, + "loss": 1.3423, + "theoretical_loss": 3.5444146740306124, + "tokens_seen": 1500643328 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004407141424272819, + "loss": 1.3615, + "theoretical_loss": 3.5444011372512616, + "tokens_seen": 1500708864 + }, + { + "epoch": 0.45, + "objective/train/docs_used": 557476, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.355412483215332, + "objective/train/theoretical_loss": 3.5443876012285638, + "objective/train/tokens_used": 1521234400, + "theoretical_loss": 3.5443876012285638, + "tokens_seen": 1500774400 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044069809428284857, + "loss": 1.3554, + "theoretical_loss": 3.5443876012285638, + "tokens_seen": 1500774400 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004406820461384153, + "loss": 1.3034, + "theoretical_loss": 3.544374065962443, + "tokens_seen": 1500839936 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044066599799398197, + "loss": 1.2873, + "theoretical_loss": 3.544360531452825, + "tokens_seen": 1500905472 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004406499498495487, + "loss": 1.3067, + "theoretical_loss": 3.544346997699633, + "tokens_seen": 1500971008 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044063390170511537, + "loss": 1.3291, + "theoretical_loss": 3.5443334647027935, + "tokens_seen": 1501036544 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004406178535606821, + "loss": 1.2883, + "theoretical_loss": 3.54431993246223, + "tokens_seen": 1501102080 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044060180541624877, + "loss": 1.3102, + "theoretical_loss": 3.5443064009778675, + "tokens_seen": 1501167616 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004405857572718155, + "loss": 1.3101, + "theoretical_loss": 3.54429287024963, + "tokens_seen": 1501233152 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044056970912738217, + "loss": 1.2608, + "theoretical_loss": 3.544279340277444, + "tokens_seen": 1501298688 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004405536609829489, + "loss": 1.3472, + "theoretical_loss": 3.5442658110612326, + "tokens_seen": 1501364224 + }, + { + "epoch": 0.45, + "learning_rate": 0.00044053761283851557, + "loss": 1.4191, + "theoretical_loss": 3.5442522826009215, + "tokens_seen": 1501429760 + }, + { + "epoch": 0.45, + "learning_rate": 0.0004405215646940823, + "loss": 1.3428, + "theoretical_loss": 3.544238754896435, + "tokens_seen": 1501495296 + }, + { + "epoch": 0.46, + "learning_rate": 0.00044050551654964897, + "loss": 1.3141, + "theoretical_loss": 3.544225227947698, + "tokens_seen": 1501560832 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004404894684052157, + "loss": 1.4784, + "theoretical_loss": 3.544211701754636, + "tokens_seen": 1501626368 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004404734202607824, + "loss": 1.2663, + "theoretical_loss": 3.5441981763171726, + "tokens_seen": 1501691904 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004404573721163491, + "loss": 1.3035, + "theoretical_loss": 3.5441846516352333, + "tokens_seen": 1501757440 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004404413239719158, + "loss": 1.2916, + "theoretical_loss": 3.5441711277087427, + "tokens_seen": 1501822976 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004404252758274825, + "loss": 1.2052, + "theoretical_loss": 3.5441576045376255, + "tokens_seen": 1501888512 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004404092276830492, + "loss": 1.3459, + "theoretical_loss": 3.5441440821218073, + "tokens_seen": 1501954048 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004403931795386159, + "loss": 1.3185, + "theoretical_loss": 3.544130560461212, + "tokens_seen": 1502019584 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004403771313941826, + "loss": 1.3586, + "theoretical_loss": 3.5441170395557653, + "tokens_seen": 1502085120 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004403610832497493, + "loss": 1.3831, + "theoretical_loss": 3.5441035194053914, + "tokens_seen": 1502150656 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004403450351053159, + "loss": 1.2729, + "theoretical_loss": 3.5440900000100157, + "tokens_seen": 1502216192 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004403289869608827, + "loss": 1.2942, + "theoretical_loss": 3.5440764813695633, + "tokens_seen": 1502281728 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004403129388164493, + "loss": 1.267, + "theoretical_loss": 3.5440629634839578, + "tokens_seen": 1502347264 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004402968906720161, + "loss": 1.3837, + "theoretical_loss": 3.544049446353126, + "tokens_seen": 1502412800 + }, + { + "epoch": 0.46, + "learning_rate": 0.00044028084252758273, + "loss": 1.2939, + "theoretical_loss": 3.5440359299769915, + "tokens_seen": 1502478336 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004402647943831495, + "loss": 1.358, + "theoretical_loss": 3.5440224143554793, + "tokens_seen": 1502543872 + }, + { + "epoch": 0.46, + "learning_rate": 0.00044024874623871613, + "loss": 1.2986, + "theoretical_loss": 3.544008899488515, + "tokens_seen": 1502609408 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004402326980942829, + "loss": 1.2407, + "theoretical_loss": 3.5439953853760233, + "tokens_seen": 1502674944 + }, + { + "epoch": 0.46, + "learning_rate": 0.00044021664994984953, + "loss": 1.2598, + "theoretical_loss": 3.543981872017929, + "tokens_seen": 1502740480 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004402006018054163, + "loss": 1.3193, + "theoretical_loss": 3.5439683594141576, + "tokens_seen": 1502806016 + }, + { + "epoch": 0.46, + "learning_rate": 0.00044018455366098293, + "loss": 1.3534, + "theoretical_loss": 3.543954847564633, + "tokens_seen": 1502871552 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004401685055165497, + "loss": 1.3338, + "theoretical_loss": 3.543941336469282, + "tokens_seen": 1502937088 + }, + { + "epoch": 0.46, + "learning_rate": 0.00044015245737211633, + "loss": 1.4294, + "theoretical_loss": 3.5439278261280274, + "tokens_seen": 1503002624 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004401364092276831, + "loss": 1.2593, + "theoretical_loss": 3.5439143165407962, + "tokens_seen": 1503068160 + }, + { + "epoch": 0.46, + "learning_rate": 0.00044012036108324973, + "loss": 1.3072, + "theoretical_loss": 3.5439008077075123, + "tokens_seen": 1503133696 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004401043129388165, + "loss": 1.3393, + "theoretical_loss": 3.5438872996281012, + "tokens_seen": 1503199232 + }, + { + "epoch": 0.46, + "learning_rate": 0.00044008826479438314, + "loss": 1.3916, + "theoretical_loss": 3.543873792302488, + "tokens_seen": 1503264768 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004400722166499499, + "loss": 1.3307, + "theoretical_loss": 3.5438602857305974, + "tokens_seen": 1503330304 + }, + { + "epoch": 0.46, + "learning_rate": 0.00044005616850551654, + "loss": 1.2868, + "theoretical_loss": 3.5438467799123545, + "tokens_seen": 1503395840 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004400401203610833, + "loss": 1.4233, + "theoretical_loss": 3.5438332748476853, + "tokens_seen": 1503461376 + }, + { + "epoch": 0.46, + "learning_rate": 0.00044002407221664994, + "loss": 1.3196, + "theoretical_loss": 3.5438197705365138, + "tokens_seen": 1503526912 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004400080240722167, + "loss": 1.4014, + "theoretical_loss": 3.5438062669787653, + "tokens_seen": 1503592448 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043999197592778334, + "loss": 1.2711, + "theoretical_loss": 3.5437927641743654, + "tokens_seen": 1503657984 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004399759277833501, + "loss": 1.2907, + "theoretical_loss": 3.5437792621232393, + "tokens_seen": 1503723520 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043995987963891674, + "loss": 1.2926, + "theoretical_loss": 3.5437657608253117, + "tokens_seen": 1503789056 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004399438314944835, + "loss": 1.3212, + "theoretical_loss": 3.543752260280508, + "tokens_seen": 1503854592 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043992778335005014, + "loss": 1.2637, + "theoretical_loss": 3.543738760488753, + "tokens_seen": 1503920128 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004399117352056169, + "loss": 1.3608, + "theoretical_loss": 3.543725261449972, + "tokens_seen": 1503985664 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043989568706118354, + "loss": 1.2298, + "theoretical_loss": 3.543711763164091, + "tokens_seen": 1504051200 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004398796389167503, + "loss": 1.1932, + "theoretical_loss": 3.5436982656310345, + "tokens_seen": 1504116736 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043986359077231694, + "loss": 1.3902, + "theoretical_loss": 3.5436847688507274, + "tokens_seen": 1504182272 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004398475426278837, + "loss": 1.3955, + "theoretical_loss": 3.5436712728230955, + "tokens_seen": 1504247808 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043983149448345035, + "loss": 1.3158, + "theoretical_loss": 3.543657777548064, + "tokens_seen": 1504313344 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004398154463390171, + "loss": 1.3667, + "theoretical_loss": 3.543644283025558, + "tokens_seen": 1504378880 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043979939819458375, + "loss": 1.2875, + "theoretical_loss": 3.5436307892555026, + "tokens_seen": 1504444416 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004397833500501505, + "loss": 1.3186, + "theoretical_loss": 3.543617296237823, + "tokens_seen": 1504509952 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043976730190571715, + "loss": 1.348, + "theoretical_loss": 3.543603803972445, + "tokens_seen": 1504575488 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004397512537612839, + "loss": 1.2596, + "theoretical_loss": 3.543590312459294, + "tokens_seen": 1504641024 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043973520561685055, + "loss": 1.3399, + "theoretical_loss": 3.543576821698294, + "tokens_seen": 1504706560 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043971915747241725, + "loss": 1.3144, + "theoretical_loss": 3.543563331689372, + "tokens_seen": 1504772096 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043970310932798395, + "loss": 1.3475, + "theoretical_loss": 3.5435498424324523, + "tokens_seen": 1504837632 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043968706118355065, + "loss": 1.4498, + "theoretical_loss": 3.5435363539274602, + "tokens_seen": 1504903168 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043967101303911735, + "loss": 1.3647, + "theoretical_loss": 3.543522866174322, + "tokens_seen": 1504968704 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043965496489468405, + "loss": 1.2882, + "theoretical_loss": 3.5435093791729617, + "tokens_seen": 1505034240 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004396389167502508, + "loss": 1.2727, + "theoretical_loss": 3.543495892923305, + "tokens_seen": 1505099776 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043962286860581745, + "loss": 1.3077, + "theoretical_loss": 3.5434824074252784, + "tokens_seen": 1505165312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004396068204613842, + "loss": 1.3261, + "theoretical_loss": 3.543468922678806, + "tokens_seen": 1505230848 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043959077231695085, + "loss": 1.3639, + "theoretical_loss": 3.543455438683814, + "tokens_seen": 1505296384 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004395747241725176, + "loss": 1.2544, + "theoretical_loss": 3.5434419554402274, + "tokens_seen": 1505361920 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043955867602808426, + "loss": 1.3563, + "theoretical_loss": 3.543428472947972, + "tokens_seen": 1505427456 + }, + { + "epoch": 0.46, + "learning_rate": 0.000439542627883651, + "loss": 1.2359, + "theoretical_loss": 3.5434149912069723, + "tokens_seen": 1505492992 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043952657973921766, + "loss": 1.35, + "theoretical_loss": 3.543401510217155, + "tokens_seen": 1505558528 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004395105315947844, + "loss": 1.3203, + "theoretical_loss": 3.5433880299784444, + "tokens_seen": 1505624064 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043949448345035106, + "loss": 1.3044, + "theoretical_loss": 3.5433745504907668, + "tokens_seen": 1505689600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004394784353059178, + "loss": 1.3532, + "theoretical_loss": 3.543361071754047, + "tokens_seen": 1505755136 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043946238716148446, + "loss": 1.3491, + "theoretical_loss": 3.5433475937682113, + "tokens_seen": 1505820672 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004394463390170512, + "loss": 1.3241, + "theoretical_loss": 3.543334116533184, + "tokens_seen": 1505886208 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043943029087261786, + "loss": 1.187, + "theoretical_loss": 3.543320640048892, + "tokens_seen": 1505951744 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004394142427281846, + "loss": 1.2335, + "theoretical_loss": 3.54330716431526, + "tokens_seen": 1506017280 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043939819458375126, + "loss": 1.2517, + "theoretical_loss": 3.543293689332214, + "tokens_seen": 1506082816 + }, + { + "epoch": 0.46, + "learning_rate": 0.000439382146439318, + "loss": 1.2852, + "theoretical_loss": 3.5432802150996787, + "tokens_seen": 1506148352 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043936609829488466, + "loss": 1.3905, + "theoretical_loss": 3.5432667416175803, + "tokens_seen": 1506213888 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004393500501504514, + "loss": 1.3209, + "theoretical_loss": 3.5432532688858442, + "tokens_seen": 1506279424 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043933400200601806, + "loss": 1.3718, + "theoretical_loss": 3.543239796904396, + "tokens_seen": 1506344960 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004393179538615848, + "loss": 1.3426, + "theoretical_loss": 3.543226325673161, + "tokens_seen": 1506410496 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043930190571715147, + "loss": 1.2067, + "theoretical_loss": 3.543212855192065, + "tokens_seen": 1506476032 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004392858575727182, + "loss": 1.1901, + "theoretical_loss": 3.543199385461034, + "tokens_seen": 1506541568 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043926980942828487, + "loss": 1.3036, + "theoretical_loss": 3.5431859164799935, + "tokens_seen": 1506607104 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004392537612838516, + "loss": 1.3298, + "theoretical_loss": 3.5431724482488685, + "tokens_seen": 1506672640 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043923771313941827, + "loss": 1.3144, + "theoretical_loss": 3.543158980767585, + "tokens_seen": 1506738176 + }, + { + "epoch": 0.46, + "learning_rate": 0.000439221664994985, + "loss": 1.323, + "theoretical_loss": 3.543145514036069, + "tokens_seen": 1506803712 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043920561685055167, + "loss": 1.2807, + "theoretical_loss": 3.543132048054245, + "tokens_seen": 1506869248 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004391895687061184, + "loss": 1.3798, + "theoretical_loss": 3.54311858282204, + "tokens_seen": 1506934784 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043917352056168507, + "loss": 1.4383, + "theoretical_loss": 3.543105118339379, + "tokens_seen": 1507000320 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004391574724172518, + "loss": 1.3505, + "theoretical_loss": 3.5430916546061875, + "tokens_seen": 1507065856 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043914142427281847, + "loss": 1.3569, + "theoretical_loss": 3.543078191622392, + "tokens_seen": 1507131392 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043912537612838523, + "loss": 1.3148, + "theoretical_loss": 3.5430647293879174, + "tokens_seen": 1507196928 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004391093279839519, + "loss": 1.2617, + "theoretical_loss": 3.5430512679026895, + "tokens_seen": 1507262464 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 559732, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2389800548553467, + "objective/train/theoretical_loss": 3.543037807166635, + "objective/train/tokens_used": 1527788000, + "theoretical_loss": 3.543037807166635, + "tokens_seen": 1507328000 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004390932798395186, + "loss": 1.239, + "theoretical_loss": 3.543037807166635, + "tokens_seen": 1507328000 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004390772316950853, + "loss": 1.3347, + "theoretical_loss": 3.5430243471796783, + "tokens_seen": 1507393536 + }, + { + "epoch": 0.46, + "learning_rate": 0.000439061183550652, + "loss": 1.3719, + "theoretical_loss": 3.543010887941746, + "tokens_seen": 1507459072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004390451354062187, + "loss": 1.2393, + "theoretical_loss": 3.5429974294527633, + "tokens_seen": 1507524608 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004390290872617854, + "loss": 1.4821, + "theoretical_loss": 3.5429839717126566, + "tokens_seen": 1507590144 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004390130391173521, + "loss": 1.374, + "theoretical_loss": 3.5429705147213513, + "tokens_seen": 1507655680 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004389969909729188, + "loss": 1.2828, + "theoretical_loss": 3.542957058478773, + "tokens_seen": 1507721216 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004389809428284855, + "loss": 1.3029, + "theoretical_loss": 3.5429436029848476, + "tokens_seen": 1507786752 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004389648946840522, + "loss": 1.2202, + "theoretical_loss": 3.542930148239501, + "tokens_seen": 1507852288 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004389488465396188, + "loss": 1.2912, + "theoretical_loss": 3.5429166942426598, + "tokens_seen": 1507917824 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004389327983951856, + "loss": 1.3624, + "theoretical_loss": 3.5429032409942485, + "tokens_seen": 1507983360 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043891675025075223, + "loss": 1.3954, + "theoretical_loss": 3.5428897884941932, + "tokens_seen": 1508048896 + }, + { + "epoch": 0.46, + "learning_rate": 0.000438900702106319, + "loss": 1.3432, + "theoretical_loss": 3.5428763367424203, + "tokens_seen": 1508114432 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043888465396188563, + "loss": 1.3276, + "theoretical_loss": 3.542862885738856, + "tokens_seen": 1508179968 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004388686058174524, + "loss": 1.2615, + "theoretical_loss": 3.542849435483425, + "tokens_seen": 1508245504 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043885255767301903, + "loss": 1.2976, + "theoretical_loss": 3.542835985976054, + "tokens_seen": 1508311040 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004388365095285858, + "loss": 1.2381, + "theoretical_loss": 3.5428225372166686, + "tokens_seen": 1508376576 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043882046138415243, + "loss": 1.3304, + "theoretical_loss": 3.542809089205195, + "tokens_seen": 1508442112 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004388044132397192, + "loss": 1.3693, + "theoretical_loss": 3.5427956419415585, + "tokens_seen": 1508507648 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043878836509528583, + "loss": 1.3042, + "theoretical_loss": 3.542782195425686, + "tokens_seen": 1508573184 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004387723169508526, + "loss": 1.3137, + "theoretical_loss": 3.542768749657503, + "tokens_seen": 1508638720 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043875626880641923, + "loss": 1.2835, + "theoretical_loss": 3.5427553046369344, + "tokens_seen": 1508704256 + }, + { + "epoch": 0.46, + "learning_rate": 0.000438740220661986, + "loss": 1.3561, + "theoretical_loss": 3.5427418603639076, + "tokens_seen": 1508769792 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043872417251755263, + "loss": 1.3355, + "theoretical_loss": 3.542728416838348, + "tokens_seen": 1508835328 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004387081243731194, + "loss": 1.3445, + "theoretical_loss": 3.542714974060182, + "tokens_seen": 1508900864 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043869207622868604, + "loss": 1.357, + "theoretical_loss": 3.542701532029335, + "tokens_seen": 1508966400 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004386760280842528, + "loss": 1.3023, + "theoretical_loss": 3.5426880907457328, + "tokens_seen": 1509031936 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043865997993981944, + "loss": 1.275, + "theoretical_loss": 3.5426746502093023, + "tokens_seen": 1509097472 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004386439317953862, + "loss": 1.3351, + "theoretical_loss": 3.542661210419969, + "tokens_seen": 1509163008 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043862788365095284, + "loss": 1.3253, + "theoretical_loss": 3.542647771377659, + "tokens_seen": 1509228544 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004386118355065196, + "loss": 1.3308, + "theoretical_loss": 3.5426343330822982, + "tokens_seen": 1509294080 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043859578736208624, + "loss": 1.3174, + "theoretical_loss": 3.5426208955338128, + "tokens_seen": 1509359616 + }, + { + "epoch": 0.46, + "learning_rate": 0.000438579739217653, + "loss": 1.3626, + "theoretical_loss": 3.542607458732129, + "tokens_seen": 1509425152 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043856369107321964, + "loss": 1.4026, + "theoretical_loss": 3.5425940226771724, + "tokens_seen": 1509490688 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004385476429287864, + "loss": 1.3105, + "theoretical_loss": 3.54258058736887, + "tokens_seen": 1509556224 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043853159478435304, + "loss": 1.3516, + "theoretical_loss": 3.5425671528071465, + "tokens_seen": 1509621760 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004385155466399198, + "loss": 1.2895, + "theoretical_loss": 3.542553718991929, + "tokens_seen": 1509687296 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043849949849548644, + "loss": 1.3758, + "theoretical_loss": 3.5425402859231436, + "tokens_seen": 1509752832 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004384834503510532, + "loss": 1.2039, + "theoretical_loss": 3.542526853600717, + "tokens_seen": 1509818368 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043846740220661984, + "loss": 1.3522, + "theoretical_loss": 3.5425134220245735, + "tokens_seen": 1509883904 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004384513540621866, + "loss": 1.2966, + "theoretical_loss": 3.5424999911946404, + "tokens_seen": 1509949440 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004384353059177533, + "loss": 1.3354, + "theoretical_loss": 3.5424865611108443, + "tokens_seen": 1510014976 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043841925777332, + "loss": 1.2282, + "theoretical_loss": 3.542473131773111, + "tokens_seen": 1510080512 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004384032096288867, + "loss": 1.417, + "theoretical_loss": 3.542459703181366, + "tokens_seen": 1510146048 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004383871614844534, + "loss": 1.3059, + "theoretical_loss": 3.5424462753355366, + "tokens_seen": 1510211584 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004383711133400201, + "loss": 1.3196, + "theoretical_loss": 3.542432848235548, + "tokens_seen": 1510277120 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004383550651955868, + "loss": 1.3107, + "theoretical_loss": 3.5424194218813265, + "tokens_seen": 1510342656 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004383390170511535, + "loss": 1.3324, + "theoretical_loss": 3.542405996272799, + "tokens_seen": 1510408192 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043832296890672015, + "loss": 1.2653, + "theoretical_loss": 3.5423925714098914, + "tokens_seen": 1510473728 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004383069207622869, + "loss": 1.2831, + "theoretical_loss": 3.5423791472925306, + "tokens_seen": 1510539264 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043829087261785355, + "loss": 1.3406, + "theoretical_loss": 3.542365723920641, + "tokens_seen": 1510604800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004382748244734203, + "loss": 1.2874, + "theoretical_loss": 3.5423523012941507, + "tokens_seen": 1510670336 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043825877632898695, + "loss": 1.2536, + "theoretical_loss": 3.5423388794129855, + "tokens_seen": 1510735872 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004382427281845537, + "loss": 1.2863, + "theoretical_loss": 3.5423254582770713, + "tokens_seen": 1510801408 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043822668004012035, + "loss": 1.239, + "theoretical_loss": 3.542312037886335, + "tokens_seen": 1510866944 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004382106318956871, + "loss": 1.3203, + "theoretical_loss": 3.542298618240702, + "tokens_seen": 1510932480 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043819458375125376, + "loss": 1.4292, + "theoretical_loss": 3.542285199340099, + "tokens_seen": 1510998016 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004381785356068205, + "loss": 1.3797, + "theoretical_loss": 3.5422717811844526, + "tokens_seen": 1511063552 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043816248746238716, + "loss": 1.2788, + "theoretical_loss": 3.5422583637736893, + "tokens_seen": 1511129088 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004381464393179539, + "loss": 1.2823, + "theoretical_loss": 3.5422449471077346, + "tokens_seen": 1511194624 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043813039117352056, + "loss": 1.3775, + "theoretical_loss": 3.542231531186516, + "tokens_seen": 1511260160 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004381143430290873, + "loss": 1.3337, + "theoretical_loss": 3.542218116009959, + "tokens_seen": 1511325696 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043809829488465396, + "loss": 1.2218, + "theoretical_loss": 3.5422047015779903, + "tokens_seen": 1511391232 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004380822467402207, + "loss": 1.3095, + "theoretical_loss": 3.542191287890536, + "tokens_seen": 1511456768 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043806619859578736, + "loss": 1.3707, + "theoretical_loss": 3.542177874947523, + "tokens_seen": 1511522304 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004380501504513541, + "loss": 1.2623, + "theoretical_loss": 3.542164462748877, + "tokens_seen": 1511587840 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043803410230692076, + "loss": 1.2594, + "theoretical_loss": 3.5421510512945256, + "tokens_seen": 1511653376 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004380180541624875, + "loss": 1.375, + "theoretical_loss": 3.542137640584394, + "tokens_seen": 1511718912 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043800200601805416, + "loss": 1.2619, + "theoretical_loss": 3.542124230618409, + "tokens_seen": 1511784448 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004379859578736209, + "loss": 1.2481, + "theoretical_loss": 3.5421108213964976, + "tokens_seen": 1511849984 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043796990972918756, + "loss": 1.34, + "theoretical_loss": 3.5420974129185856, + "tokens_seen": 1511915520 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004379538615847543, + "loss": 1.3013, + "theoretical_loss": 3.5420840051845994, + "tokens_seen": 1511981056 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043793781344032097, + "loss": 1.3628, + "theoretical_loss": 3.5420705981944662, + "tokens_seen": 1512046592 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004379217652958877, + "loss": 1.2879, + "theoretical_loss": 3.5420571919481123, + "tokens_seen": 1512112128 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043790571715145437, + "loss": 1.3669, + "theoretical_loss": 3.542043786445464, + "tokens_seen": 1512177664 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004378896690070211, + "loss": 1.3365, + "theoretical_loss": 3.5420303816864473, + "tokens_seen": 1512243200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043787362086258777, + "loss": 1.2688, + "theoretical_loss": 3.5420169776709898, + "tokens_seen": 1512308736 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004378575727181545, + "loss": 1.4151, + "theoretical_loss": 3.542003574399017, + "tokens_seen": 1512374272 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043784152457372117, + "loss": 1.3078, + "theoretical_loss": 3.5419901718704563, + "tokens_seen": 1512439808 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004378254764292879, + "loss": 1.4139, + "theoretical_loss": 3.541976770085234, + "tokens_seen": 1512505344 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043780942828485457, + "loss": 1.2758, + "theoretical_loss": 3.541963369043276, + "tokens_seen": 1512570880 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004377933801404213, + "loss": 1.3809, + "theoretical_loss": 3.5419499687445093, + "tokens_seen": 1512636416 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043777733199598797, + "loss": 1.3225, + "theoretical_loss": 3.541936569188861, + "tokens_seen": 1512701952 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004377612838515547, + "loss": 1.3334, + "theoretical_loss": 3.5419231703762577, + "tokens_seen": 1512767488 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004377452357071214, + "loss": 1.348, + "theoretical_loss": 3.5419097723066253, + "tokens_seen": 1512833024 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043772918756268813, + "loss": 1.3472, + "theoretical_loss": 3.54189637497989, + "tokens_seen": 1512898560 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004377131394182548, + "loss": 1.2799, + "theoretical_loss": 3.5418829783959804, + "tokens_seen": 1512964096 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004376970912738215, + "loss": 1.3083, + "theoretical_loss": 3.5418695825548214, + "tokens_seen": 1513029632 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004376810431293882, + "loss": 1.3996, + "theoretical_loss": 3.54185618745634, + "tokens_seen": 1513095168 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004376649949849549, + "loss": 1.284, + "theoretical_loss": 3.541842793100463, + "tokens_seen": 1513160704 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004376489468405216, + "loss": 1.3144, + "theoretical_loss": 3.541829399487117, + "tokens_seen": 1513226240 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004376328986960883, + "loss": 1.3828, + "theoretical_loss": 3.541816006616229, + "tokens_seen": 1513291776 + }, + { + "epoch": 0.46, + "learning_rate": 0.000437616850551655, + "loss": 1.2937, + "theoretical_loss": 3.5418026144877257, + "tokens_seen": 1513357312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004376008024072217, + "loss": 1.2959, + "theoretical_loss": 3.5417892231015333, + "tokens_seen": 1513422848 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004375847542627884, + "loss": 1.3707, + "theoretical_loss": 3.5417758324575788, + "tokens_seen": 1513488384 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004375687061183551, + "loss": 1.3478, + "theoretical_loss": 3.5417624425557888, + "tokens_seen": 1513553920 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004375526579739217, + "loss": 1.3122, + "theoretical_loss": 3.54174905339609, + "tokens_seen": 1513619456 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004375366098294885, + "loss": 1.2648, + "theoretical_loss": 3.5417356649784097, + "tokens_seen": 1513684992 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043752056168505513, + "loss": 1.3638, + "theoretical_loss": 3.541722277302674, + "tokens_seen": 1513750528 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004375045135406219, + "loss": 1.3221, + "theoretical_loss": 3.5417088903688096, + "tokens_seen": 1513816064 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 562169, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3687608242034912, + "objective/train/theoretical_loss": 3.541695504176744, + "objective/train/tokens_used": 1534341600, + "theoretical_loss": 3.541695504176744, + "tokens_seen": 1513881600 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043748846539618853, + "loss": 1.3688, + "theoretical_loss": 3.541695504176744, + "tokens_seen": 1513881600 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004374724172517553, + "loss": 1.3041, + "theoretical_loss": 3.5416821187264036, + "tokens_seen": 1513947136 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043745636910732193, + "loss": 1.3307, + "theoretical_loss": 3.541668734017715, + "tokens_seen": 1514012672 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004374403209628887, + "loss": 1.3094, + "theoretical_loss": 3.541655350050605, + "tokens_seen": 1514078208 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043742427281845533, + "loss": 1.2625, + "theoretical_loss": 3.5416419668250008, + "tokens_seen": 1514143744 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004374082246740221, + "loss": 1.2463, + "theoretical_loss": 3.5416285843408293, + "tokens_seen": 1514209280 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043739217652958873, + "loss": 1.2948, + "theoretical_loss": 3.5416152025980163, + "tokens_seen": 1514274816 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004373761283851555, + "loss": 1.1963, + "theoretical_loss": 3.54160182159649, + "tokens_seen": 1514340352 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043736008024072213, + "loss": 1.3098, + "theoretical_loss": 3.5415884413361765, + "tokens_seen": 1514405888 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004373440320962889, + "loss": 1.284, + "theoretical_loss": 3.541575061817003, + "tokens_seen": 1514471424 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043732798395185554, + "loss": 1.3759, + "theoretical_loss": 3.5415616830388954, + "tokens_seen": 1514536960 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004373119358074223, + "loss": 1.3697, + "theoretical_loss": 3.541548305001782, + "tokens_seen": 1514602496 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043729588766298894, + "loss": 1.294, + "theoretical_loss": 3.541534927705589, + "tokens_seen": 1514668032 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004372798395185557, + "loss": 1.3298, + "theoretical_loss": 3.5415215511502436, + "tokens_seen": 1514733568 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043726379137412234, + "loss": 1.2644, + "theoretical_loss": 3.5415081753356725, + "tokens_seen": 1514799104 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004372477432296891, + "loss": 1.3377, + "theoretical_loss": 3.5414948002618023, + "tokens_seen": 1514864640 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043723169508525585, + "loss": 1.4491, + "theoretical_loss": 3.54148142592856, + "tokens_seen": 1514930176 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004372156469408225, + "loss": 1.3298, + "theoretical_loss": 3.5414680523358735, + "tokens_seen": 1514995712 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043719959879638925, + "loss": 1.3706, + "theoretical_loss": 3.541454679483669, + "tokens_seen": 1515061248 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004371835506519559, + "loss": 1.2893, + "theoretical_loss": 3.5414413073718736, + "tokens_seen": 1515126784 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043716750250752265, + "loss": 1.3279, + "theoretical_loss": 3.5414279360004137, + "tokens_seen": 1515192320 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004371514543630893, + "loss": 1.3126, + "theoretical_loss": 3.5414145653692177, + "tokens_seen": 1515257856 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043713540621865605, + "loss": 1.2649, + "theoretical_loss": 3.541401195478211, + "tokens_seen": 1515323392 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004371193580742227, + "loss": 1.2889, + "theoretical_loss": 3.5413878263273215, + "tokens_seen": 1515388928 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043710330992978945, + "loss": 1.3004, + "theoretical_loss": 3.5413744579164765, + "tokens_seen": 1515454464 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004370872617853561, + "loss": 1.2888, + "theoretical_loss": 3.5413610902456023, + "tokens_seen": 1515520000 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043707121364092285, + "loss": 1.3666, + "theoretical_loss": 3.5413477233146264, + "tokens_seen": 1515585536 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004370551654964895, + "loss": 1.2165, + "theoretical_loss": 3.5413343571234757, + "tokens_seen": 1515651072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004370391173520562, + "loss": 1.3861, + "theoretical_loss": 3.5413209916720776, + "tokens_seen": 1515716608 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004370230692076229, + "loss": 1.2746, + "theoretical_loss": 3.5413076269603585, + "tokens_seen": 1515782144 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004370070210631896, + "loss": 1.4138, + "theoretical_loss": 3.541294262988246, + "tokens_seen": 1515847680 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004369909729187563, + "loss": 1.3104, + "theoretical_loss": 3.541280899755667, + "tokens_seen": 1515913216 + }, + { + "epoch": 0.46, + "learning_rate": 0.000436974924774323, + "loss": 1.3216, + "theoretical_loss": 3.5412675372625486, + "tokens_seen": 1515978752 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004369588766298897, + "loss": 1.3026, + "theoretical_loss": 3.5412541755088176, + "tokens_seen": 1516044288 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004369428284854564, + "loss": 1.2333, + "theoretical_loss": 3.541240814494402, + "tokens_seen": 1516109824 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043692678034102305, + "loss": 1.3196, + "theoretical_loss": 3.5412274542192286, + "tokens_seen": 1516175360 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004369107321965898, + "loss": 1.264, + "theoretical_loss": 3.541214094683224, + "tokens_seen": 1516240896 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043689468405215645, + "loss": 1.1837, + "theoretical_loss": 3.541200735886316, + "tokens_seen": 1516306432 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004368786359077232, + "loss": 1.3081, + "theoretical_loss": 3.541187377828431, + "tokens_seen": 1516371968 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043686258776328985, + "loss": 1.2069, + "theoretical_loss": 3.541174020509497, + "tokens_seen": 1516437504 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004368465396188566, + "loss": 1.3593, + "theoretical_loss": 3.541160663929441, + "tokens_seen": 1516503040 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043683049147442326, + "loss": 1.3538, + "theoretical_loss": 3.54114730808819, + "tokens_seen": 1516568576 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043681444332999, + "loss": 1.3487, + "theoretical_loss": 3.5411339529856716, + "tokens_seen": 1516634112 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043679839518555666, + "loss": 1.2678, + "theoretical_loss": 3.5411205986218124, + "tokens_seen": 1516699648 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004367823470411234, + "loss": 1.3593, + "theoretical_loss": 3.5411072449965397, + "tokens_seen": 1516765184 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043676629889669006, + "loss": 1.3674, + "theoretical_loss": 3.5410938921097816, + "tokens_seen": 1516830720 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004367502507522568, + "loss": 1.3536, + "theoretical_loss": 3.5410805399614644, + "tokens_seen": 1516896256 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043673420260782346, + "loss": 1.3687, + "theoretical_loss": 3.5410671885515157, + "tokens_seen": 1516961792 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004367181544633902, + "loss": 1.3959, + "theoretical_loss": 3.541053837879863, + "tokens_seen": 1517027328 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043670210631895686, + "loss": 1.2219, + "theoretical_loss": 3.5410404879464332, + "tokens_seen": 1517092864 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004366860581745236, + "loss": 1.2869, + "theoretical_loss": 3.5410271387511534, + "tokens_seen": 1517158400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043667001003009026, + "loss": 1.3412, + "theoretical_loss": 3.5410137902939516, + "tokens_seen": 1517223936 + }, + { + "epoch": 0.46, + "learning_rate": 0.000436653961885657, + "loss": 1.2253, + "theoretical_loss": 3.541000442574755, + "tokens_seen": 1517289472 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043663791374122366, + "loss": 1.2625, + "theoretical_loss": 3.54098709559349, + "tokens_seen": 1517355008 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004366218655967904, + "loss": 1.3888, + "theoretical_loss": 3.5409737493500852, + "tokens_seen": 1517420544 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043660581745235706, + "loss": 1.38, + "theoretical_loss": 3.5409604038444673, + "tokens_seen": 1517486080 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004365897693079238, + "loss": 1.2224, + "theoretical_loss": 3.5409470590765633, + "tokens_seen": 1517551616 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043657372116349047, + "loss": 1.2892, + "theoretical_loss": 3.540933715046301, + "tokens_seen": 1517617152 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004365576730190572, + "loss": 1.2873, + "theoretical_loss": 3.540920371753608, + "tokens_seen": 1517682688 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043654162487462387, + "loss": 1.2437, + "theoretical_loss": 3.540907029198412, + "tokens_seen": 1517748224 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004365255767301906, + "loss": 1.377, + "theoretical_loss": 3.540893687380639, + "tokens_seen": 1517813760 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043650952858575727, + "loss": 1.322, + "theoretical_loss": 3.540880346300218, + "tokens_seen": 1517879296 + }, + { + "epoch": 0.46, + "learning_rate": 0.000436493480441324, + "loss": 1.241, + "theoretical_loss": 3.5408670059570753, + "tokens_seen": 1517944832 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043647743229689067, + "loss": 1.4043, + "theoretical_loss": 3.540853666351138, + "tokens_seen": 1518010368 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004364613841524574, + "loss": 1.3245, + "theoretical_loss": 3.5408403274823352, + "tokens_seen": 1518075904 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043644533600802407, + "loss": 1.2506, + "theoretical_loss": 3.540826989350593, + "tokens_seen": 1518141440 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004364292878635908, + "loss": 1.4079, + "theoretical_loss": 3.5408136519558395, + "tokens_seen": 1518206976 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043641323971915747, + "loss": 1.3937, + "theoretical_loss": 3.5408003152980014, + "tokens_seen": 1518272512 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004363971915747242, + "loss": 1.2941, + "theoretical_loss": 3.540786979377007, + "tokens_seen": 1518338048 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004363811434302909, + "loss": 1.3353, + "theoretical_loss": 3.5407736441927833, + "tokens_seen": 1518403584 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043636509528585763, + "loss": 1.2473, + "theoretical_loss": 3.540760309745258, + "tokens_seen": 1518469120 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004363490471414243, + "loss": 1.3111, + "theoretical_loss": 3.5407469760343586, + "tokens_seen": 1518534656 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043633299899699103, + "loss": 1.3555, + "theoretical_loss": 3.5407336430600127, + "tokens_seen": 1518600192 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004363169508525577, + "loss": 1.1974, + "theoretical_loss": 3.5407203108221474, + "tokens_seen": 1518665728 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043630090270812443, + "loss": 1.2956, + "theoretical_loss": 3.540706979320691, + "tokens_seen": 1518731264 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004362848545636911, + "loss": 1.3499, + "theoretical_loss": 3.5406936485555702, + "tokens_seen": 1518796800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004362688064192578, + "loss": 1.2558, + "theoretical_loss": 3.5406803185267135, + "tokens_seen": 1518862336 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004362527582748245, + "loss": 1.4133, + "theoretical_loss": 3.5406669892340474, + "tokens_seen": 1518927872 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004362367101303912, + "loss": 1.3609, + "theoretical_loss": 3.5406536606775, + "tokens_seen": 1518993408 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004362206619859579, + "loss": 1.2518, + "theoretical_loss": 3.540640332856999, + "tokens_seen": 1519058944 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004362046138415246, + "loss": 1.3562, + "theoretical_loss": 3.540627005772472, + "tokens_seen": 1519124480 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004361885656970913, + "loss": 1.2686, + "theoretical_loss": 3.5406136794238465, + "tokens_seen": 1519190016 + }, + { + "epoch": 0.46, + "learning_rate": 0.000436172517552658, + "loss": 1.3023, + "theoretical_loss": 3.5406003538110506, + "tokens_seen": 1519255552 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043615646940822463, + "loss": 1.3083, + "theoretical_loss": 3.540587028934011, + "tokens_seen": 1519321088 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004361404212637914, + "loss": 1.3705, + "theoretical_loss": 3.5405737047926555, + "tokens_seen": 1519386624 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043612437311935803, + "loss": 1.4041, + "theoretical_loss": 3.540560381386913, + "tokens_seen": 1519452160 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004361083249749248, + "loss": 1.31, + "theoretical_loss": 3.5405470587167094, + "tokens_seen": 1519517696 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043609227683049143, + "loss": 1.2848, + "theoretical_loss": 3.540533736781973, + "tokens_seen": 1519583232 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004360762286860582, + "loss": 1.3461, + "theoretical_loss": 3.5405204155826326, + "tokens_seen": 1519648768 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043606018054162483, + "loss": 1.2552, + "theoretical_loss": 3.5405070951186146, + "tokens_seen": 1519714304 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004360441323971916, + "loss": 1.2935, + "theoretical_loss": 3.540493775389847, + "tokens_seen": 1519779840 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043602808425275834, + "loss": 1.2725, + "theoretical_loss": 3.5404804563962573, + "tokens_seen": 1519845376 + }, + { + "epoch": 0.46, + "learning_rate": 0.000436012036108325, + "loss": 1.3492, + "theoretical_loss": 3.5404671381377737, + "tokens_seen": 1519910912 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043599598796389174, + "loss": 1.3314, + "theoretical_loss": 3.5404538206143243, + "tokens_seen": 1519976448 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004359799398194584, + "loss": 1.3149, + "theoretical_loss": 3.5404405038258355, + "tokens_seen": 1520041984 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043596389167502514, + "loss": 1.3427, + "theoretical_loss": 3.540427187772236, + "tokens_seen": 1520107520 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004359478435305918, + "loss": 1.2469, + "theoretical_loss": 3.540413872453454, + "tokens_seen": 1520173056 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043593179538615855, + "loss": 1.2276, + "theoretical_loss": 3.5404005578694164, + "tokens_seen": 1520238592 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004359157472417252, + "loss": 1.5243, + "theoretical_loss": 3.5403872440200512, + "tokens_seen": 1520304128 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043589969909729195, + "loss": 1.2893, + "theoretical_loss": 3.540373930905286, + "tokens_seen": 1520369664 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 564927, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3084349632263184, + "objective/train/theoretical_loss": 3.5403606185250496, + "objective/train/tokens_used": 1540895200, + "theoretical_loss": 3.5403606185250496, + "tokens_seen": 1520435200 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004358836509528586, + "loss": 1.3084, + "theoretical_loss": 3.5403606185250496, + "tokens_seen": 1520435200 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043586760280842535, + "loss": 1.2678, + "theoretical_loss": 3.5403473068792684, + "tokens_seen": 1520500736 + }, + { + "epoch": 0.46, + "learning_rate": 0.000435851554663992, + "loss": 1.2409, + "theoretical_loss": 3.5403339959678712, + "tokens_seen": 1520566272 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043583550651955875, + "loss": 1.3809, + "theoretical_loss": 3.5403206857907854, + "tokens_seen": 1520631808 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004358194583751254, + "loss": 1.3079, + "theoretical_loss": 3.540307376347939, + "tokens_seen": 1520697344 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043580341023069215, + "loss": 1.3889, + "theoretical_loss": 3.54029406763926, + "tokens_seen": 1520762880 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004357873620862588, + "loss": 1.2658, + "theoretical_loss": 3.540280759664676, + "tokens_seen": 1520828416 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043577131394182555, + "loss": 1.334, + "theoretical_loss": 3.540267452424115, + "tokens_seen": 1520893952 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004357552657973922, + "loss": 1.2861, + "theoretical_loss": 3.540254145917505, + "tokens_seen": 1520959488 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043573921765295895, + "loss": 1.3172, + "theoretical_loss": 3.540240840144774, + "tokens_seen": 1521025024 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004357231695085256, + "loss": 1.4118, + "theoretical_loss": 3.540227535105849, + "tokens_seen": 1521090560 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043570712136409235, + "loss": 1.2364, + "theoretical_loss": 3.5402142308006592, + "tokens_seen": 1521156096 + }, + { + "epoch": 0.46, + "learning_rate": 0.000435691073219659, + "loss": 1.3337, + "theoretical_loss": 3.540200927229132, + "tokens_seen": 1521221632 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043567502507522576, + "loss": 1.4125, + "theoretical_loss": 3.540187624391195, + "tokens_seen": 1521287168 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004356589769307924, + "loss": 1.2402, + "theoretical_loss": 3.5401743222867763, + "tokens_seen": 1521352704 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004356429287863591, + "loss": 1.4215, + "theoretical_loss": 3.540161020915804, + "tokens_seen": 1521418240 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004356268806419258, + "loss": 1.3818, + "theoretical_loss": 3.540147720278206, + "tokens_seen": 1521483776 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004356108324974925, + "loss": 1.3169, + "theoretical_loss": 3.540134420373911, + "tokens_seen": 1521549312 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004355947843530592, + "loss": 1.2886, + "theoretical_loss": 3.5401211212028456, + "tokens_seen": 1521614848 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004355787362086259, + "loss": 1.2894, + "theoretical_loss": 3.5401078227649387, + "tokens_seen": 1521680384 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004355626880641926, + "loss": 1.3345, + "theoretical_loss": 3.5400945250601183, + "tokens_seen": 1521745920 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004355466399197593, + "loss": 1.3185, + "theoretical_loss": 3.540081228088312, + "tokens_seen": 1521811456 + }, + { + "epoch": 0.46, + "learning_rate": 0.000435530591775326, + "loss": 1.2857, + "theoretical_loss": 3.540067931849448, + "tokens_seen": 1521876992 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004355145436308927, + "loss": 1.3318, + "theoretical_loss": 3.540054636343455, + "tokens_seen": 1521942528 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043549849548645935, + "loss": 1.2888, + "theoretical_loss": 3.5400413415702596, + "tokens_seen": 1522008064 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004354824473420261, + "loss": 1.298, + "theoretical_loss": 3.5400280475297916, + "tokens_seen": 1522073600 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043546639919759276, + "loss": 1.3916, + "theoretical_loss": 3.5400147542219775, + "tokens_seen": 1522139136 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004354503510531595, + "loss": 1.3136, + "theoretical_loss": 3.5400014616467463, + "tokens_seen": 1522204672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043543430290872616, + "loss": 1.2597, + "theoretical_loss": 3.539988169804026, + "tokens_seen": 1522270208 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004354182547642929, + "loss": 1.273, + "theoretical_loss": 3.5399748786937444, + "tokens_seen": 1522335744 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043540220661985956, + "loss": 1.3017, + "theoretical_loss": 3.5399615883158297, + "tokens_seen": 1522401280 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004353861584754263, + "loss": 1.2329, + "theoretical_loss": 3.53994829867021, + "tokens_seen": 1522466816 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043537011033099296, + "loss": 1.3872, + "theoretical_loss": 3.5399350097568143, + "tokens_seen": 1522532352 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004353540621865597, + "loss": 1.316, + "theoretical_loss": 3.5399217215755696, + "tokens_seen": 1522597888 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043533801404212636, + "loss": 1.3934, + "theoretical_loss": 3.5399084341264038, + "tokens_seen": 1522663424 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004353219658976931, + "loss": 1.3452, + "theoretical_loss": 3.539895147409246, + "tokens_seen": 1522728960 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043530591775325976, + "loss": 1.2892, + "theoretical_loss": 3.539881861424025, + "tokens_seen": 1522794496 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004352898696088265, + "loss": 1.3802, + "theoretical_loss": 3.539868576170667, + "tokens_seen": 1522860032 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043527382146439316, + "loss": 1.2984, + "theoretical_loss": 3.5398552916491015, + "tokens_seen": 1522925568 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004352577733199599, + "loss": 1.3288, + "theoretical_loss": 3.539842007859256, + "tokens_seen": 1522991104 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043524172517552656, + "loss": 1.3253, + "theoretical_loss": 3.5398287248010596, + "tokens_seen": 1523056640 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004352256770310933, + "loss": 1.3859, + "theoretical_loss": 3.53981544247444, + "tokens_seen": 1523122176 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043520962888665997, + "loss": 1.2566, + "theoretical_loss": 3.5398021608793258, + "tokens_seen": 1523187712 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004351935807422267, + "loss": 1.3369, + "theoretical_loss": 3.539788880015644, + "tokens_seen": 1523253248 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043517753259779337, + "loss": 1.2799, + "theoretical_loss": 3.539775599883325, + "tokens_seen": 1523318784 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004351614844533601, + "loss": 1.2928, + "theoretical_loss": 3.5397623204822946, + "tokens_seen": 1523384320 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043514543630892677, + "loss": 1.2932, + "theoretical_loss": 3.5397490418124833, + "tokens_seen": 1523449856 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004351293881644935, + "loss": 1.3489, + "theoretical_loss": 3.539735763873818, + "tokens_seen": 1523515392 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043511334002006017, + "loss": 1.2201, + "theoretical_loss": 3.5397224866662276, + "tokens_seen": 1523580928 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004350972918756269, + "loss": 1.2537, + "theoretical_loss": 3.53970921018964, + "tokens_seen": 1523646464 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043508124373119357, + "loss": 1.408, + "theoretical_loss": 3.5396959344439836, + "tokens_seen": 1523712000 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004350651955867603, + "loss": 1.3345, + "theoretical_loss": 3.5396826594291873, + "tokens_seen": 1523777536 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043504914744232697, + "loss": 1.314, + "theoretical_loss": 3.539669385145179, + "tokens_seen": 1523843072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004350330992978937, + "loss": 1.4522, + "theoretical_loss": 3.5396561115918863, + "tokens_seen": 1523908608 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004350170511534604, + "loss": 1.2626, + "theoretical_loss": 3.5396428387692387, + "tokens_seen": 1523974144 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043500100300902713, + "loss": 1.2825, + "theoretical_loss": 3.5396295666771644, + "tokens_seen": 1524039680 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004349849548645938, + "loss": 1.3315, + "theoretical_loss": 3.539616295315591, + "tokens_seen": 1524105216 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043496890672016053, + "loss": 1.3626, + "theoretical_loss": 3.539603024684448, + "tokens_seen": 1524170752 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004349528585757272, + "loss": 1.2354, + "theoretical_loss": 3.5395897547836626, + "tokens_seen": 1524236288 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043493681043129393, + "loss": 1.2883, + "theoretical_loss": 3.539576485613164, + "tokens_seen": 1524301824 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004349207622868606, + "loss": 1.3433, + "theoretical_loss": 3.539563217172881, + "tokens_seen": 1524367360 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043490471414242733, + "loss": 1.3393, + "theoretical_loss": 3.5395499494627405, + "tokens_seen": 1524432896 + }, + { + "epoch": 0.46, + "learning_rate": 0.000434888665997994, + "loss": 1.362, + "theoretical_loss": 3.539536682482672, + "tokens_seen": 1524498432 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004348726178535607, + "loss": 1.2891, + "theoretical_loss": 3.539523416232605, + "tokens_seen": 1524563968 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004348565697091274, + "loss": 1.4272, + "theoretical_loss": 3.5395101507124656, + "tokens_seen": 1524629504 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004348405215646941, + "loss": 1.3866, + "theoretical_loss": 3.5394968859221834, + "tokens_seen": 1524695040 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043482447342026083, + "loss": 1.3045, + "theoretical_loss": 3.539483621861687, + "tokens_seen": 1524760576 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004348084252758275, + "loss": 1.2687, + "theoretical_loss": 3.539470358530905, + "tokens_seen": 1524826112 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043479237713139424, + "loss": 1.2997, + "theoretical_loss": 3.539457095929766, + "tokens_seen": 1524891648 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004347763289869609, + "loss": 1.2833, + "theoretical_loss": 3.539443834058198, + "tokens_seen": 1524957184 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043476028084252764, + "loss": 1.2336, + "theoretical_loss": 3.539430572916129, + "tokens_seen": 1525022720 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004347442326980943, + "loss": 1.2955, + "theoretical_loss": 3.539417312503489, + "tokens_seen": 1525088256 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043472818455366104, + "loss": 1.2605, + "theoretical_loss": 3.5394040528202058, + "tokens_seen": 1525153792 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004347121364092277, + "loss": 1.2721, + "theoretical_loss": 3.5393907938662075, + "tokens_seen": 1525219328 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043469608826479444, + "loss": 1.3672, + "theoretical_loss": 3.5393775356414237, + "tokens_seen": 1525284864 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004346800401203611, + "loss": 1.2798, + "theoretical_loss": 3.5393642781457815, + "tokens_seen": 1525350400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043466399197592784, + "loss": 1.3032, + "theoretical_loss": 3.5393510213792108, + "tokens_seen": 1525415936 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004346479438314945, + "loss": 1.2757, + "theoretical_loss": 3.5393377653416396, + "tokens_seen": 1525481472 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043463189568706124, + "loss": 1.4102, + "theoretical_loss": 3.539324510032997, + "tokens_seen": 1525547008 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004346158475426279, + "loss": 1.2793, + "theoretical_loss": 3.5393112554532102, + "tokens_seen": 1525612544 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043459979939819464, + "loss": 1.4139, + "theoretical_loss": 3.5392980016022095, + "tokens_seen": 1525678080 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004345837512537613, + "loss": 1.4629, + "theoretical_loss": 3.539284748479923, + "tokens_seen": 1525743616 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043456770310932804, + "loss": 1.3134, + "theoretical_loss": 3.539271496086279, + "tokens_seen": 1525809152 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004345516549648947, + "loss": 1.3127, + "theoretical_loss": 3.539258244421206, + "tokens_seen": 1525874688 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043453560682046145, + "loss": 1.2238, + "theoretical_loss": 3.5392449934846333, + "tokens_seen": 1525940224 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004345195586760281, + "loss": 1.288, + "theoretical_loss": 3.5392317432764893, + "tokens_seen": 1526005760 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043450351053159485, + "loss": 1.338, + "theoretical_loss": 3.539218493796702, + "tokens_seen": 1526071296 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004344874623871615, + "loss": 1.3848, + "theoretical_loss": 3.539205245045202, + "tokens_seen": 1526136832 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043447141424272825, + "loss": 1.3114, + "theoretical_loss": 3.5391919970219154, + "tokens_seen": 1526202368 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004344553660982949, + "loss": 1.3619, + "theoretical_loss": 3.539178749726773, + "tokens_seen": 1526267904 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043443931795386165, + "loss": 1.3485, + "theoretical_loss": 3.5391655031597025, + "tokens_seen": 1526333440 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004344232698094283, + "loss": 1.3449, + "theoretical_loss": 3.539152257320633, + "tokens_seen": 1526398976 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043440722166499505, + "loss": 1.4136, + "theoretical_loss": 3.5391390122094926, + "tokens_seen": 1526464512 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004343911735205617, + "loss": 1.3379, + "theoretical_loss": 3.539125767826211, + "tokens_seen": 1526530048 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043437512537612845, + "loss": 1.4611, + "theoretical_loss": 3.5391125241707164, + "tokens_seen": 1526595584 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004343590772316951, + "loss": 1.356, + "theoretical_loss": 3.5390992812429376, + "tokens_seen": 1526661120 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043434302908726185, + "loss": 1.4392, + "theoretical_loss": 3.539086039042804, + "tokens_seen": 1526726656 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004343269809428285, + "loss": 1.3229, + "theoretical_loss": 3.5390727975702427, + "tokens_seen": 1526792192 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043431093279839526, + "loss": 1.368, + "theoretical_loss": 3.539059556825184, + "tokens_seen": 1526857728 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004342948846539619, + "loss": 1.2884, + "theoretical_loss": 3.539046316807557, + "tokens_seen": 1526923264 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 567239, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.37321937084198, + "objective/train/theoretical_loss": 3.5390330775172893, + "objective/train/tokens_used": 1547448800, + "theoretical_loss": 3.5390330775172893, + "tokens_seen": 1526988800 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043427883650952866, + "loss": 1.3732, + "theoretical_loss": 3.5390330775172893, + "tokens_seen": 1526988800 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004342627883650953, + "loss": 1.3813, + "theoretical_loss": 3.53901983895431, + "tokens_seen": 1527054336 + }, + { + "epoch": 0.46, + "learning_rate": 0.000434246740220662, + "loss": 1.2038, + "theoretical_loss": 3.5390066011185493, + "tokens_seen": 1527119872 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004342306920762287, + "loss": 1.346, + "theoretical_loss": 3.538993364009934, + "tokens_seen": 1527185408 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004342146439317954, + "loss": 1.2844, + "theoretical_loss": 3.538980127628394, + "tokens_seen": 1527250944 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004341985957873621, + "loss": 1.355, + "theoretical_loss": 3.5389668919738586, + "tokens_seen": 1527316480 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004341825476429288, + "loss": 1.291, + "theoretical_loss": 3.5389536570462554, + "tokens_seen": 1527382016 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004341664994984955, + "loss": 1.278, + "theoretical_loss": 3.5389404228455144, + "tokens_seen": 1527447552 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004341504513540622, + "loss": 1.3061, + "theoretical_loss": 3.538927189371564, + "tokens_seen": 1527513088 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004341344032096289, + "loss": 1.3821, + "theoretical_loss": 3.5389139566243335, + "tokens_seen": 1527578624 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004341183550651956, + "loss": 1.3665, + "theoretical_loss": 3.538900724603752, + "tokens_seen": 1527644160 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043410230692076225, + "loss": 1.2788, + "theoretical_loss": 3.538887493309747, + "tokens_seen": 1527709696 + }, + { + "epoch": 0.46, + "learning_rate": 0.000434086258776329, + "loss": 1.2567, + "theoretical_loss": 3.538874262742249, + "tokens_seen": 1527775232 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043407021063189566, + "loss": 1.3677, + "theoretical_loss": 3.5388610329011865, + "tokens_seen": 1527840768 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004340541624874624, + "loss": 1.349, + "theoretical_loss": 3.5388478037864877, + "tokens_seen": 1527906304 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043403811434302906, + "loss": 1.4149, + "theoretical_loss": 3.5388345753980825, + "tokens_seen": 1527971840 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004340220661985958, + "loss": 1.2075, + "theoretical_loss": 3.5388213477359, + "tokens_seen": 1528037376 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043400601805416246, + "loss": 1.4031, + "theoretical_loss": 3.5388081207998683, + "tokens_seen": 1528102912 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004339899699097292, + "loss": 1.3715, + "theoretical_loss": 3.538794894589917, + "tokens_seen": 1528168448 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043397392176529586, + "loss": 1.357, + "theoretical_loss": 3.5387816691059752, + "tokens_seen": 1528233984 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004339578736208626, + "loss": 1.2952, + "theoretical_loss": 3.5387684443479714, + "tokens_seen": 1528299520 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043394182547642926, + "loss": 1.2399, + "theoretical_loss": 3.538755220315835, + "tokens_seen": 1528365056 + }, + { + "epoch": 0.46, + "learning_rate": 0.000433925777331996, + "loss": 1.2761, + "theoretical_loss": 3.538741997009495, + "tokens_seen": 1528430592 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043390972918756266, + "loss": 1.293, + "theoretical_loss": 3.53872877442888, + "tokens_seen": 1528496128 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004338936810431294, + "loss": 1.3912, + "theoretical_loss": 3.53871555257392, + "tokens_seen": 1528561664 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043387763289869606, + "loss": 1.252, + "theoretical_loss": 3.5387023314445436, + "tokens_seen": 1528627200 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004338615847542628, + "loss": 1.2787, + "theoretical_loss": 3.538689111040679, + "tokens_seen": 1528692736 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043384553660982947, + "loss": 1.3346, + "theoretical_loss": 3.5386758913622565, + "tokens_seen": 1528758272 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004338294884653962, + "loss": 1.2668, + "theoretical_loss": 3.538662672409205, + "tokens_seen": 1528823808 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043381344032096287, + "loss": 1.3826, + "theoretical_loss": 3.5386494541814533, + "tokens_seen": 1528889344 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004337973921765296, + "loss": 1.3496, + "theoretical_loss": 3.5386362366789306, + "tokens_seen": 1528954880 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043378134403209627, + "loss": 1.3147, + "theoretical_loss": 3.538623019901566, + "tokens_seen": 1529020416 + }, + { + "epoch": 0.46, + "learning_rate": 0.000433765295887663, + "loss": 1.3699, + "theoretical_loss": 3.5386098038492886, + "tokens_seen": 1529085952 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043374924774322967, + "loss": 1.4893, + "theoretical_loss": 3.5385965885220276, + "tokens_seen": 1529151488 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004337331995987964, + "loss": 1.4124, + "theoretical_loss": 3.538583373919712, + "tokens_seen": 1529217024 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043371715145436307, + "loss": 1.3214, + "theoretical_loss": 3.5385701600422714, + "tokens_seen": 1529282560 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004337011033099298, + "loss": 1.3014, + "theoretical_loss": 3.538556946889635, + "tokens_seen": 1529348096 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043368505516549647, + "loss": 1.3505, + "theoretical_loss": 3.538543734461731, + "tokens_seen": 1529413632 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004336690070210632, + "loss": 1.2789, + "theoretical_loss": 3.5385305227584896, + "tokens_seen": 1529479168 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043365295887662987, + "loss": 1.4128, + "theoretical_loss": 3.53851731177984, + "tokens_seen": 1529544704 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043363691073219663, + "loss": 1.2745, + "theoretical_loss": 3.538504101525711, + "tokens_seen": 1529610240 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043362086258776333, + "loss": 1.3083, + "theoretical_loss": 3.5384908919960316, + "tokens_seen": 1529675776 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043360481444333003, + "loss": 1.2761, + "theoretical_loss": 3.538477683190732, + "tokens_seen": 1529741312 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043358876629889673, + "loss": 1.2022, + "theoretical_loss": 3.5384644751097403, + "tokens_seen": 1529806848 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043357271815446343, + "loss": 1.5015, + "theoretical_loss": 3.5384512677529862, + "tokens_seen": 1529872384 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043355667001003013, + "loss": 1.2489, + "theoretical_loss": 3.5384380611203987, + "tokens_seen": 1529937920 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043354062186559683, + "loss": 1.2591, + "theoretical_loss": 3.5384248552119084, + "tokens_seen": 1530003456 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043352457372116353, + "loss": 1.3435, + "theoretical_loss": 3.538411650027443, + "tokens_seen": 1530068992 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043350852557673023, + "loss": 1.3412, + "theoretical_loss": 3.5383984455669326, + "tokens_seen": 1530134528 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043349247743229693, + "loss": 1.2566, + "theoretical_loss": 3.538385241830306, + "tokens_seen": 1530200064 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004334764292878636, + "loss": 1.4831, + "theoretical_loss": 3.5383720388174935, + "tokens_seen": 1530265600 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043346038114343033, + "loss": 1.2658, + "theoretical_loss": 3.538358836528423, + "tokens_seen": 1530331136 + }, + { + "epoch": 0.46, + "learning_rate": 0.000433444332998997, + "loss": 1.4922, + "theoretical_loss": 3.538345634963025, + "tokens_seen": 1530396672 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043342828485456374, + "loss": 1.4012, + "theoretical_loss": 3.5383324341212283, + "tokens_seen": 1530462208 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004334122367101304, + "loss": 1.272, + "theoretical_loss": 3.5383192340029623, + "tokens_seen": 1530527744 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043339618856569714, + "loss": 1.3002, + "theoretical_loss": 3.5383060346081567, + "tokens_seen": 1530593280 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004333801404212638, + "loss": 1.3805, + "theoretical_loss": 3.5382928359367405, + "tokens_seen": 1530658816 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043336409227683054, + "loss": 1.276, + "theoretical_loss": 3.5382796379886434, + "tokens_seen": 1530724352 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004333480441323972, + "loss": 1.4593, + "theoretical_loss": 3.5382664407637945, + "tokens_seen": 1530789888 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043333199598796394, + "loss": 1.3107, + "theoretical_loss": 3.538253244262123, + "tokens_seen": 1530855424 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004333159478435306, + "loss": 1.4196, + "theoretical_loss": 3.538240048483559, + "tokens_seen": 1530920960 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043329989969909734, + "loss": 1.2528, + "theoretical_loss": 3.5382268534280317, + "tokens_seen": 1530986496 + }, + { + "epoch": 0.46, + "learning_rate": 0.000433283851554664, + "loss": 1.479, + "theoretical_loss": 3.53821365909547, + "tokens_seen": 1531052032 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043326780341023074, + "loss": 1.3259, + "theoretical_loss": 3.5382004654858035, + "tokens_seen": 1531117568 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004332517552657974, + "loss": 1.3661, + "theoretical_loss": 3.5381872725989627, + "tokens_seen": 1531183104 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043323570712136414, + "loss": 1.2796, + "theoretical_loss": 3.5381740804348754, + "tokens_seen": 1531248640 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004332196589769308, + "loss": 1.4289, + "theoretical_loss": 3.538160888993472, + "tokens_seen": 1531314176 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043320361083249754, + "loss": 1.3491, + "theoretical_loss": 3.538147698274683, + "tokens_seen": 1531379712 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004331875626880642, + "loss": 1.4402, + "theoretical_loss": 3.5381345082784357, + "tokens_seen": 1531445248 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043317151454363095, + "loss": 1.313, + "theoretical_loss": 3.538121319004661, + "tokens_seen": 1531510784 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004331554663991976, + "loss": 1.3925, + "theoretical_loss": 3.538108130453288, + "tokens_seen": 1531576320 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043313941825476435, + "loss": 1.359, + "theoretical_loss": 3.5380949426242463, + "tokens_seen": 1531641856 + }, + { + "epoch": 0.46, + "learning_rate": 0.000433123370110331, + "loss": 1.2767, + "theoretical_loss": 3.538081755517466, + "tokens_seen": 1531707392 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043310732196589775, + "loss": 1.3202, + "theoretical_loss": 3.5380685691328755, + "tokens_seen": 1531772928 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004330912738214644, + "loss": 1.2242, + "theoretical_loss": 3.5380553834704056, + "tokens_seen": 1531838464 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043307522567703115, + "loss": 1.2359, + "theoretical_loss": 3.5380421985299844, + "tokens_seen": 1531904000 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004330591775325978, + "loss": 1.3996, + "theoretical_loss": 3.538029014311543, + "tokens_seen": 1531969536 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043304312938816455, + "loss": 1.3842, + "theoretical_loss": 3.5380158308150103, + "tokens_seen": 1532035072 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004330270812437312, + "loss": 1.3338, + "theoretical_loss": 3.5380026480403153, + "tokens_seen": 1532100608 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043301103309929795, + "loss": 1.3933, + "theoretical_loss": 3.5379894659873887, + "tokens_seen": 1532166144 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004329949849548646, + "loss": 1.343, + "theoretical_loss": 3.5379762846561595, + "tokens_seen": 1532231680 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043297893681043135, + "loss": 1.3512, + "theoretical_loss": 3.5379631040465576, + "tokens_seen": 1532297216 + }, + { + "epoch": 0.46, + "learning_rate": 0.000432962888665998, + "loss": 1.2549, + "theoretical_loss": 3.5379499241585117, + "tokens_seen": 1532362752 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043294684052156475, + "loss": 1.4055, + "theoretical_loss": 3.5379367449919528, + "tokens_seen": 1532428288 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004329307923771314, + "loss": 1.3669, + "theoretical_loss": 3.53792356654681, + "tokens_seen": 1532493824 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043291474423269816, + "loss": 1.3434, + "theoretical_loss": 3.537910388823013, + "tokens_seen": 1532559360 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004328986960882648, + "loss": 1.3619, + "theoretical_loss": 3.537897211820491, + "tokens_seen": 1532624896 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043288264794383156, + "loss": 1.3885, + "theoretical_loss": 3.5378840355391743, + "tokens_seen": 1532690432 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004328665997993982, + "loss": 1.3555, + "theoretical_loss": 3.537870859978992, + "tokens_seen": 1532755968 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004328505516549649, + "loss": 1.4773, + "theoretical_loss": 3.5378576851398744, + "tokens_seen": 1532821504 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004328345035105316, + "loss": 1.2226, + "theoretical_loss": 3.537844511021751, + "tokens_seen": 1532887040 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004328184553660983, + "loss": 1.3339, + "theoretical_loss": 3.5378313376245516, + "tokens_seen": 1532952576 + }, + { + "epoch": 0.46, + "learning_rate": 0.000432802407221665, + "loss": 1.315, + "theoretical_loss": 3.537818164948206, + "tokens_seen": 1533018112 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004327863590772317, + "loss": 1.3048, + "theoretical_loss": 3.537804992992643, + "tokens_seen": 1533083648 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004327703109327984, + "loss": 1.2924, + "theoretical_loss": 3.5377918217577937, + "tokens_seen": 1533149184 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004327542627883651, + "loss": 1.3231, + "theoretical_loss": 3.5377786512435874, + "tokens_seen": 1533214720 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004327382146439318, + "loss": 1.3267, + "theoretical_loss": 3.537765481449954, + "tokens_seen": 1533280256 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004327221664994985, + "loss": 1.2622, + "theoretical_loss": 3.5377523123768224, + "tokens_seen": 1533345792 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043270611835506516, + "loss": 1.4712, + "theoretical_loss": 3.5377391440241226, + "tokens_seen": 1533411328 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004326900702106319, + "loss": 1.3107, + "theoretical_loss": 3.5377259763917857, + "tokens_seen": 1533476864 + }, + { + "epoch": 0.46, + "objective/train/docs_used": 569719, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3017504215240479, + "objective/train/theoretical_loss": 3.53771280947974, + "objective/train/tokens_used": 1554002400, + "theoretical_loss": 3.53771280947974, + "tokens_seen": 1533542400 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043267402206619856, + "loss": 1.3018, + "theoretical_loss": 3.53771280947974, + "tokens_seen": 1533542400 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004326579739217653, + "loss": 1.321, + "theoretical_loss": 3.5376996432879166, + "tokens_seen": 1533607936 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043264192577733196, + "loss": 1.4112, + "theoretical_loss": 3.5376864778162442, + "tokens_seen": 1533673472 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004326258776328987, + "loss": 1.3399, + "theoretical_loss": 3.537673313064653, + "tokens_seen": 1533739008 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043260982948846536, + "loss": 1.3319, + "theoretical_loss": 3.5376601490330737, + "tokens_seen": 1533804544 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004325937813440321, + "loss": 1.3889, + "theoretical_loss": 3.5376469857214348, + "tokens_seen": 1533870080 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043257773319959876, + "loss": 1.3466, + "theoretical_loss": 3.537633823129667, + "tokens_seen": 1533935616 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004325616850551655, + "loss": 1.2611, + "theoretical_loss": 3.5376206612576997, + "tokens_seen": 1534001152 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043254563691073216, + "loss": 1.374, + "theoretical_loss": 3.537607500105463, + "tokens_seen": 1534066688 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004325295887662989, + "loss": 1.3939, + "theoretical_loss": 3.5375943396728875, + "tokens_seen": 1534132224 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043251354062186556, + "loss": 1.3691, + "theoretical_loss": 3.5375811799599015, + "tokens_seen": 1534197760 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004324974924774323, + "loss": 1.3501, + "theoretical_loss": 3.537568020966437, + "tokens_seen": 1534263296 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043248144433299896, + "loss": 1.3612, + "theoretical_loss": 3.537554862692422, + "tokens_seen": 1534328832 + }, + { + "epoch": 0.46, + "learning_rate": 0.0004324653961885657, + "loss": 1.4062, + "theoretical_loss": 3.5375417051377873, + "tokens_seen": 1534394368 + }, + { + "epoch": 0.46, + "learning_rate": 0.00043244934804413237, + "loss": 1.266, + "theoretical_loss": 3.537528548302463, + "tokens_seen": 1534459904 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004324332998996991, + "loss": 1.3307, + "theoretical_loss": 3.5375153921863793, + "tokens_seen": 1534525440 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004324172517552659, + "loss": 1.5011, + "theoretical_loss": 3.5375022367894653, + "tokens_seen": 1534590976 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004324012036108325, + "loss": 1.2058, + "theoretical_loss": 3.537489082111651, + "tokens_seen": 1534656512 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004323851554663993, + "loss": 1.3519, + "theoretical_loss": 3.537475928152867, + "tokens_seen": 1534722048 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004323691073219659, + "loss": 1.3372, + "theoretical_loss": 3.5374627749130436, + "tokens_seen": 1534787584 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004323530591775327, + "loss": 1.2622, + "theoretical_loss": 3.53744962239211, + "tokens_seen": 1534853120 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004323370110330993, + "loss": 1.3093, + "theoretical_loss": 3.537436470589996, + "tokens_seen": 1534918656 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004323209628886661, + "loss": 1.4196, + "theoretical_loss": 3.5374233195066327, + "tokens_seen": 1534984192 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004323049147442327, + "loss": 1.2648, + "theoretical_loss": 3.5374101691419497, + "tokens_seen": 1535049728 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004322888665997995, + "loss": 1.3639, + "theoretical_loss": 3.5373970194958764, + "tokens_seen": 1535115264 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043227281845536613, + "loss": 1.2198, + "theoretical_loss": 3.5373838705683434, + "tokens_seen": 1535180800 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004322567703109329, + "loss": 1.4222, + "theoretical_loss": 3.537370722359281, + "tokens_seen": 1535246336 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043224072216649953, + "loss": 1.2701, + "theoretical_loss": 3.5373575748686186, + "tokens_seen": 1535311872 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043222467402206623, + "loss": 1.2658, + "theoretical_loss": 3.5373444280962874, + "tokens_seen": 1535377408 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043220862587763293, + "loss": 1.3173, + "theoretical_loss": 3.5373312820422163, + "tokens_seen": 1535442944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043219257773319963, + "loss": 1.3347, + "theoretical_loss": 3.537318136706336, + "tokens_seen": 1535508480 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043217652958876633, + "loss": 1.4506, + "theoretical_loss": 3.5373049920885764, + "tokens_seen": 1535574016 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043216048144433303, + "loss": 1.2993, + "theoretical_loss": 3.5372918481888673, + "tokens_seen": 1535639552 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043214443329989973, + "loss": 1.2896, + "theoretical_loss": 3.53727870500714, + "tokens_seen": 1535705088 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043212838515546643, + "loss": 1.2683, + "theoretical_loss": 3.5372655625433236, + "tokens_seen": 1535770624 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043211233701103313, + "loss": 1.3009, + "theoretical_loss": 3.5372524207973486, + "tokens_seen": 1535836160 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043209628886659983, + "loss": 1.2939, + "theoretical_loss": 3.537239279769145, + "tokens_seen": 1535901696 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004320802407221665, + "loss": 1.3564, + "theoretical_loss": 3.5372261394586433, + "tokens_seen": 1535967232 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043206419257773324, + "loss": 1.3489, + "theoretical_loss": 3.5372129998657735, + "tokens_seen": 1536032768 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004320481444332999, + "loss": 1.232, + "theoretical_loss": 3.537199860990466, + "tokens_seen": 1536098304 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043203209628886664, + "loss": 1.4069, + "theoretical_loss": 3.53718672283265, + "tokens_seen": 1536163840 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004320160481444333, + "loss": 1.3783, + "theoretical_loss": 3.5371735853922566, + "tokens_seen": 1536229376 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043200000000000004, + "loss": 1.5331, + "theoretical_loss": 3.5371604486692165, + "tokens_seen": 1536294912 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004319839518555667, + "loss": 1.4607, + "theoretical_loss": 3.5371473126634587, + "tokens_seen": 1536360448 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043196790371113344, + "loss": 1.3166, + "theoretical_loss": 3.5371341773749143, + "tokens_seen": 1536425984 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004319518555667001, + "loss": 1.3292, + "theoretical_loss": 3.5371210428035136, + "tokens_seen": 1536491520 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043193580742226684, + "loss": 1.3369, + "theoretical_loss": 3.537107908949186, + "tokens_seen": 1536557056 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004319197592778335, + "loss": 1.2081, + "theoretical_loss": 3.537094775811863, + "tokens_seen": 1536622592 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043190371113340024, + "loss": 1.3214, + "theoretical_loss": 3.537081643391474, + "tokens_seen": 1536688128 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004318876629889669, + "loss": 1.2565, + "theoretical_loss": 3.5370685116879494, + "tokens_seen": 1536753664 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043187161484453364, + "loss": 1.4042, + "theoretical_loss": 3.5370553807012195, + "tokens_seen": 1536819200 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004318555667001003, + "loss": 1.2638, + "theoretical_loss": 3.537042250431215, + "tokens_seen": 1536884736 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043183951855566704, + "loss": 1.3119, + "theoretical_loss": 3.5370291208778655, + "tokens_seen": 1536950272 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004318234704112337, + "loss": 1.3758, + "theoretical_loss": 3.537015992041103, + "tokens_seen": 1537015808 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043180742226680045, + "loss": 1.3545, + "theoretical_loss": 3.537002863920855, + "tokens_seen": 1537081344 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004317913741223671, + "loss": 1.3837, + "theoretical_loss": 3.5369897365170546, + "tokens_seen": 1537146880 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043177532597793385, + "loss": 1.3302, + "theoretical_loss": 3.5369766098296305, + "tokens_seen": 1537212416 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004317592778335005, + "loss": 1.428, + "theoretical_loss": 3.536963483858514, + "tokens_seen": 1537277952 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043174322968906725, + "loss": 1.3593, + "theoretical_loss": 3.536950358603635, + "tokens_seen": 1537343488 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004317271815446339, + "loss": 1.2849, + "theoretical_loss": 3.5369372340649234, + "tokens_seen": 1537409024 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043171113340020065, + "loss": 1.3709, + "theoretical_loss": 3.5369241102423103, + "tokens_seen": 1537474560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004316950852557673, + "loss": 1.2455, + "theoretical_loss": 3.5369109871357267, + "tokens_seen": 1537540096 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043167903711133405, + "loss": 1.3471, + "theoretical_loss": 3.536897864745101, + "tokens_seen": 1537605632 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004316629889669007, + "loss": 1.3576, + "theoretical_loss": 3.536884743070366, + "tokens_seen": 1537671168 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043164694082246745, + "loss": 1.2881, + "theoretical_loss": 3.536871622111451, + "tokens_seen": 1537736704 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004316308926780341, + "loss": 1.2567, + "theoretical_loss": 3.536858501868286, + "tokens_seen": 1537802240 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043161484453360085, + "loss": 1.303, + "theoretical_loss": 3.536845382340802, + "tokens_seen": 1537867776 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004315987963891675, + "loss": 1.2741, + "theoretical_loss": 3.5368322635289298, + "tokens_seen": 1537933312 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043158274824473425, + "loss": 1.2959, + "theoretical_loss": 3.536819145432599, + "tokens_seen": 1537998848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004315667001003009, + "loss": 1.2979, + "theoretical_loss": 3.536806028051741, + "tokens_seen": 1538064384 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043155065195586766, + "loss": 1.3902, + "theoretical_loss": 3.5367929113862853, + "tokens_seen": 1538129920 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004315346038114343, + "loss": 1.2574, + "theoretical_loss": 3.5367797954361633, + "tokens_seen": 1538195456 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043151855566700106, + "loss": 1.3173, + "theoretical_loss": 3.536766680201305, + "tokens_seen": 1538260992 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004315025075225677, + "loss": 1.3843, + "theoretical_loss": 3.536753565681641, + "tokens_seen": 1538326528 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043148645937813446, + "loss": 1.3083, + "theoretical_loss": 3.5367404518771024, + "tokens_seen": 1538392064 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004314704112337011, + "loss": 1.3041, + "theoretical_loss": 3.5367273387876192, + "tokens_seen": 1538457600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004314543630892678, + "loss": 1.3427, + "theoretical_loss": 3.536714226413122, + "tokens_seen": 1538523136 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004314383149448345, + "loss": 1.3547, + "theoretical_loss": 3.5367011147535408, + "tokens_seen": 1538588672 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004314222668004012, + "loss": 1.3757, + "theoretical_loss": 3.5366880038088073, + "tokens_seen": 1538654208 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004314062186559679, + "loss": 1.3969, + "theoretical_loss": 3.5366748935788515, + "tokens_seen": 1538719744 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004313901705115346, + "loss": 1.3928, + "theoretical_loss": 3.5366617840636043, + "tokens_seen": 1538785280 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004313741223671013, + "loss": 1.4042, + "theoretical_loss": 3.5366486752629953, + "tokens_seen": 1538850816 + }, + { + "epoch": 0.47, + "learning_rate": 0.000431358074222668, + "loss": 1.3318, + "theoretical_loss": 3.5366355671769565, + "tokens_seen": 1538916352 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004313420260782347, + "loss": 1.3243, + "theoretical_loss": 3.5366224598054177, + "tokens_seen": 1538981888 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004313259779338014, + "loss": 1.3353, + "theoretical_loss": 3.5366093531483096, + "tokens_seen": 1539047424 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043130992978936806, + "loss": 1.3204, + "theoretical_loss": 3.536596247205563, + "tokens_seen": 1539112960 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004312938816449348, + "loss": 1.379, + "theoretical_loss": 3.5365831419771085, + "tokens_seen": 1539178496 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043127783350050146, + "loss": 1.3958, + "theoretical_loss": 3.5365700374628766, + "tokens_seen": 1539244032 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004312617853560682, + "loss": 1.3188, + "theoretical_loss": 3.5365569336627987, + "tokens_seen": 1539309568 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043124573721163486, + "loss": 1.4255, + "theoretical_loss": 3.5365438305768038, + "tokens_seen": 1539375104 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004312296890672016, + "loss": 1.3501, + "theoretical_loss": 3.5365307282048244, + "tokens_seen": 1539440640 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043121364092276837, + "loss": 1.2556, + "theoretical_loss": 3.5365176265467904, + "tokens_seen": 1539506176 + }, + { + "epoch": 0.47, + "learning_rate": 0.000431197592778335, + "loss": 1.467, + "theoretical_loss": 3.5365045256026324, + "tokens_seen": 1539571712 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043118154463390177, + "loss": 1.3823, + "theoretical_loss": 3.5364914253722812, + "tokens_seen": 1539637248 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004311654964894684, + "loss": 1.4141, + "theoretical_loss": 3.5364783258556685, + "tokens_seen": 1539702784 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043114944834503517, + "loss": 1.4863, + "theoretical_loss": 3.536465227052723, + "tokens_seen": 1539768320 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004311334002006018, + "loss": 1.3249, + "theoretical_loss": 3.5364521289633775, + "tokens_seen": 1539833856 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004311173520561686, + "loss": 1.2305, + "theoretical_loss": 3.5364390315875616, + "tokens_seen": 1539899392 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004311013039117352, + "loss": 1.3682, + "theoretical_loss": 3.536425934925206, + "tokens_seen": 1539964928 + }, + { + "epoch": 0.47, + "learning_rate": 0.000431085255767302, + "loss": 1.2972, + "theoretical_loss": 3.5364128389762417, + "tokens_seen": 1540030464 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 571968, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2907238006591797, + "objective/train/theoretical_loss": 3.5363997437406, + "objective/train/tokens_used": 1560556000, + "theoretical_loss": 3.5363997437406, + "tokens_seen": 1540096000 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004310692076228686, + "loss": 1.2907, + "theoretical_loss": 3.5363997437406, + "tokens_seen": 1540096000 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004310531594784354, + "loss": 1.3386, + "theoretical_loss": 3.5363866492182114, + "tokens_seen": 1540161536 + }, + { + "epoch": 0.47, + "learning_rate": 0.000431037111334002, + "loss": 1.3539, + "theoretical_loss": 3.536373555409006, + "tokens_seen": 1540227072 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004310210631895688, + "loss": 1.4476, + "theoretical_loss": 3.536360462312916, + "tokens_seen": 1540292608 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004310050150451354, + "loss": 1.329, + "theoretical_loss": 3.536347369929871, + "tokens_seen": 1540358144 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004309889669007022, + "loss": 1.2859, + "theoretical_loss": 3.536334278259802, + "tokens_seen": 1540423680 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004309729187562688, + "loss": 1.3982, + "theoretical_loss": 3.536321187302641, + "tokens_seen": 1540489216 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004309568706118356, + "loss": 1.3368, + "theoretical_loss": 3.536308097058317, + "tokens_seen": 1540554752 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004309408224674022, + "loss": 1.4073, + "theoretical_loss": 3.5362950075267623, + "tokens_seen": 1540620288 + }, + { + "epoch": 0.47, + "learning_rate": 0.000430924774322969, + "loss": 1.3891, + "theoretical_loss": 3.536281918707907, + "tokens_seen": 1540685824 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004309087261785356, + "loss": 1.3591, + "theoretical_loss": 3.5362688306016823, + "tokens_seen": 1540751360 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004308926780341024, + "loss": 1.292, + "theoretical_loss": 3.5362557432080193, + "tokens_seen": 1540816896 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043087662988966903, + "loss": 1.3007, + "theoretical_loss": 3.5362426565268485, + "tokens_seen": 1540882432 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004308605817452358, + "loss": 1.4265, + "theoretical_loss": 3.536229570558101, + "tokens_seen": 1540947968 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043084453360080243, + "loss": 1.34, + "theoretical_loss": 3.5362164853017077, + "tokens_seen": 1541013504 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004308284854563692, + "loss": 1.3878, + "theoretical_loss": 3.5362034007575995, + "tokens_seen": 1541079040 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043081243731193583, + "loss": 1.3891, + "theoretical_loss": 3.5361903169257074, + "tokens_seen": 1541144576 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043079638916750253, + "loss": 1.3267, + "theoretical_loss": 3.5361772338059625, + "tokens_seen": 1541210112 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043078034102306923, + "loss": 1.31, + "theoretical_loss": 3.536164151398295, + "tokens_seen": 1541275648 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043076429287863593, + "loss": 1.3318, + "theoretical_loss": 3.536151069702637, + "tokens_seen": 1541341184 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043074824473420263, + "loss": 1.3482, + "theoretical_loss": 3.5361379887189184, + "tokens_seen": 1541406720 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043073219658976933, + "loss": 1.2994, + "theoretical_loss": 3.5361249084470714, + "tokens_seen": 1541472256 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043071614844533603, + "loss": 1.3688, + "theoretical_loss": 3.5361118288870257, + "tokens_seen": 1541537792 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043070010030090274, + "loss": 1.3657, + "theoretical_loss": 3.536098750038713, + "tokens_seen": 1541603328 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004306840521564694, + "loss": 1.308, + "theoretical_loss": 3.5360856719020646, + "tokens_seen": 1541668864 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043066800401203614, + "loss": 1.2918, + "theoretical_loss": 3.5360725944770106, + "tokens_seen": 1541734400 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004306519558676028, + "loss": 1.3281, + "theoretical_loss": 3.5360595177634826, + "tokens_seen": 1541799936 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043063590772316954, + "loss": 1.2541, + "theoretical_loss": 3.536046441761412, + "tokens_seen": 1541865472 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004306198595787362, + "loss": 1.3532, + "theoretical_loss": 3.5360333664707295, + "tokens_seen": 1541931008 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043060381143430294, + "loss": 1.4041, + "theoretical_loss": 3.536020291891366, + "tokens_seen": 1541996544 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004305877632898696, + "loss": 1.3291, + "theoretical_loss": 3.5360072180232525, + "tokens_seen": 1542062080 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043057171514543634, + "loss": 1.3008, + "theoretical_loss": 3.5359941448663204, + "tokens_seen": 1542127616 + }, + { + "epoch": 0.47, + "learning_rate": 0.000430555667001003, + "loss": 1.3209, + "theoretical_loss": 3.535981072420501, + "tokens_seen": 1542193152 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043053961885656974, + "loss": 1.2617, + "theoretical_loss": 3.535968000685725, + "tokens_seen": 1542258688 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004305235707121364, + "loss": 1.3045, + "theoretical_loss": 3.5359549296619237, + "tokens_seen": 1542324224 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043050752256770314, + "loss": 1.3012, + "theoretical_loss": 3.5359418593490277, + "tokens_seen": 1542389760 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004304914744232698, + "loss": 1.4003, + "theoretical_loss": 3.535928789746969, + "tokens_seen": 1542455296 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043047542627883654, + "loss": 1.3405, + "theoretical_loss": 3.5359157208556775, + "tokens_seen": 1542520832 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004304593781344032, + "loss": 1.3431, + "theoretical_loss": 3.5359026526750865, + "tokens_seen": 1542586368 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043044332998996995, + "loss": 1.2575, + "theoretical_loss": 3.5358895852051244, + "tokens_seen": 1542651904 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004304272818455366, + "loss": 1.3744, + "theoretical_loss": 3.5358765184457246, + "tokens_seen": 1542717440 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043041123370110335, + "loss": 1.312, + "theoretical_loss": 3.5358634523968178, + "tokens_seen": 1542782976 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043039518555667, + "loss": 1.3421, + "theoretical_loss": 3.535850387058334, + "tokens_seen": 1542848512 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043037913741223675, + "loss": 1.3874, + "theoretical_loss": 3.535837322430206, + "tokens_seen": 1542914048 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004303630892678034, + "loss": 1.3314, + "theoretical_loss": 3.5358242585123634, + "tokens_seen": 1542979584 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043034704112337015, + "loss": 1.372, + "theoretical_loss": 3.5358111953047393, + "tokens_seen": 1543045120 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004303309929789368, + "loss": 1.3774, + "theoretical_loss": 3.535798132807263, + "tokens_seen": 1543110656 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043031494483450355, + "loss": 1.4146, + "theoretical_loss": 3.535785071019867, + "tokens_seen": 1543176192 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004302988966900702, + "loss": 1.3132, + "theoretical_loss": 3.535772009942482, + "tokens_seen": 1543241728 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043028284854563695, + "loss": 1.3692, + "theoretical_loss": 3.53575894957504, + "tokens_seen": 1543307264 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004302668004012036, + "loss": 1.3294, + "theoretical_loss": 3.5357458899174716, + "tokens_seen": 1543372800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043025075225677035, + "loss": 1.3881, + "theoretical_loss": 3.5357328309697076, + "tokens_seen": 1543438336 + }, + { + "epoch": 0.47, + "learning_rate": 0.000430234704112337, + "loss": 1.3947, + "theoretical_loss": 3.5357197727316807, + "tokens_seen": 1543503872 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043021865596790375, + "loss": 1.3284, + "theoretical_loss": 3.535706715203321, + "tokens_seen": 1543569408 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004302026078234704, + "loss": 1.3502, + "theoretical_loss": 3.5356936583845595, + "tokens_seen": 1543634944 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043018655967903716, + "loss": 1.2334, + "theoretical_loss": 3.5356806022753284, + "tokens_seen": 1543700480 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004301705115346038, + "loss": 1.3401, + "theoretical_loss": 3.5356675468755596, + "tokens_seen": 1543766016 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043015446339017056, + "loss": 1.4042, + "theoretical_loss": 3.5356544921851834, + "tokens_seen": 1543831552 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004301384152457372, + "loss": 1.3063, + "theoretical_loss": 3.535641438204131, + "tokens_seen": 1543897088 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043012236710130396, + "loss": 1.3473, + "theoretical_loss": 3.535628384932334, + "tokens_seen": 1543962624 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004301063189568706, + "loss": 1.2896, + "theoretical_loss": 3.5356153323697246, + "tokens_seen": 1544028160 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043009027081243736, + "loss": 1.4297, + "theoretical_loss": 3.5356022805162333, + "tokens_seen": 1544093696 + }, + { + "epoch": 0.47, + "learning_rate": 0.000430074222668004, + "loss": 1.3268, + "theoretical_loss": 3.5355892293717908, + "tokens_seen": 1544159232 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043005817452357076, + "loss": 1.2478, + "theoretical_loss": 3.5355761789363305, + "tokens_seen": 1544224768 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004300421263791374, + "loss": 1.3242, + "theoretical_loss": 3.5355631292097813, + "tokens_seen": 1544290304 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004300260782347041, + "loss": 1.2298, + "theoretical_loss": 3.535550080192077, + "tokens_seen": 1544355840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00043001003009027086, + "loss": 1.4182, + "theoretical_loss": 3.535537031883148, + "tokens_seen": 1544421376 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004299939819458375, + "loss": 1.401, + "theoretical_loss": 3.535523984282925, + "tokens_seen": 1544486912 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042997793380140426, + "loss": 1.3713, + "theoretical_loss": 3.535510937391341, + "tokens_seen": 1544552448 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004299618856569709, + "loss": 1.3841, + "theoretical_loss": 3.5354978912083257, + "tokens_seen": 1544617984 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042994583751253767, + "loss": 1.2659, + "theoretical_loss": 3.5354848457338117, + "tokens_seen": 1544683520 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004299297893681043, + "loss": 1.4154, + "theoretical_loss": 3.5354718009677306, + "tokens_seen": 1544749056 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042991374122367107, + "loss": 1.3058, + "theoretical_loss": 3.535458756910013, + "tokens_seen": 1544814592 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004298976930792377, + "loss": 1.2624, + "theoretical_loss": 3.5354457135605912, + "tokens_seen": 1544880128 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042988164493480447, + "loss": 1.3587, + "theoretical_loss": 3.5354326709193957, + "tokens_seen": 1544945664 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004298655967903711, + "loss": 1.3808, + "theoretical_loss": 3.5354196289863595, + "tokens_seen": 1545011200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042984954864593787, + "loss": 1.359, + "theoretical_loss": 3.535406587761413, + "tokens_seen": 1545076736 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004298335005015045, + "loss": 1.2978, + "theoretical_loss": 3.535393547244488, + "tokens_seen": 1545142272 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042981745235707127, + "loss": 1.3753, + "theoretical_loss": 3.535380507435516, + "tokens_seen": 1545207808 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004298014042126379, + "loss": 1.246, + "theoretical_loss": 3.535367468334428, + "tokens_seen": 1545273344 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042978535606820467, + "loss": 1.3725, + "theoretical_loss": 3.5353544299411572, + "tokens_seen": 1545338880 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004297693079237713, + "loss": 1.358, + "theoretical_loss": 3.535341392255633, + "tokens_seen": 1545404416 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042975325977933807, + "loss": 1.2821, + "theoretical_loss": 3.5353283552777888, + "tokens_seen": 1545469952 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004297372116349047, + "loss": 1.3576, + "theoretical_loss": 3.5353153190075552, + "tokens_seen": 1545535488 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004297211634904715, + "loss": 1.3873, + "theoretical_loss": 3.5353022834448637, + "tokens_seen": 1545601024 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004297051153460381, + "loss": 1.2965, + "theoretical_loss": 3.535289248589647, + "tokens_seen": 1545666560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004296890672016049, + "loss": 1.3659, + "theoretical_loss": 3.5352762144418355, + "tokens_seen": 1545732096 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004296730190571715, + "loss": 1.3976, + "theoretical_loss": 3.535263181001361, + "tokens_seen": 1545797632 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004296569709127383, + "loss": 1.3286, + "theoretical_loss": 3.5352501482681555, + "tokens_seen": 1545863168 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004296409227683049, + "loss": 1.3178, + "theoretical_loss": 3.535237116242151, + "tokens_seen": 1545928704 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004296248746238717, + "loss": 1.3522, + "theoretical_loss": 3.5352240849232786, + "tokens_seen": 1545994240 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004296088264794383, + "loss": 1.3544, + "theoretical_loss": 3.5352110543114694, + "tokens_seen": 1546059776 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004295927783350051, + "loss": 1.3872, + "theoretical_loss": 3.535198024406656, + "tokens_seen": 1546125312 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004295767301905717, + "loss": 1.2333, + "theoretical_loss": 3.5351849952087697, + "tokens_seen": 1546190848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004295606820461385, + "loss": 1.29, + "theoretical_loss": 3.5351719667177424, + "tokens_seen": 1546256384 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004295446339017051, + "loss": 1.2095, + "theoretical_loss": 3.535158938933506, + "tokens_seen": 1546321920 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004295285857572719, + "loss": 1.2501, + "theoretical_loss": 3.5351459118559916, + "tokens_seen": 1546387456 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042951253761283853, + "loss": 1.3067, + "theoretical_loss": 3.5351328854851305, + "tokens_seen": 1546452992 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004294964894684053, + "loss": 1.3847, + "theoretical_loss": 3.535119859820856, + "tokens_seen": 1546518528 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042948044132397193, + "loss": 1.2648, + "theoretical_loss": 3.5351068348630985, + "tokens_seen": 1546584064 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 574309, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2947477102279663, + "objective/train/theoretical_loss": 3.53509381061179, + "objective/train/tokens_used": 1567109600, + "theoretical_loss": 3.53509381061179, + "tokens_seen": 1546649600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004294643931795387, + "loss": 1.2947, + "theoretical_loss": 3.53509381061179, + "tokens_seen": 1546649600 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042944834503510533, + "loss": 1.3313, + "theoretical_loss": 3.535080787066863, + "tokens_seen": 1546715136 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004294322968906721, + "loss": 1.2272, + "theoretical_loss": 3.5350677642282484, + "tokens_seen": 1546780672 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042941624874623873, + "loss": 1.464, + "theoretical_loss": 3.5350547420958778, + "tokens_seen": 1546846208 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042940020060180543, + "loss": 1.3859, + "theoretical_loss": 3.5350417206696836, + "tokens_seen": 1546911744 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042938415245737213, + "loss": 1.2952, + "theoretical_loss": 3.535028699949598, + "tokens_seen": 1546977280 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042936810431293883, + "loss": 1.3252, + "theoretical_loss": 3.535015679935552, + "tokens_seen": 1547042816 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042935205616850553, + "loss": 1.3501, + "theoretical_loss": 3.535002660627477, + "tokens_seen": 1547108352 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042933600802407223, + "loss": 1.289, + "theoretical_loss": 3.534989642025306, + "tokens_seen": 1547173888 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042931995987963894, + "loss": 1.3494, + "theoretical_loss": 3.53497662412897, + "tokens_seen": 1547239424 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042930391173520564, + "loss": 1.3312, + "theoretical_loss": 3.534963606938401, + "tokens_seen": 1547304960 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042928786359077234, + "loss": 1.3881, + "theoretical_loss": 3.5349505904535317, + "tokens_seen": 1547370496 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042927181544633904, + "loss": 1.3705, + "theoretical_loss": 3.5349375746742924, + "tokens_seen": 1547436032 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004292557673019057, + "loss": 1.259, + "theoretical_loss": 3.5349245596006162, + "tokens_seen": 1547501568 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042923971915747244, + "loss": 1.4479, + "theoretical_loss": 3.5349115452324344, + "tokens_seen": 1547567104 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004292236710130391, + "loss": 1.2838, + "theoretical_loss": 3.534898531569679, + "tokens_seen": 1547632640 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042920762286860584, + "loss": 1.3049, + "theoretical_loss": 3.534885518612282, + "tokens_seen": 1547698176 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004291915747241725, + "loss": 1.2918, + "theoretical_loss": 3.5348725063601756, + "tokens_seen": 1547763712 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042917552657973924, + "loss": 1.2207, + "theoretical_loss": 3.5348594948132908, + "tokens_seen": 1547829248 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004291594784353059, + "loss": 1.2766, + "theoretical_loss": 3.5348464839715605, + "tokens_seen": 1547894784 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042914343029087264, + "loss": 1.4161, + "theoretical_loss": 3.534833473834916, + "tokens_seen": 1547960320 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004291273821464393, + "loss": 1.3006, + "theoretical_loss": 3.534820464403289, + "tokens_seen": 1548025856 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042911133400200604, + "loss": 1.3121, + "theoretical_loss": 3.5348074556766127, + "tokens_seen": 1548091392 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004290952858575727, + "loss": 1.2318, + "theoretical_loss": 3.534794447654818, + "tokens_seen": 1548156928 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042907923771313945, + "loss": 1.2638, + "theoretical_loss": 3.534781440337836, + "tokens_seen": 1548222464 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004290631895687061, + "loss": 1.4016, + "theoretical_loss": 3.5347684337256013, + "tokens_seen": 1548288000 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042904714142427285, + "loss": 1.3319, + "theoretical_loss": 3.5347554278180437, + "tokens_seen": 1548353536 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004290310932798395, + "loss": 1.3386, + "theoretical_loss": 3.534742422615096, + "tokens_seen": 1548419072 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042901504513540625, + "loss": 1.2987, + "theoretical_loss": 3.53472941811669, + "tokens_seen": 1548484608 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004289989969909729, + "loss": 1.3537, + "theoretical_loss": 3.534716414322758, + "tokens_seen": 1548550144 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042898294884653965, + "loss": 1.302, + "theoretical_loss": 3.534703411233232, + "tokens_seen": 1548615680 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004289669007021063, + "loss": 1.3257, + "theoretical_loss": 3.5346904088480433, + "tokens_seen": 1548681216 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042895085255767305, + "loss": 1.3814, + "theoretical_loss": 3.534677407167125, + "tokens_seen": 1548746752 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004289348044132397, + "loss": 1.2775, + "theoretical_loss": 3.5346644061904082, + "tokens_seen": 1548812288 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042891875626880645, + "loss": 1.266, + "theoretical_loss": 3.534651405917826, + "tokens_seen": 1548877824 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004289027081243731, + "loss": 1.4325, + "theoretical_loss": 3.5346384063493095, + "tokens_seen": 1548943360 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042888665997993985, + "loss": 1.2949, + "theoretical_loss": 3.534625407484791, + "tokens_seen": 1549008896 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004288706118355065, + "loss": 1.3037, + "theoretical_loss": 3.534612409324203, + "tokens_seen": 1549074432 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042885456369107325, + "loss": 1.3005, + "theoretical_loss": 3.534599411867478, + "tokens_seen": 1549139968 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004288385155466399, + "loss": 1.3413, + "theoretical_loss": 3.5345864151145467, + "tokens_seen": 1549205504 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042882246740220666, + "loss": 1.3398, + "theoretical_loss": 3.5345734190653424, + "tokens_seen": 1549271040 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004288064192577734, + "loss": 1.2713, + "theoretical_loss": 3.534560423719797, + "tokens_seen": 1549336576 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042879037111334006, + "loss": 1.3343, + "theoretical_loss": 3.5345474290778416, + "tokens_seen": 1549402112 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042877432296890676, + "loss": 1.3018, + "theoretical_loss": 3.5345344351394097, + "tokens_seen": 1549467648 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042875827482447346, + "loss": 1.3226, + "theoretical_loss": 3.5345214419044333, + "tokens_seen": 1549533184 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042874222668004016, + "loss": 1.1917, + "theoretical_loss": 3.534508449372844, + "tokens_seen": 1549598720 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042872617853560686, + "loss": 1.3299, + "theoretical_loss": 3.5344954575445744, + "tokens_seen": 1549664256 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042871013039117356, + "loss": 1.2926, + "theoretical_loss": 3.5344824664195564, + "tokens_seen": 1549729792 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042869408224674026, + "loss": 1.3575, + "theoretical_loss": 3.534469475997722, + "tokens_seen": 1549795328 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042867803410230696, + "loss": 1.2653, + "theoretical_loss": 3.534456486279004, + "tokens_seen": 1549860864 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042866198595787366, + "loss": 1.2908, + "theoretical_loss": 3.534443497263335, + "tokens_seen": 1549926400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042864593781344036, + "loss": 1.3247, + "theoretical_loss": 3.5344305089506456, + "tokens_seen": 1549991936 + }, + { + "epoch": 0.47, + "learning_rate": 0.000428629889669007, + "loss": 1.3039, + "theoretical_loss": 3.5344175213408695, + "tokens_seen": 1550057472 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042861384152457376, + "loss": 1.3614, + "theoretical_loss": 3.534404534433938, + "tokens_seen": 1550123008 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004285977933801404, + "loss": 1.4359, + "theoretical_loss": 3.534391548229784, + "tokens_seen": 1550188544 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042858174523570716, + "loss": 1.4046, + "theoretical_loss": 3.5343785627283397, + "tokens_seen": 1550254080 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004285656970912738, + "loss": 1.3202, + "theoretical_loss": 3.5343655779295373, + "tokens_seen": 1550319616 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042854964894684057, + "loss": 1.2455, + "theoretical_loss": 3.5343525938333085, + "tokens_seen": 1550385152 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004285336008024072, + "loss": 1.3346, + "theoretical_loss": 3.534339610439586, + "tokens_seen": 1550450688 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042851755265797397, + "loss": 1.3858, + "theoretical_loss": 3.534326627748303, + "tokens_seen": 1550516224 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004285015045135406, + "loss": 1.3342, + "theoretical_loss": 3.5343136457593904, + "tokens_seen": 1550581760 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042848545636910737, + "loss": 1.412, + "theoretical_loss": 3.5343006644727817, + "tokens_seen": 1550647296 + }, + { + "epoch": 0.47, + "learning_rate": 0.000428469408224674, + "loss": 1.4561, + "theoretical_loss": 3.534287683888408, + "tokens_seen": 1550712832 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042845336008024077, + "loss": 1.2629, + "theoretical_loss": 3.5342747040062026, + "tokens_seen": 1550778368 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004284373119358074, + "loss": 1.3012, + "theoretical_loss": 3.534261724826097, + "tokens_seen": 1550843904 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042842126379137417, + "loss": 1.3772, + "theoretical_loss": 3.5342487463480246, + "tokens_seen": 1550909440 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004284052156469408, + "loss": 1.3112, + "theoretical_loss": 3.534235768571917, + "tokens_seen": 1550974976 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042838916750250757, + "loss": 1.228, + "theoretical_loss": 3.534222791497707, + "tokens_seen": 1551040512 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004283731193580742, + "loss": 1.263, + "theoretical_loss": 3.534209815125327, + "tokens_seen": 1551106048 + }, + { + "epoch": 0.47, + "learning_rate": 0.000428357071213641, + "loss": 1.2658, + "theoretical_loss": 3.5341968394547085, + "tokens_seen": 1551171584 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004283410230692076, + "loss": 1.3024, + "theoretical_loss": 3.534183864485785, + "tokens_seen": 1551237120 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004283249749247744, + "loss": 1.4252, + "theoretical_loss": 3.5341708902184887, + "tokens_seen": 1551302656 + }, + { + "epoch": 0.47, + "learning_rate": 0.000428308926780341, + "loss": 1.371, + "theoretical_loss": 3.5341579166527515, + "tokens_seen": 1551368192 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004282928786359078, + "loss": 1.3024, + "theoretical_loss": 3.5341449437885064, + "tokens_seen": 1551433728 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004282768304914744, + "loss": 1.3074, + "theoretical_loss": 3.534131971625685, + "tokens_seen": 1551499264 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004282607823470412, + "loss": 1.2648, + "theoretical_loss": 3.534119000164221, + "tokens_seen": 1551564800 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004282447342026078, + "loss": 1.3033, + "theoretical_loss": 3.5341060294040463, + "tokens_seen": 1551630336 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004282286860581746, + "loss": 1.245, + "theoretical_loss": 3.5340930593450928, + "tokens_seen": 1551695872 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004282126379137412, + "loss": 1.2366, + "theoretical_loss": 3.5340800899872935, + "tokens_seen": 1551761408 + }, + { + "epoch": 0.47, + "learning_rate": 0.000428196589769308, + "loss": 1.2983, + "theoretical_loss": 3.5340671213305814, + "tokens_seen": 1551826944 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004281805416248746, + "loss": 1.2811, + "theoretical_loss": 3.5340541533748877, + "tokens_seen": 1551892480 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004281644934804414, + "loss": 1.3105, + "theoretical_loss": 3.534041186120146, + "tokens_seen": 1551958016 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042814844533600803, + "loss": 1.4229, + "theoretical_loss": 3.534028219566289, + "tokens_seen": 1552023552 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004281323971915748, + "loss": 1.392, + "theoretical_loss": 3.5340152537132474, + "tokens_seen": 1552089088 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042811634904714143, + "loss": 1.2724, + "theoretical_loss": 3.5340022885609557, + "tokens_seen": 1552154624 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004281003009027082, + "loss": 1.2444, + "theoretical_loss": 3.5339893241093456, + "tokens_seen": 1552220160 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042808425275827483, + "loss": 1.2122, + "theoretical_loss": 3.5339763603583503, + "tokens_seen": 1552285696 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004280682046138416, + "loss": 1.3829, + "theoretical_loss": 3.533963397307901, + "tokens_seen": 1552351232 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042805215646940823, + "loss": 1.348, + "theoretical_loss": 3.533950434957932, + "tokens_seen": 1552416768 + }, + { + "epoch": 0.47, + "learning_rate": 0.000428036108324975, + "loss": 1.2895, + "theoretical_loss": 3.5339374733083746, + "tokens_seen": 1552482304 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042802006018054163, + "loss": 1.3438, + "theoretical_loss": 3.5339245123591616, + "tokens_seen": 1552547840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042800401203610833, + "loss": 1.2297, + "theoretical_loss": 3.533911552110226, + "tokens_seen": 1552613376 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042798796389167503, + "loss": 1.3885, + "theoretical_loss": 3.5338985925615, + "tokens_seen": 1552678912 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042797191574724173, + "loss": 1.3515, + "theoretical_loss": 3.533885633712917, + "tokens_seen": 1552744448 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042795586760280844, + "loss": 1.2562, + "theoretical_loss": 3.5338726755644085, + "tokens_seen": 1552809984 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042793981945837514, + "loss": 1.3731, + "theoretical_loss": 3.5338597181159077, + "tokens_seen": 1552875520 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042792377131394184, + "loss": 1.3534, + "theoretical_loss": 3.5338467613673474, + "tokens_seen": 1552941056 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042790772316950854, + "loss": 1.2935, + "theoretical_loss": 3.53383380531866, + "tokens_seen": 1553006592 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042789167502507524, + "loss": 1.3975, + "theoretical_loss": 3.5338208499697785, + "tokens_seen": 1553072128 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042787562688064194, + "loss": 1.3396, + "theoretical_loss": 3.5338078953206358, + "tokens_seen": 1553137664 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 577157, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3988282680511475, + "objective/train/theoretical_loss": 3.533794941371163, + "objective/train/tokens_used": 1573663200, + "theoretical_loss": 3.533794941371163, + "tokens_seen": 1553203200 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004278595787362086, + "loss": 1.3988, + "theoretical_loss": 3.533794941371163, + "tokens_seen": 1553203200 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042784353059177534, + "loss": 1.3819, + "theoretical_loss": 3.5337819881212944, + "tokens_seen": 1553268736 + }, + { + "epoch": 0.47, + "learning_rate": 0.000427827482447342, + "loss": 1.4001, + "theoretical_loss": 3.533769035570962, + "tokens_seen": 1553334272 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042781143430290874, + "loss": 1.3175, + "theoretical_loss": 3.5337560837200988, + "tokens_seen": 1553399808 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004277953861584754, + "loss": 1.3796, + "theoretical_loss": 3.5337431325686377, + "tokens_seen": 1553465344 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042777933801404214, + "loss": 1.364, + "theoretical_loss": 3.5337301821165106, + "tokens_seen": 1553530880 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004277632898696088, + "loss": 1.3315, + "theoretical_loss": 3.533717232363651, + "tokens_seen": 1553596416 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042774724172517554, + "loss": 1.3925, + "theoretical_loss": 3.533704283309992, + "tokens_seen": 1553661952 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004277311935807422, + "loss": 1.3614, + "theoretical_loss": 3.5336913349554653, + "tokens_seen": 1553727488 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042771514543630894, + "loss": 1.3485, + "theoretical_loss": 3.533678387300004, + "tokens_seen": 1553793024 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004276990972918756, + "loss": 1.3697, + "theoretical_loss": 3.5336654403435417, + "tokens_seen": 1553858560 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042768304914744235, + "loss": 1.2836, + "theoretical_loss": 3.5336524940860103, + "tokens_seen": 1553924096 + }, + { + "epoch": 0.47, + "learning_rate": 0.000427667001003009, + "loss": 1.3411, + "theoretical_loss": 3.533639548527342, + "tokens_seen": 1553989632 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042765095285857575, + "loss": 1.4185, + "theoretical_loss": 3.5336266036674715, + "tokens_seen": 1554055168 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004276349047141424, + "loss": 1.3132, + "theoretical_loss": 3.53361365950633, + "tokens_seen": 1554120704 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042761885656970915, + "loss": 1.4239, + "theoretical_loss": 3.5336007160438507, + "tokens_seen": 1554186240 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004276028084252759, + "loss": 1.3372, + "theoretical_loss": 3.533587773279967, + "tokens_seen": 1554251776 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042758676028084255, + "loss": 1.3082, + "theoretical_loss": 3.533574831214611, + "tokens_seen": 1554317312 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004275707121364093, + "loss": 1.4359, + "theoretical_loss": 3.533561889847716, + "tokens_seen": 1554382848 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042755466399197595, + "loss": 1.2222, + "theoretical_loss": 3.533548949179215, + "tokens_seen": 1554448384 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004275386158475427, + "loss": 1.3979, + "theoretical_loss": 3.5335360092090404, + "tokens_seen": 1554513920 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042752256770310935, + "loss": 1.4148, + "theoretical_loss": 3.533523069937125, + "tokens_seen": 1554579456 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004275065195586761, + "loss": 1.3185, + "theoretical_loss": 3.5335101313634025, + "tokens_seen": 1554644992 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042749047141424275, + "loss": 1.3322, + "theoretical_loss": 3.533497193487805, + "tokens_seen": 1554710528 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004274744232698095, + "loss": 1.2128, + "theoretical_loss": 3.5334842563102655, + "tokens_seen": 1554776064 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042745837512537616, + "loss": 1.3147, + "theoretical_loss": 3.533471319830717, + "tokens_seen": 1554841600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004274423269809429, + "loss": 1.3432, + "theoretical_loss": 3.533458384049093, + "tokens_seen": 1554907136 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042742627883650956, + "loss": 1.3548, + "theoretical_loss": 3.5334454489653258, + "tokens_seen": 1554972672 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004274102306920763, + "loss": 1.4434, + "theoretical_loss": 3.5334325145793484, + "tokens_seen": 1555038208 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042739418254764296, + "loss": 1.33, + "theoretical_loss": 3.5334195808910938, + "tokens_seen": 1555103744 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042737813440320966, + "loss": 1.3699, + "theoretical_loss": 3.533406647900495, + "tokens_seen": 1555169280 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042736208625877636, + "loss": 1.2221, + "theoretical_loss": 3.5333937156074846, + "tokens_seen": 1555234816 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042734603811434306, + "loss": 1.4027, + "theoretical_loss": 3.533380784011997, + "tokens_seen": 1555300352 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042732998996990976, + "loss": 1.2791, + "theoretical_loss": 3.5333678531139627, + "tokens_seen": 1555365888 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042731394182547646, + "loss": 1.343, + "theoretical_loss": 3.5333549229133174, + "tokens_seen": 1555431424 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042729789368104316, + "loss": 1.3781, + "theoretical_loss": 3.5333419934099926, + "tokens_seen": 1555496960 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042728184553660986, + "loss": 1.3656, + "theoretical_loss": 3.533329064603921, + "tokens_seen": 1555562496 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042726579739217656, + "loss": 1.2938, + "theoretical_loss": 3.5333161364950367, + "tokens_seen": 1555628032 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042724974924774326, + "loss": 1.3245, + "theoretical_loss": 3.5333032090832717, + "tokens_seen": 1555693568 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004272337011033099, + "loss": 1.2821, + "theoretical_loss": 3.53329028236856, + "tokens_seen": 1555759104 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042721765295887666, + "loss": 1.3192, + "theoretical_loss": 3.533277356350834, + "tokens_seen": 1555824640 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004272016048144433, + "loss": 1.3279, + "theoretical_loss": 3.5332644310300276, + "tokens_seen": 1555890176 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042718555667001007, + "loss": 1.3471, + "theoretical_loss": 3.5332515064060726, + "tokens_seen": 1555955712 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004271695085255767, + "loss": 1.3693, + "theoretical_loss": 3.5332385824789028, + "tokens_seen": 1556021248 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042715346038114347, + "loss": 1.2136, + "theoretical_loss": 3.5332256592484512, + "tokens_seen": 1556086784 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004271374122367101, + "loss": 1.3391, + "theoretical_loss": 3.5332127367146513, + "tokens_seen": 1556152320 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042712136409227687, + "loss": 1.2902, + "theoretical_loss": 3.5331998148774355, + "tokens_seen": 1556217856 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004271053159478435, + "loss": 1.2537, + "theoretical_loss": 3.533186893736737, + "tokens_seen": 1556283392 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042708926780341027, + "loss": 1.2418, + "theoretical_loss": 3.53317397329249, + "tokens_seen": 1556348928 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004270732196589769, + "loss": 1.3236, + "theoretical_loss": 3.5331610535446263, + "tokens_seen": 1556414464 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042705717151454367, + "loss": 1.2817, + "theoretical_loss": 3.5331481344930795, + "tokens_seen": 1556480000 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004270411233701103, + "loss": 1.3003, + "theoretical_loss": 3.533135216137783, + "tokens_seen": 1556545536 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042702507522567707, + "loss": 1.3593, + "theoretical_loss": 3.53312229847867, + "tokens_seen": 1556611072 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004270090270812437, + "loss": 1.4099, + "theoretical_loss": 3.533109381515673, + "tokens_seen": 1556676608 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004269929789368105, + "loss": 1.3086, + "theoretical_loss": 3.5330964652487253, + "tokens_seen": 1556742144 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004269769307923771, + "loss": 1.2893, + "theoretical_loss": 3.5330835496777615, + "tokens_seen": 1556807680 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004269608826479439, + "loss": 1.3315, + "theoretical_loss": 3.533070634802713, + "tokens_seen": 1556873216 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004269448345035105, + "loss": 1.3698, + "theoretical_loss": 3.533057720623514, + "tokens_seen": 1556938752 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004269287863590773, + "loss": 1.2682, + "theoretical_loss": 3.5330448071400973, + "tokens_seen": 1557004288 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004269127382146439, + "loss": 1.4004, + "theoretical_loss": 3.5330318943523964, + "tokens_seen": 1557069824 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004268966900702107, + "loss": 1.34, + "theoretical_loss": 3.5330189822603444, + "tokens_seen": 1557135360 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004268806419257773, + "loss": 1.3807, + "theoretical_loss": 3.5330060708638746, + "tokens_seen": 1557200896 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004268645937813441, + "loss": 1.4068, + "theoretical_loss": 3.5329931601629205, + "tokens_seen": 1557266432 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004268485456369107, + "loss": 1.2702, + "theoretical_loss": 3.5329802501574146, + "tokens_seen": 1557331968 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004268324974924775, + "loss": 1.2668, + "theoretical_loss": 3.532967340847291, + "tokens_seen": 1557397504 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004268164493480441, + "loss": 1.2819, + "theoretical_loss": 3.532954432232483, + "tokens_seen": 1557463040 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004268004012036109, + "loss": 1.3598, + "theoretical_loss": 3.532941524312923, + "tokens_seen": 1557528576 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042678435305917753, + "loss": 1.2847, + "theoretical_loss": 3.5329286170885448, + "tokens_seen": 1557594112 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004267683049147443, + "loss": 1.2232, + "theoretical_loss": 3.532915710559282, + "tokens_seen": 1557659648 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042675225677031093, + "loss": 1.4368, + "theoretical_loss": 3.532902804725068, + "tokens_seen": 1557725184 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004267362086258777, + "loss": 1.2853, + "theoretical_loss": 3.532889899585835, + "tokens_seen": 1557790720 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042672016048144433, + "loss": 1.3233, + "theoretical_loss": 3.532876995141518, + "tokens_seen": 1557856256 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004267041123370111, + "loss": 1.3065, + "theoretical_loss": 3.5328640913920495, + "tokens_seen": 1557921792 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042668806419257773, + "loss": 1.2417, + "theoretical_loss": 3.5328511883373626, + "tokens_seen": 1557987328 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004266720160481445, + "loss": 1.346, + "theoretical_loss": 3.532838285977391, + "tokens_seen": 1558052864 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042665596790371113, + "loss": 1.2801, + "theoretical_loss": 3.5328253843120674, + "tokens_seen": 1558118400 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004266399197592779, + "loss": 1.2729, + "theoretical_loss": 3.532812483341327, + "tokens_seen": 1558183936 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042662387161484453, + "loss": 1.3227, + "theoretical_loss": 3.532799583065101, + "tokens_seen": 1558249472 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042660782347041123, + "loss": 1.371, + "theoretical_loss": 3.532786683483324, + "tokens_seen": 1558315008 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042659177532597794, + "loss": 1.2453, + "theoretical_loss": 3.5327737845959293, + "tokens_seen": 1558380544 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042657572718154464, + "loss": 1.3035, + "theoretical_loss": 3.5327608864028504, + "tokens_seen": 1558446080 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042655967903711134, + "loss": 1.3123, + "theoretical_loss": 3.5327479889040205, + "tokens_seen": 1558511616 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042654363089267804, + "loss": 1.3252, + "theoretical_loss": 3.532735092099373, + "tokens_seen": 1558577152 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042652758274824474, + "loss": 1.2923, + "theoretical_loss": 3.5327221959888417, + "tokens_seen": 1558642688 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042651153460381144, + "loss": 1.3727, + "theoretical_loss": 3.5327093005723595, + "tokens_seen": 1558708224 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042649548645937814, + "loss": 1.3464, + "theoretical_loss": 3.53269640584986, + "tokens_seen": 1558773760 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042647943831494484, + "loss": 1.2618, + "theoretical_loss": 3.5326835118212774, + "tokens_seen": 1558839296 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004264633901705115, + "loss": 1.2041, + "theoretical_loss": 3.532670618486544, + "tokens_seen": 1558904832 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042644734202607824, + "loss": 1.3197, + "theoretical_loss": 3.5326577258455947, + "tokens_seen": 1558970368 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004264312938816449, + "loss": 1.3874, + "theoretical_loss": 3.5326448338983623, + "tokens_seen": 1559035904 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042641524573721164, + "loss": 1.4201, + "theoretical_loss": 3.5326319426447794, + "tokens_seen": 1559101440 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004263991975927784, + "loss": 1.3364, + "theoretical_loss": 3.5326190520847813, + "tokens_seen": 1559166976 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042638314944834504, + "loss": 1.3448, + "theoretical_loss": 3.5326061622183, + "tokens_seen": 1559232512 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004263671013039118, + "loss": 1.3191, + "theoretical_loss": 3.53259327304527, + "tokens_seen": 1559298048 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042635105315947844, + "loss": 1.3496, + "theoretical_loss": 3.532580384565624, + "tokens_seen": 1559363584 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004263350050150452, + "loss": 1.3014, + "theoretical_loss": 3.5325674967792966, + "tokens_seen": 1559429120 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042631895687061185, + "loss": 1.3383, + "theoretical_loss": 3.532554609686221, + "tokens_seen": 1559494656 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004263029087261786, + "loss": 1.2708, + "theoretical_loss": 3.53254172328633, + "tokens_seen": 1559560192 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042628686058174525, + "loss": 1.3134, + "theoretical_loss": 3.5325288375795583, + "tokens_seen": 1559625728 + }, + { + "epoch": 0.47, + "learning_rate": 0.000426270812437312, + "loss": 1.3414, + "theoretical_loss": 3.5325159525658387, + "tokens_seen": 1559691264 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 579478, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3584932088851929, + "objective/train/theoretical_loss": 3.5325030682451053, + "objective/train/tokens_used": 1580216800, + "theoretical_loss": 3.5325030682451053, + "tokens_seen": 1559756800 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042625476429287865, + "loss": 1.3585, + "theoretical_loss": 3.5325030682451053, + "tokens_seen": 1559756800 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004262387161484454, + "loss": 1.2855, + "theoretical_loss": 3.5324901846172914, + "tokens_seen": 1559822336 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042622266800401205, + "loss": 1.2996, + "theoretical_loss": 3.5324773016823308, + "tokens_seen": 1559887872 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004262066198595788, + "loss": 1.3471, + "theoretical_loss": 3.5324644194401573, + "tokens_seen": 1559953408 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042619057171514545, + "loss": 1.3724, + "theoretical_loss": 3.532451537890704, + "tokens_seen": 1560018944 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004261745235707122, + "loss": 1.3548, + "theoretical_loss": 3.532438657033905, + "tokens_seen": 1560084480 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042615847542627885, + "loss": 1.3257, + "theoretical_loss": 3.532425776869694, + "tokens_seen": 1560150016 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004261424272818456, + "loss": 1.4542, + "theoretical_loss": 3.532412897398004, + "tokens_seen": 1560215552 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042612637913741225, + "loss": 1.3297, + "theoretical_loss": 3.5324000186187696, + "tokens_seen": 1560281088 + }, + { + "epoch": 0.47, + "learning_rate": 0.000426110330992979, + "loss": 1.3347, + "theoretical_loss": 3.532387140531924, + "tokens_seen": 1560346624 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042609428284854565, + "loss": 1.2666, + "theoretical_loss": 3.5323742631374007, + "tokens_seen": 1560412160 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004260782347041124, + "loss": 1.2382, + "theoretical_loss": 3.5323613864351344, + "tokens_seen": 1560477696 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042606218655967906, + "loss": 1.353, + "theoretical_loss": 3.5323485104250576, + "tokens_seen": 1560543232 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004260461384152458, + "loss": 1.3461, + "theoretical_loss": 3.5323356351071045, + "tokens_seen": 1560608768 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042603009027081246, + "loss": 1.4145, + "theoretical_loss": 3.532322760481209, + "tokens_seen": 1560674304 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004260140421263792, + "loss": 1.3073, + "theoretical_loss": 3.5323098865473046, + "tokens_seen": 1560739840 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042599799398194586, + "loss": 1.2629, + "theoretical_loss": 3.532297013305325, + "tokens_seen": 1560805376 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042598194583751256, + "loss": 1.3611, + "theoretical_loss": 3.532284140755204, + "tokens_seen": 1560870912 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042596589769307926, + "loss": 1.3521, + "theoretical_loss": 3.532271268896876, + "tokens_seen": 1560936448 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042594984954864596, + "loss": 1.2738, + "theoretical_loss": 3.532258397730274, + "tokens_seen": 1561001984 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042593380140421266, + "loss": 1.319, + "theoretical_loss": 3.532245527255332, + "tokens_seen": 1561067520 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042591775325977936, + "loss": 1.409, + "theoretical_loss": 3.532232657471984, + "tokens_seen": 1561133056 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042590170511534606, + "loss": 1.3495, + "theoretical_loss": 3.5322197883801634, + "tokens_seen": 1561198592 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042588565697091276, + "loss": 1.2733, + "theoretical_loss": 3.5322069199798043, + "tokens_seen": 1561264128 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042586960882647946, + "loss": 1.272, + "theoretical_loss": 3.5321940522708406, + "tokens_seen": 1561329664 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042585356068204616, + "loss": 1.3875, + "theoretical_loss": 3.532181185253206, + "tokens_seen": 1561395200 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004258375125376128, + "loss": 1.2789, + "theoretical_loss": 3.5321683189268342, + "tokens_seen": 1561460736 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042582146439317957, + "loss": 1.3146, + "theoretical_loss": 3.5321554532916597, + "tokens_seen": 1561526272 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004258054162487462, + "loss": 1.3041, + "theoretical_loss": 3.5321425883476154, + "tokens_seen": 1561591808 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042578936810431297, + "loss": 1.4013, + "theoretical_loss": 3.5321297240946357, + "tokens_seen": 1561657344 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004257733199598796, + "loss": 1.3486, + "theoretical_loss": 3.5321168605326543, + "tokens_seen": 1561722880 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042575727181544637, + "loss": 1.3587, + "theoretical_loss": 3.532103997661606, + "tokens_seen": 1561788416 + }, + { + "epoch": 0.47, + "learning_rate": 0.000425741223671013, + "loss": 1.396, + "theoretical_loss": 3.532091135481423, + "tokens_seen": 1561853952 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042572517552657977, + "loss": 1.3423, + "theoretical_loss": 3.5320782739920404, + "tokens_seen": 1561919488 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004257091273821464, + "loss": 1.3897, + "theoretical_loss": 3.5320654131933917, + "tokens_seen": 1561985024 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042569307923771317, + "loss": 1.4014, + "theoretical_loss": 3.5320525530854114, + "tokens_seen": 1562050560 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004256770310932798, + "loss": 1.3203, + "theoretical_loss": 3.532039693668033, + "tokens_seen": 1562116096 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042566098294884657, + "loss": 1.3763, + "theoretical_loss": 3.53202683494119, + "tokens_seen": 1562181632 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004256449348044132, + "loss": 1.3229, + "theoretical_loss": 3.532013976904817, + "tokens_seen": 1562247168 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042562888665998, + "loss": 1.3767, + "theoretical_loss": 3.5320011195588483, + "tokens_seen": 1562312704 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004256128385155466, + "loss": 1.2623, + "theoretical_loss": 3.5319882629032167, + "tokens_seen": 1562378240 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004255967903711134, + "loss": 1.3586, + "theoretical_loss": 3.531975406937857, + "tokens_seen": 1562443776 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042558074222668, + "loss": 1.3244, + "theoretical_loss": 3.5319625516627027, + "tokens_seen": 1562509312 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004255646940822468, + "loss": 1.3577, + "theoretical_loss": 3.531949697077688, + "tokens_seen": 1562574848 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004255486459378134, + "loss": 1.4012, + "theoretical_loss": 3.5319368431827476, + "tokens_seen": 1562640384 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004255325977933802, + "loss": 1.2698, + "theoretical_loss": 3.5319239899778143, + "tokens_seen": 1562705920 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004255165496489468, + "loss": 1.2356, + "theoretical_loss": 3.5319111374628234, + "tokens_seen": 1562771456 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004255005015045136, + "loss": 1.2645, + "theoretical_loss": 3.5318982856377077, + "tokens_seen": 1562836992 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004254844533600802, + "loss": 1.3096, + "theoretical_loss": 3.531885434502402, + "tokens_seen": 1562902528 + }, + { + "epoch": 0.47, + "learning_rate": 0.000425468405215647, + "loss": 1.2982, + "theoretical_loss": 3.53187258405684, + "tokens_seen": 1562968064 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004254523570712136, + "loss": 1.3395, + "theoretical_loss": 3.5318597343009563, + "tokens_seen": 1563033600 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004254363089267804, + "loss": 1.3466, + "theoretical_loss": 3.5318468852346845, + "tokens_seen": 1563099136 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042542026078234703, + "loss": 1.3699, + "theoretical_loss": 3.5318340368579584, + "tokens_seen": 1563164672 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004254042126379138, + "loss": 1.2697, + "theoretical_loss": 3.531821189170713, + "tokens_seen": 1563230208 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042538816449348043, + "loss": 1.2914, + "theoretical_loss": 3.531808342172882, + "tokens_seen": 1563295744 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004253721163490472, + "loss": 1.2716, + "theoretical_loss": 3.5317954958643987, + "tokens_seen": 1563361280 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042535606820461383, + "loss": 1.266, + "theoretical_loss": 3.531782650245198, + "tokens_seen": 1563426816 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004253400200601806, + "loss": 1.4037, + "theoretical_loss": 3.5317698053152142, + "tokens_seen": 1563492352 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042532397191574723, + "loss": 1.2372, + "theoretical_loss": 3.5317569610743815, + "tokens_seen": 1563557888 + }, + { + "epoch": 0.47, + "learning_rate": 0.000425307923771314, + "loss": 1.3439, + "theoretical_loss": 3.531744117522633, + "tokens_seen": 1563623424 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042529187562688063, + "loss": 1.3012, + "theoretical_loss": 3.5317312746599034, + "tokens_seen": 1563688960 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004252758274824474, + "loss": 1.3042, + "theoretical_loss": 3.5317184324861275, + "tokens_seen": 1563754496 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042525977933801403, + "loss": 1.3557, + "theoretical_loss": 3.531705591001239, + "tokens_seen": 1563820032 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004252437311935808, + "loss": 1.3695, + "theoretical_loss": 3.531692750205172, + "tokens_seen": 1563885568 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042522768304914743, + "loss": 1.2856, + "theoretical_loss": 3.5316799100978606, + "tokens_seen": 1563951104 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042521163490471414, + "loss": 1.2965, + "theoretical_loss": 3.531667070679239, + "tokens_seen": 1564016640 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004251955867602809, + "loss": 1.27, + "theoretical_loss": 3.5316542319492417, + "tokens_seen": 1564082176 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042517953861584754, + "loss": 1.2162, + "theoretical_loss": 3.531641393907803, + "tokens_seen": 1564147712 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004251634904714143, + "loss": 1.2384, + "theoretical_loss": 3.5316285565548564, + "tokens_seen": 1564213248 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042514744232698094, + "loss": 1.422, + "theoretical_loss": 3.531615719890337, + "tokens_seen": 1564278784 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004251313941825477, + "loss": 1.336, + "theoretical_loss": 3.5316028839141786, + "tokens_seen": 1564344320 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042511534603811434, + "loss": 1.3491, + "theoretical_loss": 3.5315900486263154, + "tokens_seen": 1564409856 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004250992978936811, + "loss": 1.3099, + "theoretical_loss": 3.5315772140266817, + "tokens_seen": 1564475392 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042508324974924774, + "loss": 1.2904, + "theoretical_loss": 3.5315643801152117, + "tokens_seen": 1564540928 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004250672016048145, + "loss": 1.2795, + "theoretical_loss": 3.5315515468918406, + "tokens_seen": 1564606464 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042505115346038114, + "loss": 1.3345, + "theoretical_loss": 3.5315387143565014, + "tokens_seen": 1564672000 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004250351053159479, + "loss": 1.342, + "theoretical_loss": 3.5315258825091282, + "tokens_seen": 1564737536 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042501905717151454, + "loss": 1.2804, + "theoretical_loss": 3.531513051349657, + "tokens_seen": 1564803072 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004250030090270813, + "loss": 1.3628, + "theoretical_loss": 3.531500220878021, + "tokens_seen": 1564868608 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042498696088264794, + "loss": 1.2715, + "theoretical_loss": 3.531487391094154, + "tokens_seen": 1564934144 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004249709127382147, + "loss": 1.2915, + "theoretical_loss": 3.5314745619979915, + "tokens_seen": 1564999680 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042495486459378135, + "loss": 1.3147, + "theoretical_loss": 3.5314617335894676, + "tokens_seen": 1565065216 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004249388164493481, + "loss": 1.3489, + "theoretical_loss": 3.5314489058685155, + "tokens_seen": 1565130752 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042492276830491475, + "loss": 1.218, + "theoretical_loss": 3.531436078835071, + "tokens_seen": 1565196288 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004249067201604815, + "loss": 1.3055, + "theoretical_loss": 3.531423252489068, + "tokens_seen": 1565261824 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042489067201604815, + "loss": 1.3797, + "theoretical_loss": 3.5314104268304405, + "tokens_seen": 1565327360 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004248746238716149, + "loss": 1.3413, + "theoretical_loss": 3.531397601859123, + "tokens_seen": 1565392896 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042485857572718155, + "loss": 1.316, + "theoretical_loss": 3.5313847775750506, + "tokens_seen": 1565458432 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004248425275827483, + "loss": 1.2467, + "theoretical_loss": 3.5313719539781565, + "tokens_seen": 1565523968 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042482647943831495, + "loss": 1.27, + "theoretical_loss": 3.5313591310683763, + "tokens_seen": 1565589504 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004248104312938817, + "loss": 1.2808, + "theoretical_loss": 3.5313463088456434, + "tokens_seen": 1565655040 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042479438314944835, + "loss": 1.3858, + "theoretical_loss": 3.5313334873098934, + "tokens_seen": 1565720576 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004247783350050151, + "loss": 1.388, + "theoretical_loss": 3.5313206664610597, + "tokens_seen": 1565786112 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042476228686058175, + "loss": 1.2598, + "theoretical_loss": 3.531307846299077, + "tokens_seen": 1565851648 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004247462387161485, + "loss": 1.337, + "theoretical_loss": 3.53129502682388, + "tokens_seen": 1565917184 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042473019057171515, + "loss": 1.2547, + "theoretical_loss": 3.531282208035403, + "tokens_seen": 1565982720 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004247141424272819, + "loss": 1.3081, + "theoretical_loss": 3.5312693899335805, + "tokens_seen": 1566048256 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042469809428284856, + "loss": 1.3389, + "theoretical_loss": 3.5312565725183473, + "tokens_seen": 1566113792 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004246820461384153, + "loss": 1.3612, + "theoretical_loss": 3.531243755789637, + "tokens_seen": 1566179328 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042466599799398196, + "loss": 1.2756, + "theoretical_loss": 3.5312309397473856, + "tokens_seen": 1566244864 + }, + { + "epoch": 0.47, + "objective/train/docs_used": 581739, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3337347507476807, + "objective/train/theoretical_loss": 3.531218124391526, + "objective/train/tokens_used": 1586770400, + "theoretical_loss": 3.531218124391526, + "tokens_seen": 1566310400 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004246499498495487, + "loss": 1.3337, + "theoretical_loss": 3.531218124391526, + "tokens_seen": 1566310400 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042463390170511536, + "loss": 1.3055, + "theoretical_loss": 3.5312053097219938, + "tokens_seen": 1566375936 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004246178535606821, + "loss": 1.3059, + "theoretical_loss": 3.531192495738723, + "tokens_seen": 1566441472 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042460180541624876, + "loss": 1.3877, + "theoretical_loss": 3.531179682441648, + "tokens_seen": 1566507008 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004245857572718155, + "loss": 1.2944, + "theoretical_loss": 3.531166869830704, + "tokens_seen": 1566572544 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042456970912738216, + "loss": 1.3545, + "theoretical_loss": 3.5311540579058254, + "tokens_seen": 1566638080 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042455366098294886, + "loss": 1.2794, + "theoretical_loss": 3.5311412466669463, + "tokens_seen": 1566703616 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042453761283851556, + "loss": 1.3283, + "theoretical_loss": 3.5311284361140016, + "tokens_seen": 1566769152 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042452156469408226, + "loss": 1.3804, + "theoretical_loss": 3.531115626246926, + "tokens_seen": 1566834688 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042450551654964896, + "loss": 1.4299, + "theoretical_loss": 3.531102817065654, + "tokens_seen": 1566900224 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042448946840521566, + "loss": 1.3753, + "theoretical_loss": 3.5310900085701196, + "tokens_seen": 1566965760 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042447342026078236, + "loss": 1.3784, + "theoretical_loss": 3.5310772007602584, + "tokens_seen": 1567031296 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042445737211634907, + "loss": 1.3711, + "theoretical_loss": 3.5310643936360044, + "tokens_seen": 1567096832 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004244413239719157, + "loss": 1.2906, + "theoretical_loss": 3.5310515871972927, + "tokens_seen": 1567162368 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042442527582748247, + "loss": 1.4127, + "theoretical_loss": 3.5310387814440576, + "tokens_seen": 1567227904 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004244092276830491, + "loss": 1.317, + "theoretical_loss": 3.5310259763762333, + "tokens_seen": 1567293440 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042439317953861587, + "loss": 1.2368, + "theoretical_loss": 3.5310131719937554, + "tokens_seen": 1567358976 + }, + { + "epoch": 0.47, + "learning_rate": 0.0004243771313941825, + "loss": 1.3033, + "theoretical_loss": 3.531000368296558, + "tokens_seen": 1567424512 + }, + { + "epoch": 0.47, + "learning_rate": 0.00042436108324974927, + "loss": 1.3814, + "theoretical_loss": 3.5309875652845757, + "tokens_seen": 1567490048 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004243450351053159, + "loss": 1.3262, + "theoretical_loss": 3.5309747629577437, + "tokens_seen": 1567555584 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042432898696088267, + "loss": 1.198, + "theoretical_loss": 3.530961961315996, + "tokens_seen": 1567621120 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004243129388164493, + "loss": 1.304, + "theoretical_loss": 3.5309491603592678, + "tokens_seen": 1567686656 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042429689067201607, + "loss": 1.3275, + "theoretical_loss": 3.5309363600874937, + "tokens_seen": 1567752192 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004242808425275827, + "loss": 1.2743, + "theoretical_loss": 3.530923560500608, + "tokens_seen": 1567817728 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042426479438314947, + "loss": 1.3596, + "theoretical_loss": 3.5309107615985464, + "tokens_seen": 1567883264 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004242487462387161, + "loss": 1.3218, + "theoretical_loss": 3.5308979633812427, + "tokens_seen": 1567948800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004242326980942829, + "loss": 1.4042, + "theoretical_loss": 3.530885165848632, + "tokens_seen": 1568014336 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004242166499498495, + "loss": 1.4633, + "theoretical_loss": 3.530872369000649, + "tokens_seen": 1568079872 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004242006018054163, + "loss": 1.3248, + "theoretical_loss": 3.5308595728372296, + "tokens_seen": 1568145408 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004241845536609829, + "loss": 1.2717, + "theoretical_loss": 3.530846777358306, + "tokens_seen": 1568210944 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004241685055165497, + "loss": 1.3016, + "theoretical_loss": 3.5308339825638155, + "tokens_seen": 1568276480 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004241524573721163, + "loss": 1.2056, + "theoretical_loss": 3.5308211884536913, + "tokens_seen": 1568342016 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004241364092276831, + "loss": 1.3571, + "theoretical_loss": 3.5308083950278686, + "tokens_seen": 1568407552 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004241203610832497, + "loss": 1.3045, + "theoretical_loss": 3.5307956022862825, + "tokens_seen": 1568473088 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004241043129388165, + "loss": 1.3032, + "theoretical_loss": 3.5307828102288683, + "tokens_seen": 1568538624 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004240882647943831, + "loss": 1.2892, + "theoretical_loss": 3.5307700188555593, + "tokens_seen": 1568604160 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004240722166499499, + "loss": 1.355, + "theoretical_loss": 3.530757228166292, + "tokens_seen": 1568669696 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004240561685055165, + "loss": 1.3031, + "theoretical_loss": 3.5307444381610003, + "tokens_seen": 1568735232 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004240401203610833, + "loss": 1.2905, + "theoretical_loss": 3.5307316488396188, + "tokens_seen": 1568800768 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042402407221664993, + "loss": 1.3837, + "theoretical_loss": 3.530718860202083, + "tokens_seen": 1568866304 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004240080240722167, + "loss": 1.3499, + "theoretical_loss": 3.5307060722483277, + "tokens_seen": 1568931840 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042399197592778344, + "loss": 1.3332, + "theoretical_loss": 3.530693284978288, + "tokens_seen": 1568997376 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004239759277833501, + "loss": 1.4413, + "theoretical_loss": 3.530680498391898, + "tokens_seen": 1569062912 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042395987963891684, + "loss": 1.2884, + "theoretical_loss": 3.530667712489093, + "tokens_seen": 1569128448 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004239438314944835, + "loss": 1.2304, + "theoretical_loss": 3.5306549272698082, + "tokens_seen": 1569193984 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004239277833500502, + "loss": 1.2504, + "theoretical_loss": 3.5306421427339783, + "tokens_seen": 1569259520 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004239117352056169, + "loss": 1.3663, + "theoretical_loss": 3.530629358881538, + "tokens_seen": 1569325056 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004238956870611836, + "loss": 1.3217, + "theoretical_loss": 3.5306165757124224, + "tokens_seen": 1569390592 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004238796389167503, + "loss": 1.2521, + "theoretical_loss": 3.5306037932265664, + "tokens_seen": 1569456128 + }, + { + "epoch": 0.48, + "learning_rate": 0.000423863590772317, + "loss": 1.2052, + "theoretical_loss": 3.5305910114239056, + "tokens_seen": 1569521664 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004238475426278837, + "loss": 1.3591, + "theoretical_loss": 3.5305782303043736, + "tokens_seen": 1569587200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004238314944834504, + "loss": 1.3706, + "theoretical_loss": 3.5305654498679067, + "tokens_seen": 1569652736 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004238154463390171, + "loss": 1.2031, + "theoretical_loss": 3.5305526701144387, + "tokens_seen": 1569718272 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004237993981945838, + "loss": 1.2849, + "theoretical_loss": 3.5305398910439054, + "tokens_seen": 1569783808 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042378335005015044, + "loss": 1.3205, + "theoretical_loss": 3.5305271126562423, + "tokens_seen": 1569849344 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004237673019057172, + "loss": 1.2038, + "theoretical_loss": 3.5305143349513832, + "tokens_seen": 1569914880 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042375125376128384, + "loss": 1.2941, + "theoretical_loss": 3.530501557929264, + "tokens_seen": 1569980416 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004237352056168506, + "loss": 1.3796, + "theoretical_loss": 3.5304887815898187, + "tokens_seen": 1570045952 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042371915747241724, + "loss": 1.3356, + "theoretical_loss": 3.530476005932983, + "tokens_seen": 1570111488 + }, + { + "epoch": 0.48, + "learning_rate": 0.000423703109327984, + "loss": 1.273, + "theoretical_loss": 3.5304632309586923, + "tokens_seen": 1570177024 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042368706118355064, + "loss": 1.3019, + "theoretical_loss": 3.5304504566668813, + "tokens_seen": 1570242560 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004236710130391174, + "loss": 1.4629, + "theoretical_loss": 3.5304376830574844, + "tokens_seen": 1570308096 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042365496489468404, + "loss": 1.3279, + "theoretical_loss": 3.530424910130438, + "tokens_seen": 1570373632 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004236389167502508, + "loss": 1.2912, + "theoretical_loss": 3.5304121378856763, + "tokens_seen": 1570439168 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042362286860581744, + "loss": 1.286, + "theoretical_loss": 3.5303993663231346, + "tokens_seen": 1570504704 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004236068204613842, + "loss": 1.3581, + "theoretical_loss": 3.5303865954427476, + "tokens_seen": 1570570240 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042359077231695085, + "loss": 1.2882, + "theoretical_loss": 3.530373825244451, + "tokens_seen": 1570635776 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004235747241725176, + "loss": 1.2555, + "theoretical_loss": 3.5303610557281795, + "tokens_seen": 1570701312 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042355867602808425, + "loss": 1.3025, + "theoretical_loss": 3.5303482868938687, + "tokens_seen": 1570766848 + }, + { + "epoch": 0.48, + "learning_rate": 0.000423542627883651, + "loss": 1.451, + "theoretical_loss": 3.5303355187414533, + "tokens_seen": 1570832384 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042352657973921765, + "loss": 1.3041, + "theoretical_loss": 3.5303227512708686, + "tokens_seen": 1570897920 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004235105315947844, + "loss": 1.4518, + "theoretical_loss": 3.530309984482049, + "tokens_seen": 1570963456 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042349448345035105, + "loss": 1.3283, + "theoretical_loss": 3.5302972183749306, + "tokens_seen": 1571028992 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004234784353059178, + "loss": 1.3622, + "theoretical_loss": 3.530284452949449, + "tokens_seen": 1571094528 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042346238716148445, + "loss": 1.3538, + "theoretical_loss": 3.5302716882055383, + "tokens_seen": 1571160064 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004234463390170512, + "loss": 1.3148, + "theoretical_loss": 3.530258924143134, + "tokens_seen": 1571225600 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042343029087261785, + "loss": 1.3017, + "theoretical_loss": 3.5302461607621716, + "tokens_seen": 1571291136 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004234142427281846, + "loss": 1.4906, + "theoretical_loss": 3.5302333980625855, + "tokens_seen": 1571356672 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042339819458375125, + "loss": 1.3225, + "theoretical_loss": 3.5302206360443122, + "tokens_seen": 1571422208 + }, + { + "epoch": 0.48, + "learning_rate": 0.000423382146439318, + "loss": 1.3263, + "theoretical_loss": 3.5302078747072856, + "tokens_seen": 1571487744 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042336609829488465, + "loss": 1.3394, + "theoretical_loss": 3.530195114051442, + "tokens_seen": 1571553280 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004233500501504514, + "loss": 1.2739, + "theoretical_loss": 3.5301823540767154, + "tokens_seen": 1571618816 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042333400200601806, + "loss": 1.2316, + "theoretical_loss": 3.5301695947830423, + "tokens_seen": 1571684352 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004233179538615848, + "loss": 1.3224, + "theoretical_loss": 3.5301568361703577, + "tokens_seen": 1571749888 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042330190571715146, + "loss": 1.4414, + "theoretical_loss": 3.530144078238596, + "tokens_seen": 1571815424 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004232858575727182, + "loss": 1.3544, + "theoretical_loss": 3.5301313209876937, + "tokens_seen": 1571880960 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042326980942828486, + "loss": 1.3667, + "theoretical_loss": 3.530118564417585, + "tokens_seen": 1571946496 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004232537612838516, + "loss": 1.3269, + "theoretical_loss": 3.530105808528206, + "tokens_seen": 1572012032 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042323771313941826, + "loss": 1.2699, + "theoretical_loss": 3.5300930533194914, + "tokens_seen": 1572077568 + }, + { + "epoch": 0.48, + "learning_rate": 0.000423221664994985, + "loss": 1.2987, + "theoretical_loss": 3.5300802987913764, + "tokens_seen": 1572143104 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042320561685055166, + "loss": 1.2436, + "theoretical_loss": 3.5300675449437975, + "tokens_seen": 1572208640 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004231895687061184, + "loss": 1.2421, + "theoretical_loss": 3.5300547917766885, + "tokens_seen": 1572274176 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042317352056168506, + "loss": 1.2977, + "theoretical_loss": 3.530042039289986, + "tokens_seen": 1572339712 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042315747241725176, + "loss": 1.2997, + "theoretical_loss": 3.5300292874836243, + "tokens_seen": 1572405248 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042314142427281846, + "loss": 1.2531, + "theoretical_loss": 3.5300165363575395, + "tokens_seen": 1572470784 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042312537612838516, + "loss": 1.3758, + "theoretical_loss": 3.530003785911666, + "tokens_seen": 1572536320 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042310932798395186, + "loss": 1.3176, + "theoretical_loss": 3.5299910361459403, + "tokens_seen": 1572601856 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042309327983951856, + "loss": 1.3405, + "theoretical_loss": 3.529978287060298, + "tokens_seen": 1572667392 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042307723169508527, + "loss": 1.2449, + "theoretical_loss": 3.5299655386546727, + "tokens_seen": 1572732928 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042306118355065197, + "loss": 1.3129, + "theoretical_loss": 3.5299527909290016, + "tokens_seen": 1572798464 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 584050, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2666208744049072, + "objective/train/theoretical_loss": 3.529940043883219, + "objective/train/tokens_used": 1593324000, + "theoretical_loss": 3.529940043883219, + "tokens_seen": 1572864000 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042304513540621867, + "loss": 1.2666, + "theoretical_loss": 3.529940043883219, + "tokens_seen": 1572864000 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042302908726178537, + "loss": 1.3048, + "theoretical_loss": 3.529927297517261, + "tokens_seen": 1572929536 + }, + { + "epoch": 0.48, + "learning_rate": 0.000423013039117352, + "loss": 1.2511, + "theoretical_loss": 3.5299145518310624, + "tokens_seen": 1572995072 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042299699097291877, + "loss": 1.3294, + "theoretical_loss": 3.5299018068245593, + "tokens_seen": 1573060608 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004229809428284854, + "loss": 1.3453, + "theoretical_loss": 3.529889062497687, + "tokens_seen": 1573126144 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042296489468405217, + "loss": 1.2624, + "theoretical_loss": 3.5298763188503806, + "tokens_seen": 1573191680 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004229488465396188, + "loss": 1.3722, + "theoretical_loss": 3.529863575882575, + "tokens_seen": 1573257216 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042293279839518557, + "loss": 1.2499, + "theoretical_loss": 3.529850833594207, + "tokens_seen": 1573322752 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004229167502507522, + "loss": 1.3273, + "theoretical_loss": 3.5298380919852113, + "tokens_seen": 1573388288 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042290070210631897, + "loss": 1.3617, + "theoretical_loss": 3.529825351055524, + "tokens_seen": 1573453824 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004228846539618856, + "loss": 1.3713, + "theoretical_loss": 3.529812610805079, + "tokens_seen": 1573519360 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004228686058174524, + "loss": 1.3325, + "theoretical_loss": 3.529799871233814, + "tokens_seen": 1573584896 + }, + { + "epoch": 0.48, + "learning_rate": 0.000422852557673019, + "loss": 1.33, + "theoretical_loss": 3.529787132341663, + "tokens_seen": 1573650432 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004228365095285858, + "loss": 1.3208, + "theoretical_loss": 3.529774394128562, + "tokens_seen": 1573715968 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042282046138415253, + "loss": 1.4488, + "theoretical_loss": 3.529761656594447, + "tokens_seen": 1573781504 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004228044132397192, + "loss": 1.2967, + "theoretical_loss": 3.5297489197392524, + "tokens_seen": 1573847040 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042278836509528593, + "loss": 1.4383, + "theoretical_loss": 3.529736183562914, + "tokens_seen": 1573912576 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004227723169508526, + "loss": 1.4457, + "theoretical_loss": 3.5297234480653685, + "tokens_seen": 1573978112 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042275626880641933, + "loss": 1.3046, + "theoretical_loss": 3.5297107132465504, + "tokens_seen": 1574043648 + }, + { + "epoch": 0.48, + "learning_rate": 0.000422740220661986, + "loss": 1.3734, + "theoretical_loss": 3.529697979106395, + "tokens_seen": 1574109184 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042272417251755273, + "loss": 1.2636, + "theoretical_loss": 3.5296852456448393, + "tokens_seen": 1574174720 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004227081243731194, + "loss": 1.2374, + "theoretical_loss": 3.5296725128618176, + "tokens_seen": 1574240256 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042269207622868614, + "loss": 1.4067, + "theoretical_loss": 3.5296597807572656, + "tokens_seen": 1574305792 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004226760280842528, + "loss": 1.3246, + "theoretical_loss": 3.5296470493311194, + "tokens_seen": 1574371328 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042265997993981954, + "loss": 1.3827, + "theoretical_loss": 3.529634318583315, + "tokens_seen": 1574436864 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004226439317953862, + "loss": 1.3282, + "theoretical_loss": 3.5296215885137867, + "tokens_seen": 1574502400 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042262788365095294, + "loss": 1.2387, + "theoretical_loss": 3.5296088591224715, + "tokens_seen": 1574567936 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004226118355065196, + "loss": 1.3213, + "theoretical_loss": 3.529596130409304, + "tokens_seen": 1574633472 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042259578736208634, + "loss": 1.275, + "theoretical_loss": 3.5295834023742203, + "tokens_seen": 1574699008 + }, + { + "epoch": 0.48, + "learning_rate": 0.000422579739217653, + "loss": 1.3467, + "theoretical_loss": 3.529570675017156, + "tokens_seen": 1574764544 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042256369107321974, + "loss": 1.2644, + "theoretical_loss": 3.529557948338047, + "tokens_seen": 1574830080 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004225476429287864, + "loss": 1.3252, + "theoretical_loss": 3.5295452223368287, + "tokens_seen": 1574895616 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004225315947843531, + "loss": 1.3232, + "theoretical_loss": 3.529532497013437, + "tokens_seen": 1574961152 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004225155466399198, + "loss": 1.3936, + "theoretical_loss": 3.5295197723678076, + "tokens_seen": 1575026688 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004224994984954865, + "loss": 1.243, + "theoretical_loss": 3.5295070483998754, + "tokens_seen": 1575092224 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004224834503510532, + "loss": 1.3552, + "theoretical_loss": 3.529494325109577, + "tokens_seen": 1575157760 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004224674022066199, + "loss": 1.3231, + "theoretical_loss": 3.5294816024968485, + "tokens_seen": 1575223296 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004224513540621866, + "loss": 1.2873, + "theoretical_loss": 3.529468880561624, + "tokens_seen": 1575288832 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004224353059177533, + "loss": 1.3436, + "theoretical_loss": 3.529456159303841, + "tokens_seen": 1575354368 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042241925777332, + "loss": 1.2563, + "theoretical_loss": 3.5294434387234346, + "tokens_seen": 1575419904 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004224032096288867, + "loss": 1.2438, + "theoretical_loss": 3.52943071882034, + "tokens_seen": 1575485440 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042238716148445334, + "loss": 1.3426, + "theoretical_loss": 3.529417999594494, + "tokens_seen": 1575550976 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004223711133400201, + "loss": 1.3055, + "theoretical_loss": 3.529405281045831, + "tokens_seen": 1575616512 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042235506519558674, + "loss": 1.314, + "theoretical_loss": 3.529392563174288, + "tokens_seen": 1575682048 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004223390170511535, + "loss": 1.3746, + "theoretical_loss": 3.5293798459798005, + "tokens_seen": 1575747584 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042232296890672014, + "loss": 1.2564, + "theoretical_loss": 3.529367129462304, + "tokens_seen": 1575813120 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004223069207622869, + "loss": 1.394, + "theoretical_loss": 3.5293544136217347, + "tokens_seen": 1575878656 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042229087261785354, + "loss": 1.3663, + "theoretical_loss": 3.529341698458028, + "tokens_seen": 1575944192 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004222748244734203, + "loss": 1.3653, + "theoretical_loss": 3.52932898397112, + "tokens_seen": 1576009728 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042225877632898694, + "loss": 1.38, + "theoretical_loss": 3.529316270160946, + "tokens_seen": 1576075264 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004222427281845537, + "loss": 1.3641, + "theoretical_loss": 3.529303557027443, + "tokens_seen": 1576140800 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042222668004012035, + "loss": 1.3033, + "theoretical_loss": 3.5292908445705455, + "tokens_seen": 1576206336 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004222106318956871, + "loss": 1.3928, + "theoretical_loss": 3.5292781327901905, + "tokens_seen": 1576271872 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042219458375125375, + "loss": 1.3682, + "theoretical_loss": 3.5292654216863135, + "tokens_seen": 1576337408 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004221785356068205, + "loss": 1.4174, + "theoretical_loss": 3.5292527112588497, + "tokens_seen": 1576402944 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042216248746238715, + "loss": 1.335, + "theoretical_loss": 3.529240001507736, + "tokens_seen": 1576468480 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004221464393179539, + "loss": 1.2271, + "theoretical_loss": 3.529227292432908, + "tokens_seen": 1576534016 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042213039117352055, + "loss": 1.2711, + "theoretical_loss": 3.5292145840343006, + "tokens_seen": 1576599552 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004221143430290873, + "loss": 1.34, + "theoretical_loss": 3.529201876311851, + "tokens_seen": 1576665088 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042209829488465395, + "loss": 1.3374, + "theoretical_loss": 3.5291891692654946, + "tokens_seen": 1576730624 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004220822467402207, + "loss": 1.321, + "theoretical_loss": 3.529176462895168, + "tokens_seen": 1576796160 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042206619859578735, + "loss": 1.3253, + "theoretical_loss": 3.529163757200806, + "tokens_seen": 1576861696 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004220501504513541, + "loss": 1.2801, + "theoretical_loss": 3.5291510521823453, + "tokens_seen": 1576927232 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042203410230692075, + "loss": 1.3328, + "theoretical_loss": 3.529138347839721, + "tokens_seen": 1576992768 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004220180541624875, + "loss": 1.2826, + "theoretical_loss": 3.5291256441728702, + "tokens_seen": 1577058304 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042200200601805415, + "loss": 1.3815, + "theoretical_loss": 3.5291129411817286, + "tokens_seen": 1577123840 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004219859578736209, + "loss": 1.3317, + "theoretical_loss": 3.5291002388662323, + "tokens_seen": 1577189376 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042196990972918756, + "loss": 1.2136, + "theoretical_loss": 3.5290875372263164, + "tokens_seen": 1577254912 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004219538615847543, + "loss": 1.3231, + "theoretical_loss": 3.529074836261917, + "tokens_seen": 1577320448 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042193781344032096, + "loss": 1.32, + "theoretical_loss": 3.5290621359729712, + "tokens_seen": 1577385984 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004219217652958877, + "loss": 1.3707, + "theoretical_loss": 3.5290494363594145, + "tokens_seen": 1577451520 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042190571715145436, + "loss": 1.2402, + "theoretical_loss": 3.5290367374211824, + "tokens_seen": 1577517056 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004218896690070211, + "loss": 1.2562, + "theoretical_loss": 3.5290240391582115, + "tokens_seen": 1577582592 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042187362086258776, + "loss": 1.3859, + "theoretical_loss": 3.5290113415704383, + "tokens_seen": 1577648128 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004218575727181545, + "loss": 1.3273, + "theoretical_loss": 3.5289986446577974, + "tokens_seen": 1577713664 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042184152457372116, + "loss": 1.3851, + "theoretical_loss": 3.528985948420226, + "tokens_seen": 1577779200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004218254764292879, + "loss": 1.3259, + "theoretical_loss": 3.52897325285766, + "tokens_seen": 1577844736 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042180942828485456, + "loss": 1.3317, + "theoretical_loss": 3.528960557970035, + "tokens_seen": 1577910272 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004217933801404213, + "loss": 1.3028, + "theoretical_loss": 3.5289478637572875, + "tokens_seen": 1577975808 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042177733199598796, + "loss": 1.3683, + "theoretical_loss": 3.528935170219354, + "tokens_seen": 1578041344 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042176128385155466, + "loss": 1.2529, + "theoretical_loss": 3.5289224773561703, + "tokens_seen": 1578106880 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042174523570712136, + "loss": 1.2188, + "theoretical_loss": 3.528909785167672, + "tokens_seen": 1578172416 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042172918756268806, + "loss": 1.3758, + "theoretical_loss": 3.528897093653795, + "tokens_seen": 1578237952 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042171313941825477, + "loss": 1.2987, + "theoretical_loss": 3.5288844028144766, + "tokens_seen": 1578303488 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042169709127382147, + "loss": 1.2511, + "theoretical_loss": 3.5288717126496523, + "tokens_seen": 1578369024 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042168104312938817, + "loss": 1.3187, + "theoretical_loss": 3.5288590231592583, + "tokens_seen": 1578434560 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042166499498495487, + "loss": 1.3929, + "theoretical_loss": 3.528846334343231, + "tokens_seen": 1578500096 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042164894684052157, + "loss": 1.3415, + "theoretical_loss": 3.528833646201506, + "tokens_seen": 1578565632 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042163289869608827, + "loss": 1.2815, + "theoretical_loss": 3.52882095873402, + "tokens_seen": 1578631168 + }, + { + "epoch": 0.48, + "learning_rate": 0.000421616850551655, + "loss": 1.3573, + "theoretical_loss": 3.528808271940709, + "tokens_seen": 1578696704 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042160080240722167, + "loss": 1.2086, + "theoretical_loss": 3.5287955858215083, + "tokens_seen": 1578762240 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004215847542627884, + "loss": 1.3041, + "theoretical_loss": 3.5287829003763562, + "tokens_seen": 1578827776 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042156870611835507, + "loss": 1.2911, + "theoretical_loss": 3.5287702156051868, + "tokens_seen": 1578893312 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004215526579739218, + "loss": 1.2865, + "theoretical_loss": 3.528757531507938, + "tokens_seen": 1578958848 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042153660982948847, + "loss": 1.3767, + "theoretical_loss": 3.5287448480845445, + "tokens_seen": 1579024384 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042152056168505523, + "loss": 1.2893, + "theoretical_loss": 3.5287321653349437, + "tokens_seen": 1579089920 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004215045135406219, + "loss": 1.4101, + "theoretical_loss": 3.5287194832590707, + "tokens_seen": 1579155456 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042148846539618863, + "loss": 1.3034, + "theoretical_loss": 3.5287068018568633, + "tokens_seen": 1579220992 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004214724172517553, + "loss": 1.2687, + "theoretical_loss": 3.5286941211282565, + "tokens_seen": 1579286528 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042145636910732203, + "loss": 1.2349, + "theoretical_loss": 3.5286814410731875, + "tokens_seen": 1579352064 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 586437, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2834954261779785, + "objective/train/theoretical_loss": 3.5286687616915913, + "objective/train/tokens_used": 1599877600, + "theoretical_loss": 3.5286687616915913, + "tokens_seen": 1579417600 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004214403209628887, + "loss": 1.2835, + "theoretical_loss": 3.5286687616915913, + "tokens_seen": 1579417600 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042142427281845543, + "loss": 1.3017, + "theoretical_loss": 3.5286560829834057, + "tokens_seen": 1579483136 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004214082246740221, + "loss": 1.3582, + "theoretical_loss": 3.5286434049485655, + "tokens_seen": 1579548672 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042139217652958883, + "loss": 1.2152, + "theoretical_loss": 3.528630727587008, + "tokens_seen": 1579614208 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004213761283851555, + "loss": 1.2982, + "theoretical_loss": 3.52861805089867, + "tokens_seen": 1579679744 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042136008024072223, + "loss": 1.2917, + "theoretical_loss": 3.5286053748834862, + "tokens_seen": 1579745280 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004213440320962889, + "loss": 1.222, + "theoretical_loss": 3.5285926995413943, + "tokens_seen": 1579810816 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042132798395185563, + "loss": 1.3392, + "theoretical_loss": 3.52858002487233, + "tokens_seen": 1579876352 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004213119358074223, + "loss": 1.2554, + "theoretical_loss": 3.5285673508762296, + "tokens_seen": 1579941888 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042129588766298904, + "loss": 1.307, + "theoretical_loss": 3.52855467755303, + "tokens_seen": 1580007424 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004212798395185557, + "loss": 1.322, + "theoretical_loss": 3.528542004902667, + "tokens_seen": 1580072960 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042126379137412244, + "loss": 1.3281, + "theoretical_loss": 3.528529332925077, + "tokens_seen": 1580138496 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004212477432296891, + "loss": 1.3088, + "theoretical_loss": 3.5285166616201966, + "tokens_seen": 1580204032 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042123169508525584, + "loss": 1.3281, + "theoretical_loss": 3.5285039909879625, + "tokens_seen": 1580269568 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004212156469408225, + "loss": 1.2537, + "theoretical_loss": 3.5284913210283113, + "tokens_seen": 1580335104 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042119959879638924, + "loss": 1.2006, + "theoretical_loss": 3.528478651741178, + "tokens_seen": 1580400640 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004211835506519559, + "loss": 1.256, + "theoretical_loss": 3.5284659831265, + "tokens_seen": 1580466176 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042116750250752264, + "loss": 1.2834, + "theoretical_loss": 3.528453315184214, + "tokens_seen": 1580531712 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004211514543630893, + "loss": 1.3827, + "theoretical_loss": 3.528440647914256, + "tokens_seen": 1580597248 + }, + { + "epoch": 0.48, + "learning_rate": 0.000421135406218656, + "loss": 1.298, + "theoretical_loss": 3.528427981316562, + "tokens_seen": 1580662784 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004211193580742227, + "loss": 1.3531, + "theoretical_loss": 3.5284153153910696, + "tokens_seen": 1580728320 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004211033099297894, + "loss": 1.294, + "theoretical_loss": 3.528402650137714, + "tokens_seen": 1580793856 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004210872617853561, + "loss": 1.3045, + "theoretical_loss": 3.528389985556433, + "tokens_seen": 1580859392 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004210712136409228, + "loss": 1.2778, + "theoretical_loss": 3.528377321647162, + "tokens_seen": 1580924928 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004210551654964895, + "loss": 1.2996, + "theoretical_loss": 3.5283646584098376, + "tokens_seen": 1580990464 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004210391173520562, + "loss": 1.3461, + "theoretical_loss": 3.528351995844397, + "tokens_seen": 1581056000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004210230692076229, + "loss": 1.3763, + "theoretical_loss": 3.5283393339507763, + "tokens_seen": 1581121536 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004210070210631896, + "loss": 1.3895, + "theoretical_loss": 3.5283266727289115, + "tokens_seen": 1581187072 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042099097291875624, + "loss": 1.3256, + "theoretical_loss": 3.5283140121787397, + "tokens_seen": 1581252608 + }, + { + "epoch": 0.48, + "learning_rate": 0.000420974924774323, + "loss": 1.3118, + "theoretical_loss": 3.5283013523001974, + "tokens_seen": 1581318144 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042095887662988964, + "loss": 1.3778, + "theoretical_loss": 3.528288693093221, + "tokens_seen": 1581383680 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004209428284854564, + "loss": 1.24, + "theoretical_loss": 3.528276034557747, + "tokens_seen": 1581449216 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042092678034102304, + "loss": 1.3254, + "theoretical_loss": 3.5282633766937126, + "tokens_seen": 1581514752 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004209107321965898, + "loss": 1.3124, + "theoretical_loss": 3.5282507195010533, + "tokens_seen": 1581580288 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042089468405215644, + "loss": 1.2652, + "theoretical_loss": 3.528238062979706, + "tokens_seen": 1581645824 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004208786359077232, + "loss": 1.3802, + "theoretical_loss": 3.5282254071296077, + "tokens_seen": 1581711360 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042086258776328984, + "loss": 1.325, + "theoretical_loss": 3.528212751950695, + "tokens_seen": 1581776896 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004208465396188566, + "loss": 1.3784, + "theoretical_loss": 3.528200097442904, + "tokens_seen": 1581842432 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042083049147442325, + "loss": 1.3746, + "theoretical_loss": 3.5281874436061713, + "tokens_seen": 1581907968 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042081444332999, + "loss": 1.3256, + "theoretical_loss": 3.5281747904404344, + "tokens_seen": 1581973504 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042079839518555665, + "loss": 1.3358, + "theoretical_loss": 3.528162137945629, + "tokens_seen": 1582039040 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004207823470411234, + "loss": 1.3135, + "theoretical_loss": 3.528149486121692, + "tokens_seen": 1582104576 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042076629889669005, + "loss": 1.3064, + "theoretical_loss": 3.52813683496856, + "tokens_seen": 1582170112 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004207502507522568, + "loss": 1.2834, + "theoretical_loss": 3.52812418448617, + "tokens_seen": 1582235648 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042073420260782345, + "loss": 1.3562, + "theoretical_loss": 3.528111534674458, + "tokens_seen": 1582301184 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004207181544633902, + "loss": 1.3778, + "theoretical_loss": 3.5280988855333613, + "tokens_seen": 1582366720 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042070210631895685, + "loss": 1.4375, + "theoretical_loss": 3.5280862370628165, + "tokens_seen": 1582432256 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004206860581745236, + "loss": 1.3089, + "theoretical_loss": 3.5280735892627595, + "tokens_seen": 1582497792 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042067001003009025, + "loss": 1.27, + "theoretical_loss": 3.528060942133128, + "tokens_seen": 1582563328 + }, + { + "epoch": 0.48, + "learning_rate": 0.000420653961885657, + "loss": 1.3274, + "theoretical_loss": 3.528048295673859, + "tokens_seen": 1582628864 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042063791374122365, + "loss": 1.234, + "theoretical_loss": 3.5280356498848873, + "tokens_seen": 1582694400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004206218655967904, + "loss": 1.3874, + "theoretical_loss": 3.528023004766151, + "tokens_seen": 1582759936 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042060581745235705, + "loss": 1.3804, + "theoretical_loss": 3.5280103603175874, + "tokens_seen": 1582825472 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004205897693079238, + "loss": 1.3994, + "theoretical_loss": 3.527997716539132, + "tokens_seen": 1582891008 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042057372116349046, + "loss": 1.3643, + "theoretical_loss": 3.527985073430722, + "tokens_seen": 1582956544 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004205576730190572, + "loss": 1.2669, + "theoretical_loss": 3.5279724309922944, + "tokens_seen": 1583022080 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042054162487462386, + "loss": 1.3695, + "theoretical_loss": 3.527959789223786, + "tokens_seen": 1583087616 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004205255767301906, + "loss": 1.248, + "theoretical_loss": 3.5279471481251328, + "tokens_seen": 1583153152 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042050952858575726, + "loss": 1.3518, + "theoretical_loss": 3.5279345076962727, + "tokens_seen": 1583218688 + }, + { + "epoch": 0.48, + "learning_rate": 0.000420493480441324, + "loss": 1.3196, + "theoretical_loss": 3.527921867937142, + "tokens_seen": 1583284224 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042047743229689066, + "loss": 1.3646, + "theoretical_loss": 3.527909228847676, + "tokens_seen": 1583349760 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004204613841524574, + "loss": 1.3256, + "theoretical_loss": 3.5278965904278143, + "tokens_seen": 1583415296 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042044533600802406, + "loss": 1.3344, + "theoretical_loss": 3.5278839526774917, + "tokens_seen": 1583480832 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004204292878635908, + "loss": 1.2877, + "theoretical_loss": 3.527871315596646, + "tokens_seen": 1583546368 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004204132397191575, + "loss": 1.2645, + "theoretical_loss": 3.527858679185213, + "tokens_seen": 1583611904 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004203971915747242, + "loss": 1.3215, + "theoretical_loss": 3.527846043443131, + "tokens_seen": 1583677440 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004203811434302909, + "loss": 1.2919, + "theoretical_loss": 3.527833408370336, + "tokens_seen": 1583742976 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042036509528585756, + "loss": 1.4344, + "theoretical_loss": 3.527820773966764, + "tokens_seen": 1583808512 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004203490471414243, + "loss": 1.3252, + "theoretical_loss": 3.5278081402323536, + "tokens_seen": 1583874048 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042033299899699097, + "loss": 1.2702, + "theoretical_loss": 3.5277955071670406, + "tokens_seen": 1583939584 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004203169508525577, + "loss": 1.3587, + "theoretical_loss": 3.5277828747707622, + "tokens_seen": 1584005120 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042030090270812437, + "loss": 1.2734, + "theoretical_loss": 3.527770243043455, + "tokens_seen": 1584070656 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004202848545636911, + "loss": 1.3073, + "theoretical_loss": 3.5277576119850567, + "tokens_seen": 1584136192 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042026880641925777, + "loss": 1.23, + "theoretical_loss": 3.527744981595503, + "tokens_seen": 1584201728 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004202527582748245, + "loss": 1.3311, + "theoretical_loss": 3.5277323518747314, + "tokens_seen": 1584267264 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042023671013039117, + "loss": 1.3151, + "theoretical_loss": 3.5277197228226793, + "tokens_seen": 1584332800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004202206619859579, + "loss": 1.2316, + "theoretical_loss": 3.5277070944392825, + "tokens_seen": 1584398336 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042020461384152457, + "loss": 1.2725, + "theoretical_loss": 3.5276944667244794, + "tokens_seen": 1584463872 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004201885656970913, + "loss": 1.3644, + "theoretical_loss": 3.527681839678206, + "tokens_seen": 1584529408 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042017251755265797, + "loss": 1.2839, + "theoretical_loss": 3.527669213300399, + "tokens_seen": 1584594944 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004201564694082247, + "loss": 1.2589, + "theoretical_loss": 3.5276565875909967, + "tokens_seen": 1584660480 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004201404212637914, + "loss": 1.2974, + "theoretical_loss": 3.5276439625499343, + "tokens_seen": 1584726016 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042012437311935813, + "loss": 1.3448, + "theoretical_loss": 3.5276313381771502, + "tokens_seen": 1584791552 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004201083249749248, + "loss": 1.3524, + "theoretical_loss": 3.5276187144725806, + "tokens_seen": 1584857088 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042009227683049153, + "loss": 1.3019, + "theoretical_loss": 3.527606091436163, + "tokens_seen": 1584922624 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004200762286860582, + "loss": 1.3561, + "theoretical_loss": 3.527593469067834, + "tokens_seen": 1584988160 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042006018054162493, + "loss": 1.3409, + "theoretical_loss": 3.5275808473675307, + "tokens_seen": 1585053696 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004200441323971916, + "loss": 1.3741, + "theoretical_loss": 3.52756822633519, + "tokens_seen": 1585119232 + }, + { + "epoch": 0.48, + "learning_rate": 0.00042002808425275833, + "loss": 1.253, + "theoretical_loss": 3.52755560597075, + "tokens_seen": 1585184768 + }, + { + "epoch": 0.48, + "learning_rate": 0.000420012036108325, + "loss": 1.3916, + "theoretical_loss": 3.5275429862741463, + "tokens_seen": 1585250304 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041999598796389173, + "loss": 1.2705, + "theoretical_loss": 3.5275303672453164, + "tokens_seen": 1585315840 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004199799398194584, + "loss": 1.2498, + "theoretical_loss": 3.5275177488841982, + "tokens_seen": 1585381376 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041996389167502513, + "loss": 1.2603, + "theoretical_loss": 3.5275051311907277, + "tokens_seen": 1585446912 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004199478435305918, + "loss": 1.3169, + "theoretical_loss": 3.5274925141648423, + "tokens_seen": 1585512448 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041993179538615854, + "loss": 1.4256, + "theoretical_loss": 3.5274798978064794, + "tokens_seen": 1585577984 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004199157472417252, + "loss": 1.3848, + "theoretical_loss": 3.527467282115576, + "tokens_seen": 1585643520 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041989969909729194, + "loss": 1.3088, + "theoretical_loss": 3.5274546670920683, + "tokens_seen": 1585709056 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004198836509528586, + "loss": 1.2542, + "theoretical_loss": 3.5274420527358945, + "tokens_seen": 1585774592 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041986760280842534, + "loss": 1.2191, + "theoretical_loss": 3.5274294390469914, + "tokens_seen": 1585840128 + }, + { + "epoch": 0.48, + "learning_rate": 0.000419851554663992, + "loss": 1.2554, + "theoretical_loss": 3.5274168260252967, + "tokens_seen": 1585905664 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 588943, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.287621259689331, + "objective/train/theoretical_loss": 3.527404213670746, + "objective/train/tokens_used": 1606431200, + "theoretical_loss": 3.527404213670746, + "tokens_seen": 1585971200 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041983550651955874, + "loss": 1.2876, + "theoretical_loss": 3.527404213670746, + "tokens_seen": 1585971200 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004198194583751254, + "loss": 1.3695, + "theoretical_loss": 3.527391601983278, + "tokens_seen": 1586036736 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041980341023069214, + "loss": 1.2826, + "theoretical_loss": 3.527378990962829, + "tokens_seen": 1586102272 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004197873620862588, + "loss": 1.2875, + "theoretical_loss": 3.5273663806093367, + "tokens_seen": 1586167808 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041977131394182554, + "loss": 1.3633, + "theoretical_loss": 3.527353770922738, + "tokens_seen": 1586233344 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004197552657973922, + "loss": 1.3142, + "theoretical_loss": 3.52734116190297, + "tokens_seen": 1586298880 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004197392176529589, + "loss": 1.2971, + "theoretical_loss": 3.5273285535499697, + "tokens_seen": 1586364416 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004197231695085256, + "loss": 1.2324, + "theoretical_loss": 3.5273159458636747, + "tokens_seen": 1586429952 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004197071213640923, + "loss": 1.3826, + "theoretical_loss": 3.527303338844022, + "tokens_seen": 1586495488 + }, + { + "epoch": 0.48, + "learning_rate": 0.000419691073219659, + "loss": 1.3842, + "theoretical_loss": 3.527290732490949, + "tokens_seen": 1586561024 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004196750250752257, + "loss": 1.311, + "theoretical_loss": 3.5272781268043927, + "tokens_seen": 1586626560 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004196589769307924, + "loss": 1.2892, + "theoretical_loss": 3.5272655217842908, + "tokens_seen": 1586692096 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004196429287863591, + "loss": 1.3726, + "theoretical_loss": 3.5272529174305802, + "tokens_seen": 1586757632 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004196268806419258, + "loss": 1.325, + "theoretical_loss": 3.5272403137431976, + "tokens_seen": 1586823168 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004196108324974925, + "loss": 1.2967, + "theoretical_loss": 3.527227710722081, + "tokens_seen": 1586888704 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041959478435305914, + "loss": 1.3747, + "theoretical_loss": 3.5272151083671677, + "tokens_seen": 1586954240 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004195787362086259, + "loss": 1.4032, + "theoretical_loss": 3.5272025066783943, + "tokens_seen": 1587019776 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041956268806419254, + "loss": 1.2848, + "theoretical_loss": 3.527189905655699, + "tokens_seen": 1587085312 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004195466399197593, + "loss": 1.3219, + "theoretical_loss": 3.527177305299018, + "tokens_seen": 1587150848 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041953059177532594, + "loss": 1.4009, + "theoretical_loss": 3.5271647056082895, + "tokens_seen": 1587216384 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004195145436308927, + "loss": 1.3308, + "theoretical_loss": 3.5271521065834506, + "tokens_seen": 1587281920 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041949849548645934, + "loss": 1.3512, + "theoretical_loss": 3.5271395082244386, + "tokens_seen": 1587347456 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004194824473420261, + "loss": 1.3937, + "theoretical_loss": 3.52712691053119, + "tokens_seen": 1587412992 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041946639919759275, + "loss": 1.3933, + "theoretical_loss": 3.5271143135036436, + "tokens_seen": 1587478528 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004194503510531595, + "loss": 1.2842, + "theoretical_loss": 3.5271017171417363, + "tokens_seen": 1587544064 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041943430290872615, + "loss": 1.2621, + "theoretical_loss": 3.5270891214454045, + "tokens_seen": 1587609600 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004194182547642929, + "loss": 1.2614, + "theoretical_loss": 3.5270765264145867, + "tokens_seen": 1587675136 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041940220661985955, + "loss": 1.2908, + "theoretical_loss": 3.527063932049219, + "tokens_seen": 1587740672 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004193861584754263, + "loss": 1.2792, + "theoretical_loss": 3.5270513383492403, + "tokens_seen": 1587806208 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041937011033099295, + "loss": 1.3476, + "theoretical_loss": 3.5270387453145875, + "tokens_seen": 1587871744 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004193540621865597, + "loss": 1.4119, + "theoretical_loss": 3.527026152945197, + "tokens_seen": 1587937280 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041933801404212635, + "loss": 1.3528, + "theoretical_loss": 3.5270135612410076, + "tokens_seen": 1588002816 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004193219658976931, + "loss": 1.3456, + "theoretical_loss": 3.5270009702019562, + "tokens_seen": 1588068352 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041930591775325975, + "loss": 1.3317, + "theoretical_loss": 3.5269883798279795, + "tokens_seen": 1588133888 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004192898696088265, + "loss": 1.3463, + "theoretical_loss": 3.5269757901190157, + "tokens_seen": 1588199424 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041927382146439315, + "loss": 1.3669, + "theoretical_loss": 3.5269632010750023, + "tokens_seen": 1588264960 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004192577733199599, + "loss": 1.3518, + "theoretical_loss": 3.526950612695876, + "tokens_seen": 1588330496 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041924172517552655, + "loss": 1.2739, + "theoretical_loss": 3.5269380249815754, + "tokens_seen": 1588396032 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004192256770310933, + "loss": 1.329, + "theoretical_loss": 3.5269254379320367, + "tokens_seen": 1588461568 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041920962888666006, + "loss": 1.2787, + "theoretical_loss": 3.5269128515471984, + "tokens_seen": 1588527104 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004191935807422267, + "loss": 1.246, + "theoretical_loss": 3.5269002658269972, + "tokens_seen": 1588592640 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041917753259779347, + "loss": 1.3675, + "theoretical_loss": 3.526887680771371, + "tokens_seen": 1588658176 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004191614844533601, + "loss": 1.2414, + "theoretical_loss": 3.526875096380257, + "tokens_seen": 1588723712 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041914543630892687, + "loss": 1.2701, + "theoretical_loss": 3.5268625126535937, + "tokens_seen": 1588789248 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004191293881644935, + "loss": 1.2704, + "theoretical_loss": 3.526849929591317, + "tokens_seen": 1588854784 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041911334002006027, + "loss": 1.3256, + "theoretical_loss": 3.526837347193366, + "tokens_seen": 1588920320 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004190972918756269, + "loss": 1.3113, + "theoretical_loss": 3.5268247654596774, + "tokens_seen": 1588985856 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004190812437311936, + "loss": 1.3287, + "theoretical_loss": 3.5268121843901885, + "tokens_seen": 1589051392 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004190651955867603, + "loss": 1.3691, + "theoretical_loss": 3.526799603984837, + "tokens_seen": 1589116928 + }, + { + "epoch": 0.48, + "learning_rate": 0.000419049147442327, + "loss": 1.406, + "theoretical_loss": 3.526787024243561, + "tokens_seen": 1589182464 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004190330992978937, + "loss": 1.2516, + "theoretical_loss": 3.5267744451662977, + "tokens_seen": 1589248000 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004190170511534604, + "loss": 1.304, + "theoretical_loss": 3.5267618667529845, + "tokens_seen": 1589313536 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004190010030090271, + "loss": 1.3301, + "theoretical_loss": 3.5267492890035594, + "tokens_seen": 1589379072 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004189849548645938, + "loss": 1.3119, + "theoretical_loss": 3.52673671191796, + "tokens_seen": 1589444608 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041896890672016047, + "loss": 1.252, + "theoretical_loss": 3.5267241354961225, + "tokens_seen": 1589510144 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004189528585757272, + "loss": 1.3636, + "theoretical_loss": 3.5267115597379863, + "tokens_seen": 1589575680 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041893681043129387, + "loss": 1.3415, + "theoretical_loss": 3.5266989846434886, + "tokens_seen": 1589641216 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004189207622868606, + "loss": 1.3117, + "theoretical_loss": 3.526686410212567, + "tokens_seen": 1589706752 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041890471414242727, + "loss": 1.2508, + "theoretical_loss": 3.526673836445158, + "tokens_seen": 1589772288 + }, + { + "epoch": 0.48, + "learning_rate": 0.000418888665997994, + "loss": 1.3127, + "theoretical_loss": 3.5266612633412007, + "tokens_seen": 1589837824 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041887261785356067, + "loss": 1.2989, + "theoretical_loss": 3.526648690900632, + "tokens_seen": 1589903360 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004188565697091274, + "loss": 1.2637, + "theoretical_loss": 3.52663611912339, + "tokens_seen": 1589968896 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041884052156469407, + "loss": 1.3204, + "theoretical_loss": 3.526623548009412, + "tokens_seen": 1590034432 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004188244734202608, + "loss": 1.2925, + "theoretical_loss": 3.5266109775586356, + "tokens_seen": 1590099968 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041880842527582747, + "loss": 1.1811, + "theoretical_loss": 3.5265984077709986, + "tokens_seen": 1590165504 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004187923771313942, + "loss": 1.3335, + "theoretical_loss": 3.526585838646439, + "tokens_seen": 1590231040 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004187763289869609, + "loss": 1.3356, + "theoretical_loss": 3.526573270184894, + "tokens_seen": 1590296576 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041876028084252763, + "loss": 1.349, + "theoretical_loss": 3.526560702386302, + "tokens_seen": 1590362112 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004187442326980943, + "loss": 1.3598, + "theoretical_loss": 3.5265481352505996, + "tokens_seen": 1590427648 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041872818455366103, + "loss": 1.3674, + "theoretical_loss": 3.5265355687777253, + "tokens_seen": 1590493184 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004187121364092277, + "loss": 1.378, + "theoretical_loss": 3.5265230029676173, + "tokens_seen": 1590558720 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041869608826479443, + "loss": 1.3376, + "theoretical_loss": 3.526510437820212, + "tokens_seen": 1590624256 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004186800401203611, + "loss": 1.3281, + "theoretical_loss": 3.5264978733354484, + "tokens_seen": 1590689792 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041866399197592783, + "loss": 1.3002, + "theoretical_loss": 3.526485309513264, + "tokens_seen": 1590755328 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004186479438314945, + "loss": 1.4147, + "theoretical_loss": 3.526472746353596, + "tokens_seen": 1590820864 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041863189568706123, + "loss": 1.237, + "theoretical_loss": 3.5264601838563823, + "tokens_seen": 1590886400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004186158475426279, + "loss": 1.3399, + "theoretical_loss": 3.526447622021561, + "tokens_seen": 1590951936 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041859979939819463, + "loss": 1.3163, + "theoretical_loss": 3.52643506084907, + "tokens_seen": 1591017472 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004185837512537613, + "loss": 1.2974, + "theoretical_loss": 3.526422500338846, + "tokens_seen": 1591083008 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041856770310932804, + "loss": 1.2598, + "theoretical_loss": 3.5264099404908285, + "tokens_seen": 1591148544 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004185516549648947, + "loss": 1.3827, + "theoretical_loss": 3.5263973813049545, + "tokens_seen": 1591214080 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041853560682046144, + "loss": 1.2357, + "theoretical_loss": 3.5263848227811616, + "tokens_seen": 1591279616 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004185195586760281, + "loss": 1.2101, + "theoretical_loss": 3.526372264919388, + "tokens_seen": 1591345152 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041850351053159484, + "loss": 1.2136, + "theoretical_loss": 3.526359707719571, + "tokens_seen": 1591410688 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004184874623871615, + "loss": 1.3127, + "theoretical_loss": 3.5263471511816493, + "tokens_seen": 1591476224 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041847141424272824, + "loss": 1.2839, + "theoretical_loss": 3.52633459530556, + "tokens_seen": 1591541760 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004184553660982949, + "loss": 1.3111, + "theoretical_loss": 3.5263220400912414, + "tokens_seen": 1591607296 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041843931795386164, + "loss": 1.2805, + "theoretical_loss": 3.526309485538631, + "tokens_seen": 1591672832 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004184232698094283, + "loss": 1.2986, + "theoretical_loss": 3.5262969316476673, + "tokens_seen": 1591738368 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041840722166499504, + "loss": 1.2732, + "theoretical_loss": 3.5262843784182873, + "tokens_seen": 1591803904 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004183911735205617, + "loss": 1.4415, + "theoretical_loss": 3.5262718258504298, + "tokens_seen": 1591869440 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041837512537612844, + "loss": 1.2069, + "theoretical_loss": 3.5262592739440324, + "tokens_seen": 1591934976 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004183590772316951, + "loss": 1.3339, + "theoretical_loss": 3.5262467226990326, + "tokens_seen": 1592000512 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041834302908726184, + "loss": 1.3374, + "theoretical_loss": 3.5262341721153687, + "tokens_seen": 1592066048 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004183269809428285, + "loss": 1.4158, + "theoretical_loss": 3.5262216221929785, + "tokens_seen": 1592131584 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004183109327983952, + "loss": 1.2691, + "theoretical_loss": 3.5262090729318, + "tokens_seen": 1592197120 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004182948846539619, + "loss": 1.327, + "theoretical_loss": 3.5261965243317714, + "tokens_seen": 1592262656 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004182788365095286, + "loss": 1.3436, + "theoretical_loss": 3.52618397639283, + "tokens_seen": 1592328192 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004182627883650953, + "loss": 1.3218, + "theoretical_loss": 3.526171429114915, + "tokens_seen": 1592393728 + }, + { + "epoch": 0.48, + "learning_rate": 0.000418246740220662, + "loss": 1.408, + "theoretical_loss": 3.526158882497963, + "tokens_seen": 1592459264 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 591321, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3383864164352417, + "objective/train/theoretical_loss": 3.5261463365419123, + "objective/train/tokens_used": 1612984800, + "theoretical_loss": 3.5261463365419123, + "tokens_seen": 1592524800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004182306920762287, + "loss": 1.3384, + "theoretical_loss": 3.5261463365419123, + "tokens_seen": 1592524800 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004182146439317954, + "loss": 1.3451, + "theoretical_loss": 3.5261337912467017, + "tokens_seen": 1592590336 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041819859578736204, + "loss": 1.2207, + "theoretical_loss": 3.5261212466122682, + "tokens_seen": 1592655872 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004181825476429288, + "loss": 1.3379, + "theoretical_loss": 3.5261087026385507, + "tokens_seen": 1592721408 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041816649949849544, + "loss": 1.2967, + "theoretical_loss": 3.526096159325486, + "tokens_seen": 1592786944 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004181504513540622, + "loss": 1.357, + "theoretical_loss": 3.526083616673014, + "tokens_seen": 1592852480 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041813440320962884, + "loss": 1.2599, + "theoretical_loss": 3.526071074681071, + "tokens_seen": 1592918016 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004181183550651956, + "loss": 1.3213, + "theoretical_loss": 3.5260585333495955, + "tokens_seen": 1592983552 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041810230692076225, + "loss": 1.2666, + "theoretical_loss": 3.526045992678526, + "tokens_seen": 1593049088 + }, + { + "epoch": 0.48, + "learning_rate": 0.000418086258776329, + "loss": 1.1966, + "theoretical_loss": 3.5260334526678, + "tokens_seen": 1593114624 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041807021063189565, + "loss": 1.3158, + "theoretical_loss": 3.526020913317356, + "tokens_seen": 1593180160 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004180541624874624, + "loss": 1.3764, + "theoretical_loss": 3.526008374627132, + "tokens_seen": 1593245696 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041803811434302905, + "loss": 1.3585, + "theoretical_loss": 3.525995836597066, + "tokens_seen": 1593311232 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004180220661985958, + "loss": 1.3073, + "theoretical_loss": 3.5259832992270956, + "tokens_seen": 1593376768 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041800601805416256, + "loss": 1.3278, + "theoretical_loss": 3.52597076251716, + "tokens_seen": 1593442304 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004179899699097292, + "loss": 1.3676, + "theoretical_loss": 3.5259582264671963, + "tokens_seen": 1593507840 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041797392176529596, + "loss": 1.3002, + "theoretical_loss": 3.525945691077143, + "tokens_seen": 1593573376 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004179578736208626, + "loss": 1.2793, + "theoretical_loss": 3.5259331563469387, + "tokens_seen": 1593638912 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041794182547642936, + "loss": 1.344, + "theoretical_loss": 3.525920622276521, + "tokens_seen": 1593704448 + }, + { + "epoch": 0.48, + "learning_rate": 0.000417925777331996, + "loss": 1.3026, + "theoretical_loss": 3.525908088865828, + "tokens_seen": 1593769984 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041790972918756276, + "loss": 1.2824, + "theoretical_loss": 3.5258955561147975, + "tokens_seen": 1593835520 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004178936810431294, + "loss": 1.3426, + "theoretical_loss": 3.5258830240233685, + "tokens_seen": 1593901056 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041787763289869616, + "loss": 1.2772, + "theoretical_loss": 3.525870492591479, + "tokens_seen": 1593966592 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004178615847542628, + "loss": 1.3424, + "theoretical_loss": 3.5258579618190664, + "tokens_seen": 1594032128 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041784553660982956, + "loss": 1.2876, + "theoretical_loss": 3.5258454317060695, + "tokens_seen": 1594097664 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004178294884653962, + "loss": 1.2681, + "theoretical_loss": 3.525832902252427, + "tokens_seen": 1594163200 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041781344032096297, + "loss": 1.3903, + "theoretical_loss": 3.5258203734580764, + "tokens_seen": 1594228736 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004177973921765296, + "loss": 1.3244, + "theoretical_loss": 3.5258078453229555, + "tokens_seen": 1594294272 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041778134403209637, + "loss": 1.2566, + "theoretical_loss": 3.5257953178470034, + "tokens_seen": 1594359808 + }, + { + "epoch": 0.48, + "learning_rate": 0.000417765295887663, + "loss": 1.2349, + "theoretical_loss": 3.5257827910301582, + "tokens_seen": 1594425344 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041774924774322977, + "loss": 1.3647, + "theoretical_loss": 3.5257702648723575, + "tokens_seen": 1594490880 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004177331995987964, + "loss": 1.3691, + "theoretical_loss": 3.5257577393735406, + "tokens_seen": 1594556416 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041771715145436317, + "loss": 1.2912, + "theoretical_loss": 3.5257452145336443, + "tokens_seen": 1594621952 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004177011033099298, + "loss": 1.4021, + "theoretical_loss": 3.525732690352608, + "tokens_seen": 1594687488 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004176850551654965, + "loss": 1.2753, + "theoretical_loss": 3.5257201668303697, + "tokens_seen": 1594753024 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004176690070210632, + "loss": 1.271, + "theoretical_loss": 3.525707643966868, + "tokens_seen": 1594818560 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004176529588766299, + "loss": 1.2395, + "theoretical_loss": 3.5256951217620403, + "tokens_seen": 1594884096 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004176369107321966, + "loss": 1.2615, + "theoretical_loss": 3.5256826002158252, + "tokens_seen": 1594949632 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004176208625877633, + "loss": 1.2972, + "theoretical_loss": 3.5256700793281617, + "tokens_seen": 1595015168 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041760481444333, + "loss": 1.3576, + "theoretical_loss": 3.5256575590989874, + "tokens_seen": 1595080704 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004175887662988967, + "loss": 1.3646, + "theoretical_loss": 3.5256450395282406, + "tokens_seen": 1595146240 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004175727181544634, + "loss": 1.3517, + "theoretical_loss": 3.52563252061586, + "tokens_seen": 1595211776 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004175566700100301, + "loss": 1.236, + "theoretical_loss": 3.5256200023617836, + "tokens_seen": 1595277312 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041754062186559677, + "loss": 1.3408, + "theoretical_loss": 3.52560748476595, + "tokens_seen": 1595342848 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004175245737211635, + "loss": 1.2425, + "theoretical_loss": 3.5255949678282974, + "tokens_seen": 1595408384 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041750852557673017, + "loss": 1.3148, + "theoretical_loss": 3.5255824515487646, + "tokens_seen": 1595473920 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004174924774322969, + "loss": 1.3848, + "theoretical_loss": 3.5255699359272894, + "tokens_seen": 1595539456 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041747642928786357, + "loss": 1.267, + "theoretical_loss": 3.52555742096381, + "tokens_seen": 1595604992 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004174603811434303, + "loss": 1.2142, + "theoretical_loss": 3.5255449066582654, + "tokens_seen": 1595670528 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041744433299899697, + "loss": 1.3215, + "theoretical_loss": 3.525532393010594, + "tokens_seen": 1595736064 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004174282848545637, + "loss": 1.2947, + "theoretical_loss": 3.5255198800207332, + "tokens_seen": 1595801600 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041741223671013037, + "loss": 1.2356, + "theoretical_loss": 3.525507367688623, + "tokens_seen": 1595867136 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041739618856569713, + "loss": 1.2874, + "theoretical_loss": 3.5254948560142, + "tokens_seen": 1595932672 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004173801404212638, + "loss": 1.2166, + "theoretical_loss": 3.525482344997404, + "tokens_seen": 1595998208 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041736409227683053, + "loss": 1.251, + "theoretical_loss": 3.5254698346381734, + "tokens_seen": 1596063744 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004173480441323972, + "loss": 1.3453, + "theoretical_loss": 3.525457324936446, + "tokens_seen": 1596129280 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041733199598796393, + "loss": 1.293, + "theoretical_loss": 3.5254448158921603, + "tokens_seen": 1596194816 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004173159478435306, + "loss": 1.2848, + "theoretical_loss": 3.5254323075052554, + "tokens_seen": 1596260352 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041729989969909733, + "loss": 1.3092, + "theoretical_loss": 3.525419799775669, + "tokens_seen": 1596325888 + }, + { + "epoch": 0.48, + "learning_rate": 0.000417283851554664, + "loss": 1.3667, + "theoretical_loss": 3.52540729270334, + "tokens_seen": 1596391424 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041726780341023073, + "loss": 1.303, + "theoretical_loss": 3.5253947862882065, + "tokens_seen": 1596456960 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004172517552657974, + "loss": 1.2352, + "theoretical_loss": 3.5253822805302075, + "tokens_seen": 1596522496 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041723570712136413, + "loss": 1.3134, + "theoretical_loss": 3.525369775429281, + "tokens_seen": 1596588032 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004172196589769308, + "loss": 1.3988, + "theoretical_loss": 3.525357270985366, + "tokens_seen": 1596653568 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041720361083249754, + "loss": 1.3491, + "theoretical_loss": 3.5253447671984013, + "tokens_seen": 1596719104 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004171875626880642, + "loss": 1.3041, + "theoretical_loss": 3.5253322640683242, + "tokens_seen": 1596784640 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041717151454363094, + "loss": 1.2904, + "theoretical_loss": 3.5253197615950747, + "tokens_seen": 1596850176 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004171554663991976, + "loss": 1.3045, + "theoretical_loss": 3.5253072597785895, + "tokens_seen": 1596915712 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041713941825476434, + "loss": 1.289, + "theoretical_loss": 3.5252947586188093, + "tokens_seen": 1596981248 + }, + { + "epoch": 0.48, + "learning_rate": 0.000417123370110331, + "loss": 1.3831, + "theoretical_loss": 3.525282258115671, + "tokens_seen": 1597046784 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041710732196589774, + "loss": 1.2038, + "theoretical_loss": 3.525269758269114, + "tokens_seen": 1597112320 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004170912738214644, + "loss": 1.3905, + "theoretical_loss": 3.5252572590790763, + "tokens_seen": 1597177856 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041707522567703114, + "loss": 1.2612, + "theoretical_loss": 3.5252447605454966, + "tokens_seen": 1597243392 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004170591775325978, + "loss": 1.3009, + "theoretical_loss": 3.5252322626683146, + "tokens_seen": 1597308928 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041704312938816454, + "loss": 1.4105, + "theoretical_loss": 3.525219765447467, + "tokens_seen": 1597374464 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004170270812437312, + "loss": 1.2694, + "theoretical_loss": 3.525207268882894, + "tokens_seen": 1597440000 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041701103309929794, + "loss": 1.3103, + "theoretical_loss": 3.5251947729745337, + "tokens_seen": 1597505536 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004169949849548646, + "loss": 1.319, + "theoretical_loss": 3.5251822777223243, + "tokens_seen": 1597571072 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041697893681043134, + "loss": 1.2836, + "theoretical_loss": 3.525169783126205, + "tokens_seen": 1597636608 + }, + { + "epoch": 0.48, + "learning_rate": 0.000416962888665998, + "loss": 1.2807, + "theoretical_loss": 3.5251572891861143, + "tokens_seen": 1597702144 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041694684052156475, + "loss": 1.3745, + "theoretical_loss": 3.5251447959019906, + "tokens_seen": 1597767680 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004169307923771314, + "loss": 1.2954, + "theoretical_loss": 3.525132303273773, + "tokens_seen": 1597833216 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004169147442326981, + "loss": 1.4053, + "theoretical_loss": 3.5251198113013995, + "tokens_seen": 1597898752 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004168986960882648, + "loss": 1.349, + "theoretical_loss": 3.5251073199848095, + "tokens_seen": 1597964288 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004168826479438315, + "loss": 1.2516, + "theoretical_loss": 3.5250948293239412, + "tokens_seen": 1598029824 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004168665997993982, + "loss": 1.3364, + "theoretical_loss": 3.5250823393187334, + "tokens_seen": 1598095360 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004168505516549649, + "loss": 1.3124, + "theoretical_loss": 3.5250698499691246, + "tokens_seen": 1598160896 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004168345035105316, + "loss": 1.2718, + "theoretical_loss": 3.5250573612750538, + "tokens_seen": 1598226432 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004168184553660983, + "loss": 1.2821, + "theoretical_loss": 3.52504487323646, + "tokens_seen": 1598291968 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041680240722166505, + "loss": 1.4539, + "theoretical_loss": 3.5250323858532813, + "tokens_seen": 1598357504 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004167863590772317, + "loss": 1.251, + "theoretical_loss": 3.5250198991254567, + "tokens_seen": 1598423040 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041677031093279845, + "loss": 1.3566, + "theoretical_loss": 3.5250074130529248, + "tokens_seen": 1598488576 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004167542627883651, + "loss": 1.3411, + "theoretical_loss": 3.5249949276356247, + "tokens_seen": 1598554112 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041673821464393185, + "loss": 1.4306, + "theoretical_loss": 3.524982442873495, + "tokens_seen": 1598619648 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004167221664994985, + "loss": 1.2856, + "theoretical_loss": 3.5249699587664742, + "tokens_seen": 1598685184 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041670611835506525, + "loss": 1.3559, + "theoretical_loss": 3.524957475314501, + "tokens_seen": 1598750720 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004166900702106319, + "loss": 1.3565, + "theoretical_loss": 3.524944992517515, + "tokens_seen": 1598816256 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041667402206619866, + "loss": 1.317, + "theoretical_loss": 3.5249325103754545, + "tokens_seen": 1598881792 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004166579739217653, + "loss": 1.2849, + "theoretical_loss": 3.524920028888258, + "tokens_seen": 1598947328 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041664192577733206, + "loss": 1.3211, + "theoretical_loss": 3.5249075480558645, + "tokens_seen": 1599012864 + }, + { + "epoch": 0.48, + "objective/train/docs_used": 593753, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3243303298950195, + "objective/train/theoretical_loss": 3.5248950678782127, + "objective/train/tokens_used": 1619538400, + "theoretical_loss": 3.5248950678782127, + "tokens_seen": 1599078400 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004166258776328987, + "loss": 1.3243, + "theoretical_loss": 3.5248950678782127, + "tokens_seen": 1599078400 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041660982948846546, + "loss": 1.3531, + "theoretical_loss": 3.5248825883552417, + "tokens_seen": 1599143936 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004165937813440321, + "loss": 1.2676, + "theoretical_loss": 3.5248701094868906, + "tokens_seen": 1599209472 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041657773319959886, + "loss": 1.3807, + "theoretical_loss": 3.5248576312730973, + "tokens_seen": 1599275008 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004165616850551655, + "loss": 1.3256, + "theoretical_loss": 3.5248451537138012, + "tokens_seen": 1599340544 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041654563691073226, + "loss": 1.357, + "theoretical_loss": 3.5248326768089413, + "tokens_seen": 1599406080 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004165295887662989, + "loss": 1.3344, + "theoretical_loss": 3.524820200558456, + "tokens_seen": 1599471616 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041651354062186566, + "loss": 1.2772, + "theoretical_loss": 3.524807724962285, + "tokens_seen": 1599537152 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004164974924774323, + "loss": 1.2675, + "theoretical_loss": 3.5247952500203663, + "tokens_seen": 1599602688 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041648144433299906, + "loss": 1.2705, + "theoretical_loss": 3.5247827757326395, + "tokens_seen": 1599668224 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004164653961885657, + "loss": 1.2822, + "theoretical_loss": 3.524770302099043, + "tokens_seen": 1599733760 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041644934804413247, + "loss": 1.2832, + "theoretical_loss": 3.5247578291195154, + "tokens_seen": 1599799296 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004164332998996991, + "loss": 1.2863, + "theoretical_loss": 3.5247453567939964, + "tokens_seen": 1599864832 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041641725175526587, + "loss": 1.3362, + "theoretical_loss": 3.5247328851224244, + "tokens_seen": 1599930368 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004164012036108325, + "loss": 1.2708, + "theoretical_loss": 3.5247204141047392, + "tokens_seen": 1599995904 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041638515546639927, + "loss": 1.2662, + "theoretical_loss": 3.524707943740878, + "tokens_seen": 1600061440 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004163691073219659, + "loss": 1.3497, + "theoretical_loss": 3.524695474030781, + "tokens_seen": 1600126976 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041635305917753267, + "loss": 1.4103, + "theoretical_loss": 3.5246830049743876, + "tokens_seen": 1600192512 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004163370110330993, + "loss": 1.24, + "theoretical_loss": 3.5246705365716355, + "tokens_seen": 1600258048 + }, + { + "epoch": 0.48, + "learning_rate": 0.00041632096288866607, + "loss": 1.2809, + "theoretical_loss": 3.5246580688224647, + "tokens_seen": 1600323584 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004163049147442327, + "loss": 1.2633, + "theoretical_loss": 3.524645601726813, + "tokens_seen": 1600389120 + }, + { + "epoch": 0.48, + "learning_rate": 0.0004162888665997994, + "loss": 1.3407, + "theoretical_loss": 3.5246331352846205, + "tokens_seen": 1600454656 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004162728184553661, + "loss": 1.3323, + "theoretical_loss": 3.524620669495826, + "tokens_seen": 1600520192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004162567703109328, + "loss": 1.3551, + "theoretical_loss": 3.524608204360368, + "tokens_seen": 1600585728 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004162407221664995, + "loss": 1.2992, + "theoretical_loss": 3.524595739878186, + "tokens_seen": 1600651264 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004162246740220662, + "loss": 1.2494, + "theoretical_loss": 3.524583276049219, + "tokens_seen": 1600716800 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004162086258776329, + "loss": 1.4061, + "theoretical_loss": 3.5245708128734057, + "tokens_seen": 1600782336 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004161925777331996, + "loss": 1.3562, + "theoretical_loss": 3.524558350350685, + "tokens_seen": 1600847872 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004161765295887663, + "loss": 1.4394, + "theoretical_loss": 3.524545888480996, + "tokens_seen": 1600913408 + }, + { + "epoch": 0.49, + "learning_rate": 0.000416160481444333, + "loss": 1.294, + "theoretical_loss": 3.524533427264279, + "tokens_seen": 1600978944 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041614443329989967, + "loss": 1.2533, + "theoretical_loss": 3.5245209667004715, + "tokens_seen": 1601044480 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004161283851554664, + "loss": 1.3094, + "theoretical_loss": 3.524508506789513, + "tokens_seen": 1601110016 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041611233701103307, + "loss": 1.2869, + "theoretical_loss": 3.5244960475313425, + "tokens_seen": 1601175552 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004160962888665998, + "loss": 1.2924, + "theoretical_loss": 3.5244835889259, + "tokens_seen": 1601241088 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041608024072216647, + "loss": 1.2201, + "theoretical_loss": 3.524471130973123, + "tokens_seen": 1601306624 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004160641925777332, + "loss": 1.2874, + "theoretical_loss": 3.524458673672952, + "tokens_seen": 1601372160 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041604814443329987, + "loss": 1.2541, + "theoretical_loss": 3.524446217025326, + "tokens_seen": 1601437696 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041603209628886663, + "loss": 1.3184, + "theoretical_loss": 3.524433761030183, + "tokens_seen": 1601503232 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004160160481444333, + "loss": 1.3093, + "theoretical_loss": 3.524421305687463, + "tokens_seen": 1601568768 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041600000000000003, + "loss": 1.2712, + "theoretical_loss": 3.524408850997105, + "tokens_seen": 1601634304 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004159839518555667, + "loss": 1.3962, + "theoretical_loss": 3.5243963969590477, + "tokens_seen": 1601699840 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041596790371113343, + "loss": 1.4236, + "theoretical_loss": 3.524383943573231, + "tokens_seen": 1601765376 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004159518555667001, + "loss": 1.3366, + "theoretical_loss": 3.5243714908395933, + "tokens_seen": 1601830912 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041593580742226683, + "loss": 1.2945, + "theoretical_loss": 3.5243590387580745, + "tokens_seen": 1601896448 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004159197592778335, + "loss": 1.3159, + "theoretical_loss": 3.5243465873286137, + "tokens_seen": 1601961984 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041590371113340023, + "loss": 1.237, + "theoretical_loss": 3.5243341365511496, + "tokens_seen": 1602027520 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004158876629889669, + "loss": 1.3009, + "theoretical_loss": 3.5243216864256213, + "tokens_seen": 1602093056 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041587161484453363, + "loss": 1.3357, + "theoretical_loss": 3.524309236951969, + "tokens_seen": 1602158592 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004158555667001003, + "loss": 1.3925, + "theoretical_loss": 3.5242967881301306, + "tokens_seen": 1602224128 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041583951855566703, + "loss": 1.4386, + "theoretical_loss": 3.5242843399600456, + "tokens_seen": 1602289664 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004158234704112337, + "loss": 1.2973, + "theoretical_loss": 3.524271892441654, + "tokens_seen": 1602355200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041580742226680044, + "loss": 1.3078, + "theoretical_loss": 3.5242594455748946, + "tokens_seen": 1602420736 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004157913741223671, + "loss": 1.3035, + "theoretical_loss": 3.5242469993597068, + "tokens_seen": 1602486272 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041577532597793384, + "loss": 1.2434, + "theoretical_loss": 3.5242345537960293, + "tokens_seen": 1602551808 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004157592778335005, + "loss": 1.3245, + "theoretical_loss": 3.5242221088838015, + "tokens_seen": 1602617344 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041574322968906724, + "loss": 1.2838, + "theoretical_loss": 3.5242096646229633, + "tokens_seen": 1602682880 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004157271815446339, + "loss": 1.3432, + "theoretical_loss": 3.5241972210134533, + "tokens_seen": 1602748416 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041571113340020064, + "loss": 1.3614, + "theoretical_loss": 3.5241847780552114, + "tokens_seen": 1602813952 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004156950852557673, + "loss": 1.1529, + "theoretical_loss": 3.524172335748176, + "tokens_seen": 1602879488 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041567903711133404, + "loss": 1.305, + "theoretical_loss": 3.5241598940922874, + "tokens_seen": 1602945024 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004156629889669007, + "loss": 1.3915, + "theoretical_loss": 3.524147453087484, + "tokens_seen": 1603010560 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041564694082246744, + "loss": 1.2397, + "theoretical_loss": 3.524135012733706, + "tokens_seen": 1603076096 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004156308926780341, + "loss": 1.3367, + "theoretical_loss": 3.5241225730308914, + "tokens_seen": 1603141632 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041561484453360084, + "loss": 1.2844, + "theoretical_loss": 3.524110133978981, + "tokens_seen": 1603207168 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041559879638916754, + "loss": 1.3781, + "theoretical_loss": 3.5240976955779137, + "tokens_seen": 1603272704 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041558274824473425, + "loss": 1.1878, + "theoretical_loss": 3.524085257827628, + "tokens_seen": 1603338240 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041556670010030095, + "loss": 1.3054, + "theoretical_loss": 3.5240728207280645, + "tokens_seen": 1603403776 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041555065195586765, + "loss": 1.444, + "theoretical_loss": 3.5240603842791614, + "tokens_seen": 1603469312 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041553460381143435, + "loss": 1.3499, + "theoretical_loss": 3.5240479484808587, + "tokens_seen": 1603534848 + }, + { + "epoch": 0.49, + "learning_rate": 0.000415518555667001, + "loss": 1.2966, + "theoretical_loss": 3.524035513333096, + "tokens_seen": 1603600384 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041550250752256775, + "loss": 1.4559, + "theoretical_loss": 3.5240230788358122, + "tokens_seen": 1603665920 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004154864593781344, + "loss": 1.3121, + "theoretical_loss": 3.5240106449889472, + "tokens_seen": 1603731456 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041547041123370115, + "loss": 1.3264, + "theoretical_loss": 3.5239982117924398, + "tokens_seen": 1603796992 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004154543630892678, + "loss": 1.2611, + "theoretical_loss": 3.52398577924623, + "tokens_seen": 1603862528 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041543831494483455, + "loss": 1.3001, + "theoretical_loss": 3.5239733473502564, + "tokens_seen": 1603928064 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004154222668004012, + "loss": 1.2245, + "theoretical_loss": 3.5239609161044596, + "tokens_seen": 1603993600 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041540621865596795, + "loss": 1.3141, + "theoretical_loss": 3.5239484855087775, + "tokens_seen": 1604059136 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004153901705115346, + "loss": 1.4154, + "theoretical_loss": 3.523936055563151, + "tokens_seen": 1604124672 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041537412236710135, + "loss": 1.2991, + "theoretical_loss": 3.5239236262675186, + "tokens_seen": 1604190208 + }, + { + "epoch": 0.49, + "learning_rate": 0.000415358074222668, + "loss": 1.3686, + "theoretical_loss": 3.5239111976218203, + "tokens_seen": 1604255744 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041534202607823475, + "loss": 1.2509, + "theoretical_loss": 3.523898769625996, + "tokens_seen": 1604321280 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004153259779338014, + "loss": 1.3472, + "theoretical_loss": 3.523886342279984, + "tokens_seen": 1604386816 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041530992978936816, + "loss": 1.3334, + "theoretical_loss": 3.523873915583724, + "tokens_seen": 1604452352 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004152938816449348, + "loss": 1.3049, + "theoretical_loss": 3.5238614895371563, + "tokens_seen": 1604517888 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041527783350050156, + "loss": 1.2306, + "theoretical_loss": 3.5238490641402196, + "tokens_seen": 1604583424 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004152617853560682, + "loss": 1.3472, + "theoretical_loss": 3.523836639392854, + "tokens_seen": 1604648960 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041524573721163496, + "loss": 1.275, + "theoretical_loss": 3.5238242152949986, + "tokens_seen": 1604714496 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004152296890672016, + "loss": 1.4297, + "theoretical_loss": 3.5238117918465934, + "tokens_seen": 1604780032 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041521364092276836, + "loss": 1.2217, + "theoretical_loss": 3.5237993690475773, + "tokens_seen": 1604845568 + }, + { + "epoch": 0.49, + "learning_rate": 0.000415197592778335, + "loss": 1.2468, + "theoretical_loss": 3.52378694689789, + "tokens_seen": 1604911104 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041518154463390176, + "loss": 1.244, + "theoretical_loss": 3.5237745253974717, + "tokens_seen": 1604976640 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004151654964894684, + "loss": 1.3767, + "theoretical_loss": 3.523762104546261, + "tokens_seen": 1605042176 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041514944834503516, + "loss": 1.3375, + "theoretical_loss": 3.523749684344198, + "tokens_seen": 1605107712 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004151334002006018, + "loss": 1.371, + "theoretical_loss": 3.5237372647912224, + "tokens_seen": 1605173248 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041511735205616856, + "loss": 1.3145, + "theoretical_loss": 3.5237248458872736, + "tokens_seen": 1605238784 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004151013039117352, + "loss": 1.3816, + "theoretical_loss": 3.5237124276322906, + "tokens_seen": 1605304320 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041508525576730196, + "loss": 1.322, + "theoretical_loss": 3.523700010026214, + "tokens_seen": 1605369856 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004150692076228686, + "loss": 1.2975, + "theoretical_loss": 3.5236875930689826, + "tokens_seen": 1605435392 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041505315947843537, + "loss": 1.2956, + "theoretical_loss": 3.5236751767605368, + "tokens_seen": 1605500928 + }, + { + "epoch": 0.49, + "learning_rate": 0.000415037111334002, + "loss": 1.2893, + "theoretical_loss": 3.5236627611008156, + "tokens_seen": 1605566464 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 596166, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3440099954605103, + "objective/train/theoretical_loss": 3.5236503460897586, + "objective/train/tokens_used": 1626092000, + "theoretical_loss": 3.5236503460897586, + "tokens_seen": 1605632000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041502106318956877, + "loss": 1.344, + "theoretical_loss": 3.5236503460897586, + "tokens_seen": 1605632000 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004150050150451354, + "loss": 1.3337, + "theoretical_loss": 3.523637931727306, + "tokens_seen": 1605697536 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041498896690070217, + "loss": 1.2942, + "theoretical_loss": 3.5236255180133966, + "tokens_seen": 1605763072 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004149729187562688, + "loss": 1.3567, + "theoretical_loss": 3.5236131049479713, + "tokens_seen": 1605828608 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041495687061183557, + "loss": 1.2589, + "theoretical_loss": 3.5236006925309686, + "tokens_seen": 1605894144 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004149408224674022, + "loss": 1.1811, + "theoretical_loss": 3.523588280762328, + "tokens_seen": 1605959680 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041492477432296897, + "loss": 1.3011, + "theoretical_loss": 3.5235758696419905, + "tokens_seen": 1606025216 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004149087261785356, + "loss": 1.2886, + "theoretical_loss": 3.5235634591698948, + "tokens_seen": 1606090752 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004148926780341023, + "loss": 1.2475, + "theoretical_loss": 3.523551049345981, + "tokens_seen": 1606156288 + }, + { + "epoch": 0.49, + "learning_rate": 0.000414876629889669, + "loss": 1.3177, + "theoretical_loss": 3.5235386401701883, + "tokens_seen": 1606221824 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004148605817452357, + "loss": 1.2989, + "theoretical_loss": 3.5235262316424567, + "tokens_seen": 1606287360 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004148445336008024, + "loss": 1.2832, + "theoretical_loss": 3.523513823762726, + "tokens_seen": 1606352896 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004148284854563691, + "loss": 1.3336, + "theoretical_loss": 3.5235014165309364, + "tokens_seen": 1606418432 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004148124373119358, + "loss": 1.3393, + "theoretical_loss": 3.5234890099470264, + "tokens_seen": 1606483968 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004147963891675025, + "loss": 1.3018, + "theoretical_loss": 3.523476604010937, + "tokens_seen": 1606549504 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004147803410230692, + "loss": 1.3263, + "theoretical_loss": 3.523464198722607, + "tokens_seen": 1606615040 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004147642928786359, + "loss": 1.3437, + "theoretical_loss": 3.5234517940819767, + "tokens_seen": 1606680576 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041474824473420257, + "loss": 1.3911, + "theoretical_loss": 3.5234393900889858, + "tokens_seen": 1606746112 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004147321965897693, + "loss": 1.4899, + "theoretical_loss": 3.5234269867435737, + "tokens_seen": 1606811648 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041471614844533597, + "loss": 1.3443, + "theoretical_loss": 3.523414584045681, + "tokens_seen": 1606877184 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004147001003009027, + "loss": 1.3171, + "theoretical_loss": 3.5234021819952464, + "tokens_seen": 1606942720 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041468405215646937, + "loss": 1.2512, + "theoretical_loss": 3.5233897805922103, + "tokens_seen": 1607008256 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004146680040120361, + "loss": 1.3593, + "theoretical_loss": 3.523377379836513, + "tokens_seen": 1607073792 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004146519558676028, + "loss": 1.1819, + "theoretical_loss": 3.5233649797280933, + "tokens_seen": 1607139328 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041463590772316953, + "loss": 1.2692, + "theoretical_loss": 3.5233525802668915, + "tokens_seen": 1607204864 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004146198595787362, + "loss": 1.3711, + "theoretical_loss": 3.5233401814528476, + "tokens_seen": 1607270400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041460381143430293, + "loss": 1.2832, + "theoretical_loss": 3.523327783285901, + "tokens_seen": 1607335936 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004145877632898696, + "loss": 1.3053, + "theoretical_loss": 3.5233153857659922, + "tokens_seen": 1607401472 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041457171514543633, + "loss": 1.254, + "theoretical_loss": 3.523302988893061, + "tokens_seen": 1607467008 + }, + { + "epoch": 0.49, + "learning_rate": 0.000414555667001003, + "loss": 1.3299, + "theoretical_loss": 3.5232905926670464, + "tokens_seen": 1607532544 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041453961885656973, + "loss": 1.26, + "theoretical_loss": 3.5232781970878886, + "tokens_seen": 1607598080 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004145235707121364, + "loss": 1.2866, + "theoretical_loss": 3.523265802155528, + "tokens_seen": 1607663616 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041450752256770313, + "loss": 1.3408, + "theoretical_loss": 3.5232534078699036, + "tokens_seen": 1607729152 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004144914744232698, + "loss": 1.2831, + "theoretical_loss": 3.5232410142309565, + "tokens_seen": 1607794688 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041447542627883653, + "loss": 1.2817, + "theoretical_loss": 3.523228621238626, + "tokens_seen": 1607860224 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004144593781344032, + "loss": 1.4042, + "theoretical_loss": 3.523216228892852, + "tokens_seen": 1607925760 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041444332998996994, + "loss": 1.2647, + "theoretical_loss": 3.5232038371935737, + "tokens_seen": 1607991296 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004144272818455366, + "loss": 1.3015, + "theoretical_loss": 3.523191446140732, + "tokens_seen": 1608056832 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041441123370110334, + "loss": 1.279, + "theoretical_loss": 3.523179055734267, + "tokens_seen": 1608122368 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004143951855566701, + "loss": 1.2798, + "theoretical_loss": 3.523166665974118, + "tokens_seen": 1608187904 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041437913741223674, + "loss": 1.3222, + "theoretical_loss": 3.523154276860225, + "tokens_seen": 1608253440 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004143630892678035, + "loss": 1.3846, + "theoretical_loss": 3.5231418883925283, + "tokens_seen": 1608318976 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041434704112337014, + "loss": 1.351, + "theoretical_loss": 3.523129500570967, + "tokens_seen": 1608384512 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004143309929789369, + "loss": 1.2386, + "theoretical_loss": 3.5231171133954824, + "tokens_seen": 1608450048 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041431494483450354, + "loss": 1.3292, + "theoretical_loss": 3.5231047268660136, + "tokens_seen": 1608515584 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004142988966900703, + "loss": 1.4214, + "theoretical_loss": 3.523092340982501, + "tokens_seen": 1608581120 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041428284854563694, + "loss": 1.3436, + "theoretical_loss": 3.523079955744884, + "tokens_seen": 1608646656 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041426680040120364, + "loss": 1.3216, + "theoretical_loss": 3.5230675711531036, + "tokens_seen": 1608712192 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041425075225677034, + "loss": 1.3284, + "theoretical_loss": 3.523055187207099, + "tokens_seen": 1608777728 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041423470411233704, + "loss": 1.3553, + "theoretical_loss": 3.5230428039068102, + "tokens_seen": 1608843264 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041421865596790374, + "loss": 1.3913, + "theoretical_loss": 3.5230304212521775, + "tokens_seen": 1608908800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041420260782347045, + "loss": 1.2824, + "theoretical_loss": 3.523018039243141, + "tokens_seen": 1608974336 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041418655967903715, + "loss": 1.38, + "theoretical_loss": 3.523005657879641, + "tokens_seen": 1609039872 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041417051153460385, + "loss": 1.3561, + "theoretical_loss": 3.522993277161617, + "tokens_seen": 1609105408 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041415446339017055, + "loss": 1.3698, + "theoretical_loss": 3.522980897089009, + "tokens_seen": 1609170944 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041413841524573725, + "loss": 1.3179, + "theoretical_loss": 3.5229685176617576, + "tokens_seen": 1609236480 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004141223671013039, + "loss": 1.423, + "theoretical_loss": 3.5229561388798025, + "tokens_seen": 1609302016 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041410631895687065, + "loss": 1.2993, + "theoretical_loss": 3.5229437607430842, + "tokens_seen": 1609367552 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004140902708124373, + "loss": 1.2842, + "theoretical_loss": 3.5229313832515423, + "tokens_seen": 1609433088 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041407422266800405, + "loss": 1.4084, + "theoretical_loss": 3.5229190064051172, + "tokens_seen": 1609498624 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004140581745235707, + "loss": 1.387, + "theoretical_loss": 3.522906630203749, + "tokens_seen": 1609564160 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041404212637913745, + "loss": 1.2902, + "theoretical_loss": 3.522894254647378, + "tokens_seen": 1609629696 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004140260782347041, + "loss": 1.2974, + "theoretical_loss": 3.522881879735943, + "tokens_seen": 1609695232 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041401003009027085, + "loss": 1.2413, + "theoretical_loss": 3.5228695054693864, + "tokens_seen": 1609760768 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004139939819458375, + "loss": 1.3193, + "theoretical_loss": 3.5228571318476467, + "tokens_seen": 1609826304 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041397793380140425, + "loss": 1.353, + "theoretical_loss": 3.522844758870664, + "tokens_seen": 1609891840 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004139618856569709, + "loss": 1.2807, + "theoretical_loss": 3.5228323865383793, + "tokens_seen": 1609957376 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041394583751253766, + "loss": 1.3022, + "theoretical_loss": 3.5228200148507325, + "tokens_seen": 1610022912 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004139297893681043, + "loss": 1.2939, + "theoretical_loss": 3.5228076438076643, + "tokens_seen": 1610088448 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041391374122367106, + "loss": 1.358, + "theoretical_loss": 3.5227952734091135, + "tokens_seen": 1610153984 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004138976930792377, + "loss": 1.2543, + "theoretical_loss": 3.5227829036550213, + "tokens_seen": 1610219520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041388164493480446, + "loss": 1.2213, + "theoretical_loss": 3.5227705345453275, + "tokens_seen": 1610285056 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004138655967903711, + "loss": 1.3373, + "theoretical_loss": 3.5227581660799725, + "tokens_seen": 1610350592 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041384954864593786, + "loss": 1.2058, + "theoretical_loss": 3.5227457982588968, + "tokens_seen": 1610416128 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004138335005015045, + "loss": 1.3691, + "theoretical_loss": 3.52273343108204, + "tokens_seen": 1610481664 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041381745235707126, + "loss": 1.3292, + "theoretical_loss": 3.522721064549343, + "tokens_seen": 1610547200 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004138014042126379, + "loss": 1.2435, + "theoretical_loss": 3.5227086986607454, + "tokens_seen": 1610612736 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041378535606820466, + "loss": 1.3545, + "theoretical_loss": 3.522696333416188, + "tokens_seen": 1610678272 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004137693079237713, + "loss": 1.3835, + "theoretical_loss": 3.5226839688156106, + "tokens_seen": 1610743808 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041375325977933806, + "loss": 1.2476, + "theoretical_loss": 3.522671604858953, + "tokens_seen": 1610809344 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004137372116349047, + "loss": 1.2594, + "theoretical_loss": 3.5226592415461573, + "tokens_seen": 1610874880 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041372116349047146, + "loss": 1.3359, + "theoretical_loss": 3.522646878877162, + "tokens_seen": 1610940416 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004137051153460381, + "loss": 1.3922, + "theoretical_loss": 3.522634516851908, + "tokens_seen": 1611005952 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041368906720160487, + "loss": 1.2265, + "theoretical_loss": 3.5226221554703354, + "tokens_seen": 1611071488 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004136730190571715, + "loss": 1.2951, + "theoretical_loss": 3.522609794732385, + "tokens_seen": 1611137024 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041365697091273827, + "loss": 1.3219, + "theoretical_loss": 3.5225974346379965, + "tokens_seen": 1611202560 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004136409227683049, + "loss": 1.366, + "theoretical_loss": 3.52258507518711, + "tokens_seen": 1611268096 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041362487462387167, + "loss": 1.3464, + "theoretical_loss": 3.5225727163796674, + "tokens_seen": 1611333632 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004136088264794383, + "loss": 1.3116, + "theoretical_loss": 3.5225603582156078, + "tokens_seen": 1611399168 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041359277833500507, + "loss": 1.2393, + "theoretical_loss": 3.5225480006948713, + "tokens_seen": 1611464704 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004135767301905717, + "loss": 1.3399, + "theoretical_loss": 3.5225356438173985, + "tokens_seen": 1611530240 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041356068204613847, + "loss": 1.2636, + "theoretical_loss": 3.5225232875831303, + "tokens_seen": 1611595776 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004135446339017051, + "loss": 1.3509, + "theoretical_loss": 3.5225109319920067, + "tokens_seen": 1611661312 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041352858575727187, + "loss": 1.283, + "theoretical_loss": 3.522498577043968, + "tokens_seen": 1611726848 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004135125376128385, + "loss": 1.3057, + "theoretical_loss": 3.522486222738954, + "tokens_seen": 1611792384 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004134964894684052, + "loss": 1.3007, + "theoretical_loss": 3.522473869076906, + "tokens_seen": 1611857920 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004134804413239719, + "loss": 1.253, + "theoretical_loss": 3.5224615160577644, + "tokens_seen": 1611923456 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004134643931795386, + "loss": 1.245, + "theoretical_loss": 3.5224491636814697, + "tokens_seen": 1611988992 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004134483450351053, + "loss": 1.3438, + "theoretical_loss": 3.522436811947961, + "tokens_seen": 1612054528 + }, + { + "epoch": 0.49, + "learning_rate": 0.000413432296890672, + "loss": 1.2346, + "theoretical_loss": 3.522424460857181, + "tokens_seen": 1612120064 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 598464, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3062889575958252, + "objective/train/theoretical_loss": 3.5224121104090678, + "objective/train/tokens_used": 1632645600, + "theoretical_loss": 3.5224121104090678, + "tokens_seen": 1612185600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004134162487462387, + "loss": 1.3063, + "theoretical_loss": 3.5224121104090678, + "tokens_seen": 1612185600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004134002006018054, + "loss": 1.3063, + "theoretical_loss": 3.5223997606035624, + "tokens_seen": 1612251136 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004133841524573721, + "loss": 1.2844, + "theoretical_loss": 3.5223874114406066, + "tokens_seen": 1612316672 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004133681043129388, + "loss": 1.3339, + "theoretical_loss": 3.5223750629201396, + "tokens_seen": 1612382208 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041335205616850547, + "loss": 1.2957, + "theoretical_loss": 3.522362715042102, + "tokens_seen": 1612447744 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004133360080240722, + "loss": 1.2561, + "theoretical_loss": 3.5223503678064345, + "tokens_seen": 1612513280 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041331995987963887, + "loss": 1.2496, + "theoretical_loss": 3.522338021213078, + "tokens_seen": 1612578816 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004133039117352056, + "loss": 1.4899, + "theoretical_loss": 3.522325675261972, + "tokens_seen": 1612644352 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004132878635907723, + "loss": 1.3274, + "theoretical_loss": 3.5223133299530573, + "tokens_seen": 1612709888 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041327181544633903, + "loss": 1.2804, + "theoretical_loss": 3.522300985286275, + "tokens_seen": 1612775424 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004132557673019057, + "loss": 1.277, + "theoretical_loss": 3.5222886412615653, + "tokens_seen": 1612840960 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041323971915747243, + "loss": 1.2764, + "theoretical_loss": 3.522276297878869, + "tokens_seen": 1612906496 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004132236710130391, + "loss": 1.3037, + "theoretical_loss": 3.5222639551381256, + "tokens_seen": 1612972032 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041320762286860583, + "loss": 1.1914, + "theoretical_loss": 3.5222516130392765, + "tokens_seen": 1613037568 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004131915747241726, + "loss": 1.3265, + "theoretical_loss": 3.522239271582262, + "tokens_seen": 1613103104 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041317552657973923, + "loss": 1.3648, + "theoretical_loss": 3.5222269307670224, + "tokens_seen": 1613168640 + }, + { + "epoch": 0.49, + "learning_rate": 0.000413159478435306, + "loss": 1.3114, + "theoretical_loss": 3.522214590593499, + "tokens_seen": 1613234176 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041314343029087263, + "loss": 1.3164, + "theoretical_loss": 3.5222022510616315, + "tokens_seen": 1613299712 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004131273821464394, + "loss": 1.2365, + "theoretical_loss": 3.5221899121713616, + "tokens_seen": 1613365248 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041311133400200603, + "loss": 1.2953, + "theoretical_loss": 3.522177573922628, + "tokens_seen": 1613430784 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004130952858575728, + "loss": 1.2702, + "theoretical_loss": 3.5221652363153733, + "tokens_seen": 1613496320 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041307923771313944, + "loss": 1.3604, + "theoretical_loss": 3.5221528993495372, + "tokens_seen": 1613561856 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004130631895687062, + "loss": 1.3794, + "theoretical_loss": 3.5221405630250597, + "tokens_seen": 1613627392 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041304714142427284, + "loss": 1.3788, + "theoretical_loss": 3.5221282273418826, + "tokens_seen": 1613692928 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004130310932798396, + "loss": 1.2355, + "theoretical_loss": 3.5221158922999463, + "tokens_seen": 1613758464 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041301504513540624, + "loss": 1.2896, + "theoretical_loss": 3.5221035578991904, + "tokens_seen": 1613824000 + }, + { + "epoch": 0.49, + "learning_rate": 0.000412998996990973, + "loss": 1.3806, + "theoretical_loss": 3.5220912241395563, + "tokens_seen": 1613889536 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041298294884653964, + "loss": 1.3418, + "theoretical_loss": 3.522078891020985, + "tokens_seen": 1613955072 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004129669007021064, + "loss": 1.2499, + "theoretical_loss": 3.5220665585434165, + "tokens_seen": 1614020608 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041295085255767304, + "loss": 1.4165, + "theoretical_loss": 3.522054226706792, + "tokens_seen": 1614086144 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004129348044132398, + "loss": 1.2754, + "theoretical_loss": 3.522041895511051, + "tokens_seen": 1614151680 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041291875626880644, + "loss": 1.2528, + "theoretical_loss": 3.522029564956136, + "tokens_seen": 1614217216 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004129027081243732, + "loss": 1.2977, + "theoretical_loss": 3.522017235041986, + "tokens_seen": 1614282752 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041288665997993984, + "loss": 1.2257, + "theoretical_loss": 3.5220049057685427, + "tokens_seen": 1614348288 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004128706118355066, + "loss": 1.2402, + "theoretical_loss": 3.5219925771357463, + "tokens_seen": 1614413824 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041285456369107324, + "loss": 1.3335, + "theoretical_loss": 3.521980249143538, + "tokens_seen": 1614479360 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041283851554663995, + "loss": 1.3117, + "theoretical_loss": 3.521967921791858, + "tokens_seen": 1614544896 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041282246740220665, + "loss": 1.2312, + "theoretical_loss": 3.521955595080647, + "tokens_seen": 1614610432 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041280641925777335, + "loss": 1.3229, + "theoretical_loss": 3.5219432690098467, + "tokens_seen": 1614675968 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041279037111334005, + "loss": 1.2647, + "theoretical_loss": 3.521930943579397, + "tokens_seen": 1614741504 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041277432296890675, + "loss": 1.3492, + "theoretical_loss": 3.521918618789238, + "tokens_seen": 1614807040 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041275827482447345, + "loss": 1.3226, + "theoretical_loss": 3.521906294639312, + "tokens_seen": 1614872576 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041274222668004015, + "loss": 1.3891, + "theoretical_loss": 3.5218939711295585, + "tokens_seen": 1614938112 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004127261785356068, + "loss": 1.3382, + "theoretical_loss": 3.521881648259919, + "tokens_seen": 1615003648 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041271013039117355, + "loss": 1.2959, + "theoretical_loss": 3.521869326030334, + "tokens_seen": 1615069184 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004126940822467402, + "loss": 1.2887, + "theoretical_loss": 3.521857004440744, + "tokens_seen": 1615134720 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041267803410230695, + "loss": 1.2081, + "theoretical_loss": 3.52184468349109, + "tokens_seen": 1615200256 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004126619859578736, + "loss": 1.1824, + "theoretical_loss": 3.5218323631813133, + "tokens_seen": 1615265792 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041264593781344035, + "loss": 1.3043, + "theoretical_loss": 3.5218200435113545, + "tokens_seen": 1615331328 + }, + { + "epoch": 0.49, + "learning_rate": 0.000412629889669007, + "loss": 1.3377, + "theoretical_loss": 3.521807724481154, + "tokens_seen": 1615396864 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041261384152457375, + "loss": 1.3719, + "theoretical_loss": 3.5217954060906527, + "tokens_seen": 1615462400 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004125977933801404, + "loss": 1.3738, + "theoretical_loss": 3.5217830883397916, + "tokens_seen": 1615527936 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041258174523570716, + "loss": 1.2998, + "theoretical_loss": 3.521770771228512, + "tokens_seen": 1615593472 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004125656970912738, + "loss": 1.2582, + "theoretical_loss": 3.5217584547567538, + "tokens_seen": 1615659008 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041254964894684056, + "loss": 1.3256, + "theoretical_loss": 3.521746138924458, + "tokens_seen": 1615724544 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004125336008024072, + "loss": 1.3163, + "theoretical_loss": 3.5217338237315667, + "tokens_seen": 1615790080 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041251755265797396, + "loss": 1.337, + "theoretical_loss": 3.521721509178019, + "tokens_seen": 1615855616 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004125015045135406, + "loss": 1.3218, + "theoretical_loss": 3.5217091952637567, + "tokens_seen": 1615921152 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041248545636910736, + "loss": 1.4177, + "theoretical_loss": 3.5216968819887216, + "tokens_seen": 1615986688 + }, + { + "epoch": 0.49, + "learning_rate": 0.000412469408224674, + "loss": 1.3471, + "theoretical_loss": 3.5216845693528525, + "tokens_seen": 1616052224 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041245336008024076, + "loss": 1.4305, + "theoretical_loss": 3.5216722573560917, + "tokens_seen": 1616117760 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004124373119358074, + "loss": 1.308, + "theoretical_loss": 3.5216599459983806, + "tokens_seen": 1616183296 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041242126379137416, + "loss": 1.3254, + "theoretical_loss": 3.5216476352796584, + "tokens_seen": 1616248832 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004124052156469408, + "loss": 1.3989, + "theoretical_loss": 3.5216353251998673, + "tokens_seen": 1616314368 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041238916750250756, + "loss": 1.2405, + "theoretical_loss": 3.5216230157589483, + "tokens_seen": 1616379904 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004123731193580742, + "loss": 1.2526, + "theoretical_loss": 3.5216107069568414, + "tokens_seen": 1616445440 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041235707121364096, + "loss": 1.3551, + "theoretical_loss": 3.5215983987934885, + "tokens_seen": 1616510976 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004123410230692076, + "loss": 1.3558, + "theoretical_loss": 3.52158609126883, + "tokens_seen": 1616576512 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041232497492477437, + "loss": 1.398, + "theoretical_loss": 3.5215737843828068, + "tokens_seen": 1616642048 + }, + { + "epoch": 0.49, + "learning_rate": 0.000412308926780341, + "loss": 1.2875, + "theoretical_loss": 3.521561478135361, + "tokens_seen": 1616707584 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041229287863590777, + "loss": 1.3071, + "theoretical_loss": 3.5215491725264316, + "tokens_seen": 1616773120 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004122768304914744, + "loss": 1.3571, + "theoretical_loss": 3.5215368675559615, + "tokens_seen": 1616838656 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041226078234704117, + "loss": 1.3426, + "theoretical_loss": 3.52152456322389, + "tokens_seen": 1616904192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004122447342026078, + "loss": 1.2887, + "theoretical_loss": 3.52151225953016, + "tokens_seen": 1616969728 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041222868605817457, + "loss": 1.3276, + "theoretical_loss": 3.521499956474711, + "tokens_seen": 1617035264 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004122126379137412, + "loss": 1.3056, + "theoretical_loss": 3.521487654057484, + "tokens_seen": 1617100800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041219658976930797, + "loss": 1.3411, + "theoretical_loss": 3.5214753522784212, + "tokens_seen": 1617166336 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004121805416248746, + "loss": 1.3344, + "theoretical_loss": 3.521463051137463, + "tokens_seen": 1617231872 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041216449348044137, + "loss": 1.2563, + "theoretical_loss": 3.5214507506345507, + "tokens_seen": 1617297408 + }, + { + "epoch": 0.49, + "learning_rate": 0.000412148445336008, + "loss": 1.2905, + "theoretical_loss": 3.5214384507696246, + "tokens_seen": 1617362944 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004121323971915748, + "loss": 1.28, + "theoretical_loss": 3.5214261515426264, + "tokens_seen": 1617428480 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004121163490471414, + "loss": 1.2497, + "theoretical_loss": 3.521413852953497, + "tokens_seen": 1617494016 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004121003009027082, + "loss": 1.4484, + "theoretical_loss": 3.5214015550021776, + "tokens_seen": 1617559552 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004120842527582748, + "loss": 1.2971, + "theoretical_loss": 3.5213892576886088, + "tokens_seen": 1617625088 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004120682046138415, + "loss": 1.3962, + "theoretical_loss": 3.5213769610127326, + "tokens_seen": 1617690624 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004120521564694082, + "loss": 1.2819, + "theoretical_loss": 3.521364664974489, + "tokens_seen": 1617756160 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004120361083249749, + "loss": 1.2369, + "theoretical_loss": 3.52135236957382, + "tokens_seen": 1617821696 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004120200601805416, + "loss": 1.4045, + "theoretical_loss": 3.5213400748106665, + "tokens_seen": 1617887232 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004120040120361083, + "loss": 1.2287, + "theoretical_loss": 3.5213277806849694, + "tokens_seen": 1617952768 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004119879638916751, + "loss": 1.2199, + "theoretical_loss": 3.5213154871966696, + "tokens_seen": 1618018304 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004119719157472417, + "loss": 1.1511, + "theoretical_loss": 3.521303194345709, + "tokens_seen": 1618083840 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004119558676028085, + "loss": 1.3709, + "theoretical_loss": 3.5212909021320282, + "tokens_seen": 1618149376 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004119398194583751, + "loss": 1.2566, + "theoretical_loss": 3.5212786105555685, + "tokens_seen": 1618214912 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004119237713139419, + "loss": 1.3167, + "theoretical_loss": 3.521266319616271, + "tokens_seen": 1618280448 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041190772316950853, + "loss": 1.2688, + "theoretical_loss": 3.521254029314077, + "tokens_seen": 1618345984 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004118916750250753, + "loss": 1.2461, + "theoretical_loss": 3.5212417396489273, + "tokens_seen": 1618411520 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041187562688064193, + "loss": 1.2937, + "theoretical_loss": 3.5212294506207638, + "tokens_seen": 1618477056 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004118595787362087, + "loss": 1.3476, + "theoretical_loss": 3.5212171622295267, + "tokens_seen": 1618542592 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041184353059177533, + "loss": 1.3942, + "theoretical_loss": 3.521204874475158, + "tokens_seen": 1618608128 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004118274824473421, + "loss": 1.2067, + "theoretical_loss": 3.521192587357599, + "tokens_seen": 1618673664 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 601131, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.302499771118164, + "objective/train/theoretical_loss": 3.5211803008767903, + "objective/train/tokens_used": 1639199200, + "theoretical_loss": 3.5211803008767903, + "tokens_seen": 1618739200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041181143430290873, + "loss": 1.3025, + "theoretical_loss": 3.5211803008767903, + "tokens_seen": 1618739200 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004117953861584755, + "loss": 1.2981, + "theoretical_loss": 3.521168015032674, + "tokens_seen": 1618804736 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041177933801404213, + "loss": 1.3456, + "theoretical_loss": 3.52115572982519, + "tokens_seen": 1618870272 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004117632898696089, + "loss": 1.2913, + "theoretical_loss": 3.521143445254281, + "tokens_seen": 1618935808 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041174724172517553, + "loss": 1.1506, + "theoretical_loss": 3.521131161319887, + "tokens_seen": 1619001344 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004117311935807423, + "loss": 1.2159, + "theoretical_loss": 3.52111887802195, + "tokens_seen": 1619066880 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041171514543630894, + "loss": 1.2995, + "theoretical_loss": 3.5211065953604104, + "tokens_seen": 1619132416 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004116990972918757, + "loss": 1.3966, + "theoretical_loss": 3.521094313335211, + "tokens_seen": 1619197952 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041168304914744234, + "loss": 1.3589, + "theoretical_loss": 3.5210820319462917, + "tokens_seen": 1619263488 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004116670010030091, + "loss": 1.2526, + "theoretical_loss": 3.521069751193594, + "tokens_seen": 1619329024 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041165095285857574, + "loss": 1.4052, + "theoretical_loss": 3.521057471077061, + "tokens_seen": 1619394560 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004116349047141425, + "loss": 1.3682, + "theoretical_loss": 3.521045191596631, + "tokens_seen": 1619460096 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041161885656970914, + "loss": 1.3246, + "theoretical_loss": 3.521032912752247, + "tokens_seen": 1619525632 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004116028084252759, + "loss": 1.41, + "theoretical_loss": 3.521020634543851, + "tokens_seen": 1619591168 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041158676028084254, + "loss": 1.2822, + "theoretical_loss": 3.5210083569713824, + "tokens_seen": 1619656704 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004115707121364093, + "loss": 1.3655, + "theoretical_loss": 3.5209960800347844, + "tokens_seen": 1619722240 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041155466399197594, + "loss": 1.3181, + "theoretical_loss": 3.520983803733997, + "tokens_seen": 1619787776 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004115386158475427, + "loss": 1.311, + "theoretical_loss": 3.520971528068962, + "tokens_seen": 1619853312 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041152256770310934, + "loss": 1.3383, + "theoretical_loss": 3.5209592530396217, + "tokens_seen": 1619918848 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004115065195586761, + "loss": 1.213, + "theoretical_loss": 3.5209469786459158, + "tokens_seen": 1619984384 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041149047141424274, + "loss": 1.3484, + "theoretical_loss": 3.5209347048877864, + "tokens_seen": 1620049920 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004114744232698095, + "loss": 1.2824, + "theoretical_loss": 3.5209224317651753, + "tokens_seen": 1620115456 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041145837512537615, + "loss": 1.3482, + "theoretical_loss": 3.5209101592780234, + "tokens_seen": 1620180992 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041144232698094285, + "loss": 1.2786, + "theoretical_loss": 3.520897887426272, + "tokens_seen": 1620246528 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041142627883650955, + "loss": 1.319, + "theoretical_loss": 3.5208856162098634, + "tokens_seen": 1620312064 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041141023069207625, + "loss": 1.351, + "theoretical_loss": 3.520873345628738, + "tokens_seen": 1620377600 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041139418254764295, + "loss": 1.3499, + "theoretical_loss": 3.5208610756828373, + "tokens_seen": 1620443136 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041137813440320965, + "loss": 1.2355, + "theoretical_loss": 3.5208488063721033, + "tokens_seen": 1620508672 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041136208625877635, + "loss": 1.2458, + "theoretical_loss": 3.5208365376964768, + "tokens_seen": 1620574208 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041134603811434305, + "loss": 1.2683, + "theoretical_loss": 3.5208242696558996, + "tokens_seen": 1620639744 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041132998996990975, + "loss": 1.4061, + "theoretical_loss": 3.520812002250314, + "tokens_seen": 1620705280 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041131394182547645, + "loss": 1.3574, + "theoretical_loss": 3.5207997354796596, + "tokens_seen": 1620770816 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004112978936810431, + "loss": 1.3415, + "theoretical_loss": 3.5207874693438788, + "tokens_seen": 1620836352 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041128184553660985, + "loss": 1.4411, + "theoretical_loss": 3.5207752038429136, + "tokens_seen": 1620901888 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004112657973921765, + "loss": 1.3495, + "theoretical_loss": 3.520762938976705, + "tokens_seen": 1620967424 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041124974924774325, + "loss": 1.3333, + "theoretical_loss": 3.5207506747451944, + "tokens_seen": 1621032960 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004112337011033099, + "loss": 1.2865, + "theoretical_loss": 3.520738411148323, + "tokens_seen": 1621098496 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041121765295887666, + "loss": 1.2173, + "theoretical_loss": 3.520726148186033, + "tokens_seen": 1621164032 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004112016048144433, + "loss": 1.2975, + "theoretical_loss": 3.520713885858266, + "tokens_seen": 1621229568 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041118555667001006, + "loss": 1.3056, + "theoretical_loss": 3.5207016241649627, + "tokens_seen": 1621295104 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004111695085255767, + "loss": 1.4252, + "theoretical_loss": 3.520689363106065, + "tokens_seen": 1621360640 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041115346038114346, + "loss": 1.2302, + "theoretical_loss": 3.520677102681515, + "tokens_seen": 1621426176 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004111374122367101, + "loss": 1.3505, + "theoretical_loss": 3.520664842891253, + "tokens_seen": 1621491712 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041112136409227686, + "loss": 1.2975, + "theoretical_loss": 3.5206525837352216, + "tokens_seen": 1621557248 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004111053159478435, + "loss": 1.3487, + "theoretical_loss": 3.5206403252133622, + "tokens_seen": 1621622784 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041108926780341026, + "loss": 1.274, + "theoretical_loss": 3.5206280673256156, + "tokens_seen": 1621688320 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004110732196589769, + "loss": 1.422, + "theoretical_loss": 3.5206158100719245, + "tokens_seen": 1621753856 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041105717151454366, + "loss": 1.2168, + "theoretical_loss": 3.52060355345223, + "tokens_seen": 1621819392 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004110411233701103, + "loss": 1.3159, + "theoretical_loss": 3.5205912974664737, + "tokens_seen": 1621884928 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041102507522567706, + "loss": 1.2973, + "theoretical_loss": 3.520579042114597, + "tokens_seen": 1621950464 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004110090270812437, + "loss": 1.2925, + "theoretical_loss": 3.5205667873965414, + "tokens_seen": 1622016000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041099297893681046, + "loss": 1.2583, + "theoretical_loss": 3.520554533312249, + "tokens_seen": 1622081536 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004109769307923771, + "loss": 1.2733, + "theoretical_loss": 3.5205422798616617, + "tokens_seen": 1622147072 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041096088264794387, + "loss": 1.2901, + "theoretical_loss": 3.52053002704472, + "tokens_seen": 1622212608 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004109448345035105, + "loss": 1.2543, + "theoretical_loss": 3.520517774861366, + "tokens_seen": 1622278144 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041092878635907727, + "loss": 1.2811, + "theoretical_loss": 3.520505523311542, + "tokens_seen": 1622343680 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004109127382146439, + "loss": 1.2864, + "theoretical_loss": 3.520493272395189, + "tokens_seen": 1622409216 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041089669007021067, + "loss": 1.2722, + "theoretical_loss": 3.5204810221122487, + "tokens_seen": 1622474752 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004108806419257773, + "loss": 1.3908, + "theoretical_loss": 3.520468772462663, + "tokens_seen": 1622540288 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041086459378134407, + "loss": 1.2985, + "theoretical_loss": 3.5204565234463736, + "tokens_seen": 1622605824 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004108485456369107, + "loss": 1.212, + "theoretical_loss": 3.520444275063322, + "tokens_seen": 1622671360 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041083249749247747, + "loss": 1.2981, + "theoretical_loss": 3.5204320273134493, + "tokens_seen": 1622736896 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004108164493480441, + "loss": 1.2536, + "theoretical_loss": 3.5204197801966983, + "tokens_seen": 1622802432 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041080040120361087, + "loss": 1.3169, + "theoretical_loss": 3.5204075337130103, + "tokens_seen": 1622867968 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041078435305917757, + "loss": 1.324, + "theoretical_loss": 3.5203952878623266, + "tokens_seen": 1622933504 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041076830491474427, + "loss": 1.3284, + "theoretical_loss": 3.52038304264459, + "tokens_seen": 1622999040 + }, + { + "epoch": 0.49, + "learning_rate": 0.000410752256770311, + "loss": 1.3194, + "theoretical_loss": 3.5203707980597407, + "tokens_seen": 1623064576 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004107362086258777, + "loss": 1.2878, + "theoretical_loss": 3.5203585541077214, + "tokens_seen": 1623130112 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004107201604814444, + "loss": 1.2703, + "theoretical_loss": 3.520346310788474, + "tokens_seen": 1623195648 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004107041123370111, + "loss": 1.3466, + "theoretical_loss": 3.5203340681019397, + "tokens_seen": 1623261184 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004106880641925778, + "loss": 1.2654, + "theoretical_loss": 3.5203218260480607, + "tokens_seen": 1623326720 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004106720160481444, + "loss": 1.3745, + "theoretical_loss": 3.5203095846267782, + "tokens_seen": 1623392256 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004106559679037112, + "loss": 1.2925, + "theoretical_loss": 3.5202973438380347, + "tokens_seen": 1623457792 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004106399197592778, + "loss": 1.3136, + "theoretical_loss": 3.5202851036817715, + "tokens_seen": 1623523328 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004106238716148446, + "loss": 1.3791, + "theoretical_loss": 3.5202728641579304, + "tokens_seen": 1623588864 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004106078234704112, + "loss": 1.2878, + "theoretical_loss": 3.5202606252664532, + "tokens_seen": 1623654400 + }, + { + "epoch": 0.49, + "learning_rate": 0.000410591775325978, + "loss": 1.292, + "theoretical_loss": 3.520248387007282, + "tokens_seen": 1623719936 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004105757271815446, + "loss": 1.3805, + "theoretical_loss": 3.5202361493803584, + "tokens_seen": 1623785472 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004105596790371114, + "loss": 1.2644, + "theoretical_loss": 3.520223912385624, + "tokens_seen": 1623851008 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041054363089267803, + "loss": 1.348, + "theoretical_loss": 3.5202116760230213, + "tokens_seen": 1623916544 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004105275827482448, + "loss": 1.2888, + "theoretical_loss": 3.5201994402924917, + "tokens_seen": 1623982080 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041051153460381143, + "loss": 1.3102, + "theoretical_loss": 3.5201872051939764, + "tokens_seen": 1624047616 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004104954864593782, + "loss": 1.3343, + "theoretical_loss": 3.5201749707274184, + "tokens_seen": 1624113152 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041047943831494483, + "loss": 1.2777, + "theoretical_loss": 3.5201627368927593, + "tokens_seen": 1624178688 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004104633901705116, + "loss": 1.2648, + "theoretical_loss": 3.5201505036899405, + "tokens_seen": 1624244224 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041044734202607823, + "loss": 1.3611, + "theoretical_loss": 3.520138271118904, + "tokens_seen": 1624309760 + }, + { + "epoch": 0.49, + "learning_rate": 0.000410431293881645, + "loss": 1.3124, + "theoretical_loss": 3.520126039179592, + "tokens_seen": 1624375296 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041041524573721163, + "loss": 1.3052, + "theoretical_loss": 3.520113807871946, + "tokens_seen": 1624440832 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004103991975927784, + "loss": 1.2148, + "theoretical_loss": 3.5201015771959083, + "tokens_seen": 1624506368 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041038314944834503, + "loss": 1.298, + "theoretical_loss": 3.52008934715142, + "tokens_seen": 1624571904 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004103671013039118, + "loss": 1.2942, + "theoretical_loss": 3.520077117738424, + "tokens_seen": 1624637440 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041035105315947844, + "loss": 1.3142, + "theoretical_loss": 3.5200648889568624, + "tokens_seen": 1624702976 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004103350050150452, + "loss": 1.3296, + "theoretical_loss": 3.520052660806676, + "tokens_seen": 1624768512 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041031895687061184, + "loss": 1.2968, + "theoretical_loss": 3.5200404332878072, + "tokens_seen": 1624834048 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004103029087261786, + "loss": 1.3407, + "theoretical_loss": 3.520028206400198, + "tokens_seen": 1624899584 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041028686058174524, + "loss": 1.351, + "theoretical_loss": 3.5200159801437905, + "tokens_seen": 1624965120 + }, + { + "epoch": 0.49, + "learning_rate": 0.000410270812437312, + "loss": 1.3142, + "theoretical_loss": 3.5200037545185268, + "tokens_seen": 1625030656 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041025476429287864, + "loss": 1.3699, + "theoretical_loss": 3.519991529524348, + "tokens_seen": 1625096192 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004102387161484454, + "loss": 1.3945, + "theoretical_loss": 3.5199793051611974, + "tokens_seen": 1625161728 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041022266800401204, + "loss": 1.1894, + "theoretical_loss": 3.519967081429016, + "tokens_seen": 1625227264 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 603543, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3595436811447144, + "objective/train/theoretical_loss": 3.519954858327746, + "objective/train/tokens_used": 1645752800, + "theoretical_loss": 3.519954858327746, + "tokens_seen": 1625292800 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004102066198595788, + "loss": 1.3595, + "theoretical_loss": 3.519954858327746, + "tokens_seen": 1625292800 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041019057171514544, + "loss": 1.2527, + "theoretical_loss": 3.5199426358573294, + "tokens_seen": 1625358336 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004101745235707122, + "loss": 1.2874, + "theoretical_loss": 3.5199304140177086, + "tokens_seen": 1625423872 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041015847542627884, + "loss": 1.4749, + "theoretical_loss": 3.5199181928088246, + "tokens_seen": 1625489408 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004101424272818456, + "loss": 1.2645, + "theoretical_loss": 3.519905972230621, + "tokens_seen": 1625554944 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041012637913741224, + "loss": 1.3568, + "theoretical_loss": 3.5198937522830382, + "tokens_seen": 1625620480 + }, + { + "epoch": 0.49, + "learning_rate": 0.000410110330992979, + "loss": 1.2923, + "theoretical_loss": 3.5198815329660196, + "tokens_seen": 1625686016 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041009428284854565, + "loss": 1.1764, + "theoretical_loss": 3.519869314279506, + "tokens_seen": 1625751552 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004100782347041124, + "loss": 1.279, + "theoretical_loss": 3.51985709622344, + "tokens_seen": 1625817088 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041006218655967905, + "loss": 1.2665, + "theoretical_loss": 3.5198448787977643, + "tokens_seen": 1625882624 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041004613841524575, + "loss": 1.3414, + "theoretical_loss": 3.51983266200242, + "tokens_seen": 1625948160 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041003009027081245, + "loss": 1.2196, + "theoretical_loss": 3.51982044583735, + "tokens_seen": 1626013696 + }, + { + "epoch": 0.49, + "learning_rate": 0.00041001404212637915, + "loss": 1.3051, + "theoretical_loss": 3.5198082303024956, + "tokens_seen": 1626079232 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040999799398194585, + "loss": 1.3423, + "theoretical_loss": 3.519796015397799, + "tokens_seen": 1626144768 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040998194583751255, + "loss": 1.3264, + "theoretical_loss": 3.5197838011232028, + "tokens_seen": 1626210304 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040996589769307925, + "loss": 1.2551, + "theoretical_loss": 3.5197715874786493, + "tokens_seen": 1626275840 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040994984954864595, + "loss": 1.1833, + "theoretical_loss": 3.51975937446408, + "tokens_seen": 1626341376 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040993380140421265, + "loss": 1.2824, + "theoretical_loss": 3.5197471620794367, + "tokens_seen": 1626406912 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040991775325977935, + "loss": 1.3333, + "theoretical_loss": 3.519734950324662, + "tokens_seen": 1626472448 + }, + { + "epoch": 0.49, + "learning_rate": 0.000409901705115346, + "loss": 1.2011, + "theoretical_loss": 3.519722739199698, + "tokens_seen": 1626537984 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040988565697091275, + "loss": 1.1238, + "theoretical_loss": 3.519710528704487, + "tokens_seen": 1626603520 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004098696088264794, + "loss": 1.2983, + "theoretical_loss": 3.5196983188389717, + "tokens_seen": 1626669056 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040985356068204615, + "loss": 1.3, + "theoretical_loss": 3.519686109603093, + "tokens_seen": 1626734592 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004098375125376128, + "loss": 1.3, + "theoretical_loss": 3.5196739009967937, + "tokens_seen": 1626800128 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040982146439317956, + "loss": 1.3319, + "theoretical_loss": 3.519661693020016, + "tokens_seen": 1626865664 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004098054162487462, + "loss": 1.4139, + "theoretical_loss": 3.5196494856727023, + "tokens_seen": 1626931200 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040978936810431296, + "loss": 1.3372, + "theoretical_loss": 3.519637278954794, + "tokens_seen": 1626996736 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004097733199598796, + "loss": 1.299, + "theoretical_loss": 3.5196250728662344, + "tokens_seen": 1627062272 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040975727181544636, + "loss": 1.3184, + "theoretical_loss": 3.5196128674069644, + "tokens_seen": 1627127808 + }, + { + "epoch": 0.49, + "learning_rate": 0.000409741223671013, + "loss": 1.3046, + "theoretical_loss": 3.519600662576927, + "tokens_seen": 1627193344 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040972517552657976, + "loss": 1.3192, + "theoretical_loss": 3.519588458376065, + "tokens_seen": 1627258880 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004097091273821464, + "loss": 1.2035, + "theoretical_loss": 3.5195762548043197, + "tokens_seen": 1627324416 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040969307923771316, + "loss": 1.2926, + "theoretical_loss": 3.5195640518616336, + "tokens_seen": 1627389952 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004096770310932798, + "loss": 1.2789, + "theoretical_loss": 3.5195518495479488, + "tokens_seen": 1627455488 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040966098294884656, + "loss": 1.227, + "theoretical_loss": 3.5195396478632075, + "tokens_seen": 1627521024 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004096449348044132, + "loss": 1.3345, + "theoretical_loss": 3.519527446807353, + "tokens_seen": 1627586560 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040962888665997996, + "loss": 1.3201, + "theoretical_loss": 3.519515246380326, + "tokens_seen": 1627652096 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004096128385155466, + "loss": 1.2483, + "theoretical_loss": 3.519503046582069, + "tokens_seen": 1627717632 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040959679037111336, + "loss": 1.2746, + "theoretical_loss": 3.519490847412526, + "tokens_seen": 1627783168 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004095807422266801, + "loss": 1.3951, + "theoretical_loss": 3.5194786488716376, + "tokens_seen": 1627848704 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040956469408224677, + "loss": 1.1878, + "theoretical_loss": 3.5194664509593463, + "tokens_seen": 1627914240 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004095486459378135, + "loss": 1.3509, + "theoretical_loss": 3.519454253675595, + "tokens_seen": 1627979776 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040953259779338017, + "loss": 1.3021, + "theoretical_loss": 3.519442057020325, + "tokens_seen": 1628045312 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004095165496489469, + "loss": 1.2881, + "theoretical_loss": 3.51942986099348, + "tokens_seen": 1628110848 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040950050150451357, + "loss": 1.3774, + "theoretical_loss": 3.519417665595002, + "tokens_seen": 1628176384 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004094844533600803, + "loss": 1.2941, + "theoretical_loss": 3.5194054708248323, + "tokens_seen": 1628241920 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040946840521564697, + "loss": 1.28, + "theoretical_loss": 3.519393276682914, + "tokens_seen": 1628307456 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004094523570712137, + "loss": 1.334, + "theoretical_loss": 3.5193810831691894, + "tokens_seen": 1628372992 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040943630892678037, + "loss": 1.2563, + "theoretical_loss": 3.5193688902836007, + "tokens_seen": 1628438528 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040942026078234707, + "loss": 1.2859, + "theoretical_loss": 3.5193566980260904, + "tokens_seen": 1628504064 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040940421263791377, + "loss": 1.429, + "theoretical_loss": 3.5193445063966013, + "tokens_seen": 1628569600 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004093881644934805, + "loss": 1.2965, + "theoretical_loss": 3.519332315395075, + "tokens_seen": 1628635136 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004093721163490472, + "loss": 1.2881, + "theoretical_loss": 3.519320125021454, + "tokens_seen": 1628700672 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004093560682046139, + "loss": 1.2449, + "theoretical_loss": 3.5193079352756813, + "tokens_seen": 1628766208 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004093400200601806, + "loss": 1.2865, + "theoretical_loss": 3.519295746157699, + "tokens_seen": 1628831744 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004093239719157473, + "loss": 1.3866, + "theoretical_loss": 3.519283557667449, + "tokens_seen": 1628897280 + }, + { + "epoch": 0.49, + "learning_rate": 0.000409307923771314, + "loss": 1.2794, + "theoretical_loss": 3.519271369804874, + "tokens_seen": 1628962816 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004092918756268807, + "loss": 1.3191, + "theoretical_loss": 3.5192591825699173, + "tokens_seen": 1629028352 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004092758274824473, + "loss": 1.3968, + "theoretical_loss": 3.5192469959625203, + "tokens_seen": 1629093888 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004092597793380141, + "loss": 1.2709, + "theoretical_loss": 3.5192348099826254, + "tokens_seen": 1629159424 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004092437311935807, + "loss": 1.3167, + "theoretical_loss": 3.5192226246301757, + "tokens_seen": 1629224960 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004092276830491475, + "loss": 1.2631, + "theoretical_loss": 3.5192104399051134, + "tokens_seen": 1629290496 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004092116349047141, + "loss": 1.2839, + "theoretical_loss": 3.519198255807381, + "tokens_seen": 1629356032 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004091955867602809, + "loss": 1.325, + "theoretical_loss": 3.5191860723369204, + "tokens_seen": 1629421568 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040917953861584753, + "loss": 1.361, + "theoretical_loss": 3.5191738894936746, + "tokens_seen": 1629487104 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004091634904714143, + "loss": 1.3187, + "theoretical_loss": 3.5191617072775863, + "tokens_seen": 1629552640 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040914744232698093, + "loss": 1.3533, + "theoretical_loss": 3.519149525688598, + "tokens_seen": 1629618176 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004091313941825477, + "loss": 1.2811, + "theoretical_loss": 3.519137344726652, + "tokens_seen": 1629683712 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040911534603811433, + "loss": 1.2868, + "theoretical_loss": 3.5191251643916903, + "tokens_seen": 1629749248 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004090992978936811, + "loss": 1.3646, + "theoretical_loss": 3.519112984683656, + "tokens_seen": 1629814784 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040908324974924773, + "loss": 1.2901, + "theoretical_loss": 3.5191008056024917, + "tokens_seen": 1629880320 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004090672016048145, + "loss": 1.3102, + "theoretical_loss": 3.5190886271481396, + "tokens_seen": 1629945856 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040905115346038113, + "loss": 1.2443, + "theoretical_loss": 3.5190764493205426, + "tokens_seen": 1630011392 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004090351053159479, + "loss": 1.3397, + "theoretical_loss": 3.5190642721196426, + "tokens_seen": 1630076928 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040901905717151453, + "loss": 1.3158, + "theoretical_loss": 3.5190520955453826, + "tokens_seen": 1630142464 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004090030090270813, + "loss": 1.2845, + "theoretical_loss": 3.519039919597706, + "tokens_seen": 1630208000 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040898696088264793, + "loss": 1.3492, + "theoretical_loss": 3.5190277442765536, + "tokens_seen": 1630273536 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004089709127382147, + "loss": 1.3213, + "theoretical_loss": 3.519015569581869, + "tokens_seen": 1630339072 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040895486459378134, + "loss": 1.3099, + "theoretical_loss": 3.5190033955135944, + "tokens_seen": 1630404608 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004089388164493481, + "loss": 1.2876, + "theoretical_loss": 3.5189912220716733, + "tokens_seen": 1630470144 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040892276830491474, + "loss": 1.3351, + "theoretical_loss": 3.518979049256047, + "tokens_seen": 1630535680 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004089067201604815, + "loss": 1.2739, + "theoretical_loss": 3.5189668770666596, + "tokens_seen": 1630601216 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040889067201604814, + "loss": 1.1681, + "theoretical_loss": 3.518954705503452, + "tokens_seen": 1630666752 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004088746238716149, + "loss": 1.2716, + "theoretical_loss": 3.5189425345663685, + "tokens_seen": 1630732288 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040885857572718154, + "loss": 1.3974, + "theoretical_loss": 3.5189303642553504, + "tokens_seen": 1630797824 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004088425275827483, + "loss": 1.4435, + "theoretical_loss": 3.5189181945703414, + "tokens_seen": 1630863360 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040882647943831494, + "loss": 1.2573, + "theoretical_loss": 3.518906025511283, + "tokens_seen": 1630928896 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004088104312938817, + "loss": 1.1632, + "theoretical_loss": 3.518893857078119, + "tokens_seen": 1630994432 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040879438314944834, + "loss": 1.3457, + "theoretical_loss": 3.518881689270791, + "tokens_seen": 1631059968 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004087783350050151, + "loss": 1.2958, + "theoretical_loss": 3.518869522089243, + "tokens_seen": 1631125504 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040876228686058174, + "loss": 1.2624, + "theoretical_loss": 3.5188573555334166, + "tokens_seen": 1631191040 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004087462387161485, + "loss": 1.2967, + "theoretical_loss": 3.518845189603254, + "tokens_seen": 1631256576 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040873019057171515, + "loss": 1.3109, + "theoretical_loss": 3.5188330242987, + "tokens_seen": 1631322112 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004087141424272819, + "loss": 1.2713, + "theoretical_loss": 3.5188208596196944, + "tokens_seen": 1631387648 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040869809428284855, + "loss": 1.2715, + "theoretical_loss": 3.5188086955661824, + "tokens_seen": 1631453184 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004086820461384153, + "loss": 1.2468, + "theoretical_loss": 3.518796532138106, + "tokens_seen": 1631518720 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040866599799398195, + "loss": 1.3511, + "theoretical_loss": 3.518784369335407, + "tokens_seen": 1631584256 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040864994984954865, + "loss": 1.3499, + "theoretical_loss": 3.5187722071580296, + "tokens_seen": 1631649792 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040863390170511535, + "loss": 1.3172, + "theoretical_loss": 3.518760045605915, + "tokens_seen": 1631715328 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040861785356068205, + "loss": 1.2788, + "theoretical_loss": 3.518747884679007, + "tokens_seen": 1631780864 + }, + { + "epoch": 0.49, + "objective/train/docs_used": 605960, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3807892799377441, + "objective/train/theoretical_loss": 3.5187357243772484, + "objective/train/tokens_used": 1652306400, + "theoretical_loss": 3.5187357243772484, + "tokens_seen": 1631846400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040860180541624875, + "loss": 1.3808, + "theoretical_loss": 3.5187357243772484, + "tokens_seen": 1631846400 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040858575727181545, + "loss": 1.2723, + "theoretical_loss": 3.518723564700581, + "tokens_seen": 1631911936 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040856970912738215, + "loss": 1.496, + "theoretical_loss": 3.518711405648949, + "tokens_seen": 1631977472 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040855366098294885, + "loss": 1.1541, + "theoretical_loss": 3.518699247222293, + "tokens_seen": 1632043008 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040853761283851555, + "loss": 1.2316, + "theoretical_loss": 3.5186870894205584, + "tokens_seen": 1632108544 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040852156469408225, + "loss": 1.3536, + "theoretical_loss": 3.518674932243686, + "tokens_seen": 1632174080 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004085055165496489, + "loss": 1.3878, + "theoretical_loss": 3.51866277569162, + "tokens_seen": 1632239616 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040848946840521565, + "loss": 1.3292, + "theoretical_loss": 3.518650619764302, + "tokens_seen": 1632305152 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004084734202607823, + "loss": 1.2883, + "theoretical_loss": 3.5186384644616755, + "tokens_seen": 1632370688 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040845737211634906, + "loss": 1.2734, + "theoretical_loss": 3.518626309783683, + "tokens_seen": 1632436224 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004084413239719157, + "loss": 1.2762, + "theoretical_loss": 3.5186141557302677, + "tokens_seen": 1632501760 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040842527582748246, + "loss": 1.3242, + "theoretical_loss": 3.518602002301372, + "tokens_seen": 1632567296 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004084092276830491, + "loss": 1.2247, + "theoretical_loss": 3.518589849496939, + "tokens_seen": 1632632832 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040839317953861586, + "loss": 1.4037, + "theoretical_loss": 3.518577697316912, + "tokens_seen": 1632698368 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004083771313941826, + "loss": 1.2479, + "theoretical_loss": 3.5185655457612324, + "tokens_seen": 1632763904 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040836108324974926, + "loss": 1.2842, + "theoretical_loss": 3.518553394829845, + "tokens_seen": 1632829440 + }, + { + "epoch": 0.49, + "learning_rate": 0.000408345035105316, + "loss": 1.4659, + "theoretical_loss": 3.518541244522691, + "tokens_seen": 1632894976 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040832898696088266, + "loss": 1.2866, + "theoretical_loss": 3.5185290948397148, + "tokens_seen": 1632960512 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004083129388164494, + "loss": 1.2387, + "theoretical_loss": 3.518516945780858, + "tokens_seen": 1633026048 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040829689067201606, + "loss": 1.3135, + "theoretical_loss": 3.518504797346064, + "tokens_seen": 1633091584 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004082808425275828, + "loss": 1.2191, + "theoretical_loss": 3.5184926495352755, + "tokens_seen": 1633157120 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040826479438314946, + "loss": 1.2961, + "theoretical_loss": 3.518480502348435, + "tokens_seen": 1633222656 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004082487462387162, + "loss": 1.2899, + "theoretical_loss": 3.518468355785487, + "tokens_seen": 1633288192 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040823269809428286, + "loss": 1.307, + "theoretical_loss": 3.518456209846373, + "tokens_seen": 1633353728 + }, + { + "epoch": 0.49, + "learning_rate": 0.0004082166499498496, + "loss": 1.3524, + "theoretical_loss": 3.5184440645310366, + "tokens_seen": 1633419264 + }, + { + "epoch": 0.49, + "learning_rate": 0.00040820060180541627, + "loss": 1.2974, + "theoretical_loss": 3.5184319198394203, + "tokens_seen": 1633484800 + }, + { + "epoch": 0.5, + "learning_rate": 0.000408184553660983, + "loss": 1.3154, + "theoretical_loss": 3.5184197757714673, + "tokens_seen": 1633550336 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040816850551654967, + "loss": 1.3626, + "theoretical_loss": 3.5184076323271203, + "tokens_seen": 1633615872 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004081524573721164, + "loss": 1.2994, + "theoretical_loss": 3.5183954895063225, + "tokens_seen": 1633681408 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040813640922768307, + "loss": 1.3429, + "theoretical_loss": 3.518383347309017, + "tokens_seen": 1633746944 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004081203610832498, + "loss": 1.2493, + "theoretical_loss": 3.5183712057351464, + "tokens_seen": 1633812480 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040810431293881647, + "loss": 1.2623, + "theoretical_loss": 3.5183590647846543, + "tokens_seen": 1633878016 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004080882647943832, + "loss": 1.2079, + "theoretical_loss": 3.5183469244574828, + "tokens_seen": 1633943552 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040807221664994987, + "loss": 1.2925, + "theoretical_loss": 3.5183347847535753, + "tokens_seen": 1634009088 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004080561685055166, + "loss": 1.2123, + "theoretical_loss": 3.5183226456728756, + "tokens_seen": 1634074624 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040804012036108327, + "loss": 1.4017, + "theoretical_loss": 3.5183105072153253, + "tokens_seen": 1634140160 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040802407221665, + "loss": 1.3126, + "theoretical_loss": 3.5182983693808683, + "tokens_seen": 1634205696 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004080080240722167, + "loss": 1.3645, + "theoretical_loss": 3.518286232169447, + "tokens_seen": 1634271232 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004079919759277834, + "loss": 1.2983, + "theoretical_loss": 3.5182740955810057, + "tokens_seen": 1634336768 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004079759277833501, + "loss": 1.3797, + "theoretical_loss": 3.5182619596154865, + "tokens_seen": 1634402304 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004079598796389168, + "loss": 1.1639, + "theoretical_loss": 3.518249824272832, + "tokens_seen": 1634467840 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004079438314944835, + "loss": 1.3202, + "theoretical_loss": 3.5182376895529863, + "tokens_seen": 1634533376 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004079277833500502, + "loss": 1.401, + "theoretical_loss": 3.518225555455892, + "tokens_seen": 1634598912 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004079117352056169, + "loss": 1.4299, + "theoretical_loss": 3.5182134219814922, + "tokens_seen": 1634664448 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004078956870611836, + "loss": 1.3603, + "theoretical_loss": 3.51820128912973, + "tokens_seen": 1634729984 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004078796389167502, + "loss": 1.2692, + "theoretical_loss": 3.518189156900548, + "tokens_seen": 1634795520 + }, + { + "epoch": 0.5, + "learning_rate": 0.000407863590772317, + "loss": 1.2982, + "theoretical_loss": 3.51817702529389, + "tokens_seen": 1634861056 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004078475426278836, + "loss": 1.2421, + "theoretical_loss": 3.5181648943096993, + "tokens_seen": 1634926592 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004078314944834504, + "loss": 1.3493, + "theoretical_loss": 3.518152763947918, + "tokens_seen": 1634992128 + }, + { + "epoch": 0.5, + "learning_rate": 0.000407815446339017, + "loss": 1.3591, + "theoretical_loss": 3.51814063420849, + "tokens_seen": 1635057664 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004077993981945838, + "loss": 1.4418, + "theoretical_loss": 3.518128505091358, + "tokens_seen": 1635123200 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040778335005015043, + "loss": 1.4044, + "theoretical_loss": 3.5181163765964656, + "tokens_seen": 1635188736 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004077673019057172, + "loss": 1.4027, + "theoretical_loss": 3.5181042487237555, + "tokens_seen": 1635254272 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040775125376128383, + "loss": 1.3643, + "theoretical_loss": 3.5180921214731713, + "tokens_seen": 1635319808 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004077352056168506, + "loss": 1.4227, + "theoretical_loss": 3.5180799948446557, + "tokens_seen": 1635385344 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040771915747241723, + "loss": 1.297, + "theoretical_loss": 3.5180678688381524, + "tokens_seen": 1635450880 + }, + { + "epoch": 0.5, + "learning_rate": 0.000407703109327984, + "loss": 1.3816, + "theoretical_loss": 3.518055743453604, + "tokens_seen": 1635516416 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040768706118355063, + "loss": 1.3336, + "theoretical_loss": 3.5180436186909536, + "tokens_seen": 1635581952 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004076710130391174, + "loss": 1.3998, + "theoretical_loss": 3.518031494550145, + "tokens_seen": 1635647488 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040765496489468403, + "loss": 1.3132, + "theoretical_loss": 3.518019371031121, + "tokens_seen": 1635713024 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004076389167502508, + "loss": 1.3768, + "theoretical_loss": 3.518007248133825, + "tokens_seen": 1635778560 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040762286860581743, + "loss": 1.3972, + "theoretical_loss": 3.5179951258582, + "tokens_seen": 1635844096 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004076068204613842, + "loss": 1.2843, + "theoretical_loss": 3.517983004204189, + "tokens_seen": 1635909632 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040759077231695084, + "loss": 1.2988, + "theoretical_loss": 3.5179708831717362, + "tokens_seen": 1635975168 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004075747241725176, + "loss": 1.3088, + "theoretical_loss": 3.5179587627607836, + "tokens_seen": 1636040704 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040755867602808424, + "loss": 1.3033, + "theoretical_loss": 3.5179466429712756, + "tokens_seen": 1636106240 + }, + { + "epoch": 0.5, + "learning_rate": 0.000407542627883651, + "loss": 1.3372, + "theoretical_loss": 3.5179345238031545, + "tokens_seen": 1636171776 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040752657973921764, + "loss": 1.4301, + "theoretical_loss": 3.517922405256364, + "tokens_seen": 1636237312 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004075105315947844, + "loss": 1.4328, + "theoretical_loss": 3.5179102873308468, + "tokens_seen": 1636302848 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040749448345035104, + "loss": 1.1861, + "theoretical_loss": 3.5178981700265473, + "tokens_seen": 1636368384 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004074784353059178, + "loss": 1.2883, + "theoretical_loss": 3.5178860533434078, + "tokens_seen": 1636433920 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040746238716148444, + "loss": 1.4231, + "theoretical_loss": 3.5178739372813723, + "tokens_seen": 1636499456 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004074463390170512, + "loss": 1.3528, + "theoretical_loss": 3.517861821840383, + "tokens_seen": 1636564992 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040743029087261784, + "loss": 1.279, + "theoretical_loss": 3.5178497070203845, + "tokens_seen": 1636630528 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004074142427281846, + "loss": 1.3365, + "theoretical_loss": 3.5178375928213192, + "tokens_seen": 1636696064 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040739819458375124, + "loss": 1.4148, + "theoretical_loss": 3.5178254792431307, + "tokens_seen": 1636761600 + }, + { + "epoch": 0.5, + "learning_rate": 0.000407382146439318, + "loss": 1.2708, + "theoretical_loss": 3.517813366285762, + "tokens_seen": 1636827136 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040736609829488464, + "loss": 1.2523, + "theoretical_loss": 3.517801253949157, + "tokens_seen": 1636892672 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004073500501504514, + "loss": 1.3287, + "theoretical_loss": 3.5177891422332594, + "tokens_seen": 1636958208 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040733400200601805, + "loss": 1.322, + "theoretical_loss": 3.5177770311380114, + "tokens_seen": 1637023744 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004073179538615848, + "loss": 1.2552, + "theoretical_loss": 3.5177649206633568, + "tokens_seen": 1637089280 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040730190571715145, + "loss": 1.3247, + "theoretical_loss": 3.5177528108092395, + "tokens_seen": 1637154816 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004072858575727182, + "loss": 1.3483, + "theoretical_loss": 3.517740701575602, + "tokens_seen": 1637220352 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040726980942828485, + "loss": 1.3807, + "theoretical_loss": 3.5177285929623885, + "tokens_seen": 1637285888 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040725376128385155, + "loss": 1.4166, + "theoretical_loss": 3.5177164849695415, + "tokens_seen": 1637351424 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040723771313941825, + "loss": 1.2436, + "theoretical_loss": 3.5177043775970054, + "tokens_seen": 1637416960 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040722166499498495, + "loss": 1.2859, + "theoretical_loss": 3.517692270844722, + "tokens_seen": 1637482496 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004072056168505517, + "loss": 1.2833, + "theoretical_loss": 3.517680164712637, + "tokens_seen": 1637548032 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040718956870611835, + "loss": 1.3461, + "theoretical_loss": 3.517668059200692, + "tokens_seen": 1637613568 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004071735205616851, + "loss": 1.4359, + "theoretical_loss": 3.5176559543088306, + "tokens_seen": 1637679104 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040715747241725175, + "loss": 1.2789, + "theoretical_loss": 3.517643850036997, + "tokens_seen": 1637744640 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004071414242728185, + "loss": 1.3127, + "theoretical_loss": 3.5176317463851343, + "tokens_seen": 1637810176 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040712537612838515, + "loss": 1.2066, + "theoretical_loss": 3.5176196433531857, + "tokens_seen": 1637875712 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004071093279839519, + "loss": 1.3076, + "theoretical_loss": 3.517607540941095, + "tokens_seen": 1637941248 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040709327983951856, + "loss": 1.3031, + "theoretical_loss": 3.5175954391488053, + "tokens_seen": 1638006784 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004070772316950853, + "loss": 1.3519, + "theoretical_loss": 3.5175833379762604, + "tokens_seen": 1638072320 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040706118355065196, + "loss": 1.3363, + "theoretical_loss": 3.5175712374234034, + "tokens_seen": 1638137856 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004070451354062187, + "loss": 1.212, + "theoretical_loss": 3.5175591374901782, + "tokens_seen": 1638203392 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040702908726178536, + "loss": 1.404, + "theoretical_loss": 3.517547038176528, + "tokens_seen": 1638268928 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004070130391173521, + "loss": 1.3993, + "theoretical_loss": 3.5175349394823963, + "tokens_seen": 1638334464 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 608340, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3364691734313965, + "objective/train/theoretical_loss": 3.5175228414077266, + "objective/train/tokens_used": 1658860000, + "theoretical_loss": 3.5175228414077266, + "tokens_seen": 1638400000 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040699699097291876, + "loss": 1.3365, + "theoretical_loss": 3.5175228414077266, + "tokens_seen": 1638400000 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004069809428284855, + "loss": 1.3842, + "theoretical_loss": 3.517510743952463, + "tokens_seen": 1638465536 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040696489468405216, + "loss": 1.3077, + "theoretical_loss": 3.5174986471165477, + "tokens_seen": 1638531072 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004069488465396189, + "loss": 1.3108, + "theoretical_loss": 3.5174865508999256, + "tokens_seen": 1638596608 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040693279839518556, + "loss": 1.3513, + "theoretical_loss": 3.5174744553025388, + "tokens_seen": 1638662144 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004069167502507523, + "loss": 1.3886, + "theoretical_loss": 3.5174623603243322, + "tokens_seen": 1638727680 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040690070210631896, + "loss": 1.3587, + "theoretical_loss": 3.517450265965249, + "tokens_seen": 1638793216 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004068846539618857, + "loss": 1.3201, + "theoretical_loss": 3.5174381722252326, + "tokens_seen": 1638858752 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040686860581745236, + "loss": 1.3327, + "theoretical_loss": 3.517426079104226, + "tokens_seen": 1638924288 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004068525576730191, + "loss": 1.2764, + "theoretical_loss": 3.5174139866021736, + "tokens_seen": 1638989824 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040683650952858577, + "loss": 1.3121, + "theoretical_loss": 3.5174018947190193, + "tokens_seen": 1639055360 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004068204613841525, + "loss": 1.2745, + "theoretical_loss": 3.517389803454705, + "tokens_seen": 1639120896 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040680441323971917, + "loss": 1.3115, + "theoretical_loss": 3.517377712809176, + "tokens_seen": 1639186432 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004067883650952859, + "loss": 1.3479, + "theoretical_loss": 3.517365622782375, + "tokens_seen": 1639251968 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040677231695085257, + "loss": 1.3171, + "theoretical_loss": 3.5173535333742456, + "tokens_seen": 1639317504 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004067562688064193, + "loss": 1.2943, + "theoretical_loss": 3.5173414445847317, + "tokens_seen": 1639383040 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040674022066198597, + "loss": 1.2869, + "theoretical_loss": 3.517329356413777, + "tokens_seen": 1639448576 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004067241725175527, + "loss": 1.3003, + "theoretical_loss": 3.5173172688613255, + "tokens_seen": 1639514112 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040670812437311937, + "loss": 1.3225, + "theoretical_loss": 3.5173051819273193, + "tokens_seen": 1639579648 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004066920762286861, + "loss": 1.3425, + "theoretical_loss": 3.517293095611704, + "tokens_seen": 1639645184 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040667602808425277, + "loss": 1.3345, + "theoretical_loss": 3.5172810099144214, + "tokens_seen": 1639710720 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004066599799398195, + "loss": 1.3024, + "theoretical_loss": 3.5172689248354168, + "tokens_seen": 1639776256 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004066439317953862, + "loss": 1.4211, + "theoretical_loss": 3.5172568403746327, + "tokens_seen": 1639841792 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040662788365095293, + "loss": 1.2981, + "theoretical_loss": 3.517244756532013, + "tokens_seen": 1639907328 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004066118355065196, + "loss": 1.3244, + "theoretical_loss": 3.517232673307502, + "tokens_seen": 1639972864 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004065957873620863, + "loss": 1.4259, + "theoretical_loss": 3.5172205907010428, + "tokens_seen": 1640038400 + }, + { + "epoch": 0.5, + "learning_rate": 0.000406579739217653, + "loss": 1.3652, + "theoretical_loss": 3.517208508712579, + "tokens_seen": 1640103936 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004065636910732197, + "loss": 1.3109, + "theoretical_loss": 3.5171964273420544, + "tokens_seen": 1640169472 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004065476429287864, + "loss": 1.3125, + "theoretical_loss": 3.5171843465894135, + "tokens_seen": 1640235008 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004065315947843531, + "loss": 1.3319, + "theoretical_loss": 3.5171722664545992, + "tokens_seen": 1640300544 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004065155466399198, + "loss": 1.3295, + "theoretical_loss": 3.517160186937555, + "tokens_seen": 1640366080 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004064994984954865, + "loss": 1.2945, + "theoretical_loss": 3.517148108038225, + "tokens_seen": 1640431616 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004064834503510531, + "loss": 1.3372, + "theoretical_loss": 3.5171360297565535, + "tokens_seen": 1640497152 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004064674022066199, + "loss": 1.3657, + "theoretical_loss": 3.517123952092483, + "tokens_seen": 1640562688 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004064513540621865, + "loss": 1.2687, + "theoretical_loss": 3.5171118750459582, + "tokens_seen": 1640628224 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004064353059177533, + "loss": 1.3603, + "theoretical_loss": 3.517099798616923, + "tokens_seen": 1640693760 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040641925777331993, + "loss": 1.3498, + "theoretical_loss": 3.51708772280532, + "tokens_seen": 1640759296 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004064032096288867, + "loss": 1.3662, + "theoretical_loss": 3.5170756476110943, + "tokens_seen": 1640824832 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040638716148445333, + "loss": 1.3026, + "theoretical_loss": 3.517063573034189, + "tokens_seen": 1640890368 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004063711133400201, + "loss": 1.2952, + "theoretical_loss": 3.517051499074548, + "tokens_seen": 1640955904 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040635506519558673, + "loss": 1.2347, + "theoretical_loss": 3.517039425732115, + "tokens_seen": 1641021440 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004063390170511535, + "loss": 1.3667, + "theoretical_loss": 3.5170273530068337, + "tokens_seen": 1641086976 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040632296890672013, + "loss": 1.2544, + "theoretical_loss": 3.517015280898648, + "tokens_seen": 1641152512 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004063069207622869, + "loss": 1.4538, + "theoretical_loss": 3.5170032094075023, + "tokens_seen": 1641218048 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040629087261785353, + "loss": 1.201, + "theoretical_loss": 3.51699113853334, + "tokens_seen": 1641283584 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004062748244734203, + "loss": 1.3778, + "theoretical_loss": 3.516979068276105, + "tokens_seen": 1641349120 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040625877632898693, + "loss": 1.3567, + "theoretical_loss": 3.5169669986357404, + "tokens_seen": 1641414656 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004062427281845537, + "loss": 1.3915, + "theoretical_loss": 3.516954929612191, + "tokens_seen": 1641480192 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040622668004012034, + "loss": 1.2119, + "theoretical_loss": 3.5169428612053997, + "tokens_seen": 1641545728 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004062106318956871, + "loss": 1.2226, + "theoretical_loss": 3.5169307934153116, + "tokens_seen": 1641611264 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040619458375125374, + "loss": 1.31, + "theoretical_loss": 3.5169187262418697, + "tokens_seen": 1641676800 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004061785356068205, + "loss": 1.3385, + "theoretical_loss": 3.5169066596850183, + "tokens_seen": 1641742336 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040616248746238714, + "loss": 1.3246, + "theoretical_loss": 3.516894593744701, + "tokens_seen": 1641807872 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004061464393179539, + "loss": 1.3474, + "theoretical_loss": 3.516882528420862, + "tokens_seen": 1641873408 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040613039117352054, + "loss": 1.3625, + "theoretical_loss": 3.5168704637134445, + "tokens_seen": 1641938944 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004061143430290873, + "loss": 1.2549, + "theoretical_loss": 3.5168583996223934, + "tokens_seen": 1642004480 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040609829488465394, + "loss": 1.2264, + "theoretical_loss": 3.516846336147652, + "tokens_seen": 1642070016 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004060822467402207, + "loss": 1.3046, + "theoretical_loss": 3.516834273289164, + "tokens_seen": 1642135552 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040606619859578734, + "loss": 1.2705, + "theoretical_loss": 3.516822211046874, + "tokens_seen": 1642201088 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004060501504513541, + "loss": 1.2613, + "theoretical_loss": 3.516810149420725, + "tokens_seen": 1642266624 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040603410230692074, + "loss": 1.3482, + "theoretical_loss": 3.5167980884106624, + "tokens_seen": 1642332160 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004060180541624875, + "loss": 1.3949, + "theoretical_loss": 3.5167860280166288, + "tokens_seen": 1642397696 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040600200601805425, + "loss": 1.2998, + "theoretical_loss": 3.516773968238569, + "tokens_seen": 1642463232 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004059859578736209, + "loss": 1.3104, + "theoretical_loss": 3.5167619090764264, + "tokens_seen": 1642528768 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004059699097291876, + "loss": 1.2205, + "theoretical_loss": 3.516749850530145, + "tokens_seen": 1642594304 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004059538615847543, + "loss": 1.2931, + "theoretical_loss": 3.5167377925996686, + "tokens_seen": 1642659840 + }, + { + "epoch": 0.5, + "learning_rate": 0.000405937813440321, + "loss": 1.2829, + "theoretical_loss": 3.516725735284942, + "tokens_seen": 1642725376 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004059217652958877, + "loss": 1.3597, + "theoretical_loss": 3.5167136785859086, + "tokens_seen": 1642790912 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004059057171514544, + "loss": 1.3346, + "theoretical_loss": 3.5167016225025125, + "tokens_seen": 1642856448 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004058896690070211, + "loss": 1.2562, + "theoretical_loss": 3.516689567034698, + "tokens_seen": 1642921984 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004058736208625878, + "loss": 1.2823, + "theoretical_loss": 3.516677512182408, + "tokens_seen": 1642987520 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004058575727181545, + "loss": 1.2755, + "theoretical_loss": 3.5166654579455887, + "tokens_seen": 1643053056 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004058415245737212, + "loss": 1.2713, + "theoretical_loss": 3.516653404324182, + "tokens_seen": 1643118592 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040582547642928785, + "loss": 1.3542, + "theoretical_loss": 3.5166413513181327, + "tokens_seen": 1643184128 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004058094282848546, + "loss": 1.3778, + "theoretical_loss": 3.516629298927385, + "tokens_seen": 1643249664 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040579338014042125, + "loss": 1.4286, + "theoretical_loss": 3.516617247151883, + "tokens_seen": 1643315200 + }, + { + "epoch": 0.5, + "learning_rate": 0.000405777331995988, + "loss": 1.1928, + "theoretical_loss": 3.5166051959915703, + "tokens_seen": 1643380736 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040576128385155465, + "loss": 1.3359, + "theoretical_loss": 3.5165931454463912, + "tokens_seen": 1643446272 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004057452357071214, + "loss": 1.4728, + "theoretical_loss": 3.5165810955162904, + "tokens_seen": 1643511808 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040572918756268806, + "loss": 1.3301, + "theoretical_loss": 3.516569046201211, + "tokens_seen": 1643577344 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004057131394182548, + "loss": 1.3159, + "theoretical_loss": 3.5165569975010973, + "tokens_seen": 1643642880 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040569709127382146, + "loss": 1.2888, + "theoretical_loss": 3.516544949415894, + "tokens_seen": 1643708416 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004056810431293882, + "loss": 1.3228, + "theoretical_loss": 3.5165329019455447, + "tokens_seen": 1643773952 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040566499498495486, + "loss": 1.3848, + "theoretical_loss": 3.5165208550899933, + "tokens_seen": 1643839488 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004056489468405216, + "loss": 1.2436, + "theoretical_loss": 3.516508808849184, + "tokens_seen": 1643905024 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040563289869608826, + "loss": 1.2612, + "theoretical_loss": 3.5164967632230617, + "tokens_seen": 1643970560 + }, + { + "epoch": 0.5, + "learning_rate": 0.000405616850551655, + "loss": 1.366, + "theoretical_loss": 3.51648471821157, + "tokens_seen": 1644036096 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040560080240722166, + "loss": 1.3539, + "theoretical_loss": 3.5164726738146532, + "tokens_seen": 1644101632 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004055847542627884, + "loss": 1.2309, + "theoretical_loss": 3.5164606300322547, + "tokens_seen": 1644167168 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040556870611835506, + "loss": 1.2705, + "theoretical_loss": 3.5164485868643194, + "tokens_seen": 1644232704 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004055526579739218, + "loss": 1.4179, + "theoretical_loss": 3.5164365443107917, + "tokens_seen": 1644298240 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040553660982948846, + "loss": 1.3049, + "theoretical_loss": 3.5164245023716147, + "tokens_seen": 1644363776 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004055205616850552, + "loss": 1.3528, + "theoretical_loss": 3.5164124610467336, + "tokens_seen": 1644429312 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040550451354062186, + "loss": 1.217, + "theoretical_loss": 3.516400420336092, + "tokens_seen": 1644494848 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004054884653961886, + "loss": 1.2568, + "theoretical_loss": 3.5163883802396345, + "tokens_seen": 1644560384 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040547241725175527, + "loss": 1.4148, + "theoretical_loss": 3.516376340757305, + "tokens_seen": 1644625920 + }, + { + "epoch": 0.5, + "learning_rate": 0.000405456369107322, + "loss": 1.2428, + "theoretical_loss": 3.516364301889048, + "tokens_seen": 1644691456 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040544032096288867, + "loss": 1.3164, + "theoretical_loss": 3.516352263634807, + "tokens_seen": 1644756992 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004054242728184554, + "loss": 1.3158, + "theoretical_loss": 3.516340225994527, + "tokens_seen": 1644822528 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040540822467402207, + "loss": 1.3231, + "theoretical_loss": 3.516328188968152, + "tokens_seen": 1644888064 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 610577, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3093676567077637, + "objective/train/theoretical_loss": 3.5163161525556266, + "objective/train/tokens_used": 1665413600, + "theoretical_loss": 3.5163161525556266, + "tokens_seen": 1644953600 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004053921765295888, + "loss": 1.3094, + "theoretical_loss": 3.5163161525556266, + "tokens_seen": 1644953600 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040537612838515547, + "loss": 1.3156, + "theoretical_loss": 3.516304116756894, + "tokens_seen": 1645019136 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004053600802407222, + "loss": 1.2874, + "theoretical_loss": 3.5162920815718994, + "tokens_seen": 1645084672 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040534403209628887, + "loss": 1.3523, + "theoretical_loss": 3.5162800470005866, + "tokens_seen": 1645150208 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004053279839518556, + "loss": 1.2845, + "theoretical_loss": 3.5162680130429003, + "tokens_seen": 1645215744 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040531193580742227, + "loss": 1.3406, + "theoretical_loss": 3.516255979698784, + "tokens_seen": 1645281280 + }, + { + "epoch": 0.5, + "learning_rate": 0.000405295887662989, + "loss": 1.342, + "theoretical_loss": 3.5162439469681828, + "tokens_seen": 1645346816 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004052798395185557, + "loss": 1.3648, + "theoretical_loss": 3.5162319148510406, + "tokens_seen": 1645412352 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040526379137412243, + "loss": 1.3106, + "theoretical_loss": 3.5162198833473015, + "tokens_seen": 1645477888 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004052477432296891, + "loss": 1.3313, + "theoretical_loss": 3.51620785245691, + "tokens_seen": 1645543424 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040523169508525583, + "loss": 1.2922, + "theoretical_loss": 3.5161958221798106, + "tokens_seen": 1645608960 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004052156469408225, + "loss": 1.3884, + "theoretical_loss": 3.516183792515948, + "tokens_seen": 1645674496 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004051995987963892, + "loss": 1.2759, + "theoretical_loss": 3.516171763465265, + "tokens_seen": 1645740032 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004051835506519559, + "loss": 1.3204, + "theoretical_loss": 3.516159735027707, + "tokens_seen": 1645805568 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004051675025075226, + "loss": 1.3595, + "theoretical_loss": 3.516147707203219, + "tokens_seen": 1645871104 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004051514543630893, + "loss": 1.2746, + "theoretical_loss": 3.516135679991744, + "tokens_seen": 1645936640 + }, + { + "epoch": 0.5, + "learning_rate": 0.000405135406218656, + "loss": 1.2397, + "theoretical_loss": 3.516123653393227, + "tokens_seen": 1646002176 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004051193580742227, + "loss": 1.289, + "theoretical_loss": 3.5161116274076125, + "tokens_seen": 1646067712 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004051033099297894, + "loss": 1.3531, + "theoretical_loss": 3.5160996020348447, + "tokens_seen": 1646133248 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004050872617853561, + "loss": 1.2904, + "theoretical_loss": 3.5160875772748676, + "tokens_seen": 1646198784 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004050712136409228, + "loss": 1.2579, + "theoretical_loss": 3.516075553127626, + "tokens_seen": 1646264320 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040505516549648943, + "loss": 1.2888, + "theoretical_loss": 3.5160635295930645, + "tokens_seen": 1646329856 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004050391173520562, + "loss": 1.3051, + "theoretical_loss": 3.5160515066711264, + "tokens_seen": 1646395392 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040502306920762283, + "loss": 1.2504, + "theoretical_loss": 3.516039484361758, + "tokens_seen": 1646460928 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004050070210631896, + "loss": 1.3896, + "theoretical_loss": 3.5160274626649017, + "tokens_seen": 1646526464 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040499097291875623, + "loss": 1.3444, + "theoretical_loss": 3.516015441580503, + "tokens_seen": 1646592000 + }, + { + "epoch": 0.5, + "learning_rate": 0.000404974924774323, + "loss": 1.2732, + "theoretical_loss": 3.5160034211085067, + "tokens_seen": 1646657536 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040495887662988963, + "loss": 1.2798, + "theoretical_loss": 3.515991401248856, + "tokens_seen": 1646723072 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004049428284854564, + "loss": 1.4088, + "theoretical_loss": 3.515979382001496, + "tokens_seen": 1646788608 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040492678034102303, + "loss": 1.3123, + "theoretical_loss": 3.5159673633663715, + "tokens_seen": 1646854144 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004049107321965898, + "loss": 1.4436, + "theoretical_loss": 3.5159553453434267, + "tokens_seen": 1646919680 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040489468405215643, + "loss": 1.3108, + "theoretical_loss": 3.5159433279326056, + "tokens_seen": 1646985216 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004048786359077232, + "loss": 1.4041, + "theoretical_loss": 3.515931311133853, + "tokens_seen": 1647050752 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040486258776328984, + "loss": 1.3422, + "theoretical_loss": 3.515919294947114, + "tokens_seen": 1647116288 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004048465396188566, + "loss": 1.4339, + "theoretical_loss": 3.515907279372332, + "tokens_seen": 1647181824 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040483049147442324, + "loss": 1.3154, + "theoretical_loss": 3.5158952644094517, + "tokens_seen": 1647247360 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040481444332999, + "loss": 1.2789, + "theoretical_loss": 3.515883250058418, + "tokens_seen": 1647312896 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040479839518555675, + "loss": 1.3577, + "theoretical_loss": 3.5158712363191755, + "tokens_seen": 1647378432 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004047823470411234, + "loss": 1.3473, + "theoretical_loss": 3.5158592231916685, + "tokens_seen": 1647443968 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040476629889669015, + "loss": 1.2823, + "theoretical_loss": 3.5158472106758416, + "tokens_seen": 1647509504 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004047502507522568, + "loss": 1.1423, + "theoretical_loss": 3.515835198771639, + "tokens_seen": 1647575040 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040473420260782355, + "loss": 1.2652, + "theoretical_loss": 3.5158231874790054, + "tokens_seen": 1647640576 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004047181544633902, + "loss": 1.3725, + "theoretical_loss": 3.5158111767978855, + "tokens_seen": 1647706112 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040470210631895695, + "loss": 1.3444, + "theoretical_loss": 3.5157991667282236, + "tokens_seen": 1647771648 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004046860581745236, + "loss": 1.3932, + "theoretical_loss": 3.5157871572699646, + "tokens_seen": 1647837184 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040467001003009035, + "loss": 1.3526, + "theoretical_loss": 3.515775148423053, + "tokens_seen": 1647902720 + }, + { + "epoch": 0.5, + "learning_rate": 0.000404653961885657, + "loss": 1.2114, + "theoretical_loss": 3.515763140187433, + "tokens_seen": 1647968256 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040463791374122375, + "loss": 1.3468, + "theoretical_loss": 3.5157511325630493, + "tokens_seen": 1648033792 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004046218655967904, + "loss": 1.3703, + "theoretical_loss": 3.5157391255498465, + "tokens_seen": 1648099328 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040460581745235715, + "loss": 1.2572, + "theoretical_loss": 3.5157271191477695, + "tokens_seen": 1648164864 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004045897693079238, + "loss": 1.3241, + "theoretical_loss": 3.5157151133567623, + "tokens_seen": 1648230400 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004045737211634905, + "loss": 1.1996, + "theoretical_loss": 3.5157031081767705, + "tokens_seen": 1648295936 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004045576730190572, + "loss": 1.3829, + "theoretical_loss": 3.515691103607738, + "tokens_seen": 1648361472 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004045416248746239, + "loss": 1.1028, + "theoretical_loss": 3.515679099649609, + "tokens_seen": 1648427008 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004045255767301906, + "loss": 1.275, + "theoretical_loss": 3.515667096302329, + "tokens_seen": 1648492544 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004045095285857573, + "loss": 1.3056, + "theoretical_loss": 3.515655093565842, + "tokens_seen": 1648558080 + }, + { + "epoch": 0.5, + "learning_rate": 0.000404493480441324, + "loss": 1.2961, + "theoretical_loss": 3.5156430914400927, + "tokens_seen": 1648623616 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004044774322968907, + "loss": 1.3906, + "theoretical_loss": 3.5156310899250265, + "tokens_seen": 1648689152 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004044613841524574, + "loss": 1.4145, + "theoretical_loss": 3.5156190890205874, + "tokens_seen": 1648754688 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004044453360080241, + "loss": 1.4084, + "theoretical_loss": 3.5156070887267195, + "tokens_seen": 1648820224 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040442928786359075, + "loss": 1.3021, + "theoretical_loss": 3.515595089043369, + "tokens_seen": 1648885760 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004044132397191575, + "loss": 1.2976, + "theoretical_loss": 3.5155830899704794, + "tokens_seen": 1648951296 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040439719157472415, + "loss": 1.327, + "theoretical_loss": 3.5155710915079954, + "tokens_seen": 1649016832 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004043811434302909, + "loss": 1.3312, + "theoretical_loss": 3.5155590936558623, + "tokens_seen": 1649082368 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040436509528585755, + "loss": 1.405, + "theoretical_loss": 3.5155470964140245, + "tokens_seen": 1649147904 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004043490471414243, + "loss": 1.4582, + "theoretical_loss": 3.5155350997824266, + "tokens_seen": 1649213440 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040433299899699096, + "loss": 1.4282, + "theoretical_loss": 3.5155231037610135, + "tokens_seen": 1649278976 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004043169508525577, + "loss": 1.2171, + "theoretical_loss": 3.5155111083497297, + "tokens_seen": 1649344512 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040430090270812436, + "loss": 1.2498, + "theoretical_loss": 3.51549911354852, + "tokens_seen": 1649410048 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004042848545636911, + "loss": 1.3462, + "theoretical_loss": 3.5154871193573296, + "tokens_seen": 1649475584 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040426880641925776, + "loss": 1.2892, + "theoretical_loss": 3.5154751257761023, + "tokens_seen": 1649541120 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004042527582748245, + "loss": 1.3793, + "theoretical_loss": 3.5154631328047836, + "tokens_seen": 1649606656 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040423671013039116, + "loss": 1.281, + "theoretical_loss": 3.515451140443318, + "tokens_seen": 1649672192 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004042206619859579, + "loss": 1.2936, + "theoretical_loss": 3.51543914869165, + "tokens_seen": 1649737728 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040420461384152456, + "loss": 1.2631, + "theoretical_loss": 3.515427157549725, + "tokens_seen": 1649803264 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004041885656970913, + "loss": 1.2776, + "theoretical_loss": 3.5154151670174874, + "tokens_seen": 1649868800 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040417251755265796, + "loss": 1.3155, + "theoretical_loss": 3.515403177094882, + "tokens_seen": 1649934336 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004041564694082247, + "loss": 1.2751, + "theoretical_loss": 3.5153911877818533, + "tokens_seen": 1649999872 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040414042126379136, + "loss": 1.3102, + "theoretical_loss": 3.515379199078347, + "tokens_seen": 1650065408 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004041243731193581, + "loss": 1.3829, + "theoretical_loss": 3.515367210984307, + "tokens_seen": 1650130944 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040410832497492477, + "loss": 1.3151, + "theoretical_loss": 3.515355223499678, + "tokens_seen": 1650196480 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004040922768304915, + "loss": 1.2732, + "theoretical_loss": 3.515343236624406, + "tokens_seen": 1650262016 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040407622868605817, + "loss": 1.3487, + "theoretical_loss": 3.5153312503584346, + "tokens_seen": 1650327552 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004040601805416249, + "loss": 1.2867, + "theoretical_loss": 3.5153192647017093, + "tokens_seen": 1650393088 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040404413239719157, + "loss": 1.2997, + "theoretical_loss": 3.5153072796541744, + "tokens_seen": 1650458624 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004040280842527583, + "loss": 1.3913, + "theoretical_loss": 3.5152952952157754, + "tokens_seen": 1650524160 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040401203610832497, + "loss": 1.3352, + "theoretical_loss": 3.515283311386457, + "tokens_seen": 1650589696 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004039959879638917, + "loss": 1.3524, + "theoretical_loss": 3.515271328166164, + "tokens_seen": 1650655232 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040397993981945837, + "loss": 1.3493, + "theoretical_loss": 3.515259345554841, + "tokens_seen": 1650720768 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004039638916750251, + "loss": 1.3954, + "theoretical_loss": 3.515247363552433, + "tokens_seen": 1650786304 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040394784353059177, + "loss": 1.3455, + "theoretical_loss": 3.515235382158885, + "tokens_seen": 1650851840 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004039317953861585, + "loss": 1.2894, + "theoretical_loss": 3.5152234013741417, + "tokens_seen": 1650917376 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040391574724172517, + "loss": 1.3466, + "theoretical_loss": 3.5152114211981487, + "tokens_seen": 1650982912 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040389969909729193, + "loss": 1.4989, + "theoretical_loss": 3.51519944163085, + "tokens_seen": 1651048448 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004038836509528586, + "loss": 1.3095, + "theoretical_loss": 3.5151874626721904, + "tokens_seen": 1651113984 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040386760280842533, + "loss": 1.31, + "theoretical_loss": 3.515175484322116, + "tokens_seen": 1651179520 + }, + { + "epoch": 0.5, + "learning_rate": 0.000403851554663992, + "loss": 1.2484, + "theoretical_loss": 3.5151635065805706, + "tokens_seen": 1651245056 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040383550651955873, + "loss": 1.4415, + "theoretical_loss": 3.5151515294474995, + "tokens_seen": 1651310592 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004038194583751254, + "loss": 1.3745, + "theoretical_loss": 3.515139552922848, + "tokens_seen": 1651376128 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004038034102306921, + "loss": 1.2912, + "theoretical_loss": 3.5151275770065604, + "tokens_seen": 1651441664 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 613332, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2950294017791748, + "objective/train/theoretical_loss": 3.5151156016985823, + "objective/train/tokens_used": 1671967200, + "theoretical_loss": 3.5151156016985823, + "tokens_seen": 1651507200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004037873620862588, + "loss": 1.295, + "theoretical_loss": 3.5151156016985823, + "tokens_seen": 1651507200 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004037713139418255, + "loss": 1.2006, + "theoretical_loss": 3.5151036269988585, + "tokens_seen": 1651572736 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004037552657973922, + "loss": 1.3494, + "theoretical_loss": 3.5150916529073335, + "tokens_seen": 1651638272 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004037392176529589, + "loss": 1.2964, + "theoretical_loss": 3.5150796794239527, + "tokens_seen": 1651703808 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004037231695085256, + "loss": 1.3076, + "theoretical_loss": 3.5150677065486615, + "tokens_seen": 1651769344 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004037071213640923, + "loss": 1.3055, + "theoretical_loss": 3.515055734281404, + "tokens_seen": 1651834880 + }, + { + "epoch": 0.5, + "learning_rate": 0.000403691073219659, + "loss": 1.2328, + "theoretical_loss": 3.515043762622125, + "tokens_seen": 1651900416 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004036750250752257, + "loss": 1.1258, + "theoretical_loss": 3.5150317915707707, + "tokens_seen": 1651965952 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040365897693079233, + "loss": 1.4605, + "theoretical_loss": 3.5150198211272854, + "tokens_seen": 1652031488 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004036429287863591, + "loss": 1.433, + "theoretical_loss": 3.5150078512916143, + "tokens_seen": 1652097024 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040362688064192573, + "loss": 1.2831, + "theoretical_loss": 3.514995882063702, + "tokens_seen": 1652162560 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004036108324974925, + "loss": 1.2364, + "theoretical_loss": 3.5149839134434946, + "tokens_seen": 1652228096 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040359478435305924, + "loss": 1.4086, + "theoretical_loss": 3.5149719454309363, + "tokens_seen": 1652293632 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004035787362086259, + "loss": 1.2612, + "theoretical_loss": 3.5149599780259715, + "tokens_seen": 1652359168 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040356268806419264, + "loss": 1.2791, + "theoretical_loss": 3.514948011228547, + "tokens_seen": 1652424704 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004035466399197593, + "loss": 1.2744, + "theoretical_loss": 3.5149360450386062, + "tokens_seen": 1652490240 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040353059177532604, + "loss": 1.3564, + "theoretical_loss": 3.514924079456095, + "tokens_seen": 1652555776 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004035145436308927, + "loss": 1.2529, + "theoretical_loss": 3.5149121144809587, + "tokens_seen": 1652621312 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040349849548645944, + "loss": 1.2497, + "theoretical_loss": 3.5149001501131423, + "tokens_seen": 1652686848 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004034824473420261, + "loss": 1.3671, + "theoretical_loss": 3.5148881863525903, + "tokens_seen": 1652752384 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040346639919759284, + "loss": 1.3041, + "theoretical_loss": 3.5148762231992476, + "tokens_seen": 1652817920 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004034503510531595, + "loss": 1.2864, + "theoretical_loss": 3.5148642606530602, + "tokens_seen": 1652883456 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040343430290872625, + "loss": 1.3044, + "theoretical_loss": 3.514852298713973, + "tokens_seen": 1652948992 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004034182547642929, + "loss": 1.3199, + "theoretical_loss": 3.514840337381931, + "tokens_seen": 1653014528 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040340220661985965, + "loss": 1.2908, + "theoretical_loss": 3.5148283766568795, + "tokens_seen": 1653080064 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004033861584754263, + "loss": 1.3177, + "theoretical_loss": 3.514816416538763, + "tokens_seen": 1653145600 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040337011033099305, + "loss": 1.3329, + "theoretical_loss": 3.5148044570275276, + "tokens_seen": 1653211136 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004033540621865597, + "loss": 1.4222, + "theoretical_loss": 3.5147924981231173, + "tokens_seen": 1653276672 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040333801404212645, + "loss": 1.241, + "theoretical_loss": 3.5147805398254786, + "tokens_seen": 1653342208 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004033219658976931, + "loss": 1.3078, + "theoretical_loss": 3.514768582134555, + "tokens_seen": 1653407744 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040330591775325985, + "loss": 1.2663, + "theoretical_loss": 3.5147566250502935, + "tokens_seen": 1653473280 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004032898696088265, + "loss": 1.2515, + "theoretical_loss": 3.514744668572638, + "tokens_seen": 1653538816 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040327382146439325, + "loss": 1.3482, + "theoretical_loss": 3.514732712701534, + "tokens_seen": 1653604352 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004032577733199599, + "loss": 1.3438, + "theoretical_loss": 3.514720757436927, + "tokens_seen": 1653669888 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040324172517552665, + "loss": 1.3217, + "theoretical_loss": 3.5147088027787623, + "tokens_seen": 1653735424 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004032256770310933, + "loss": 1.4118, + "theoretical_loss": 3.5146968487269845, + "tokens_seen": 1653800960 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040320962888666005, + "loss": 1.2965, + "theoretical_loss": 3.5146848952815386, + "tokens_seen": 1653866496 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004031935807422267, + "loss": 1.3498, + "theoretical_loss": 3.5146729424423713, + "tokens_seen": 1653932032 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004031775325977934, + "loss": 1.3104, + "theoretical_loss": 3.514660990209426, + "tokens_seen": 1653997568 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004031614844533601, + "loss": 1.2517, + "theoretical_loss": 3.5146490385826494, + "tokens_seen": 1654063104 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004031454363089268, + "loss": 1.2355, + "theoretical_loss": 3.5146370875619857, + "tokens_seen": 1654128640 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004031293881644935, + "loss": 1.3704, + "theoretical_loss": 3.5146251371473807, + "tokens_seen": 1654194176 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004031133400200602, + "loss": 1.3514, + "theoretical_loss": 3.51461318733878, + "tokens_seen": 1654259712 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004030972918756269, + "loss": 1.3512, + "theoretical_loss": 3.5146012381361276, + "tokens_seen": 1654325248 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004030812437311936, + "loss": 1.3348, + "theoretical_loss": 3.51458928953937, + "tokens_seen": 1654390784 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004030651955867603, + "loss": 1.3853, + "theoretical_loss": 3.514577341548452, + "tokens_seen": 1654456320 + }, + { + "epoch": 0.5, + "learning_rate": 0.000403049147442327, + "loss": 1.3134, + "theoretical_loss": 3.5145653941633186, + "tokens_seen": 1654521856 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040303309929789365, + "loss": 1.2569, + "theoretical_loss": 3.514553447383916, + "tokens_seen": 1654587392 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004030170511534604, + "loss": 1.3038, + "theoretical_loss": 3.5145415012101884, + "tokens_seen": 1654652928 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040300100300902705, + "loss": 1.2673, + "theoretical_loss": 3.514529555642082, + "tokens_seen": 1654718464 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004029849548645938, + "loss": 1.3009, + "theoretical_loss": 3.5145176106795417, + "tokens_seen": 1654784000 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040296890672016046, + "loss": 1.2807, + "theoretical_loss": 3.514505666322513, + "tokens_seen": 1654849536 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004029528585757272, + "loss": 1.2124, + "theoretical_loss": 3.5144937225709407, + "tokens_seen": 1654915072 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040293681043129386, + "loss": 1.3227, + "theoretical_loss": 3.514481779424771, + "tokens_seen": 1654980608 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004029207622868606, + "loss": 1.3266, + "theoretical_loss": 3.5144698368839484, + "tokens_seen": 1655046144 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040290471414242726, + "loss": 1.1699, + "theoretical_loss": 3.5144578949484186, + "tokens_seen": 1655111680 + }, + { + "epoch": 0.5, + "learning_rate": 0.000402888665997994, + "loss": 1.2696, + "theoretical_loss": 3.514445953618127, + "tokens_seen": 1655177216 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040287261785356066, + "loss": 1.5274, + "theoretical_loss": 3.5144340128930196, + "tokens_seen": 1655242752 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004028565697091274, + "loss": 1.2973, + "theoretical_loss": 3.51442207277304, + "tokens_seen": 1655308288 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040284052156469406, + "loss": 1.2178, + "theoretical_loss": 3.5144101332581354, + "tokens_seen": 1655373824 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004028244734202608, + "loss": 1.3043, + "theoretical_loss": 3.5143981943482503, + "tokens_seen": 1655439360 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040280842527582746, + "loss": 1.1962, + "theoretical_loss": 3.5143862560433305, + "tokens_seen": 1655504896 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004027923771313942, + "loss": 1.2144, + "theoretical_loss": 3.5143743183433207, + "tokens_seen": 1655570432 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040277632898696086, + "loss": 1.3432, + "theoretical_loss": 3.514362381248167, + "tokens_seen": 1655635968 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004027602808425276, + "loss": 1.4166, + "theoretical_loss": 3.514350444757815, + "tokens_seen": 1655701504 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040274423269809426, + "loss": 1.2826, + "theoretical_loss": 3.5143385088722088, + "tokens_seen": 1655767040 + }, + { + "epoch": 0.5, + "learning_rate": 0.000402728184553661, + "loss": 1.4038, + "theoretical_loss": 3.5143265735912954, + "tokens_seen": 1655832576 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040271213640922767, + "loss": 1.3746, + "theoretical_loss": 3.5143146389150193, + "tokens_seen": 1655898112 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004026960882647944, + "loss": 1.3623, + "theoretical_loss": 3.5143027048433266, + "tokens_seen": 1655963648 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040268004012036107, + "loss": 1.394, + "theoretical_loss": 3.514290771376162, + "tokens_seen": 1656029184 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004026639919759278, + "loss": 1.2828, + "theoretical_loss": 3.514278838513471, + "tokens_seen": 1656094720 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040264794383149447, + "loss": 1.2988, + "theoretical_loss": 3.5142669062552, + "tokens_seen": 1656160256 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004026318956870612, + "loss": 1.3793, + "theoretical_loss": 3.514254974601293, + "tokens_seen": 1656225792 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040261584754262787, + "loss": 1.2415, + "theoretical_loss": 3.5142430435516974, + "tokens_seen": 1656291328 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004025997993981946, + "loss": 1.3043, + "theoretical_loss": 3.514231113106357, + "tokens_seen": 1656356864 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040258375125376127, + "loss": 1.2859, + "theoretical_loss": 3.514219183265218, + "tokens_seen": 1656422400 + }, + { + "epoch": 0.5, + "learning_rate": 0.000402567703109328, + "loss": 1.305, + "theoretical_loss": 3.514207254028226, + "tokens_seen": 1656487936 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040255165496489467, + "loss": 1.2953, + "theoretical_loss": 3.5141953253953258, + "tokens_seen": 1656553472 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040253560682046143, + "loss": 1.2757, + "theoretical_loss": 3.5141833973664633, + "tokens_seen": 1656619008 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004025195586760281, + "loss": 1.2653, + "theoretical_loss": 3.5141714699415845, + "tokens_seen": 1656684544 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040250351053159483, + "loss": 1.2776, + "theoretical_loss": 3.5141595431206345, + "tokens_seen": 1656750080 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004024874623871615, + "loss": 1.2339, + "theoretical_loss": 3.514147616903559, + "tokens_seen": 1656815616 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040247141424272823, + "loss": 1.3018, + "theoretical_loss": 3.5141356912903032, + "tokens_seen": 1656881152 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004024553660982949, + "loss": 1.3255, + "theoretical_loss": 3.5141237662808127, + "tokens_seen": 1656946688 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040243931795386163, + "loss": 1.2777, + "theoretical_loss": 3.5141118418750334, + "tokens_seen": 1657012224 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004024232698094283, + "loss": 1.2286, + "theoretical_loss": 3.514099918072911, + "tokens_seen": 1657077760 + }, + { + "epoch": 0.5, + "learning_rate": 0.000402407221664995, + "loss": 1.2836, + "theoretical_loss": 3.514087994874391, + "tokens_seen": 1657143296 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040239117352056173, + "loss": 1.3027, + "theoretical_loss": 3.5140760722794178, + "tokens_seen": 1657208832 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004023751253761284, + "loss": 1.2794, + "theoretical_loss": 3.5140641502879384, + "tokens_seen": 1657274368 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040235907723169513, + "loss": 1.4099, + "theoretical_loss": 3.5140522288998977, + "tokens_seen": 1657339904 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004023430290872618, + "loss": 1.3571, + "theoretical_loss": 3.514040308115242, + "tokens_seen": 1657405440 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040232698094282854, + "loss": 1.3369, + "theoretical_loss": 3.514028387933916, + "tokens_seen": 1657470976 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004023109327983952, + "loss": 1.2841, + "theoretical_loss": 3.514016468355866, + "tokens_seen": 1657536512 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040229488465396194, + "loss": 1.3914, + "theoretical_loss": 3.5140045493810366, + "tokens_seen": 1657602048 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004022788365095286, + "loss": 1.3203, + "theoretical_loss": 3.5139926310093745, + "tokens_seen": 1657667584 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040226278836509534, + "loss": 1.2326, + "theoretical_loss": 3.5139807132408256, + "tokens_seen": 1657733120 + }, + { + "epoch": 0.5, + "learning_rate": 0.000402246740220662, + "loss": 1.2353, + "theoretical_loss": 3.513968796075334, + "tokens_seen": 1657798656 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040223069207622874, + "loss": 1.381, + "theoretical_loss": 3.513956879512847, + "tokens_seen": 1657864192 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004022146439317954, + "loss": 1.3944, + "theoretical_loss": 3.513944963553309, + "tokens_seen": 1657929728 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040219859578736214, + "loss": 1.2316, + "theoretical_loss": 3.513933048196667, + "tokens_seen": 1657995264 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 615633, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3475000858306885, + "objective/train/theoretical_loss": 3.5139211334428646, + "objective/train/tokens_used": 1678520800, + "theoretical_loss": 3.5139211334428646, + "tokens_seen": 1658060800 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004021825476429288, + "loss": 1.3475, + "theoretical_loss": 3.5139211334428646, + "tokens_seen": 1658060800 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040216649949849554, + "loss": 1.2954, + "theoretical_loss": 3.5139092192918495, + "tokens_seen": 1658126336 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004021504513540622, + "loss": 1.1916, + "theoretical_loss": 3.5138973057435665, + "tokens_seen": 1658191872 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040213440320962894, + "loss": 1.3862, + "theoretical_loss": 3.5138853927979614, + "tokens_seen": 1658257408 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004021183550651956, + "loss": 1.2284, + "theoretical_loss": 3.51387348045498, + "tokens_seen": 1658322944 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040210230692076234, + "loss": 1.3736, + "theoretical_loss": 3.513861568714568, + "tokens_seen": 1658388480 + }, + { + "epoch": 0.5, + "learning_rate": 0.000402086258776329, + "loss": 1.178, + "theoretical_loss": 3.513849657576671, + "tokens_seen": 1658454016 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040207021063189575, + "loss": 1.168, + "theoretical_loss": 3.513837747041234, + "tokens_seen": 1658519552 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004020541624874624, + "loss": 1.3149, + "theoretical_loss": 3.513825837108204, + "tokens_seen": 1658585088 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040203811434302915, + "loss": 1.4071, + "theoretical_loss": 3.513813927777526, + "tokens_seen": 1658650624 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004020220661985958, + "loss": 1.2796, + "theoretical_loss": 3.5138020190491464, + "tokens_seen": 1658716160 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040200601805416255, + "loss": 1.3253, + "theoretical_loss": 3.5137901109230096, + "tokens_seen": 1658781696 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004019899699097292, + "loss": 1.3244, + "theoretical_loss": 3.513778203399063, + "tokens_seen": 1658847232 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040197392176529595, + "loss": 1.2897, + "theoretical_loss": 3.513766296477251, + "tokens_seen": 1658912768 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004019578736208626, + "loss": 1.2967, + "theoretical_loss": 3.51375439015752, + "tokens_seen": 1658978304 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040194182547642935, + "loss": 1.3213, + "theoretical_loss": 3.5137424844398164, + "tokens_seen": 1659043840 + }, + { + "epoch": 0.5, + "learning_rate": 0.000401925777331996, + "loss": 1.2606, + "theoretical_loss": 3.5137305793240845, + "tokens_seen": 1659109376 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040190972918756275, + "loss": 1.2992, + "theoretical_loss": 3.5137186748102716, + "tokens_seen": 1659174912 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004018936810431294, + "loss": 1.2549, + "theoretical_loss": 3.513706770898322, + "tokens_seen": 1659240448 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040187763289869615, + "loss": 1.2993, + "theoretical_loss": 3.5136948675881827, + "tokens_seen": 1659305984 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004018615847542628, + "loss": 1.222, + "theoretical_loss": 3.5136829648797985, + "tokens_seen": 1659371520 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040184553660982955, + "loss": 1.2842, + "theoretical_loss": 3.5136710627731165, + "tokens_seen": 1659437056 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004018294884653962, + "loss": 1.3796, + "theoretical_loss": 3.5136591612680816, + "tokens_seen": 1659502592 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040181344032096296, + "loss": 1.3462, + "theoretical_loss": 3.51364726036464, + "tokens_seen": 1659568128 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004017973921765296, + "loss": 1.3041, + "theoretical_loss": 3.513635360062737, + "tokens_seen": 1659633664 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004017813440320963, + "loss": 1.2849, + "theoretical_loss": 3.513623460362319, + "tokens_seen": 1659699200 + }, + { + "epoch": 0.5, + "learning_rate": 0.000401765295887663, + "loss": 1.3915, + "theoretical_loss": 3.5136115612633314, + "tokens_seen": 1659764736 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004017492477432297, + "loss": 1.3716, + "theoretical_loss": 3.5135996627657207, + "tokens_seen": 1659830272 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004017331995987964, + "loss": 1.2806, + "theoretical_loss": 3.513587764869432, + "tokens_seen": 1659895808 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004017171514543631, + "loss": 1.2752, + "theoretical_loss": 3.5135758675744118, + "tokens_seen": 1659961344 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004017011033099298, + "loss": 1.2752, + "theoretical_loss": 3.5135639708806057, + "tokens_seen": 1660026880 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004016850551654965, + "loss": 1.3229, + "theoretical_loss": 3.5135520747879596, + "tokens_seen": 1660092416 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004016690070210632, + "loss": 1.2479, + "theoretical_loss": 3.51354017929642, + "tokens_seen": 1660157952 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004016529588766299, + "loss": 1.3251, + "theoretical_loss": 3.5135282844059317, + "tokens_seen": 1660223488 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040163691073219655, + "loss": 1.2685, + "theoretical_loss": 3.5135163901164406, + "tokens_seen": 1660289024 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004016208625877633, + "loss": 1.1497, + "theoretical_loss": 3.5135044964278936, + "tokens_seen": 1660354560 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040160481444332996, + "loss": 1.4785, + "theoretical_loss": 3.5134926033402367, + "tokens_seen": 1660420096 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004015887662988967, + "loss": 1.3311, + "theoretical_loss": 3.5134807108534147, + "tokens_seen": 1660485632 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040157271815446336, + "loss": 1.2948, + "theoretical_loss": 3.513468818967374, + "tokens_seen": 1660551168 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004015566700100301, + "loss": 1.3718, + "theoretical_loss": 3.5134569276820606, + "tokens_seen": 1660616704 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040154062186559676, + "loss": 1.4237, + "theoretical_loss": 3.513445036997421, + "tokens_seen": 1660682240 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004015245737211635, + "loss": 1.346, + "theoretical_loss": 3.5134331469134006, + "tokens_seen": 1660747776 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040150852557673016, + "loss": 1.374, + "theoretical_loss": 3.513421257429945, + "tokens_seen": 1660813312 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004014924774322969, + "loss": 1.2982, + "theoretical_loss": 3.5134093685470007, + "tokens_seen": 1660878848 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040147642928786356, + "loss": 1.3498, + "theoretical_loss": 3.513397480264514, + "tokens_seen": 1660944384 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004014603811434303, + "loss": 1.2652, + "theoretical_loss": 3.5133855925824298, + "tokens_seen": 1661009920 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040144433299899696, + "loss": 1.2119, + "theoretical_loss": 3.513373705500695, + "tokens_seen": 1661075456 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004014282848545637, + "loss": 1.4115, + "theoretical_loss": 3.5133618190192553, + "tokens_seen": 1661140992 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040141223671013036, + "loss": 1.3474, + "theoretical_loss": 3.513349933138057, + "tokens_seen": 1661206528 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004013961885656971, + "loss": 1.311, + "theoretical_loss": 3.5133380478570455, + "tokens_seen": 1661272064 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040138014042126376, + "loss": 1.3539, + "theoretical_loss": 3.513326163176167, + "tokens_seen": 1661337600 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004013640922768305, + "loss": 1.4478, + "theoretical_loss": 3.5133142790953684, + "tokens_seen": 1661403136 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040134804413239717, + "loss": 1.3071, + "theoretical_loss": 3.513302395614595, + "tokens_seen": 1661468672 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004013319959879639, + "loss": 1.1814, + "theoretical_loss": 3.513290512733792, + "tokens_seen": 1661534208 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040131594784353057, + "loss": 1.2842, + "theoretical_loss": 3.513278630452907, + "tokens_seen": 1661599744 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004012998996990973, + "loss": 1.3283, + "theoretical_loss": 3.513266748771885, + "tokens_seen": 1661665280 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040128385155466397, + "loss": 1.3119, + "theoretical_loss": 3.5132548676906725, + "tokens_seen": 1661730816 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004012678034102307, + "loss": 1.3254, + "theoretical_loss": 3.5132429872092157, + "tokens_seen": 1661796352 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040125175526579737, + "loss": 1.2543, + "theoretical_loss": 3.51323110732746, + "tokens_seen": 1661861888 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004012357071213641, + "loss": 1.3357, + "theoretical_loss": 3.5132192280453527, + "tokens_seen": 1661927424 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040121965897693077, + "loss": 1.2643, + "theoretical_loss": 3.5132073493628386, + "tokens_seen": 1661992960 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004012036108324975, + "loss": 1.4203, + "theoretical_loss": 3.5131954712798645, + "tokens_seen": 1662058496 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004011875626880643, + "loss": 1.3658, + "theoretical_loss": 3.513183593796376, + "tokens_seen": 1662124032 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004011715145436309, + "loss": 1.3158, + "theoretical_loss": 3.5131717169123196, + "tokens_seen": 1662189568 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040115546639919763, + "loss": 1.346, + "theoretical_loss": 3.5131598406276416, + "tokens_seen": 1662255104 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040113941825476433, + "loss": 1.2441, + "theoretical_loss": 3.513147964942288, + "tokens_seen": 1662320640 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040112337011033103, + "loss": 1.2453, + "theoretical_loss": 3.5131360898562045, + "tokens_seen": 1662386176 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040110732196589773, + "loss": 1.3145, + "theoretical_loss": 3.513124215369338, + "tokens_seen": 1662451712 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040109127382146443, + "loss": 1.3277, + "theoretical_loss": 3.5131123414816336, + "tokens_seen": 1662517248 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040107522567703113, + "loss": 1.3306, + "theoretical_loss": 3.513100468193038, + "tokens_seen": 1662582784 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040105917753259783, + "loss": 1.2756, + "theoretical_loss": 3.513088595503498, + "tokens_seen": 1662648320 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040104312938816453, + "loss": 1.2938, + "theoretical_loss": 3.5130767234129587, + "tokens_seen": 1662713856 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040102708124373123, + "loss": 1.2894, + "theoretical_loss": 3.5130648519213663, + "tokens_seen": 1662779392 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004010110330992979, + "loss": 1.2458, + "theoretical_loss": 3.513052981028668, + "tokens_seen": 1662844928 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040099498495486463, + "loss": 1.2929, + "theoretical_loss": 3.513041110734809, + "tokens_seen": 1662910464 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004009789368104313, + "loss": 1.3315, + "theoretical_loss": 3.513029241039737, + "tokens_seen": 1662976000 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040096288866599804, + "loss": 1.253, + "theoretical_loss": 3.5130173719433957, + "tokens_seen": 1663041536 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004009468405215647, + "loss": 1.3084, + "theoretical_loss": 3.5130055034457337, + "tokens_seen": 1663107072 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040093079237713144, + "loss": 1.3355, + "theoretical_loss": 3.5129936355466955, + "tokens_seen": 1663172608 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004009147442326981, + "loss": 1.335, + "theoretical_loss": 3.5129817682462283, + "tokens_seen": 1663238144 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040089869608826484, + "loss": 1.3693, + "theoretical_loss": 3.512969901544278, + "tokens_seen": 1663303680 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004008826479438315, + "loss": 1.3299, + "theoretical_loss": 3.512958035440791, + "tokens_seen": 1663369216 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040086659979939824, + "loss": 1.3435, + "theoretical_loss": 3.512946169935713, + "tokens_seen": 1663434752 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004008505516549649, + "loss": 1.295, + "theoretical_loss": 3.512934305028991, + "tokens_seen": 1663500288 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040083450351053164, + "loss": 1.2771, + "theoretical_loss": 3.512922440720571, + "tokens_seen": 1663565824 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004008184553660983, + "loss": 1.3111, + "theoretical_loss": 3.512910577010399, + "tokens_seen": 1663631360 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040080240722166504, + "loss": 1.3834, + "theoretical_loss": 3.5128987138984216, + "tokens_seen": 1663696896 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004007863590772317, + "loss": 1.3218, + "theoretical_loss": 3.512886851384585, + "tokens_seen": 1663762432 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040077031093279844, + "loss": 1.3094, + "theoretical_loss": 3.512874989468835, + "tokens_seen": 1663827968 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004007542627883651, + "loss": 1.2591, + "theoretical_loss": 3.5128631281511185, + "tokens_seen": 1663893504 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040073821464393184, + "loss": 1.4022, + "theoretical_loss": 3.512851267431382, + "tokens_seen": 1663959040 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004007221664994985, + "loss": 1.2426, + "theoretical_loss": 3.5128394073095706, + "tokens_seen": 1664024576 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040070611835506525, + "loss": 1.3208, + "theoretical_loss": 3.512827547785632, + "tokens_seen": 1664090112 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004006900702106319, + "loss": 1.3579, + "theoretical_loss": 3.512815688859512, + "tokens_seen": 1664155648 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040067402206619865, + "loss": 1.3619, + "theoretical_loss": 3.5128038305311566, + "tokens_seen": 1664221184 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004006579739217653, + "loss": 1.2646, + "theoretical_loss": 3.512791972800512, + "tokens_seen": 1664286720 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040064192577733205, + "loss": 1.3141, + "theoretical_loss": 3.5127801156675256, + "tokens_seen": 1664352256 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004006258776328987, + "loss": 1.2915, + "theoretical_loss": 3.5127682591321427, + "tokens_seen": 1664417792 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040060982948846545, + "loss": 1.2877, + "theoretical_loss": 3.5127564031943104, + "tokens_seen": 1664483328 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004005937813440321, + "loss": 1.3289, + "theoretical_loss": 3.5127445478539743, + "tokens_seen": 1664548864 + }, + { + "epoch": 0.5, + "objective/train/docs_used": 617972, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3300971984863281, + "objective/train/theoretical_loss": 3.512732693111081, + "objective/train/tokens_used": 1685074400, + "theoretical_loss": 3.512732693111081, + "tokens_seen": 1664614400 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040057773319959885, + "loss": 1.3301, + "theoretical_loss": 3.512732693111081, + "tokens_seen": 1664614400 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004005616850551655, + "loss": 1.3772, + "theoretical_loss": 3.5127208389655777, + "tokens_seen": 1664679936 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040054563691073225, + "loss": 1.1959, + "theoretical_loss": 3.51270898541741, + "tokens_seen": 1664745472 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004005295887662989, + "loss": 1.303, + "theoretical_loss": 3.5126971324665237, + "tokens_seen": 1664811008 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040051354062186565, + "loss": 1.2975, + "theoretical_loss": 3.512685280112866, + "tokens_seen": 1664876544 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004004974924774323, + "loss": 1.2779, + "theoretical_loss": 3.5126734283563836, + "tokens_seen": 1664942080 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040048144433299905, + "loss": 1.3383, + "theoretical_loss": 3.512661577197022, + "tokens_seen": 1665007616 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004004653961885657, + "loss": 1.3711, + "theoretical_loss": 3.5126497266347285, + "tokens_seen": 1665073152 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040044934804413246, + "loss": 1.3257, + "theoretical_loss": 3.512637876669449, + "tokens_seen": 1665138688 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004004332998996991, + "loss": 1.3582, + "theoretical_loss": 3.5126260273011303, + "tokens_seen": 1665204224 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040041725175526586, + "loss": 1.3733, + "theoretical_loss": 3.5126141785297182, + "tokens_seen": 1665269760 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004004012036108325, + "loss": 1.2445, + "theoretical_loss": 3.5126023303551595, + "tokens_seen": 1665335296 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004003851554663992, + "loss": 1.2579, + "theoretical_loss": 3.5125904827774006, + "tokens_seen": 1665400832 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004003691073219659, + "loss": 1.237, + "theoretical_loss": 3.5125786357963884, + "tokens_seen": 1665466368 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004003530591775326, + "loss": 1.3223, + "theoretical_loss": 3.5125667894120687, + "tokens_seen": 1665531904 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004003370110330993, + "loss": 1.3815, + "theoretical_loss": 3.5125549436243886, + "tokens_seen": 1665597440 + }, + { + "epoch": 0.5, + "learning_rate": 0.000400320962888666, + "loss": 1.3157, + "theoretical_loss": 3.512543098433294, + "tokens_seen": 1665662976 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004003049147442327, + "loss": 1.3119, + "theoretical_loss": 3.5125312538387314, + "tokens_seen": 1665728512 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004002888665997994, + "loss": 1.3575, + "theoretical_loss": 3.512519409840648, + "tokens_seen": 1665794048 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004002728184553661, + "loss": 1.3094, + "theoretical_loss": 3.5125075664389893, + "tokens_seen": 1665859584 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004002567703109328, + "loss": 1.3485, + "theoretical_loss": 3.5124957236337027, + "tokens_seen": 1665925120 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040024072216649946, + "loss": 1.2612, + "theoretical_loss": 3.512483881424734, + "tokens_seen": 1665990656 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004002246740220662, + "loss": 1.2998, + "theoretical_loss": 3.5124720398120304, + "tokens_seen": 1666056192 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040020862587763286, + "loss": 1.3006, + "theoretical_loss": 3.5124601987955377, + "tokens_seen": 1666121728 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004001925777331996, + "loss": 1.2555, + "theoretical_loss": 3.5124483583752033, + "tokens_seen": 1666187264 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040017652958876626, + "loss": 1.3524, + "theoretical_loss": 3.5124365185509725, + "tokens_seen": 1666252800 + }, + { + "epoch": 0.5, + "learning_rate": 0.000400160481444333, + "loss": 1.297, + "theoretical_loss": 3.512424679322793, + "tokens_seen": 1666318336 + }, + { + "epoch": 0.5, + "learning_rate": 0.00040014443329989966, + "loss": 1.2264, + "theoretical_loss": 3.5124128406906108, + "tokens_seen": 1666383872 + }, + { + "epoch": 0.5, + "learning_rate": 0.0004001283851554664, + "loss": 1.2897, + "theoretical_loss": 3.512401002654373, + "tokens_seen": 1666449408 + }, + { + "epoch": 0.51, + "learning_rate": 0.00040011233701103306, + "loss": 1.4256, + "theoretical_loss": 3.5123891652140253, + "tokens_seen": 1666514944 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004000962888665998, + "loss": 1.2884, + "theoretical_loss": 3.5123773283695146, + "tokens_seen": 1666580480 + }, + { + "epoch": 0.51, + "learning_rate": 0.00040008024072216646, + "loss": 1.3024, + "theoretical_loss": 3.512365492120788, + "tokens_seen": 1666646016 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004000641925777332, + "loss": 1.4027, + "theoretical_loss": 3.5123536564677913, + "tokens_seen": 1666711552 + }, + { + "epoch": 0.51, + "learning_rate": 0.00040004814443329986, + "loss": 1.4193, + "theoretical_loss": 3.5123418214104722, + "tokens_seen": 1666777088 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004000320962888666, + "loss": 1.2425, + "theoretical_loss": 3.5123299869487763, + "tokens_seen": 1666842624 + }, + { + "epoch": 0.51, + "learning_rate": 0.00040001604814443326, + "loss": 1.3089, + "theoretical_loss": 3.51231815308265, + "tokens_seen": 1666908160 + }, + { + "epoch": 0.51, + "learning_rate": 0.0004, + "loss": 1.3572, + "theoretical_loss": 3.5123063198120406, + "tokens_seen": 1666973696 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003999839518555667, + "loss": 1.2429, + "theoretical_loss": 3.5122944871368946, + "tokens_seen": 1667039232 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003999679037111334, + "loss": 1.3212, + "theoretical_loss": 3.5122826550571586, + "tokens_seen": 1667104768 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003999518555667001, + "loss": 1.1782, + "theoretical_loss": 3.5122708235727798, + "tokens_seen": 1667170304 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003999358074222668, + "loss": 1.3893, + "theoretical_loss": 3.5122589926837033, + "tokens_seen": 1667235840 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003999197592778335, + "loss": 1.3841, + "theoretical_loss": 3.512247162389877, + "tokens_seen": 1667301376 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003999037111334002, + "loss": 1.297, + "theoretical_loss": 3.5122353326912474, + "tokens_seen": 1667366912 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003998876629889669, + "loss": 1.2775, + "theoretical_loss": 3.512223503587761, + "tokens_seen": 1667432448 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003998716148445336, + "loss": 1.2391, + "theoretical_loss": 3.512211675079364, + "tokens_seen": 1667497984 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003998555667001003, + "loss": 1.2242, + "theoretical_loss": 3.512199847166004, + "tokens_seen": 1667563520 + }, + { + "epoch": 0.51, + "learning_rate": 0.000399839518555667, + "loss": 1.4005, + "theoretical_loss": 3.512188019847627, + "tokens_seen": 1667629056 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003998234704112337, + "loss": 1.3359, + "theoretical_loss": 3.5121761931241804, + "tokens_seen": 1667694592 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003998074222668004, + "loss": 1.3747, + "theoretical_loss": 3.5121643669956097, + "tokens_seen": 1667760128 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039979137412236713, + "loss": 1.3102, + "theoretical_loss": 3.512152541461863, + "tokens_seen": 1667825664 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039977532597793383, + "loss": 1.3337, + "theoretical_loss": 3.512140716522886, + "tokens_seen": 1667891200 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039975927783350053, + "loss": 1.333, + "theoretical_loss": 3.512128892178626, + "tokens_seen": 1667956736 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039974322968906723, + "loss": 1.3733, + "theoretical_loss": 3.5121170684290286, + "tokens_seen": 1668022272 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039972718154463393, + "loss": 1.3539, + "theoretical_loss": 3.5121052452740424, + "tokens_seen": 1668087808 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039971113340020063, + "loss": 1.2314, + "theoretical_loss": 3.5120934227136127, + "tokens_seen": 1668153344 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039969508525576733, + "loss": 1.3591, + "theoretical_loss": 3.512081600747687, + "tokens_seen": 1668218880 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039967903711133403, + "loss": 1.2356, + "theoretical_loss": 3.5120697793762115, + "tokens_seen": 1668284416 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039966298896690073, + "loss": 1.2293, + "theoretical_loss": 3.512057958599133, + "tokens_seen": 1668349952 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039964694082246743, + "loss": 1.2965, + "theoretical_loss": 3.512046138416399, + "tokens_seen": 1668415488 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039963089267803413, + "loss": 1.3923, + "theoretical_loss": 3.512034318827955, + "tokens_seen": 1668481024 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039961484453360083, + "loss": 1.2518, + "theoretical_loss": 3.512022499833749, + "tokens_seen": 1668546560 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003995987963891675, + "loss": 1.286, + "theoretical_loss": 3.5120106814337273, + "tokens_seen": 1668612096 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003995827482447342, + "loss": 1.3071, + "theoretical_loss": 3.511998863627837, + "tokens_seen": 1668677632 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003995667001003009, + "loss": 1.2734, + "theoretical_loss": 3.511987046416024, + "tokens_seen": 1668743168 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003995506519558676, + "loss": 1.3637, + "theoretical_loss": 3.511975229798236, + "tokens_seen": 1668808704 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003995346038114343, + "loss": 1.3393, + "theoretical_loss": 3.5119634137744193, + "tokens_seen": 1668874240 + }, + { + "epoch": 0.51, + "learning_rate": 0.000399518555667001, + "loss": 1.2152, + "theoretical_loss": 3.5119515983445213, + "tokens_seen": 1668939776 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003995025075225677, + "loss": 1.1977, + "theoretical_loss": 3.5119397835084882, + "tokens_seen": 1669005312 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003994864593781344, + "loss": 1.3381, + "theoretical_loss": 3.5119279692662673, + "tokens_seen": 1669070848 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003994704112337011, + "loss": 1.3336, + "theoretical_loss": 3.5119161556178047, + "tokens_seen": 1669136384 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003994543630892678, + "loss": 1.2661, + "theoretical_loss": 3.511904342563048, + "tokens_seen": 1669201920 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003994383149448345, + "loss": 1.292, + "theoretical_loss": 3.511892530101944, + "tokens_seen": 1669267456 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003994222668004012, + "loss": 1.356, + "theoretical_loss": 3.5118807182344396, + "tokens_seen": 1669332992 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003994062186559679, + "loss": 1.3799, + "theoretical_loss": 3.511868906960481, + "tokens_seen": 1669398528 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039939017051153464, + "loss": 1.3419, + "theoretical_loss": 3.511857096280016, + "tokens_seen": 1669464064 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039937412236710134, + "loss": 1.3069, + "theoretical_loss": 3.5118452861929907, + "tokens_seen": 1669529600 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039935807422266804, + "loss": 1.421, + "theoretical_loss": 3.5118334766993526, + "tokens_seen": 1669595136 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039934202607823475, + "loss": 1.3423, + "theoretical_loss": 3.5118216677990484, + "tokens_seen": 1669660672 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039932597793380145, + "loss": 1.256, + "theoretical_loss": 3.5118098594920246, + "tokens_seen": 1669726208 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039930992978936815, + "loss": 1.3742, + "theoretical_loss": 3.5117980517782286, + "tokens_seen": 1669791744 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039929388164493485, + "loss": 1.2818, + "theoretical_loss": 3.511786244657607, + "tokens_seen": 1669857280 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039927783350050155, + "loss": 1.3173, + "theoretical_loss": 3.5117744381301064, + "tokens_seen": 1669922816 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039926178535606825, + "loss": 1.2822, + "theoretical_loss": 3.5117626321956745, + "tokens_seen": 1669988352 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039924573721163495, + "loss": 1.3236, + "theoretical_loss": 3.5117508268542585, + "tokens_seen": 1670053888 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039922968906720165, + "loss": 1.2255, + "theoretical_loss": 3.5117390221058042, + "tokens_seen": 1670119424 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039921364092276835, + "loss": 1.3335, + "theoretical_loss": 3.511727217950259, + "tokens_seen": 1670184960 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039919759277833505, + "loss": 1.2563, + "theoretical_loss": 3.5117154143875706, + "tokens_seen": 1670250496 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039918154463390175, + "loss": 1.1974, + "theoretical_loss": 3.511703611417685, + "tokens_seen": 1670316032 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039916549648946845, + "loss": 1.277, + "theoretical_loss": 3.5116918090405496, + "tokens_seen": 1670381568 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039914944834503515, + "loss": 1.4366, + "theoretical_loss": 3.5116800072561114, + "tokens_seen": 1670447104 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039913340020060185, + "loss": 1.3768, + "theoretical_loss": 3.511668206064317, + "tokens_seen": 1670512640 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039911735205616855, + "loss": 1.3325, + "theoretical_loss": 3.5116564054651134, + "tokens_seen": 1670578176 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039910130391173525, + "loss": 1.3465, + "theoretical_loss": 3.5116446054584483, + "tokens_seen": 1670643712 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039908525576730196, + "loss": 1.305, + "theoretical_loss": 3.5116328060442683, + "tokens_seen": 1670709248 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039906920762286866, + "loss": 1.3378, + "theoretical_loss": 3.5116210072225202, + "tokens_seen": 1670774784 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039905315947843536, + "loss": 1.3392, + "theoretical_loss": 3.511609208993151, + "tokens_seen": 1670840320 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039903711133400206, + "loss": 1.2833, + "theoretical_loss": 3.5115974113561084, + "tokens_seen": 1670905856 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039902106318956876, + "loss": 1.2586, + "theoretical_loss": 3.511585614311339, + "tokens_seen": 1670971392 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039900501504513546, + "loss": 1.2866, + "theoretical_loss": 3.5115738178587894, + "tokens_seen": 1671036928 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039898896690070216, + "loss": 1.2827, + "theoretical_loss": 3.511562021998407, + "tokens_seen": 1671102464 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 620366, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.21170973777771, + "objective/train/theoretical_loss": 3.5115502267301393, + "objective/train/tokens_used": 1691628000, + "theoretical_loss": 3.5115502267301393, + "tokens_seen": 1671168000 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003989729187562688, + "loss": 1.2117, + "theoretical_loss": 3.5115502267301393, + "tokens_seen": 1671168000 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003989568706118355, + "loss": 1.3139, + "theoretical_loss": 3.5115384320539325, + "tokens_seen": 1671233536 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003989408224674022, + "loss": 1.3293, + "theoretical_loss": 3.5115266379697343, + "tokens_seen": 1671299072 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003989247743229689, + "loss": 1.2971, + "theoretical_loss": 3.5115148444774915, + "tokens_seen": 1671364608 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003989087261785356, + "loss": 1.2657, + "theoretical_loss": 3.5115030515771517, + "tokens_seen": 1671430144 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003988926780341023, + "loss": 1.3053, + "theoretical_loss": 3.511491259268661, + "tokens_seen": 1671495680 + }, + { + "epoch": 0.51, + "learning_rate": 0.000398876629889669, + "loss": 1.3331, + "theoretical_loss": 3.5114794675519674, + "tokens_seen": 1671561216 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003988605817452357, + "loss": 1.3848, + "theoretical_loss": 3.5114676764270176, + "tokens_seen": 1671626752 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003988445336008024, + "loss": 1.1918, + "theoretical_loss": 3.5114558858937586, + "tokens_seen": 1671692288 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003988284854563691, + "loss": 1.3239, + "theoretical_loss": 3.511444095952138, + "tokens_seen": 1671757824 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003988124373119358, + "loss": 1.3274, + "theoretical_loss": 3.511432306602102, + "tokens_seen": 1671823360 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003987963891675025, + "loss": 1.346, + "theoretical_loss": 3.5114205178435984, + "tokens_seen": 1671888896 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003987803410230692, + "loss": 1.3581, + "theoretical_loss": 3.511408729676575, + "tokens_seen": 1671954432 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003987642928786359, + "loss": 1.2902, + "theoretical_loss": 3.5113969421009776, + "tokens_seen": 1672019968 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003987482447342026, + "loss": 1.372, + "theoretical_loss": 3.511385155116754, + "tokens_seen": 1672085504 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003987321965897693, + "loss": 1.2284, + "theoretical_loss": 3.5113733687238513, + "tokens_seen": 1672151040 + }, + { + "epoch": 0.51, + "learning_rate": 0.000398716148445336, + "loss": 1.2222, + "theoretical_loss": 3.511361582922217, + "tokens_seen": 1672216576 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003987001003009027, + "loss": 1.314, + "theoretical_loss": 3.5113497977117976, + "tokens_seen": 1672282112 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003986840521564694, + "loss": 1.4388, + "theoretical_loss": 3.5113380130925407, + "tokens_seen": 1672347648 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003986680040120361, + "loss": 1.3369, + "theoretical_loss": 3.5113262290643936, + "tokens_seen": 1672413184 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003986519558676028, + "loss": 1.2671, + "theoretical_loss": 3.5113144456273027, + "tokens_seen": 1672478720 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003986359077231695, + "loss": 1.2777, + "theoretical_loss": 3.5113026627812163, + "tokens_seen": 1672544256 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003986198595787362, + "loss": 1.2853, + "theoretical_loss": 3.5112908805260803, + "tokens_seen": 1672609792 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003986038114343029, + "loss": 1.3776, + "theoretical_loss": 3.5112790988618436, + "tokens_seen": 1672675328 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003985877632898696, + "loss": 1.2059, + "theoretical_loss": 3.511267317788452, + "tokens_seen": 1672740864 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003985717151454363, + "loss": 1.4407, + "theoretical_loss": 3.5112555373058534, + "tokens_seen": 1672806400 + }, + { + "epoch": 0.51, + "learning_rate": 0.000398555667001003, + "loss": 1.2197, + "theoretical_loss": 3.511243757413995, + "tokens_seen": 1672871936 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003985396188565697, + "loss": 1.2025, + "theoretical_loss": 3.511231978112823, + "tokens_seen": 1672937472 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003985235707121364, + "loss": 1.3172, + "theoretical_loss": 3.5112201994022865, + "tokens_seen": 1673003008 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003985075225677031, + "loss": 1.3421, + "theoretical_loss": 3.5112084212823316, + "tokens_seen": 1673068544 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003984914744232698, + "loss": 1.3553, + "theoretical_loss": 3.5111966437529056, + "tokens_seen": 1673134080 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003984754262788365, + "loss": 1.3176, + "theoretical_loss": 3.511184866813956, + "tokens_seen": 1673199616 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003984593781344032, + "loss": 1.173, + "theoretical_loss": 3.5111730904654297, + "tokens_seen": 1673265152 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003984433299899699, + "loss": 1.3494, + "theoretical_loss": 3.5111613147072744, + "tokens_seen": 1673330688 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039842728184553663, + "loss": 1.2752, + "theoretical_loss": 3.511149539539437, + "tokens_seen": 1673396224 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039841123370110333, + "loss": 1.2713, + "theoretical_loss": 3.5111377649618656, + "tokens_seen": 1673461760 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039839518555667003, + "loss": 1.2918, + "theoretical_loss": 3.5111259909745067, + "tokens_seen": 1673527296 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039837913741223673, + "loss": 1.3447, + "theoretical_loss": 3.5111142175773074, + "tokens_seen": 1673592832 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039836308926780343, + "loss": 1.2392, + "theoretical_loss": 3.5111024447702155, + "tokens_seen": 1673658368 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039834704112337013, + "loss": 1.276, + "theoretical_loss": 3.5110906725531787, + "tokens_seen": 1673723904 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039833099297893683, + "loss": 1.3691, + "theoretical_loss": 3.5110789009261434, + "tokens_seen": 1673789440 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039831494483450353, + "loss": 1.2788, + "theoretical_loss": 3.5110671298890574, + "tokens_seen": 1673854976 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039829889669007023, + "loss": 1.3519, + "theoretical_loss": 3.511055359441868, + "tokens_seen": 1673920512 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039828284854563693, + "loss": 1.2564, + "theoretical_loss": 3.5110435895845233, + "tokens_seen": 1673986048 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039826680040120363, + "loss": 1.3112, + "theoretical_loss": 3.511031820316969, + "tokens_seen": 1674051584 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039825075225677033, + "loss": 1.3114, + "theoretical_loss": 3.511020051639154, + "tokens_seen": 1674117120 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039823470411233703, + "loss": 1.3272, + "theoretical_loss": 3.511008283551025, + "tokens_seen": 1674182656 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039821865596790374, + "loss": 1.3409, + "theoretical_loss": 3.510996516052529, + "tokens_seen": 1674248192 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003982026078234704, + "loss": 1.2925, + "theoretical_loss": 3.5109847491436144, + "tokens_seen": 1674313728 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039818655967903714, + "loss": 1.3118, + "theoretical_loss": 3.5109729828242275, + "tokens_seen": 1674379264 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039817051153460384, + "loss": 1.3231, + "theoretical_loss": 3.5109612170943163, + "tokens_seen": 1674444800 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039815446339017054, + "loss": 1.3422, + "theoretical_loss": 3.5109494519538282, + "tokens_seen": 1674510336 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039813841524573724, + "loss": 1.3117, + "theoretical_loss": 3.5109376874027105, + "tokens_seen": 1674575872 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039812236710130394, + "loss": 1.2865, + "theoretical_loss": 3.5109259234409107, + "tokens_seen": 1674641408 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039810631895687064, + "loss": 1.263, + "theoretical_loss": 3.510914160068376, + "tokens_seen": 1674706944 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039809027081243734, + "loss": 1.2506, + "theoretical_loss": 3.5109023972850535, + "tokens_seen": 1674772480 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039807422266800404, + "loss": 1.2663, + "theoretical_loss": 3.5108906350908917, + "tokens_seen": 1674838016 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039805817452357074, + "loss": 1.3802, + "theoretical_loss": 3.510878873485837, + "tokens_seen": 1674903552 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039804212637913744, + "loss": 1.36, + "theoretical_loss": 3.510867112469837, + "tokens_seen": 1674969088 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039802607823470414, + "loss": 1.2306, + "theoretical_loss": 3.51085535204284, + "tokens_seen": 1675034624 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039801003009027084, + "loss": 1.2798, + "theoretical_loss": 3.5108435922047927, + "tokens_seen": 1675100160 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039799398194583754, + "loss": 1.3088, + "theoretical_loss": 3.5108318329556427, + "tokens_seen": 1675165696 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039797793380140424, + "loss": 1.3359, + "theoretical_loss": 3.5108200742953377, + "tokens_seen": 1675231232 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039796188565697095, + "loss": 1.3143, + "theoretical_loss": 3.510808316223825, + "tokens_seen": 1675296768 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039794583751253765, + "loss": 1.3954, + "theoretical_loss": 3.5107965587410517, + "tokens_seen": 1675362304 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039792978936810435, + "loss": 1.1627, + "theoretical_loss": 3.510784801846966, + "tokens_seen": 1675427840 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039791374122367105, + "loss": 1.3401, + "theoretical_loss": 3.5107730455415145, + "tokens_seen": 1675493376 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039789769307923775, + "loss": 1.3721, + "theoretical_loss": 3.5107612898246456, + "tokens_seen": 1675558912 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039788164493480445, + "loss": 1.3861, + "theoretical_loss": 3.5107495346963065, + "tokens_seen": 1675624448 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039786559679037115, + "loss": 1.2165, + "theoretical_loss": 3.510737780156445, + "tokens_seen": 1675689984 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039784954864593785, + "loss": 1.3563, + "theoretical_loss": 3.510726026205008, + "tokens_seen": 1675755520 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039783350050150455, + "loss": 1.3418, + "theoretical_loss": 3.5107142728419434, + "tokens_seen": 1675821056 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039781745235707125, + "loss": 1.295, + "theoretical_loss": 3.5107025200671984, + "tokens_seen": 1675886592 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039780140421263795, + "loss": 1.3588, + "theoretical_loss": 3.510690767880721, + "tokens_seen": 1675952128 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039778535606820465, + "loss": 1.302, + "theoretical_loss": 3.510679016282459, + "tokens_seen": 1676017664 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039776930792377135, + "loss": 1.3948, + "theoretical_loss": 3.5106672652723594, + "tokens_seen": 1676083200 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039775325977933805, + "loss": 1.2812, + "theoretical_loss": 3.51065551485037, + "tokens_seen": 1676148736 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039773721163490475, + "loss": 1.4073, + "theoretical_loss": 3.5106437650164377, + "tokens_seen": 1676214272 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039772116349047146, + "loss": 1.356, + "theoretical_loss": 3.5106320157705113, + "tokens_seen": 1676279808 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039770511534603816, + "loss": 1.2806, + "theoretical_loss": 3.5106202671125377, + "tokens_seen": 1676345344 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039768906720160486, + "loss": 1.2849, + "theoretical_loss": 3.5106085190424645, + "tokens_seen": 1676410880 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039767301905717156, + "loss": 1.2682, + "theoretical_loss": 3.5105967715602393, + "tokens_seen": 1676476416 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039765697091273826, + "loss": 1.2839, + "theoretical_loss": 3.5105850246658097, + "tokens_seen": 1676541952 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039764092276830496, + "loss": 1.2257, + "theoretical_loss": 3.5105732783591237, + "tokens_seen": 1676607488 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039762487462387166, + "loss": 1.2178, + "theoretical_loss": 3.5105615326401285, + "tokens_seen": 1676673024 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039760882647943836, + "loss": 1.2833, + "theoretical_loss": 3.5105497875087717, + "tokens_seen": 1676738560 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039759277833500506, + "loss": 1.3606, + "theoretical_loss": 3.510538042965001, + "tokens_seen": 1676804096 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003975767301905717, + "loss": 1.3636, + "theoretical_loss": 3.510526299008764, + "tokens_seen": 1676869632 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003975606820461384, + "loss": 1.4146, + "theoretical_loss": 3.5105145556400092, + "tokens_seen": 1676935168 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003975446339017051, + "loss": 1.2531, + "theoretical_loss": 3.5105028128586824, + "tokens_seen": 1677000704 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003975285857572718, + "loss": 1.2679, + "theoretical_loss": 3.5104910706647336, + "tokens_seen": 1677066240 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003975125376128385, + "loss": 1.3169, + "theoretical_loss": 3.5104793290581084, + "tokens_seen": 1677131776 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003974964894684052, + "loss": 1.2616, + "theoretical_loss": 3.5104675880387552, + "tokens_seen": 1677197312 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003974804413239719, + "loss": 1.3, + "theoretical_loss": 3.510455847606622, + "tokens_seen": 1677262848 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003974643931795386, + "loss": 1.2668, + "theoretical_loss": 3.5104441077616566, + "tokens_seen": 1677328384 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003974483450351053, + "loss": 1.2748, + "theoretical_loss": 3.510432368503806, + "tokens_seen": 1677393920 + }, + { + "epoch": 0.51, + "learning_rate": 0.000397432296890672, + "loss": 1.4238, + "theoretical_loss": 3.5104206298330185, + "tokens_seen": 1677459456 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003974162487462387, + "loss": 1.3005, + "theoretical_loss": 3.5104088917492415, + "tokens_seen": 1677524992 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003974002006018054, + "loss": 1.2955, + "theoretical_loss": 3.510397154252423, + "tokens_seen": 1677590528 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003973841524573721, + "loss": 1.3639, + "theoretical_loss": 3.51038541734251, + "tokens_seen": 1677656064 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 622434, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.256406545639038, + "objective/train/theoretical_loss": 3.5103736810194515, + "objective/train/tokens_used": 1698181600, + "theoretical_loss": 3.5103736810194515, + "tokens_seen": 1677721600 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003973681043129388, + "loss": 1.2564, + "theoretical_loss": 3.5103736810194515, + "tokens_seen": 1677721600 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003973520561685055, + "loss": 1.3527, + "theoretical_loss": 3.5103619452831936, + "tokens_seen": 1677787136 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003973360080240722, + "loss": 1.2311, + "theoretical_loss": 3.5103502101336854, + "tokens_seen": 1677852672 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003973199598796389, + "loss": 1.4424, + "theoretical_loss": 3.510338475570874, + "tokens_seen": 1677918208 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003973039117352056, + "loss": 1.1927, + "theoretical_loss": 3.510326741594708, + "tokens_seen": 1677983744 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003972878635907723, + "loss": 1.328, + "theoretical_loss": 3.5103150082051338, + "tokens_seen": 1678049280 + }, + { + "epoch": 0.51, + "learning_rate": 0.000397271815446339, + "loss": 1.3188, + "theoretical_loss": 3.5103032754021, + "tokens_seen": 1678114816 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003972557673019057, + "loss": 1.3581, + "theoretical_loss": 3.5102915431855544, + "tokens_seen": 1678180352 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003972397191574724, + "loss": 1.2902, + "theoretical_loss": 3.5102798115554448, + "tokens_seen": 1678245888 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003972236710130391, + "loss": 1.2887, + "theoretical_loss": 3.510268080511718, + "tokens_seen": 1678311424 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003972076228686058, + "loss": 1.2504, + "theoretical_loss": 3.5102563500543233, + "tokens_seen": 1678376960 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003971915747241725, + "loss": 1.3829, + "theoretical_loss": 3.510244620183207, + "tokens_seen": 1678442496 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003971755265797392, + "loss": 1.2762, + "theoretical_loss": 3.510232890898319, + "tokens_seen": 1678508032 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003971594784353059, + "loss": 1.4101, + "theoretical_loss": 3.510221162199605, + "tokens_seen": 1678573568 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003971434302908726, + "loss": 1.3204, + "theoretical_loss": 3.5102094340870136, + "tokens_seen": 1678639104 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003971273821464393, + "loss": 1.2355, + "theoretical_loss": 3.5101977065604926, + "tokens_seen": 1678704640 + }, + { + "epoch": 0.51, + "learning_rate": 0.000397111334002006, + "loss": 1.2417, + "theoretical_loss": 3.5101859796199903, + "tokens_seen": 1678770176 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003970952858575727, + "loss": 1.282, + "theoretical_loss": 3.510174253265454, + "tokens_seen": 1678835712 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003970792377131394, + "loss": 1.3312, + "theoretical_loss": 3.5101625274968318, + "tokens_seen": 1678901248 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003970631895687061, + "loss": 1.2646, + "theoretical_loss": 3.510150802314071, + "tokens_seen": 1678966784 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039704714142427283, + "loss": 1.3078, + "theoretical_loss": 3.5101390777171204, + "tokens_seen": 1679032320 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039703109327983953, + "loss": 1.2816, + "theoretical_loss": 3.5101273537059274, + "tokens_seen": 1679097856 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039701504513540623, + "loss": 1.3581, + "theoretical_loss": 3.5101156302804393, + "tokens_seen": 1679163392 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039699899699097293, + "loss": 1.2473, + "theoretical_loss": 3.5101039074406053, + "tokens_seen": 1679228928 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003969829488465397, + "loss": 1.278, + "theoretical_loss": 3.5100921851863722, + "tokens_seen": 1679294464 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003969669007021064, + "loss": 1.3364, + "theoretical_loss": 3.5100804635176877, + "tokens_seen": 1679360000 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003969508525576731, + "loss": 1.351, + "theoretical_loss": 3.510068742434501, + "tokens_seen": 1679425536 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039693480441323973, + "loss": 1.3997, + "theoretical_loss": 3.510057021936759, + "tokens_seen": 1679491072 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039691875626880643, + "loss": 1.2306, + "theoretical_loss": 3.5100453020244093, + "tokens_seen": 1679556608 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039690270812437313, + "loss": 1.2982, + "theoretical_loss": 3.510033582697401, + "tokens_seen": 1679622144 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039688665997993983, + "loss": 1.2972, + "theoretical_loss": 3.5100218639556813, + "tokens_seen": 1679687680 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039687061183550653, + "loss": 1.3301, + "theoretical_loss": 3.510010145799198, + "tokens_seen": 1679753216 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039685456369107324, + "loss": 1.3569, + "theoretical_loss": 3.5099984282278993, + "tokens_seen": 1679818752 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039683851554663994, + "loss": 1.298, + "theoretical_loss": 3.5099867112417336, + "tokens_seen": 1679884288 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039682246740220664, + "loss": 1.3863, + "theoretical_loss": 3.509974994840648, + "tokens_seen": 1679949824 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039680641925777334, + "loss": 1.2266, + "theoretical_loss": 3.5099632790245905, + "tokens_seen": 1680015360 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039679037111334004, + "loss": 1.3356, + "theoretical_loss": 3.5099515637935097, + "tokens_seen": 1680080896 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039677432296890674, + "loss": 1.1991, + "theoretical_loss": 3.5099398491473535, + "tokens_seen": 1680146432 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039675827482447344, + "loss": 1.3355, + "theoretical_loss": 3.5099281350860694, + "tokens_seen": 1680211968 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039674222668004014, + "loss": 1.3495, + "theoretical_loss": 3.5099164216096055, + "tokens_seen": 1680277504 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039672617853560684, + "loss": 1.2001, + "theoretical_loss": 3.5099047087179107, + "tokens_seen": 1680343040 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039671013039117354, + "loss": 1.2397, + "theoretical_loss": 3.5098929964109313, + "tokens_seen": 1680408576 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039669408224674024, + "loss": 1.2931, + "theoretical_loss": 3.5098812846886167, + "tokens_seen": 1680474112 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039667803410230694, + "loss": 1.2812, + "theoretical_loss": 3.5098695735509144, + "tokens_seen": 1680539648 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039666198595787364, + "loss": 1.286, + "theoretical_loss": 3.509857862997773, + "tokens_seen": 1680605184 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039664593781344034, + "loss": 1.4955, + "theoretical_loss": 3.509846153029139, + "tokens_seen": 1680670720 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039662988966900704, + "loss": 1.2143, + "theoretical_loss": 3.509834443644962, + "tokens_seen": 1680736256 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039661384152457374, + "loss": 1.2838, + "theoretical_loss": 3.50982273484519, + "tokens_seen": 1680801792 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039659779338014045, + "loss": 1.3419, + "theoretical_loss": 3.50981102662977, + "tokens_seen": 1680867328 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039658174523570715, + "loss": 1.2098, + "theoretical_loss": 3.5097993189986507, + "tokens_seen": 1680932864 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039656569709127385, + "loss": 1.3272, + "theoretical_loss": 3.50978761195178, + "tokens_seen": 1680998400 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039654964894684055, + "loss": 1.2557, + "theoretical_loss": 3.5097759054891062, + "tokens_seen": 1681063936 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039653360080240725, + "loss": 1.3089, + "theoretical_loss": 3.5097641996105766, + "tokens_seen": 1681129472 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039651755265797395, + "loss": 1.3689, + "theoretical_loss": 3.5097524943161407, + "tokens_seen": 1681195008 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039650150451354065, + "loss": 1.4081, + "theoretical_loss": 3.5097407896057455, + "tokens_seen": 1681260544 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039648545636910735, + "loss": 1.2961, + "theoretical_loss": 3.5097290854793393, + "tokens_seen": 1681326080 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039646940822467405, + "loss": 1.3338, + "theoretical_loss": 3.50971738193687, + "tokens_seen": 1681391616 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039645336008024075, + "loss": 1.3144, + "theoretical_loss": 3.5097056789782863, + "tokens_seen": 1681457152 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039643731193580745, + "loss": 1.2343, + "theoretical_loss": 3.509693976603536, + "tokens_seen": 1681522688 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039642126379137415, + "loss": 1.2373, + "theoretical_loss": 3.509682274812567, + "tokens_seen": 1681588224 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039640521564694085, + "loss": 1.2919, + "theoretical_loss": 3.509670573605328, + "tokens_seen": 1681653760 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039638916750250755, + "loss": 1.2944, + "theoretical_loss": 3.509658872981767, + "tokens_seen": 1681719296 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039637311935807425, + "loss": 1.2703, + "theoretical_loss": 3.509647172941831, + "tokens_seen": 1681784832 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039635707121364095, + "loss": 1.2329, + "theoretical_loss": 3.50963547348547, + "tokens_seen": 1681850368 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039634102306920766, + "loss": 1.2912, + "theoretical_loss": 3.5096237746126304, + "tokens_seen": 1681915904 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039632497492477436, + "loss": 1.3769, + "theoretical_loss": 3.509612076323261, + "tokens_seen": 1681981440 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039630892678034106, + "loss": 1.2594, + "theoretical_loss": 3.5096003786173116, + "tokens_seen": 1682046976 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039629287863590776, + "loss": 1.3481, + "theoretical_loss": 3.5095886814947277, + "tokens_seen": 1682112512 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039627683049147446, + "loss": 1.2727, + "theoretical_loss": 3.509576984955459, + "tokens_seen": 1682178048 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039626078234704116, + "loss": 1.3474, + "theoretical_loss": 3.509565288999453, + "tokens_seen": 1682243584 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039624473420260786, + "loss": 1.3133, + "theoretical_loss": 3.5095535936266593, + "tokens_seen": 1682309120 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039622868605817456, + "loss": 1.292, + "theoretical_loss": 3.509541898837024, + "tokens_seen": 1682374656 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039621263791374126, + "loss": 1.2444, + "theoretical_loss": 3.5095302046304973, + "tokens_seen": 1682440192 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039619658976930796, + "loss": 1.2768, + "theoretical_loss": 3.509518511007026, + "tokens_seen": 1682505728 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039618054162487466, + "loss": 1.3193, + "theoretical_loss": 3.5095068179665585, + "tokens_seen": 1682571264 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003961644934804413, + "loss": 1.2974, + "theoretical_loss": 3.5094951255090443, + "tokens_seen": 1682636800 + }, + { + "epoch": 0.51, + "learning_rate": 0.000396148445336008, + "loss": 1.243, + "theoretical_loss": 3.50948343363443, + "tokens_seen": 1682702336 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003961323971915747, + "loss": 1.3038, + "theoretical_loss": 3.5094717423426642, + "tokens_seen": 1682767872 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003961163490471414, + "loss": 1.2899, + "theoretical_loss": 3.5094600516336962, + "tokens_seen": 1682833408 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003961003009027081, + "loss": 1.2463, + "theoretical_loss": 3.5094483615074736, + "tokens_seen": 1682898944 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003960842527582748, + "loss": 1.3809, + "theoretical_loss": 3.509436671963944, + "tokens_seen": 1682964480 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003960682046138415, + "loss": 1.3805, + "theoretical_loss": 3.5094249830030564, + "tokens_seen": 1683030016 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003960521564694082, + "loss": 1.3161, + "theoretical_loss": 3.509413294624759, + "tokens_seen": 1683095552 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003960361083249749, + "loss": 1.1383, + "theoretical_loss": 3.509401606829, + "tokens_seen": 1683161088 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003960200601805416, + "loss": 1.3234, + "theoretical_loss": 3.509389919615728, + "tokens_seen": 1683226624 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003960040120361083, + "loss": 1.2401, + "theoretical_loss": 3.509378232984891, + "tokens_seen": 1683292160 + }, + { + "epoch": 0.51, + "learning_rate": 0.000395987963891675, + "loss": 1.3015, + "theoretical_loss": 3.5093665469364366, + "tokens_seen": 1683357696 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003959719157472417, + "loss": 1.3417, + "theoretical_loss": 3.509354861470314, + "tokens_seen": 1683423232 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003959558676028084, + "loss": 1.3241, + "theoretical_loss": 3.509343176586472, + "tokens_seen": 1683488768 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003959398194583751, + "loss": 1.3019, + "theoretical_loss": 3.5093314922848577, + "tokens_seen": 1683554304 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003959237713139418, + "loss": 1.3021, + "theoretical_loss": 3.5093198085654196, + "tokens_seen": 1683619840 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003959077231695085, + "loss": 1.2202, + "theoretical_loss": 3.509308125428107, + "tokens_seen": 1683685376 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003958916750250752, + "loss": 1.3049, + "theoretical_loss": 3.5092964428728672, + "tokens_seen": 1683750912 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003958756268806419, + "loss": 1.2772, + "theoretical_loss": 3.509284760899649, + "tokens_seen": 1683816448 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003958595787362086, + "loss": 1.3689, + "theoretical_loss": 3.509273079508401, + "tokens_seen": 1683881984 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003958435305917753, + "loss": 1.3059, + "theoretical_loss": 3.5092613986990715, + "tokens_seen": 1683947520 + }, + { + "epoch": 0.51, + "learning_rate": 0.000395827482447342, + "loss": 1.26, + "theoretical_loss": 3.509249718471608, + "tokens_seen": 1684013056 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003958114343029087, + "loss": 1.4459, + "theoretical_loss": 3.50923803882596, + "tokens_seen": 1684078592 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003957953861584755, + "loss": 1.3252, + "theoretical_loss": 3.509226359762075, + "tokens_seen": 1684144128 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003957793380140422, + "loss": 1.4626, + "theoretical_loss": 3.5092146812799023, + "tokens_seen": 1684209664 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 625169, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.178755521774292, + "objective/train/theoretical_loss": 3.5092030033793895, + "objective/train/tokens_used": 1704735200, + "theoretical_loss": 3.5092030033793895, + "tokens_seen": 1684275200 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003957632898696089, + "loss": 1.1788, + "theoretical_loss": 3.5092030033793895, + "tokens_seen": 1684275200 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003957472417251756, + "loss": 1.2887, + "theoretical_loss": 3.5091913260604857, + "tokens_seen": 1684340736 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003957311935807423, + "loss": 1.307, + "theoretical_loss": 3.509179649323138, + "tokens_seen": 1684406272 + }, + { + "epoch": 0.51, + "learning_rate": 0.000395715145436309, + "loss": 1.3122, + "theoretical_loss": 3.5091679731672967, + "tokens_seen": 1684471808 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003956990972918757, + "loss": 1.2972, + "theoretical_loss": 3.5091562975929085, + "tokens_seen": 1684537344 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003956830491474424, + "loss": 1.3509, + "theoretical_loss": 3.5091446225999228, + "tokens_seen": 1684602880 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003956670010030091, + "loss": 1.3255, + "theoretical_loss": 3.509132948188287, + "tokens_seen": 1684668416 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003956509528585758, + "loss": 1.292, + "theoretical_loss": 3.5091212743579514, + "tokens_seen": 1684733952 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003956349047141425, + "loss": 1.2208, + "theoretical_loss": 3.5091096011088627, + "tokens_seen": 1684799488 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003956188565697092, + "loss": 1.3203, + "theoretical_loss": 3.5090979284409705, + "tokens_seen": 1684865024 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003956028084252759, + "loss": 1.3063, + "theoretical_loss": 3.5090862563542227, + "tokens_seen": 1684930560 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003955867602808426, + "loss": 1.3361, + "theoretical_loss": 3.509074584848568, + "tokens_seen": 1684996096 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003955707121364093, + "loss": 1.2723, + "theoretical_loss": 3.5090629139239535, + "tokens_seen": 1685061632 + }, + { + "epoch": 0.51, + "learning_rate": 0.000395554663991976, + "loss": 1.3517, + "theoretical_loss": 3.50905124358033, + "tokens_seen": 1685127168 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039553861584754263, + "loss": 1.258, + "theoretical_loss": 3.5090395738176445, + "tokens_seen": 1685192704 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039552256770310933, + "loss": 1.3065, + "theoretical_loss": 3.509027904635846, + "tokens_seen": 1685258240 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039550651955867603, + "loss": 1.2258, + "theoretical_loss": 3.509016236034883, + "tokens_seen": 1685323776 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039549047141424273, + "loss": 1.3336, + "theoretical_loss": 3.5090045680147037, + "tokens_seen": 1685389312 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039547442326980944, + "loss": 1.4345, + "theoretical_loss": 3.5089929005752563, + "tokens_seen": 1685454848 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039545837512537614, + "loss": 1.2938, + "theoretical_loss": 3.50898123371649, + "tokens_seen": 1685520384 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039544232698094284, + "loss": 1.339, + "theoretical_loss": 3.5089695674383536, + "tokens_seen": 1685585920 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039542627883650954, + "loss": 1.2881, + "theoretical_loss": 3.5089579017407946, + "tokens_seen": 1685651456 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039541023069207624, + "loss": 1.2724, + "theoretical_loss": 3.5089462366237623, + "tokens_seen": 1685716992 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039539418254764294, + "loss": 1.3765, + "theoretical_loss": 3.508934572087205, + "tokens_seen": 1685782528 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039537813440320964, + "loss": 1.2578, + "theoretical_loss": 3.5089229081310713, + "tokens_seen": 1685848064 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039536208625877634, + "loss": 1.3103, + "theoretical_loss": 3.5089112447553097, + "tokens_seen": 1685913600 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039534603811434304, + "loss": 1.292, + "theoretical_loss": 3.508899581959869, + "tokens_seen": 1685979136 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039532998996990974, + "loss": 1.299, + "theoretical_loss": 3.5088879197446974, + "tokens_seen": 1686044672 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039531394182547644, + "loss": 1.3362, + "theoretical_loss": 3.5088762581097432, + "tokens_seen": 1686110208 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039529789368104314, + "loss": 1.351, + "theoretical_loss": 3.5088645970549557, + "tokens_seen": 1686175744 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039528184553660984, + "loss": 1.2915, + "theoretical_loss": 3.5088529365802836, + "tokens_seen": 1686241280 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039526579739217654, + "loss": 1.2975, + "theoretical_loss": 3.5088412766856747, + "tokens_seen": 1686306816 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039524974924774324, + "loss": 1.2165, + "theoretical_loss": 3.5088296173710782, + "tokens_seen": 1686372352 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039523370110330995, + "loss": 1.3161, + "theoretical_loss": 3.508817958636442, + "tokens_seen": 1686437888 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039521765295887665, + "loss": 1.2816, + "theoretical_loss": 3.508806300481716, + "tokens_seen": 1686503424 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039520160481444335, + "loss": 1.4831, + "theoretical_loss": 3.5087946429068477, + "tokens_seen": 1686568960 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039518555667001005, + "loss": 1.3331, + "theoretical_loss": 3.5087829859117856, + "tokens_seen": 1686634496 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039516950852557675, + "loss": 1.3433, + "theoretical_loss": 3.5087713294964793, + "tokens_seen": 1686700032 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039515346038114345, + "loss": 1.3225, + "theoretical_loss": 3.508759673660877, + "tokens_seen": 1686765568 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039513741223671015, + "loss": 1.3431, + "theoretical_loss": 3.508748018404927, + "tokens_seen": 1686831104 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039512136409227685, + "loss": 1.3562, + "theoretical_loss": 3.5087363637285787, + "tokens_seen": 1686896640 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039510531594784355, + "loss": 1.2717, + "theoretical_loss": 3.50872470963178, + "tokens_seen": 1686962176 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039508926780341025, + "loss": 1.3428, + "theoretical_loss": 3.5087130561144795, + "tokens_seen": 1687027712 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039507321965897695, + "loss": 1.2072, + "theoretical_loss": 3.5087014031766266, + "tokens_seen": 1687093248 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039505717151454365, + "loss": 1.3277, + "theoretical_loss": 3.50868975081817, + "tokens_seen": 1687158784 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039504112337011035, + "loss": 1.3392, + "theoretical_loss": 3.5086780990390576, + "tokens_seen": 1687224320 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039502507522567705, + "loss": 1.2907, + "theoretical_loss": 3.5086664478392384, + "tokens_seen": 1687289856 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039500902708124375, + "loss": 1.4298, + "theoretical_loss": 3.5086547972186617, + "tokens_seen": 1687355392 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039499297893681045, + "loss": 1.3613, + "theoretical_loss": 3.5086431471772754, + "tokens_seen": 1687420928 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039497693079237716, + "loss": 1.4015, + "theoretical_loss": 3.5086314977150286, + "tokens_seen": 1687486464 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039496088264794386, + "loss": 1.2869, + "theoretical_loss": 3.50861984883187, + "tokens_seen": 1687552000 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039494483450351056, + "loss": 1.3575, + "theoretical_loss": 3.508608200527749, + "tokens_seen": 1687617536 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039492878635907726, + "loss": 1.3591, + "theoretical_loss": 3.5085965528026124, + "tokens_seen": 1687683072 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039491273821464396, + "loss": 1.2176, + "theoretical_loss": 3.5085849056564107, + "tokens_seen": 1687748608 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039489669007021066, + "loss": 1.3917, + "theoretical_loss": 3.508573259089092, + "tokens_seen": 1687814144 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039488064192577736, + "loss": 1.3677, + "theoretical_loss": 3.508561613100605, + "tokens_seen": 1687879680 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039486459378134406, + "loss": 1.2936, + "theoretical_loss": 3.5085499676908993, + "tokens_seen": 1687945216 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039484854563691076, + "loss": 1.3865, + "theoretical_loss": 3.5085383228599225, + "tokens_seen": 1688010752 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039483249749247746, + "loss": 1.3675, + "theoretical_loss": 3.5085266786076237, + "tokens_seen": 1688076288 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039481644934804416, + "loss": 1.3424, + "theoretical_loss": 3.5085150349339522, + "tokens_seen": 1688141824 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039480040120361086, + "loss": 1.3442, + "theoretical_loss": 3.508503391838856, + "tokens_seen": 1688207360 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039478435305917756, + "loss": 1.2923, + "theoretical_loss": 3.5084917493222845, + "tokens_seen": 1688272896 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003947683049147442, + "loss": 1.2756, + "theoretical_loss": 3.5084801073841865, + "tokens_seen": 1688338432 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003947522567703109, + "loss": 1.378, + "theoretical_loss": 3.5084684660245107, + "tokens_seen": 1688403968 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003947362086258776, + "loss": 1.3363, + "theoretical_loss": 3.5084568252432056, + "tokens_seen": 1688469504 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003947201604814443, + "loss": 1.3509, + "theoretical_loss": 3.5084451850402205, + "tokens_seen": 1688535040 + }, + { + "epoch": 0.51, + "learning_rate": 0.000394704112337011, + "loss": 1.2682, + "theoretical_loss": 3.5084335454155036, + "tokens_seen": 1688600576 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003946880641925777, + "loss": 1.2548, + "theoretical_loss": 3.508421906369004, + "tokens_seen": 1688666112 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003946720160481444, + "loss": 1.3798, + "theoretical_loss": 3.508410267900671, + "tokens_seen": 1688731648 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003946559679037111, + "loss": 1.2671, + "theoretical_loss": 3.5083986300104533, + "tokens_seen": 1688797184 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003946399197592778, + "loss": 1.3534, + "theoretical_loss": 3.508386992698299, + "tokens_seen": 1688862720 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003946238716148445, + "loss": 1.3239, + "theoretical_loss": 3.5083753559641577, + "tokens_seen": 1688928256 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003946078234704112, + "loss": 1.3776, + "theoretical_loss": 3.508363719807978, + "tokens_seen": 1688993792 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039459177532597797, + "loss": 1.2902, + "theoretical_loss": 3.508352084229709, + "tokens_seen": 1689059328 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039457572718154467, + "loss": 1.306, + "theoretical_loss": 3.5083404492292996, + "tokens_seen": 1689124864 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039455967903711137, + "loss": 1.2222, + "theoretical_loss": 3.508328814806698, + "tokens_seen": 1689190400 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039454363089267807, + "loss": 1.3215, + "theoretical_loss": 3.5083171809618534, + "tokens_seen": 1689255936 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003945275827482448, + "loss": 1.4136, + "theoretical_loss": 3.5083055476947154, + "tokens_seen": 1689321472 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003945115346038115, + "loss": 1.2998, + "theoretical_loss": 3.508293915005232, + "tokens_seen": 1689387008 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003944954864593782, + "loss": 1.256, + "theoretical_loss": 3.5082822828933526, + "tokens_seen": 1689452544 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003944794383149449, + "loss": 1.2462, + "theoretical_loss": 3.5082706513590263, + "tokens_seen": 1689518080 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003944633901705116, + "loss": 1.3587, + "theoretical_loss": 3.5082590204022015, + "tokens_seen": 1689583616 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003944473420260783, + "loss": 1.3805, + "theoretical_loss": 3.5082473900228273, + "tokens_seen": 1689649152 + }, + { + "epoch": 0.51, + "learning_rate": 0.000394431293881645, + "loss": 1.2742, + "theoretical_loss": 3.5082357602208525, + "tokens_seen": 1689714688 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003944152457372117, + "loss": 1.2484, + "theoretical_loss": 3.5082241309962265, + "tokens_seen": 1689780224 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003943991975927784, + "loss": 1.41, + "theoretical_loss": 3.508212502348898, + "tokens_seen": 1689845760 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003943831494483451, + "loss": 1.3809, + "theoretical_loss": 3.508200874278816, + "tokens_seen": 1689911296 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003943671013039118, + "loss": 1.3479, + "theoretical_loss": 3.5081892467859292, + "tokens_seen": 1689976832 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003943510531594785, + "loss": 1.2561, + "theoretical_loss": 3.5081776198701866, + "tokens_seen": 1690042368 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003943350050150452, + "loss": 1.2897, + "theoretical_loss": 3.5081659935315375, + "tokens_seen": 1690107904 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003943189568706119, + "loss": 1.2565, + "theoretical_loss": 3.5081543677699307, + "tokens_seen": 1690173440 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003943029087261786, + "loss": 1.2314, + "theoretical_loss": 3.5081427425853153, + "tokens_seen": 1690238976 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003942868605817453, + "loss": 1.369, + "theoretical_loss": 3.5081311179776398, + "tokens_seen": 1690304512 + }, + { + "epoch": 0.51, + "learning_rate": 0.000394270812437312, + "loss": 1.2417, + "theoretical_loss": 3.508119493946854, + "tokens_seen": 1690370048 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003942547642928787, + "loss": 1.3309, + "theoretical_loss": 3.5081078704929065, + "tokens_seen": 1690435584 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003942387161484454, + "loss": 1.3734, + "theoretical_loss": 3.5080962476157462, + "tokens_seen": 1690501120 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003942226680040121, + "loss": 1.3289, + "theoretical_loss": 3.5080846253153224, + "tokens_seen": 1690566656 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003942066198595788, + "loss": 1.3902, + "theoretical_loss": 3.508073003591584, + "tokens_seen": 1690632192 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003941905717151455, + "loss": 1.3534, + "theoretical_loss": 3.5080613824444793, + "tokens_seen": 1690697728 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003941745235707122, + "loss": 1.3604, + "theoretical_loss": 3.508049761873959, + "tokens_seen": 1690763264 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 627415, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.376535415649414, + "objective/train/theoretical_loss": 3.5080381418799704, + "objective/train/tokens_used": 1711288800, + "theoretical_loss": 3.5080381418799704, + "tokens_seen": 1690828800 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003941584754262789, + "loss": 1.3765, + "theoretical_loss": 3.5080381418799704, + "tokens_seen": 1690828800 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039414242728184553, + "loss": 1.2697, + "theoretical_loss": 3.5080265224624636, + "tokens_seen": 1690894336 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039412637913741223, + "loss": 1.3701, + "theoretical_loss": 3.5080149036213877, + "tokens_seen": 1690959872 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039411033099297894, + "loss": 1.3054, + "theoretical_loss": 3.508003285356691, + "tokens_seen": 1691025408 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039409428284854564, + "loss": 1.2443, + "theoretical_loss": 3.5079916676683234, + "tokens_seen": 1691090944 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039407823470411234, + "loss": 1.2257, + "theoretical_loss": 3.507980050556233, + "tokens_seen": 1691156480 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039406218655967904, + "loss": 1.3746, + "theoretical_loss": 3.50796843402037, + "tokens_seen": 1691222016 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039404613841524574, + "loss": 1.3185, + "theoretical_loss": 3.507956818060683, + "tokens_seen": 1691287552 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039403009027081244, + "loss": 1.2456, + "theoretical_loss": 3.5079452026771207, + "tokens_seen": 1691353088 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039401404212637914, + "loss": 1.354, + "theoretical_loss": 3.507933587869633, + "tokens_seen": 1691418624 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039399799398194584, + "loss": 1.2851, + "theoretical_loss": 3.5079219736381684, + "tokens_seen": 1691484160 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039398194583751254, + "loss": 1.3374, + "theoretical_loss": 3.507910359982676, + "tokens_seen": 1691549696 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039396589769307924, + "loss": 1.3284, + "theoretical_loss": 3.5078987469031055, + "tokens_seen": 1691615232 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039394984954864594, + "loss": 1.3735, + "theoretical_loss": 3.507887134399405, + "tokens_seen": 1691680768 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039393380140421264, + "loss": 1.2412, + "theoretical_loss": 3.507875522471525, + "tokens_seen": 1691746304 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039391775325977934, + "loss": 1.2712, + "theoretical_loss": 3.5078639111194136, + "tokens_seen": 1691811840 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039390170511534604, + "loss": 1.2599, + "theoretical_loss": 3.5078523003430204, + "tokens_seen": 1691877376 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039388565697091274, + "loss": 1.1852, + "theoretical_loss": 3.5078406901422943, + "tokens_seen": 1691942912 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039386960882647944, + "loss": 1.3271, + "theoretical_loss": 3.5078290805171846, + "tokens_seen": 1692008448 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039385356068204615, + "loss": 1.3376, + "theoretical_loss": 3.5078174714676407, + "tokens_seen": 1692073984 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039383751253761285, + "loss": 1.2708, + "theoretical_loss": 3.507805862993611, + "tokens_seen": 1692139520 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039382146439317955, + "loss": 1.2855, + "theoretical_loss": 3.507794255095046, + "tokens_seen": 1692205056 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039380541624874625, + "loss": 1.3181, + "theoretical_loss": 3.5077826477718936, + "tokens_seen": 1692270592 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039378936810431295, + "loss": 1.2691, + "theoretical_loss": 3.5077710410241036, + "tokens_seen": 1692336128 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039377331995987965, + "loss": 1.3142, + "theoretical_loss": 3.507759434851625, + "tokens_seen": 1692401664 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039375727181544635, + "loss": 1.3053, + "theoretical_loss": 3.5077478292544075, + "tokens_seen": 1692467200 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039374122367101305, + "loss": 1.3484, + "theoretical_loss": 3.5077362242323993, + "tokens_seen": 1692532736 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039372517552657975, + "loss": 1.2465, + "theoretical_loss": 3.5077246197855505, + "tokens_seen": 1692598272 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039370912738214645, + "loss": 1.2113, + "theoretical_loss": 3.5077130159138106, + "tokens_seen": 1692663808 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039369307923771315, + "loss": 1.3122, + "theoretical_loss": 3.507701412617128, + "tokens_seen": 1692729344 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039367703109327985, + "loss": 1.2814, + "theoretical_loss": 3.507689809895452, + "tokens_seen": 1692794880 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039366098294884655, + "loss": 1.3254, + "theoretical_loss": 3.5076782077487323, + "tokens_seen": 1692860416 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039364493480441325, + "loss": 1.3068, + "theoretical_loss": 3.507666606176918, + "tokens_seen": 1692925952 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039362888665997995, + "loss": 1.3422, + "theoretical_loss": 3.5076550051799575, + "tokens_seen": 1692991488 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039361283851554665, + "loss": 1.3273, + "theoretical_loss": 3.507643404757802, + "tokens_seen": 1693057024 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039359679037111336, + "loss": 1.3443, + "theoretical_loss": 3.5076318049103987, + "tokens_seen": 1693122560 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039358074222668006, + "loss": 1.3071, + "theoretical_loss": 3.507620205637698, + "tokens_seen": 1693188096 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039356469408224676, + "loss": 1.3582, + "theoretical_loss": 3.50760860693965, + "tokens_seen": 1693253632 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039354864593781346, + "loss": 1.3178, + "theoretical_loss": 3.507597008816202, + "tokens_seen": 1693319168 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039353259779338016, + "loss": 1.344, + "theoretical_loss": 3.5075854112673044, + "tokens_seen": 1693384704 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039351654964894686, + "loss": 1.2812, + "theoretical_loss": 3.5075738142929067, + "tokens_seen": 1693450240 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039350050150451356, + "loss": 1.3928, + "theoretical_loss": 3.507562217892958, + "tokens_seen": 1693515776 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039348445336008026, + "loss": 1.2849, + "theoretical_loss": 3.5075506220674066, + "tokens_seen": 1693581312 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039346840521564696, + "loss": 1.2924, + "theoretical_loss": 3.507539026816204, + "tokens_seen": 1693646848 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039345235707121366, + "loss": 1.4056, + "theoretical_loss": 3.5075274321392973, + "tokens_seen": 1693712384 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039343630892678036, + "loss": 1.4159, + "theoretical_loss": 3.507515838036637, + "tokens_seen": 1693777920 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039342026078234706, + "loss": 1.3076, + "theoretical_loss": 3.5075042445081728, + "tokens_seen": 1693843456 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039340421263791376, + "loss": 1.3288, + "theoretical_loss": 3.5074926515538527, + "tokens_seen": 1693908992 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039338816449348046, + "loss": 1.3452, + "theoretical_loss": 3.507481059173627, + "tokens_seen": 1693974528 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039337211634904716, + "loss": 1.3824, + "theoretical_loss": 3.5074694673674456, + "tokens_seen": 1694040064 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039335606820461387, + "loss": 1.3357, + "theoretical_loss": 3.507457876135257, + "tokens_seen": 1694105600 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039334002006018057, + "loss": 1.3072, + "theoretical_loss": 3.5074462854770103, + "tokens_seen": 1694171136 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039332397191574727, + "loss": 1.3401, + "theoretical_loss": 3.5074346953926554, + "tokens_seen": 1694236672 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039330792377131397, + "loss": 1.4514, + "theoretical_loss": 3.507423105882142, + "tokens_seen": 1694302208 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039329187562688067, + "loss": 1.2799, + "theoretical_loss": 3.5074115169454188, + "tokens_seen": 1694367744 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039327582748244737, + "loss": 1.3631, + "theoretical_loss": 3.5073999285824353, + "tokens_seen": 1694433280 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039325977933801407, + "loss": 1.1687, + "theoretical_loss": 3.507388340793142, + "tokens_seen": 1694498816 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039324373119358077, + "loss": 1.3252, + "theoretical_loss": 3.5073767535774865, + "tokens_seen": 1694564352 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039322768304914747, + "loss": 1.2501, + "theoretical_loss": 3.5073651669354193, + "tokens_seen": 1694629888 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039321163490471417, + "loss": 1.4278, + "theoretical_loss": 3.50735358086689, + "tokens_seen": 1694695424 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039319558676028087, + "loss": 1.3996, + "theoretical_loss": 3.5073419953718474, + "tokens_seen": 1694760960 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039317953861584757, + "loss": 1.3672, + "theoretical_loss": 3.507330410450242, + "tokens_seen": 1694826496 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039316349047141427, + "loss": 1.2511, + "theoretical_loss": 3.507318826102022, + "tokens_seen": 1694892032 + }, + { + "epoch": 0.51, + "learning_rate": 0.000393147442326981, + "loss": 1.2972, + "theoretical_loss": 3.507307242327137, + "tokens_seen": 1694957568 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003931313941825477, + "loss": 1.3639, + "theoretical_loss": 3.5072956591255373, + "tokens_seen": 1695023104 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003931153460381144, + "loss": 1.3168, + "theoretical_loss": 3.5072840764971716, + "tokens_seen": 1695088640 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003930992978936811, + "loss": 1.3513, + "theoretical_loss": 3.50727249444199, + "tokens_seen": 1695154176 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003930832497492478, + "loss": 1.3247, + "theoretical_loss": 3.507260912959941, + "tokens_seen": 1695219712 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003930672016048145, + "loss": 1.1519, + "theoretical_loss": 3.507249332050975, + "tokens_seen": 1695285248 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003930511534603812, + "loss": 1.3517, + "theoretical_loss": 3.5072377517150413, + "tokens_seen": 1695350784 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003930351053159479, + "loss": 1.2584, + "theoretical_loss": 3.507226171952089, + "tokens_seen": 1695416320 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003930190571715146, + "loss": 1.3361, + "theoretical_loss": 3.507214592762068, + "tokens_seen": 1695481856 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003930030090270813, + "loss": 1.3888, + "theoretical_loss": 3.5072030141449284, + "tokens_seen": 1695547392 + }, + { + "epoch": 0.51, + "learning_rate": 0.000392986960882648, + "loss": 1.2394, + "theoretical_loss": 3.507191436100618, + "tokens_seen": 1695612928 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003929709127382147, + "loss": 1.3854, + "theoretical_loss": 3.5071798586290877, + "tokens_seen": 1695678464 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003929548645937814, + "loss": 1.3317, + "theoretical_loss": 3.507168281730287, + "tokens_seen": 1695744000 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003929388164493481, + "loss": 1.3126, + "theoretical_loss": 3.5071567054041646, + "tokens_seen": 1695809536 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003929227683049148, + "loss": 1.3, + "theoretical_loss": 3.507145129650671, + "tokens_seen": 1695875072 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003929067201604815, + "loss": 1.2962, + "theoretical_loss": 3.507133554469755, + "tokens_seen": 1695940608 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003928906720160482, + "loss": 1.3176, + "theoretical_loss": 3.507121979861367, + "tokens_seen": 1696006144 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003928746238716149, + "loss": 1.2739, + "theoretical_loss": 3.507110405825455, + "tokens_seen": 1696071680 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003928585757271816, + "loss": 1.3596, + "theoretical_loss": 3.50709883236197, + "tokens_seen": 1696137216 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003928425275827483, + "loss": 1.261, + "theoretical_loss": 3.507087259470861, + "tokens_seen": 1696202752 + }, + { + "epoch": 0.51, + "learning_rate": 0.000392826479438315, + "loss": 1.3342, + "theoretical_loss": 3.5070756871520783, + "tokens_seen": 1696268288 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003928104312938817, + "loss": 1.3269, + "theoretical_loss": 3.50706411540557, + "tokens_seen": 1696333824 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003927943831494484, + "loss": 1.4192, + "theoretical_loss": 3.5070525442312874, + "tokens_seen": 1696399360 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003927783350050151, + "loss": 1.2787, + "theoretical_loss": 3.507040973629179, + "tokens_seen": 1696464896 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003927622868605818, + "loss": 1.2975, + "theoretical_loss": 3.507029403599195, + "tokens_seen": 1696530432 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003927462387161485, + "loss": 1.2511, + "theoretical_loss": 3.5070178341412843, + "tokens_seen": 1696595968 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039273019057171514, + "loss": 1.3113, + "theoretical_loss": 3.5070062652553977, + "tokens_seen": 1696661504 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039271414242728184, + "loss": 1.3661, + "theoretical_loss": 3.506994696941483, + "tokens_seen": 1696727040 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039269809428284854, + "loss": 1.3493, + "theoretical_loss": 3.5069831291994915, + "tokens_seen": 1696792576 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039268204613841524, + "loss": 1.3143, + "theoretical_loss": 3.506971562029372, + "tokens_seen": 1696858112 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039266599799398194, + "loss": 1.283, + "theoretical_loss": 3.5069599954310746, + "tokens_seen": 1696923648 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039264994984954864, + "loss": 1.4476, + "theoretical_loss": 3.5069484294045483, + "tokens_seen": 1696989184 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039263390170511534, + "loss": 1.3824, + "theoretical_loss": 3.5069368639497434, + "tokens_seen": 1697054720 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039261785356068204, + "loss": 1.287, + "theoretical_loss": 3.5069252990666095, + "tokens_seen": 1697120256 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039260180541624874, + "loss": 1.2867, + "theoretical_loss": 3.506913734755096, + "tokens_seen": 1697185792 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039258575727181544, + "loss": 1.2612, + "theoretical_loss": 3.5069021710151524, + "tokens_seen": 1697251328 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039256970912738214, + "loss": 1.2573, + "theoretical_loss": 3.506890607846729, + "tokens_seen": 1697316864 + }, + { + "epoch": 0.51, + "objective/train/docs_used": 629636, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2998379468917847, + "objective/train/theoretical_loss": 3.506879045249775, + "objective/train/tokens_used": 1717842400, + "theoretical_loss": 3.506879045249775, + "tokens_seen": 1697382400 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039255366098294884, + "loss": 1.2998, + "theoretical_loss": 3.506879045249775, + "tokens_seen": 1697382400 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039253761283851554, + "loss": 1.3701, + "theoretical_loss": 3.5068674832242404, + "tokens_seen": 1697447936 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039252156469408224, + "loss": 1.3061, + "theoretical_loss": 3.5068559217700748, + "tokens_seen": 1697513472 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039250551654964894, + "loss": 1.262, + "theoretical_loss": 3.5068443608872277, + "tokens_seen": 1697579008 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039248946840521565, + "loss": 1.3489, + "theoretical_loss": 3.5068328005756486, + "tokens_seen": 1697644544 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039247342026078235, + "loss": 1.3451, + "theoretical_loss": 3.506821240835288, + "tokens_seen": 1697710080 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039245737211634905, + "loss": 1.3044, + "theoretical_loss": 3.5068096816660956, + "tokens_seen": 1697775616 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039244132397191575, + "loss": 1.3534, + "theoretical_loss": 3.5067981230680196, + "tokens_seen": 1697841152 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039242527582748245, + "loss": 1.3756, + "theoretical_loss": 3.506786565041012, + "tokens_seen": 1697906688 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039240922768304915, + "loss": 1.3405, + "theoretical_loss": 3.506775007585021, + "tokens_seen": 1697972224 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039239317953861585, + "loss": 1.3139, + "theoretical_loss": 3.5067634506999967, + "tokens_seen": 1698037760 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039237713139418255, + "loss": 1.2637, + "theoretical_loss": 3.5067518943858893, + "tokens_seen": 1698103296 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039236108324974925, + "loss": 1.4242, + "theoretical_loss": 3.506740338642648, + "tokens_seen": 1698168832 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039234503510531595, + "loss": 1.3362, + "theoretical_loss": 3.5067287834702223, + "tokens_seen": 1698234368 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039232898696088265, + "loss": 1.393, + "theoretical_loss": 3.506717228868563, + "tokens_seen": 1698299904 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039231293881644935, + "loss": 1.3364, + "theoretical_loss": 3.506705674837619, + "tokens_seen": 1698365440 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039229689067201605, + "loss": 1.3427, + "theoretical_loss": 3.5066941213773406, + "tokens_seen": 1698430976 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039228084252758275, + "loss": 1.3796, + "theoretical_loss": 3.5066825684876775, + "tokens_seen": 1698496512 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039226479438314945, + "loss": 1.3278, + "theoretical_loss": 3.5066710161685792, + "tokens_seen": 1698562048 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039224874623871615, + "loss": 1.1775, + "theoretical_loss": 3.5066594644199958, + "tokens_seen": 1698627584 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039223269809428286, + "loss": 1.3064, + "theoretical_loss": 3.506647913241877, + "tokens_seen": 1698693120 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039221664994984956, + "loss": 1.3074, + "theoretical_loss": 3.5066363626341728, + "tokens_seen": 1698758656 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039220060180541626, + "loss": 1.2815, + "theoretical_loss": 3.506624812596833, + "tokens_seen": 1698824192 + }, + { + "epoch": 0.51, + "learning_rate": 0.000392184553660983, + "loss": 1.3365, + "theoretical_loss": 3.5066132631298075, + "tokens_seen": 1698889728 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003921685055165497, + "loss": 1.3833, + "theoretical_loss": 3.506601714233045, + "tokens_seen": 1698955264 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003921524573721164, + "loss": 1.2681, + "theoretical_loss": 3.5065901659064975, + "tokens_seen": 1699020800 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003921364092276831, + "loss": 1.3085, + "theoretical_loss": 3.5065786181501126, + "tokens_seen": 1699086336 + }, + { + "epoch": 0.51, + "learning_rate": 0.0003921203610832498, + "loss": 1.2639, + "theoretical_loss": 3.5065670709638423, + "tokens_seen": 1699151872 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039210431293881646, + "loss": 1.3361, + "theoretical_loss": 3.5065555243476347, + "tokens_seen": 1699217408 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039208826479438316, + "loss": 1.3036, + "theoretical_loss": 3.5065439783014405, + "tokens_seen": 1699282944 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039207221664994986, + "loss": 1.2908, + "theoretical_loss": 3.5065324328252094, + "tokens_seen": 1699348480 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039205616850551656, + "loss": 1.3596, + "theoretical_loss": 3.5065208879188914, + "tokens_seen": 1699414016 + }, + { + "epoch": 0.51, + "learning_rate": 0.00039204012036108326, + "loss": 1.378, + "theoretical_loss": 3.506509343582436, + "tokens_seen": 1699479552 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039202407221664996, + "loss": 1.4039, + "theoretical_loss": 3.506497799815794, + "tokens_seen": 1699545088 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039200802407221666, + "loss": 1.3936, + "theoretical_loss": 3.5064862566189143, + "tokens_seen": 1699610624 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039199197592778336, + "loss": 1.2582, + "theoretical_loss": 3.5064747139917474, + "tokens_seen": 1699676160 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039197592778335007, + "loss": 1.3685, + "theoretical_loss": 3.506463171934243, + "tokens_seen": 1699741696 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039195987963891677, + "loss": 1.3692, + "theoretical_loss": 3.5064516304463513, + "tokens_seen": 1699807232 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039194383149448347, + "loss": 1.3537, + "theoretical_loss": 3.506440089528022, + "tokens_seen": 1699872768 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039192778335005017, + "loss": 1.2712, + "theoretical_loss": 3.5064285491792044, + "tokens_seen": 1699938304 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039191173520561687, + "loss": 1.3263, + "theoretical_loss": 3.5064170093998497, + "tokens_seen": 1700003840 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039189568706118357, + "loss": 1.2625, + "theoretical_loss": 3.506405470189907, + "tokens_seen": 1700069376 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039187963891675027, + "loss": 1.4408, + "theoretical_loss": 3.5063939315493267, + "tokens_seen": 1700134912 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039186359077231697, + "loss": 1.3076, + "theoretical_loss": 3.5063823934780585, + "tokens_seen": 1700200448 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039184754262788367, + "loss": 1.3816, + "theoretical_loss": 3.506370855976052, + "tokens_seen": 1700265984 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039183149448345037, + "loss": 1.3168, + "theoretical_loss": 3.5063593190432574, + "tokens_seen": 1700331520 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039181544633901707, + "loss": 1.3274, + "theoretical_loss": 3.506347782679626, + "tokens_seen": 1700397056 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039179939819458377, + "loss": 1.3922, + "theoretical_loss": 3.5063362468851054, + "tokens_seen": 1700462592 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003917833500501505, + "loss": 1.4257, + "theoretical_loss": 3.5063247116596474, + "tokens_seen": 1700528128 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003917673019057172, + "loss": 1.3418, + "theoretical_loss": 3.506313177003202, + "tokens_seen": 1700593664 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003917512537612839, + "loss": 1.275, + "theoretical_loss": 3.5063016429157177, + "tokens_seen": 1700659200 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003917352056168506, + "loss": 1.2182, + "theoretical_loss": 3.506290109397146, + "tokens_seen": 1700724736 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003917191574724173, + "loss": 1.3154, + "theoretical_loss": 3.5062785764474365, + "tokens_seen": 1700790272 + }, + { + "epoch": 0.52, + "learning_rate": 0.000391703109327984, + "loss": 1.2994, + "theoretical_loss": 3.506267044066538, + "tokens_seen": 1700855808 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003916870611835507, + "loss": 1.2691, + "theoretical_loss": 3.506255512254403, + "tokens_seen": 1700921344 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003916710130391174, + "loss": 1.4752, + "theoretical_loss": 3.5062439810109796, + "tokens_seen": 1700986880 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003916549648946841, + "loss": 1.3396, + "theoretical_loss": 3.5062324503362183, + "tokens_seen": 1701052416 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003916389167502508, + "loss": 1.2006, + "theoretical_loss": 3.5062209202300694, + "tokens_seen": 1701117952 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003916228686058175, + "loss": 1.2828, + "theoretical_loss": 3.506209390692483, + "tokens_seen": 1701183488 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003916068204613842, + "loss": 1.3371, + "theoretical_loss": 3.5061978617234084, + "tokens_seen": 1701249024 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003915907723169509, + "loss": 1.2924, + "theoretical_loss": 3.5061863333227965, + "tokens_seen": 1701314560 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003915747241725176, + "loss": 1.3442, + "theoretical_loss": 3.5061748054905975, + "tokens_seen": 1701380096 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003915586760280843, + "loss": 1.306, + "theoretical_loss": 3.5061632782267607, + "tokens_seen": 1701445632 + }, + { + "epoch": 0.52, + "learning_rate": 0.000391542627883651, + "loss": 1.2403, + "theoretical_loss": 3.506151751531237, + "tokens_seen": 1701511168 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003915265797392177, + "loss": 1.2403, + "theoretical_loss": 3.5061402254039757, + "tokens_seen": 1701576704 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003915105315947844, + "loss": 1.3721, + "theoretical_loss": 3.5061286998449273, + "tokens_seen": 1701642240 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003914944834503511, + "loss": 1.3887, + "theoretical_loss": 3.506117174854042, + "tokens_seen": 1701707776 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003914784353059178, + "loss": 1.4169, + "theoretical_loss": 3.50610565043127, + "tokens_seen": 1701773312 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003914623871614845, + "loss": 1.3388, + "theoretical_loss": 3.506094126576561, + "tokens_seen": 1701838848 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003914463390170512, + "loss": 1.2696, + "theoretical_loss": 3.5060826032898653, + "tokens_seen": 1701904384 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003914302908726179, + "loss": 1.317, + "theoretical_loss": 3.5060710805711333, + "tokens_seen": 1701969920 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003914142427281846, + "loss": 1.232, + "theoretical_loss": 3.506059558420314, + "tokens_seen": 1702035456 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003913981945837513, + "loss": 1.3444, + "theoretical_loss": 3.5060480368373597, + "tokens_seen": 1702100992 + }, + { + "epoch": 0.52, + "learning_rate": 0.000391382146439318, + "loss": 1.3486, + "theoretical_loss": 3.5060365158222186, + "tokens_seen": 1702166528 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003913660982948847, + "loss": 1.3988, + "theoretical_loss": 3.5060249953748417, + "tokens_seen": 1702232064 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003913500501504514, + "loss": 1.2558, + "theoretical_loss": 3.506013475495179, + "tokens_seen": 1702297600 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039133400200601804, + "loss": 1.3775, + "theoretical_loss": 3.506001956183181, + "tokens_seen": 1702363136 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039131795386158474, + "loss": 1.4521, + "theoretical_loss": 3.5059904374387973, + "tokens_seen": 1702428672 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039130190571715144, + "loss": 1.3341, + "theoretical_loss": 3.5059789192619784, + "tokens_seen": 1702494208 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039128585757271814, + "loss": 1.2653, + "theoretical_loss": 3.5059674016526747, + "tokens_seen": 1702559744 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039126980942828484, + "loss": 1.2527, + "theoretical_loss": 3.5059558846108354, + "tokens_seen": 1702625280 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039125376128385154, + "loss": 1.3248, + "theoretical_loss": 3.5059443681364124, + "tokens_seen": 1702690816 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039123771313941824, + "loss": 1.3261, + "theoretical_loss": 3.5059328522293542, + "tokens_seen": 1702756352 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039122166499498494, + "loss": 1.2718, + "theoretical_loss": 3.505921336889612, + "tokens_seen": 1702821888 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039120561685055164, + "loss": 1.3084, + "theoretical_loss": 3.505909822117136, + "tokens_seen": 1702887424 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039118956870611834, + "loss": 1.2991, + "theoretical_loss": 3.5058983079118757, + "tokens_seen": 1702952960 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039117352056168504, + "loss": 1.2076, + "theoretical_loss": 3.505886794273782, + "tokens_seen": 1703018496 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039115747241725174, + "loss": 1.3353, + "theoretical_loss": 3.505875281202805, + "tokens_seen": 1703084032 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039114142427281844, + "loss": 1.2713, + "theoretical_loss": 3.5058637686988954, + "tokens_seen": 1703149568 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039112537612838514, + "loss": 1.3205, + "theoretical_loss": 3.5058522567620023, + "tokens_seen": 1703215104 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039110932798395185, + "loss": 1.2095, + "theoretical_loss": 3.505840745392077, + "tokens_seen": 1703280640 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039109327983951855, + "loss": 1.3594, + "theoretical_loss": 3.5058292345890694, + "tokens_seen": 1703346176 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039107723169508525, + "loss": 1.2871, + "theoretical_loss": 3.505817724352929, + "tokens_seen": 1703411712 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039106118355065195, + "loss": 1.2927, + "theoretical_loss": 3.505806214683608, + "tokens_seen": 1703477248 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039104513540621865, + "loss": 1.3851, + "theoretical_loss": 3.5057947055810548, + "tokens_seen": 1703542784 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039102908726178535, + "loss": 1.3997, + "theoretical_loss": 3.5057831970452207, + "tokens_seen": 1703608320 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039101303911735205, + "loss": 1.3111, + "theoretical_loss": 3.5057716890760555, + "tokens_seen": 1703673856 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039099699097291875, + "loss": 1.3352, + "theoretical_loss": 3.5057601816735096, + "tokens_seen": 1703739392 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003909809428284855, + "loss": 1.2155, + "theoretical_loss": 3.505748674837534, + "tokens_seen": 1703804928 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003909648946840522, + "loss": 1.2751, + "theoretical_loss": 3.5057371685680776, + "tokens_seen": 1703870464 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 632128, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.389420509338379, + "objective/train/theoretical_loss": 3.505725662865092, + "objective/train/tokens_used": 1724396000, + "theoretical_loss": 3.505725662865092, + "tokens_seen": 1703936000 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003909488465396189, + "loss": 1.3894, + "theoretical_loss": 3.505725662865092, + "tokens_seen": 1703936000 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003909327983951856, + "loss": 1.3831, + "theoretical_loss": 3.5057141577285273, + "tokens_seen": 1704001536 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003909167502507523, + "loss": 1.3792, + "theoretical_loss": 3.505702653158333, + "tokens_seen": 1704067072 + }, + { + "epoch": 0.52, + "learning_rate": 0.000390900702106319, + "loss": 1.351, + "theoretical_loss": 3.5056911491544605, + "tokens_seen": 1704132608 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003908846539618857, + "loss": 1.3788, + "theoretical_loss": 3.5056796457168593, + "tokens_seen": 1704198144 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003908686058174524, + "loss": 1.249, + "theoretical_loss": 3.5056681428454803, + "tokens_seen": 1704263680 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003908525576730191, + "loss": 1.223, + "theoretical_loss": 3.5056566405402743, + "tokens_seen": 1704329216 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003908365095285858, + "loss": 1.3386, + "theoretical_loss": 3.5056451388011904, + "tokens_seen": 1704394752 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003908204613841525, + "loss": 1.2824, + "theoretical_loss": 3.5056336376281796, + "tokens_seen": 1704460288 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003908044132397192, + "loss": 1.3646, + "theoretical_loss": 3.5056221370211924, + "tokens_seen": 1704525824 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003907883650952859, + "loss": 1.2837, + "theoretical_loss": 3.505610636980179, + "tokens_seen": 1704591360 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003907723169508526, + "loss": 1.3103, + "theoretical_loss": 3.50559913750509, + "tokens_seen": 1704656896 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003907562688064193, + "loss": 1.3016, + "theoretical_loss": 3.505587638595876, + "tokens_seen": 1704722432 + }, + { + "epoch": 0.52, + "learning_rate": 0.000390740220661986, + "loss": 1.1795, + "theoretical_loss": 3.5055761402524865, + "tokens_seen": 1704787968 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003907241725175527, + "loss": 1.2555, + "theoretical_loss": 3.5055646424748725, + "tokens_seen": 1704853504 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003907081243731194, + "loss": 1.3349, + "theoretical_loss": 3.5055531452629847, + "tokens_seen": 1704919040 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039069207622868606, + "loss": 1.3849, + "theoretical_loss": 3.505541648616773, + "tokens_seen": 1704984576 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039067602808425276, + "loss": 1.2441, + "theoretical_loss": 3.5055301525361884, + "tokens_seen": 1705050112 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039065997993981946, + "loss": 1.4034, + "theoretical_loss": 3.5055186570211805, + "tokens_seen": 1705115648 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039064393179538616, + "loss": 1.3597, + "theoretical_loss": 3.5055071620717007, + "tokens_seen": 1705181184 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039062788365095286, + "loss": 1.3292, + "theoretical_loss": 3.5054956676876987, + "tokens_seen": 1705246720 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039061183550651957, + "loss": 1.2557, + "theoretical_loss": 3.505484173869125, + "tokens_seen": 1705312256 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039059578736208627, + "loss": 1.4078, + "theoretical_loss": 3.5054726806159304, + "tokens_seen": 1705377792 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039057973921765297, + "loss": 1.2333, + "theoretical_loss": 3.505461187928065, + "tokens_seen": 1705443328 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039056369107321967, + "loss": 1.369, + "theoretical_loss": 3.50544969580548, + "tokens_seen": 1705508864 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039054764292878637, + "loss": 1.3424, + "theoretical_loss": 3.505438204248125, + "tokens_seen": 1705574400 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039053159478435307, + "loss": 1.323, + "theoretical_loss": 3.505426713255951, + "tokens_seen": 1705639936 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039051554663991977, + "loss": 1.29, + "theoretical_loss": 3.5054152228289084, + "tokens_seen": 1705705472 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039049949849548647, + "loss": 1.2675, + "theoretical_loss": 3.5054037329669474, + "tokens_seen": 1705771008 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039048345035105317, + "loss": 1.418, + "theoretical_loss": 3.505392243670019, + "tokens_seen": 1705836544 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039046740220661987, + "loss": 1.2765, + "theoretical_loss": 3.5053807549380736, + "tokens_seen": 1705902080 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039045135406218657, + "loss": 1.3566, + "theoretical_loss": 3.5053692667710608, + "tokens_seen": 1705967616 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039043530591775327, + "loss": 1.2293, + "theoretical_loss": 3.5053577791689325, + "tokens_seen": 1706033152 + }, + { + "epoch": 0.52, + "learning_rate": 0.00039041925777331997, + "loss": 1.278, + "theoretical_loss": 3.5053462921316383, + "tokens_seen": 1706098688 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003904032096288867, + "loss": 1.2896, + "theoretical_loss": 3.505334805659129, + "tokens_seen": 1706164224 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003903871614844534, + "loss": 1.2477, + "theoretical_loss": 3.505323319751356, + "tokens_seen": 1706229760 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003903711133400201, + "loss": 1.3528, + "theoretical_loss": 3.505311834408268, + "tokens_seen": 1706295296 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003903550651955868, + "loss": 1.2542, + "theoretical_loss": 3.505300349629817, + "tokens_seen": 1706360832 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003903390170511535, + "loss": 1.2508, + "theoretical_loss": 3.5052888654159533, + "tokens_seen": 1706426368 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003903229689067202, + "loss": 1.3095, + "theoretical_loss": 3.5052773817666267, + "tokens_seen": 1706491904 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003903069207622869, + "loss": 1.2988, + "theoretical_loss": 3.505265898681789, + "tokens_seen": 1706557440 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003902908726178536, + "loss": 1.3064, + "theoretical_loss": 3.5052544161613897, + "tokens_seen": 1706622976 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003902748244734203, + "loss": 1.314, + "theoretical_loss": 3.50524293420538, + "tokens_seen": 1706688512 + }, + { + "epoch": 0.52, + "learning_rate": 0.000390258776328987, + "loss": 1.2467, + "theoretical_loss": 3.5052314528137103, + "tokens_seen": 1706754048 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003902427281845537, + "loss": 1.4337, + "theoretical_loss": 3.5052199719863317, + "tokens_seen": 1706819584 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003902266800401204, + "loss": 1.2486, + "theoretical_loss": 3.5052084917231934, + "tokens_seen": 1706885120 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003902106318956871, + "loss": 1.3467, + "theoretical_loss": 3.5051970120242473, + "tokens_seen": 1706950656 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003901945837512538, + "loss": 1.27, + "theoretical_loss": 3.5051855328894437, + "tokens_seen": 1707016192 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003901785356068205, + "loss": 1.225, + "theoretical_loss": 3.5051740543187333, + "tokens_seen": 1707081728 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003901624874623872, + "loss": 1.3057, + "theoretical_loss": 3.5051625763120664, + "tokens_seen": 1707147264 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003901464393179539, + "loss": 1.1849, + "theoretical_loss": 3.5051510988693937, + "tokens_seen": 1707212800 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003901303911735206, + "loss": 1.3288, + "theoretical_loss": 3.505139621990666, + "tokens_seen": 1707278336 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003901143430290873, + "loss": 1.3362, + "theoretical_loss": 3.505128145675834, + "tokens_seen": 1707343872 + }, + { + "epoch": 0.52, + "learning_rate": 0.000390098294884654, + "loss": 1.3473, + "theoretical_loss": 3.505116669924848, + "tokens_seen": 1707409408 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003900822467402207, + "loss": 1.2448, + "theoretical_loss": 3.5051051947376592, + "tokens_seen": 1707474944 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003900661985957874, + "loss": 1.2845, + "theoretical_loss": 3.5050937201142176, + "tokens_seen": 1707540480 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003900501504513541, + "loss": 1.3957, + "theoretical_loss": 3.5050822460544744, + "tokens_seen": 1707606016 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003900341023069208, + "loss": 1.3314, + "theoretical_loss": 3.5050707725583803, + "tokens_seen": 1707671552 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003900180541624875, + "loss": 1.2627, + "theoretical_loss": 3.5050592996258856, + "tokens_seen": 1707737088 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003900020060180542, + "loss": 1.2528, + "theoretical_loss": 3.5050478272569414, + "tokens_seen": 1707802624 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003899859578736209, + "loss": 1.2908, + "theoretical_loss": 3.505036355451498, + "tokens_seen": 1707868160 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003899699097291876, + "loss": 1.395, + "theoretical_loss": 3.505024884209506, + "tokens_seen": 1707933696 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003899538615847543, + "loss": 1.2773, + "theoretical_loss": 3.5050134135309166, + "tokens_seen": 1707999232 + }, + { + "epoch": 0.52, + "learning_rate": 0.000389937813440321, + "loss": 1.379, + "theoretical_loss": 3.50500194341568, + "tokens_seen": 1708064768 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038992176529588764, + "loss": 1.1994, + "theoretical_loss": 3.5049904738637476, + "tokens_seen": 1708130304 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038990571715145434, + "loss": 1.3546, + "theoretical_loss": 3.5049790048750697, + "tokens_seen": 1708195840 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038988966900702104, + "loss": 1.3262, + "theoretical_loss": 3.504967536449597, + "tokens_seen": 1708261376 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038987362086258774, + "loss": 1.3314, + "theoretical_loss": 3.50495606858728, + "tokens_seen": 1708326912 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038985757271815444, + "loss": 1.2272, + "theoretical_loss": 3.50494460128807, + "tokens_seen": 1708392448 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038984152457372114, + "loss": 1.2574, + "theoretical_loss": 3.5049331345519175, + "tokens_seen": 1708457984 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038982547642928784, + "loss": 1.3265, + "theoretical_loss": 3.504921668378773, + "tokens_seen": 1708523520 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038980942828485454, + "loss": 1.3029, + "theoretical_loss": 3.504910202768588, + "tokens_seen": 1708589056 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038979338014042124, + "loss": 1.2332, + "theoretical_loss": 3.5048987377213123, + "tokens_seen": 1708654592 + }, + { + "epoch": 0.52, + "learning_rate": 0.000389777331995988, + "loss": 1.2947, + "theoretical_loss": 3.5048872732368976, + "tokens_seen": 1708720128 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003897612838515547, + "loss": 1.364, + "theoretical_loss": 3.504875809315294, + "tokens_seen": 1708785664 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003897452357071214, + "loss": 1.3527, + "theoretical_loss": 3.5048643459564524, + "tokens_seen": 1708851200 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003897291875626881, + "loss": 1.2979, + "theoretical_loss": 3.5048528831603236, + "tokens_seen": 1708916736 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003897131394182548, + "loss": 1.2297, + "theoretical_loss": 3.5048414209268586, + "tokens_seen": 1708982272 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003896970912738215, + "loss": 1.3757, + "theoretical_loss": 3.504829959256009, + "tokens_seen": 1709047808 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003896810431293882, + "loss": 1.313, + "theoretical_loss": 3.5048184981477233, + "tokens_seen": 1709113344 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003896649949849549, + "loss": 1.2637, + "theoretical_loss": 3.504807037601955, + "tokens_seen": 1709178880 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003896489468405216, + "loss": 1.3126, + "theoretical_loss": 3.504795577618653, + "tokens_seen": 1709244416 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003896328986960883, + "loss": 1.3009, + "theoretical_loss": 3.504784118197769, + "tokens_seen": 1709309952 + }, + { + "epoch": 0.52, + "learning_rate": 0.000389616850551655, + "loss": 1.382, + "theoretical_loss": 3.5047726593392534, + "tokens_seen": 1709375488 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003896008024072217, + "loss": 1.2961, + "theoretical_loss": 3.504761201043057, + "tokens_seen": 1709441024 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003895847542627884, + "loss": 1.2189, + "theoretical_loss": 3.5047497433091315, + "tokens_seen": 1709506560 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003895687061183551, + "loss": 1.3113, + "theoretical_loss": 3.504738286137427, + "tokens_seen": 1709572096 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003895526579739218, + "loss": 1.3829, + "theoretical_loss": 3.5047268295278946, + "tokens_seen": 1709637632 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003895366098294885, + "loss": 1.3524, + "theoretical_loss": 3.504715373480485, + "tokens_seen": 1709703168 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003895205616850552, + "loss": 1.3396, + "theoretical_loss": 3.5047039179951494, + "tokens_seen": 1709768704 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003895045135406219, + "loss": 1.3382, + "theoretical_loss": 3.504692463071838, + "tokens_seen": 1709834240 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003894884653961886, + "loss": 1.3682, + "theoretical_loss": 3.5046810087105027, + "tokens_seen": 1709899776 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003894724172517553, + "loss": 1.3145, + "theoretical_loss": 3.5046695549110938, + "tokens_seen": 1709965312 + }, + { + "epoch": 0.52, + "learning_rate": 0.000389456369107322, + "loss": 1.2154, + "theoretical_loss": 3.504658101673562, + "tokens_seen": 1710030848 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003894403209628887, + "loss": 1.2839, + "theoretical_loss": 3.5046466489978583, + "tokens_seen": 1710096384 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003894242728184554, + "loss": 1.3833, + "theoretical_loss": 3.504635196883934, + "tokens_seen": 1710161920 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003894082246740221, + "loss": 1.3355, + "theoretical_loss": 3.50462374533174, + "tokens_seen": 1710227456 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003893921765295888, + "loss": 1.2651, + "theoretical_loss": 3.504612294341227, + "tokens_seen": 1710292992 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003893761283851555, + "loss": 1.3337, + "theoretical_loss": 3.5046008439123453, + "tokens_seen": 1710358528 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003893600802407222, + "loss": 1.2656, + "theoretical_loss": 3.5045893940450465, + "tokens_seen": 1710424064 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 634354, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2439087629318237, + "objective/train/theoretical_loss": 3.504577944739282, + "objective/train/tokens_used": 1730949600, + "theoretical_loss": 3.504577944739282, + "tokens_seen": 1710489600 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003893440320962889, + "loss": 1.2439, + "theoretical_loss": 3.504577944739282, + "tokens_seen": 1710489600 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003893279839518556, + "loss": 1.2877, + "theoretical_loss": 3.504566495995002, + "tokens_seen": 1710555136 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003893119358074223, + "loss": 1.2217, + "theoretical_loss": 3.504555047812158, + "tokens_seen": 1710620672 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038929588766298896, + "loss": 1.3009, + "theoretical_loss": 3.5045436001907007, + "tokens_seen": 1710686208 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038927983951855566, + "loss": 1.2205, + "theoretical_loss": 3.5045321531305804, + "tokens_seen": 1710751744 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038926379137412236, + "loss": 1.365, + "theoretical_loss": 3.5045207066317494, + "tokens_seen": 1710817280 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038924774322968906, + "loss": 1.2988, + "theoretical_loss": 3.5045092606941575, + "tokens_seen": 1710882816 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038923169508525577, + "loss": 1.2528, + "theoretical_loss": 3.5044978153177566, + "tokens_seen": 1710948352 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038921564694082247, + "loss": 1.3557, + "theoretical_loss": 3.504486370502497, + "tokens_seen": 1711013888 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038919959879638917, + "loss": 1.2368, + "theoretical_loss": 3.5044749262483297, + "tokens_seen": 1711079424 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038918355065195587, + "loss": 1.2973, + "theoretical_loss": 3.504463482555207, + "tokens_seen": 1711144960 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038916750250752257, + "loss": 1.3002, + "theoretical_loss": 3.504452039423078, + "tokens_seen": 1711210496 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038915145436308927, + "loss": 1.36, + "theoretical_loss": 3.5044405968518944, + "tokens_seen": 1711276032 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038913540621865597, + "loss": 1.1863, + "theoretical_loss": 3.5044291548416076, + "tokens_seen": 1711341568 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038911935807422267, + "loss": 1.2591, + "theoretical_loss": 3.5044177133921686, + "tokens_seen": 1711407104 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038910330992978937, + "loss": 1.2476, + "theoretical_loss": 3.504406272503528, + "tokens_seen": 1711472640 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038908726178535607, + "loss": 1.3135, + "theoretical_loss": 3.5043948321756373, + "tokens_seen": 1711538176 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038907121364092277, + "loss": 1.3349, + "theoretical_loss": 3.5043833924084473, + "tokens_seen": 1711603712 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038905516549648947, + "loss": 1.3346, + "theoretical_loss": 3.5043719532019093, + "tokens_seen": 1711669248 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003890391173520562, + "loss": 1.2764, + "theoretical_loss": 3.504360514555974, + "tokens_seen": 1711734784 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003890230692076229, + "loss": 1.2847, + "theoretical_loss": 3.504349076470593, + "tokens_seen": 1711800320 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003890070210631896, + "loss": 1.3541, + "theoretical_loss": 3.5043376389457164, + "tokens_seen": 1711865856 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003889909729187563, + "loss": 1.2381, + "theoretical_loss": 3.504326201981296, + "tokens_seen": 1711931392 + }, + { + "epoch": 0.52, + "learning_rate": 0.000388974924774323, + "loss": 1.3889, + "theoretical_loss": 3.504314765577283, + "tokens_seen": 1711996928 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003889588766298897, + "loss": 1.3408, + "theoretical_loss": 3.504303329733628, + "tokens_seen": 1712062464 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003889428284854564, + "loss": 1.3082, + "theoretical_loss": 3.5042918944502826, + "tokens_seen": 1712128000 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003889267803410231, + "loss": 1.1621, + "theoretical_loss": 3.5042804597271973, + "tokens_seen": 1712193536 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003889107321965898, + "loss": 1.3652, + "theoretical_loss": 3.504269025564324, + "tokens_seen": 1712259072 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003888946840521565, + "loss": 1.1953, + "theoretical_loss": 3.504257591961613, + "tokens_seen": 1712324608 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003888786359077232, + "loss": 1.3787, + "theoretical_loss": 3.504246158919016, + "tokens_seen": 1712390144 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003888625877632899, + "loss": 1.2733, + "theoretical_loss": 3.5042347264364837, + "tokens_seen": 1712455680 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003888465396188566, + "loss": 1.2712, + "theoretical_loss": 3.5042232945139675, + "tokens_seen": 1712521216 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003888304914744233, + "loss": 1.3474, + "theoretical_loss": 3.5042118631514185, + "tokens_seen": 1712586752 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038881444332999, + "loss": 1.3083, + "theoretical_loss": 3.504200432348788, + "tokens_seen": 1712652288 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003887983951855567, + "loss": 1.2432, + "theoretical_loss": 3.5041890021060267, + "tokens_seen": 1712717824 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003887823470411234, + "loss": 1.2323, + "theoretical_loss": 3.504177572423086, + "tokens_seen": 1712783360 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003887662988966901, + "loss": 1.3174, + "theoretical_loss": 3.5041661432999174, + "tokens_seen": 1712848896 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003887502507522568, + "loss": 1.251, + "theoretical_loss": 3.504154714736472, + "tokens_seen": 1712914432 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003887502507522568, + "loss": 1.3498, + "theoretical_loss": 3.5041432867327007, + "tokens_seen": 1712979968 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003887342026078235, + "loss": 1.2397, + "theoretical_loss": 3.504131859288554, + "tokens_seen": 1713045504 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003887181544633902, + "loss": 1.2883, + "theoretical_loss": 3.504120432403984, + "tokens_seen": 1713111040 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003887021063189569, + "loss": 1.2893, + "theoretical_loss": 3.5041090060789424, + "tokens_seen": 1713176576 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003886860581745236, + "loss": 1.263, + "theoretical_loss": 3.504097580313379, + "tokens_seen": 1713242112 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003886700100300903, + "loss": 1.2107, + "theoretical_loss": 3.504086155107246, + "tokens_seen": 1713307648 + }, + { + "epoch": 0.52, + "learning_rate": 0.000388653961885657, + "loss": 1.3577, + "theoretical_loss": 3.504074730460494, + "tokens_seen": 1713373184 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003886379137412237, + "loss": 1.3325, + "theoretical_loss": 3.504063306373075, + "tokens_seen": 1713438720 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003886218655967904, + "loss": 1.3077, + "theoretical_loss": 3.50405188284494, + "tokens_seen": 1713504256 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003886058174523571, + "loss": 1.2591, + "theoretical_loss": 3.5040404598760393, + "tokens_seen": 1713569792 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003885897693079238, + "loss": 1.2594, + "theoretical_loss": 3.504029037466325, + "tokens_seen": 1713635328 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003885737211634905, + "loss": 1.3217, + "theoretical_loss": 3.504017615615748, + "tokens_seen": 1713700864 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003885576730190572, + "loss": 1.3165, + "theoretical_loss": 3.50400619432426, + "tokens_seen": 1713766400 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003885416248746239, + "loss": 1.2501, + "theoretical_loss": 3.5039947735918124, + "tokens_seen": 1713831936 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003885255767301906, + "loss": 1.3631, + "theoretical_loss": 3.5039833534183553, + "tokens_seen": 1713897472 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003885095285857573, + "loss": 1.2721, + "theoretical_loss": 3.5039719338038413, + "tokens_seen": 1713963008 + }, + { + "epoch": 0.52, + "learning_rate": 0.000388493480441324, + "loss": 1.2719, + "theoretical_loss": 3.5039605147482202, + "tokens_seen": 1714028544 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003884774322968907, + "loss": 1.1956, + "theoretical_loss": 3.503949096251445, + "tokens_seen": 1714094080 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003884613841524574, + "loss": 1.2547, + "theoretical_loss": 3.5039376783134655, + "tokens_seen": 1714159616 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003884453360080241, + "loss": 1.3354, + "theoretical_loss": 3.503926260934234, + "tokens_seen": 1714225152 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003884292878635908, + "loss": 1.3003, + "theoretical_loss": 3.5039148441137016, + "tokens_seen": 1714290688 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003884132397191575, + "loss": 1.3566, + "theoretical_loss": 3.503903427851819, + "tokens_seen": 1714356224 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003883971915747242, + "loss": 1.3439, + "theoretical_loss": 3.5038920121485386, + "tokens_seen": 1714421760 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003883811434302909, + "loss": 1.2437, + "theoretical_loss": 3.5038805970038105, + "tokens_seen": 1714487296 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003883650952858576, + "loss": 1.3334, + "theoretical_loss": 3.5038691824175867, + "tokens_seen": 1714552832 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003883490471414243, + "loss": 1.3718, + "theoretical_loss": 3.5038577683898184, + "tokens_seen": 1714618368 + }, + { + "epoch": 0.52, + "learning_rate": 0.000388332998996991, + "loss": 1.3611, + "theoretical_loss": 3.503846354920457, + "tokens_seen": 1714683904 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003883169508525577, + "loss": 1.301, + "theoretical_loss": 3.503834942009454, + "tokens_seen": 1714749440 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003883009027081244, + "loss": 1.2202, + "theoretical_loss": 3.50382352965676, + "tokens_seen": 1714814976 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003882848545636911, + "loss": 1.2411, + "theoretical_loss": 3.5038121178623274, + "tokens_seen": 1714880512 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003882688064192578, + "loss": 1.3405, + "theoretical_loss": 3.503800706626107, + "tokens_seen": 1714946048 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003882527582748245, + "loss": 1.2017, + "theoretical_loss": 3.5037892959480494, + "tokens_seen": 1715011584 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003882367101303912, + "loss": 1.1901, + "theoretical_loss": 3.5037778858281077, + "tokens_seen": 1715077120 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003882206619859579, + "loss": 1.3198, + "theoretical_loss": 3.503766476266232, + "tokens_seen": 1715142656 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003882046138415246, + "loss": 1.1917, + "theoretical_loss": 3.503755067262374, + "tokens_seen": 1715208192 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003881885656970913, + "loss": 1.2997, + "theoretical_loss": 3.503743658816485, + "tokens_seen": 1715273728 + }, + { + "epoch": 0.52, + "learning_rate": 0.000388172517552658, + "loss": 1.2756, + "theoretical_loss": 3.503732250928517, + "tokens_seen": 1715339264 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003881564694082247, + "loss": 1.3584, + "theoretical_loss": 3.5037208435984204, + "tokens_seen": 1715404800 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003881404212637914, + "loss": 1.2946, + "theoretical_loss": 3.5037094368261474, + "tokens_seen": 1715470336 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003881243731193581, + "loss": 1.2466, + "theoretical_loss": 3.503698030611649, + "tokens_seen": 1715535872 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003881083249749248, + "loss": 1.3257, + "theoretical_loss": 3.503686624954877, + "tokens_seen": 1715601408 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003880922768304915, + "loss": 1.3428, + "theoretical_loss": 3.503675219855782, + "tokens_seen": 1715666944 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003880762286860582, + "loss": 1.3696, + "theoretical_loss": 3.503663815314317, + "tokens_seen": 1715732480 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003880601805416249, + "loss": 1.2889, + "theoretical_loss": 3.5036524113304317, + "tokens_seen": 1715798016 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003880441323971916, + "loss": 1.244, + "theoretical_loss": 3.5036410079040783, + "tokens_seen": 1715863552 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003880280842527583, + "loss": 1.3188, + "theoretical_loss": 3.5036296050352087, + "tokens_seen": 1715929088 + }, + { + "epoch": 0.52, + "learning_rate": 0.000388012036108325, + "loss": 1.1425, + "theoretical_loss": 3.5036182027237732, + "tokens_seen": 1715994624 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003879959879638917, + "loss": 1.2103, + "theoretical_loss": 3.5036068009697248, + "tokens_seen": 1716060160 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003879799398194584, + "loss": 1.2096, + "theoretical_loss": 3.5035953997730136, + "tokens_seen": 1716125696 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003879638916750251, + "loss": 1.269, + "theoretical_loss": 3.5035839991335918, + "tokens_seen": 1716191232 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003879478435305918, + "loss": 1.2661, + "theoretical_loss": 3.5035725990514104, + "tokens_seen": 1716256768 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003879317953861585, + "loss": 1.3405, + "theoretical_loss": 3.5035611995264215, + "tokens_seen": 1716322304 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003879157472417252, + "loss": 1.2968, + "theoretical_loss": 3.503549800558576, + "tokens_seen": 1716387840 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038789969909729186, + "loss": 1.3628, + "theoretical_loss": 3.503538402147826, + "tokens_seen": 1716453376 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038788365095285856, + "loss": 1.2826, + "theoretical_loss": 3.5035270042941224, + "tokens_seen": 1716518912 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038786760280842527, + "loss": 1.3132, + "theoretical_loss": 3.503515606997417, + "tokens_seen": 1716584448 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038785155466399197, + "loss": 1.3277, + "theoretical_loss": 3.5035042102576615, + "tokens_seen": 1716649984 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038783550651955867, + "loss": 1.3805, + "theoretical_loss": 3.5034928140748067, + "tokens_seen": 1716715520 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038781945837512537, + "loss": 1.213, + "theoretical_loss": 3.5034814184488052, + "tokens_seen": 1716781056 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038780341023069207, + "loss": 1.2836, + "theoretical_loss": 3.5034700233796077, + "tokens_seen": 1716846592 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038778736208625877, + "loss": 1.3277, + "theoretical_loss": 3.5034586288671656, + "tokens_seen": 1716912128 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038777131394182547, + "loss": 1.3441, + "theoretical_loss": 3.5034472349114316, + "tokens_seen": 1716977664 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 636707, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2411022186279297, + "objective/train/theoretical_loss": 3.5034358415123563, + "objective/train/tokens_used": 1737503200, + "theoretical_loss": 3.5034358415123563, + "tokens_seen": 1717043200 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038775526579739217, + "loss": 1.2411, + "theoretical_loss": 3.5034358415123563, + "tokens_seen": 1717043200 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038773921765295887, + "loss": 1.1925, + "theoretical_loss": 3.5034244486698913, + "tokens_seen": 1717108736 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038772316950852557, + "loss": 1.2819, + "theoretical_loss": 3.5034130563839887, + "tokens_seen": 1717174272 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038770712136409227, + "loss": 1.1331, + "theoretical_loss": 3.5034016646545996, + "tokens_seen": 1717239808 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038769107321965897, + "loss": 1.2076, + "theoretical_loss": 3.503390273481676, + "tokens_seen": 1717305344 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038767502507522567, + "loss": 1.2763, + "theoretical_loss": 3.503378882865168, + "tokens_seen": 1717370880 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003876589769307924, + "loss": 1.2788, + "theoretical_loss": 3.5033674928050296, + "tokens_seen": 1717436416 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003876429287863591, + "loss": 1.3074, + "theoretical_loss": 3.5033561033012104, + "tokens_seen": 1717501952 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003876268806419258, + "loss": 1.2624, + "theoretical_loss": 3.5033447143536636, + "tokens_seen": 1717567488 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003876108324974925, + "loss": 1.3542, + "theoretical_loss": 3.5033333259623394, + "tokens_seen": 1717633024 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003875947843530592, + "loss": 1.2799, + "theoretical_loss": 3.50332193812719, + "tokens_seen": 1717698560 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003875787362086259, + "loss": 1.2908, + "theoretical_loss": 3.503310550848167, + "tokens_seen": 1717764096 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003875626880641926, + "loss": 1.2411, + "theoretical_loss": 3.503299164125222, + "tokens_seen": 1717829632 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003875466399197593, + "loss": 1.289, + "theoretical_loss": 3.503287777958307, + "tokens_seen": 1717895168 + }, + { + "epoch": 0.52, + "learning_rate": 0.000387530591775326, + "loss": 1.2662, + "theoretical_loss": 3.503276392347373, + "tokens_seen": 1717960704 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003875145436308927, + "loss": 1.3435, + "theoretical_loss": 3.503265007292372, + "tokens_seen": 1718026240 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003874984954864594, + "loss": 1.2898, + "theoretical_loss": 3.5032536227932556, + "tokens_seen": 1718091776 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003874824473420261, + "loss": 1.3924, + "theoretical_loss": 3.5032422388499755, + "tokens_seen": 1718157312 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003874663991975928, + "loss": 1.2274, + "theoretical_loss": 3.503230855462483, + "tokens_seen": 1718222848 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003874503510531595, + "loss": 1.2879, + "theoretical_loss": 3.5032194726307306, + "tokens_seen": 1718288384 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003874343029087262, + "loss": 1.2557, + "theoretical_loss": 3.503208090354669, + "tokens_seen": 1718353920 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003874182547642929, + "loss": 1.4238, + "theoretical_loss": 3.5031967086342504, + "tokens_seen": 1718419456 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003874022066198596, + "loss": 1.2794, + "theoretical_loss": 3.5031853274694265, + "tokens_seen": 1718484992 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038738615847542634, + "loss": 1.3083, + "theoretical_loss": 3.503173946860149, + "tokens_seen": 1718550528 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038737011033099304, + "loss": 1.3592, + "theoretical_loss": 3.5031625668063695, + "tokens_seen": 1718616064 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038735406218655974, + "loss": 1.3521, + "theoretical_loss": 3.5031511873080396, + "tokens_seen": 1718681600 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038733801404212644, + "loss": 1.2359, + "theoretical_loss": 3.5031398083651113, + "tokens_seen": 1718747136 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038732196589769314, + "loss": 1.2537, + "theoretical_loss": 3.503128429977536, + "tokens_seen": 1718812672 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038730591775325984, + "loss": 1.441, + "theoretical_loss": 3.5031170521452655, + "tokens_seen": 1718878208 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038728986960882654, + "loss": 1.3264, + "theoretical_loss": 3.5031056748682516, + "tokens_seen": 1718943744 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038727382146439324, + "loss": 1.3576, + "theoretical_loss": 3.503094298146446, + "tokens_seen": 1719009280 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003872577733199599, + "loss": 1.2877, + "theoretical_loss": 3.503082921979801, + "tokens_seen": 1719074816 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003872417251755266, + "loss": 1.3201, + "theoretical_loss": 3.5030715463682673, + "tokens_seen": 1719140352 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003872256770310933, + "loss": 1.2012, + "theoretical_loss": 3.5030601713117973, + "tokens_seen": 1719205888 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038720962888666, + "loss": 1.4248, + "theoretical_loss": 3.5030487968103428, + "tokens_seen": 1719271424 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003871935807422267, + "loss": 1.393, + "theoretical_loss": 3.503037422863855, + "tokens_seen": 1719336960 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003871775325977934, + "loss": 1.2351, + "theoretical_loss": 3.5030260494722865, + "tokens_seen": 1719402496 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003871614844533601, + "loss": 1.3529, + "theoretical_loss": 3.5030146766355887, + "tokens_seen": 1719468032 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003871454363089268, + "loss": 1.2284, + "theoretical_loss": 3.5030033043537125, + "tokens_seen": 1719533568 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003871293881644935, + "loss": 1.3066, + "theoretical_loss": 3.5029919326266112, + "tokens_seen": 1719599104 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003871133400200602, + "loss": 1.3394, + "theoretical_loss": 3.502980561454236, + "tokens_seen": 1719664640 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003870972918756269, + "loss": 1.3122, + "theoretical_loss": 3.502969190836538, + "tokens_seen": 1719730176 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003870812437311936, + "loss": 1.3278, + "theoretical_loss": 3.50295782077347, + "tokens_seen": 1719795712 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003870651955867603, + "loss": 1.3309, + "theoretical_loss": 3.5029464512649833, + "tokens_seen": 1719861248 + }, + { + "epoch": 0.52, + "learning_rate": 0.000387049147442327, + "loss": 1.2685, + "theoretical_loss": 3.50293508231103, + "tokens_seen": 1719926784 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003870330992978937, + "loss": 1.2634, + "theoretical_loss": 3.502923713911562, + "tokens_seen": 1719992320 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003870170511534604, + "loss": 1.2274, + "theoretical_loss": 3.5029123460665303, + "tokens_seen": 1720057856 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003870010030090271, + "loss": 1.3161, + "theoretical_loss": 3.502900978775888, + "tokens_seen": 1720123392 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003869849548645938, + "loss": 1.3459, + "theoretical_loss": 3.502889612039586, + "tokens_seen": 1720188928 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003869689067201605, + "loss": 1.227, + "theoretical_loss": 3.502878245857576, + "tokens_seen": 1720254464 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003869528585757272, + "loss": 1.2337, + "theoretical_loss": 3.502866880229811, + "tokens_seen": 1720320000 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003869368104312939, + "loss": 1.2817, + "theoretical_loss": 3.5028555151562415, + "tokens_seen": 1720385536 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003869207622868606, + "loss": 1.2375, + "theoretical_loss": 3.5028441506368204, + "tokens_seen": 1720451072 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003869047141424273, + "loss": 1.332, + "theoretical_loss": 3.502832786671499, + "tokens_seen": 1720516608 + }, + { + "epoch": 0.52, + "learning_rate": 0.000386888665997994, + "loss": 1.4641, + "theoretical_loss": 3.5028214232602295, + "tokens_seen": 1720582144 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003868726178535607, + "loss": 1.3129, + "theoretical_loss": 3.5028100604029637, + "tokens_seen": 1720647680 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003868565697091274, + "loss": 1.2585, + "theoretical_loss": 3.5027986980996535, + "tokens_seen": 1720713216 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003868405215646941, + "loss": 1.259, + "theoretical_loss": 3.5027873363502504, + "tokens_seen": 1720778752 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003868244734202608, + "loss": 1.2879, + "theoretical_loss": 3.502775975154707, + "tokens_seen": 1720844288 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003868084252758275, + "loss": 1.2096, + "theoretical_loss": 3.5027646145129747, + "tokens_seen": 1720909824 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003867923771313942, + "loss": 1.3729, + "theoretical_loss": 3.5027532544250057, + "tokens_seen": 1720975360 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003867763289869609, + "loss": 1.3693, + "theoretical_loss": 3.502741894890751, + "tokens_seen": 1721040896 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003867602808425276, + "loss": 1.0755, + "theoretical_loss": 3.5027305359101644, + "tokens_seen": 1721106432 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003867442326980943, + "loss": 1.2464, + "theoretical_loss": 3.502719177483196, + "tokens_seen": 1721171968 + }, + { + "epoch": 0.52, + "learning_rate": 0.000386728184553661, + "loss": 1.2695, + "theoretical_loss": 3.502707819609799, + "tokens_seen": 1721237504 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003867121364092277, + "loss": 1.3068, + "theoretical_loss": 3.502696462289925, + "tokens_seen": 1721303040 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003866960882647944, + "loss": 1.2391, + "theoretical_loss": 3.502685105523525, + "tokens_seen": 1721368576 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003866800401203611, + "loss": 1.3789, + "theoretical_loss": 3.5026737493105524, + "tokens_seen": 1721434112 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003866639919759278, + "loss": 1.3172, + "theoretical_loss": 3.502662393650958, + "tokens_seen": 1721499648 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003866479438314945, + "loss": 1.2676, + "theoretical_loss": 3.5026510385446947, + "tokens_seen": 1721565184 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003866318956870612, + "loss": 1.2595, + "theoretical_loss": 3.502639683991714, + "tokens_seen": 1721630720 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003866158475426279, + "loss": 1.2579, + "theoretical_loss": 3.5026283299919676, + "tokens_seen": 1721696256 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003865997993981946, + "loss": 1.3236, + "theoretical_loss": 3.5026169765454083, + "tokens_seen": 1721761792 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003865837512537613, + "loss": 1.3482, + "theoretical_loss": 3.5026056236519874, + "tokens_seen": 1721827328 + }, + { + "epoch": 0.52, + "learning_rate": 0.000386567703109328, + "loss": 1.2221, + "theoretical_loss": 3.502594271311657, + "tokens_seen": 1721892864 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003865516549648947, + "loss": 1.2524, + "theoretical_loss": 3.502582919524369, + "tokens_seen": 1721958400 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003865356068204614, + "loss": 1.236, + "theoretical_loss": 3.502571568290076, + "tokens_seen": 1722023936 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003865195586760281, + "loss": 1.2959, + "theoretical_loss": 3.50256021760873, + "tokens_seen": 1722089472 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003865035105315948, + "loss": 1.33, + "theoretical_loss": 3.502548867480282, + "tokens_seen": 1722155008 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038648746238716147, + "loss": 1.2745, + "theoretical_loss": 3.5025375179046847, + "tokens_seen": 1722220544 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038647141424272817, + "loss": 1.1986, + "theoretical_loss": 3.5025261688818903, + "tokens_seen": 1722286080 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038645536609829487, + "loss": 1.1634, + "theoretical_loss": 3.502514820411851, + "tokens_seen": 1722351616 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038643931795386157, + "loss": 1.2364, + "theoretical_loss": 3.502503472494518, + "tokens_seen": 1722417152 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038642326980942827, + "loss": 1.4813, + "theoretical_loss": 3.5024921251298444, + "tokens_seen": 1722482688 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038640722166499497, + "loss": 1.349, + "theoretical_loss": 3.502480778317781, + "tokens_seen": 1722548224 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038639117352056167, + "loss": 1.1708, + "theoretical_loss": 3.502469432058281, + "tokens_seen": 1722613760 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038637512537612837, + "loss": 1.2701, + "theoretical_loss": 3.5024580863512966, + "tokens_seen": 1722679296 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038635907723169507, + "loss": 1.3147, + "theoretical_loss": 3.5024467411967786, + "tokens_seen": 1722744832 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038634302908726177, + "loss": 1.3164, + "theoretical_loss": 3.50243539659468, + "tokens_seen": 1722810368 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038632698094282847, + "loss": 1.2554, + "theoretical_loss": 3.502424052544953, + "tokens_seen": 1722875904 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038631093279839517, + "loss": 1.3134, + "theoretical_loss": 3.502412709047549, + "tokens_seen": 1722941440 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003862948846539619, + "loss": 1.277, + "theoretical_loss": 3.502401366102421, + "tokens_seen": 1723006976 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003862788365095286, + "loss": 1.2146, + "theoretical_loss": 3.5023900237095202, + "tokens_seen": 1723072512 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003862627883650953, + "loss": 1.3513, + "theoretical_loss": 3.502378681868799, + "tokens_seen": 1723138048 + }, + { + "epoch": 0.52, + "learning_rate": 0.000386246740220662, + "loss": 1.3874, + "theoretical_loss": 3.50236734058021, + "tokens_seen": 1723203584 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003862306920762287, + "loss": 1.4162, + "theoretical_loss": 3.5023559998437053, + "tokens_seen": 1723269120 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003862146439317954, + "loss": 1.367, + "theoretical_loss": 3.5023446596592365, + "tokens_seen": 1723334656 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003861985957873621, + "loss": 1.342, + "theoretical_loss": 3.5023333200267555, + "tokens_seen": 1723400192 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038618254764292883, + "loss": 1.2088, + "theoretical_loss": 3.502321980946215, + "tokens_seen": 1723465728 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038616649949849553, + "loss": 1.2581, + "theoretical_loss": 3.502310642417567, + "tokens_seen": 1723531264 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 639520, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3269760608673096, + "objective/train/theoretical_loss": 3.502299304440764, + "objective/train/tokens_used": 1744056800, + "theoretical_loss": 3.502299304440764, + "tokens_seen": 1723596800 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038615045135406223, + "loss": 1.327, + "theoretical_loss": 3.502299304440764, + "tokens_seen": 1723596800 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038613440320962893, + "loss": 1.3402, + "theoretical_loss": 3.5022879670157576, + "tokens_seen": 1723662336 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038611835506519563, + "loss": 1.3168, + "theoretical_loss": 3.5022766301425, + "tokens_seen": 1723727872 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038610230692076234, + "loss": 1.2212, + "theoretical_loss": 3.502265293820944, + "tokens_seen": 1723793408 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038608625877632904, + "loss": 1.3364, + "theoretical_loss": 3.5022539580510417, + "tokens_seen": 1723858944 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038607021063189574, + "loss": 1.2548, + "theoretical_loss": 3.5022426228327443, + "tokens_seen": 1723924480 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038605416248746244, + "loss": 1.2925, + "theoretical_loss": 3.502231288166005, + "tokens_seen": 1723990016 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038603811434302914, + "loss": 1.3317, + "theoretical_loss": 3.5022199540507755, + "tokens_seen": 1724055552 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038602206619859584, + "loss": 1.2897, + "theoretical_loss": 3.502208620487008, + "tokens_seen": 1724121088 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038600601805416254, + "loss": 1.2597, + "theoretical_loss": 3.5021972874746554, + "tokens_seen": 1724186624 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038598996990972924, + "loss": 1.2896, + "theoretical_loss": 3.502185955013669, + "tokens_seen": 1724252160 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038597392176529594, + "loss": 1.2303, + "theoretical_loss": 3.5021746231040014, + "tokens_seen": 1724317696 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038595787362086264, + "loss": 1.3019, + "theoretical_loss": 3.5021632917456045, + "tokens_seen": 1724383232 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038594182547642934, + "loss": 1.2619, + "theoretical_loss": 3.502151960938431, + "tokens_seen": 1724448768 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038592577733199604, + "loss": 1.479, + "theoretical_loss": 3.502140630682433, + "tokens_seen": 1724514304 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038590972918756274, + "loss": 1.3672, + "theoretical_loss": 3.502129300977563, + "tokens_seen": 1724579840 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038589368104312944, + "loss": 1.3679, + "theoretical_loss": 3.5021179718237727, + "tokens_seen": 1724645376 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038587763289869614, + "loss": 1.3035, + "theoretical_loss": 3.5021066432210146, + "tokens_seen": 1724710912 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003858615847542628, + "loss": 1.2931, + "theoretical_loss": 3.5020953151692416, + "tokens_seen": 1724776448 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003858455366098295, + "loss": 1.3719, + "theoretical_loss": 3.5020839876684047, + "tokens_seen": 1724841984 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003858294884653962, + "loss": 1.2953, + "theoretical_loss": 3.502072660718457, + "tokens_seen": 1724907520 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003858134403209629, + "loss": 1.285, + "theoretical_loss": 3.5020613343193507, + "tokens_seen": 1724973056 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003857973921765296, + "loss": 1.3097, + "theoretical_loss": 3.5020500084710378, + "tokens_seen": 1725038592 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003857813440320963, + "loss": 1.258, + "theoretical_loss": 3.502038683173471, + "tokens_seen": 1725104128 + }, + { + "epoch": 0.52, + "learning_rate": 0.000385765295887663, + "loss": 1.375, + "theoretical_loss": 3.502027358426602, + "tokens_seen": 1725169664 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003857492477432297, + "loss": 1.3491, + "theoretical_loss": 3.502016034230384, + "tokens_seen": 1725235200 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003857331995987964, + "loss": 1.2895, + "theoretical_loss": 3.5020047105847687, + "tokens_seen": 1725300736 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003857171514543631, + "loss": 1.3203, + "theoretical_loss": 3.5019933874897085, + "tokens_seen": 1725366272 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003857011033099298, + "loss": 1.2685, + "theoretical_loss": 3.5019820649451554, + "tokens_seen": 1725431808 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003856850551654965, + "loss": 1.3784, + "theoretical_loss": 3.5019707429510625, + "tokens_seen": 1725497344 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003856690070210632, + "loss": 1.3079, + "theoretical_loss": 3.5019594215073813, + "tokens_seen": 1725562880 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003856529588766299, + "loss": 1.3996, + "theoretical_loss": 3.501948100614065, + "tokens_seen": 1725628416 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003856369107321966, + "loss": 1.3753, + "theoretical_loss": 3.5019367802710653, + "tokens_seen": 1725693952 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003856208625877633, + "loss": 1.25, + "theoretical_loss": 3.5019254604783345, + "tokens_seen": 1725759488 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038560481444333, + "loss": 1.2883, + "theoretical_loss": 3.5019141412358255, + "tokens_seen": 1725825024 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003855887662988967, + "loss": 1.3279, + "theoretical_loss": 3.5019028225434905, + "tokens_seen": 1725890560 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003855727181544634, + "loss": 1.3194, + "theoretical_loss": 3.5018915044012813, + "tokens_seen": 1725956096 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003855566700100301, + "loss": 1.331, + "theoretical_loss": 3.5018801868091507, + "tokens_seen": 1726021632 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003855406218655968, + "loss": 1.2806, + "theoretical_loss": 3.5018688697670513, + "tokens_seen": 1726087168 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003855245737211635, + "loss": 1.2671, + "theoretical_loss": 3.501857553274935, + "tokens_seen": 1726152704 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003855085255767302, + "loss": 1.2025, + "theoretical_loss": 3.501846237332755, + "tokens_seen": 1726218240 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003854924774322969, + "loss": 1.2673, + "theoretical_loss": 3.5018349219404628, + "tokens_seen": 1726283776 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003854764292878636, + "loss": 1.3299, + "theoretical_loss": 3.501823607098011, + "tokens_seen": 1726349312 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003854603811434303, + "loss": 1.3089, + "theoretical_loss": 3.5018122928053526, + "tokens_seen": 1726414848 + }, + { + "epoch": 0.52, + "learning_rate": 0.000385444332998997, + "loss": 1.2981, + "theoretical_loss": 3.5018009790624394, + "tokens_seen": 1726480384 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003854282848545637, + "loss": 1.2325, + "theoretical_loss": 3.5017896658692234, + "tokens_seen": 1726545920 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003854122367101304, + "loss": 1.3877, + "theoretical_loss": 3.5017783532256583, + "tokens_seen": 1726611456 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003853961885656971, + "loss": 1.2568, + "theoretical_loss": 3.501767041131696, + "tokens_seen": 1726676992 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003853801404212638, + "loss": 1.3348, + "theoretical_loss": 3.5017557295872885, + "tokens_seen": 1726742528 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003853640922768305, + "loss": 1.251, + "theoretical_loss": 3.5017444185923887, + "tokens_seen": 1726808064 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003853480441323972, + "loss": 1.2793, + "theoretical_loss": 3.501733108146949, + "tokens_seen": 1726873600 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003853319959879639, + "loss": 1.3228, + "theoretical_loss": 3.5017217982509212, + "tokens_seen": 1726939136 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003853159478435306, + "loss": 1.3349, + "theoretical_loss": 3.5017104889042585, + "tokens_seen": 1727004672 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003852998996990973, + "loss": 1.3805, + "theoretical_loss": 3.5016991801069137, + "tokens_seen": 1727070208 + }, + { + "epoch": 0.52, + "learning_rate": 0.000385283851554664, + "loss": 1.3648, + "theoretical_loss": 3.501687871858839, + "tokens_seen": 1727135744 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003852678034102307, + "loss": 1.2634, + "theoretical_loss": 3.501676564159986, + "tokens_seen": 1727201280 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003852517552657974, + "loss": 1.3265, + "theoretical_loss": 3.501665257010308, + "tokens_seen": 1727266816 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003852357071213641, + "loss": 1.2969, + "theoretical_loss": 3.5016539504097572, + "tokens_seen": 1727332352 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003852196589769308, + "loss": 1.2242, + "theoretical_loss": 3.501642644358286, + "tokens_seen": 1727397888 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003852036108324975, + "loss": 1.3958, + "theoretical_loss": 3.501631338855848, + "tokens_seen": 1727463424 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003851875626880642, + "loss": 1.3685, + "theoretical_loss": 3.5016200339023946, + "tokens_seen": 1727528960 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003851715145436309, + "loss": 1.2955, + "theoretical_loss": 3.501608729497878, + "tokens_seen": 1727594496 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003851554663991976, + "loss": 1.2654, + "theoretical_loss": 3.5015974256422515, + "tokens_seen": 1727660032 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003851394182547643, + "loss": 1.177, + "theoretical_loss": 3.501586122335468, + "tokens_seen": 1727725568 + }, + { + "epoch": 0.52, + "learning_rate": 0.000385123370110331, + "loss": 1.1918, + "theoretical_loss": 3.501574819577479, + "tokens_seen": 1727791104 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003851073219658977, + "loss": 1.239, + "theoretical_loss": 3.5015635173682376, + "tokens_seen": 1727856640 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038509127382146437, + "loss": 1.3048, + "theoretical_loss": 3.5015522157076964, + "tokens_seen": 1727922176 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038507522567703107, + "loss": 1.3699, + "theoretical_loss": 3.5015409145958074, + "tokens_seen": 1727987712 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038505917753259777, + "loss": 1.2257, + "theoretical_loss": 3.5015296140325236, + "tokens_seen": 1728053248 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038504312938816447, + "loss": 1.3428, + "theoretical_loss": 3.5015183140177975, + "tokens_seen": 1728118784 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038502708124373117, + "loss": 1.341, + "theoretical_loss": 3.5015070145515823, + "tokens_seen": 1728184320 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038501103309929787, + "loss": 1.3356, + "theoretical_loss": 3.5014957156338293, + "tokens_seen": 1728249856 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038499498495486457, + "loss": 1.2933, + "theoretical_loss": 3.501484417264492, + "tokens_seen": 1728315392 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003849789368104313, + "loss": 1.1973, + "theoretical_loss": 3.5014731194435225, + "tokens_seen": 1728380928 + }, + { + "epoch": 0.52, + "learning_rate": 0.000384962888665998, + "loss": 1.2363, + "theoretical_loss": 3.501461822170874, + "tokens_seen": 1728446464 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003849468405215647, + "loss": 1.1809, + "theoretical_loss": 3.5014505254464985, + "tokens_seen": 1728512000 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038493079237713143, + "loss": 1.2102, + "theoretical_loss": 3.501439229270349, + "tokens_seen": 1728577536 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038491474423269813, + "loss": 1.2877, + "theoretical_loss": 3.5014279336423777, + "tokens_seen": 1728643072 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038489869608826483, + "loss": 1.2856, + "theoretical_loss": 3.501416638562538, + "tokens_seen": 1728708608 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038488264794383153, + "loss": 1.2173, + "theoretical_loss": 3.501405344030781, + "tokens_seen": 1728774144 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038486659979939823, + "loss": 1.1944, + "theoretical_loss": 3.501394050047061, + "tokens_seen": 1728839680 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038485055165496493, + "loss": 1.3469, + "theoretical_loss": 3.5013827566113296, + "tokens_seen": 1728905216 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038483450351053163, + "loss": 1.2606, + "theoretical_loss": 3.5013714637235402, + "tokens_seen": 1728970752 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038481845536609833, + "loss": 1.3211, + "theoretical_loss": 3.501360171383645, + "tokens_seen": 1729036288 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038480240722166503, + "loss": 1.3971, + "theoretical_loss": 3.5013488795915966, + "tokens_seen": 1729101824 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038478635907723173, + "loss": 1.2739, + "theoretical_loss": 3.5013375883473477, + "tokens_seen": 1729167360 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038477031093279843, + "loss": 1.3335, + "theoretical_loss": 3.501326297650851, + "tokens_seen": 1729232896 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038475426278836513, + "loss": 1.2819, + "theoretical_loss": 3.5013150075020594, + "tokens_seen": 1729298432 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038473821464393183, + "loss": 1.3261, + "theoretical_loss": 3.5013037179009245, + "tokens_seen": 1729363968 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038472216649949854, + "loss": 1.312, + "theoretical_loss": 3.5012924288474006, + "tokens_seen": 1729429504 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038470611835506524, + "loss": 1.3279, + "theoretical_loss": 3.5012811403414394, + "tokens_seen": 1729495040 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038469007021063194, + "loss": 1.2467, + "theoretical_loss": 3.501269852382994, + "tokens_seen": 1729560576 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038467402206619864, + "loss": 1.3062, + "theoretical_loss": 3.501258564972017, + "tokens_seen": 1729626112 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038465797392176534, + "loss": 1.1661, + "theoretical_loss": 3.501247278108461, + "tokens_seen": 1729691648 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038464192577733204, + "loss": 1.2514, + "theoretical_loss": 3.501235991792279, + "tokens_seen": 1729757184 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038462587763289874, + "loss": 1.2023, + "theoretical_loss": 3.5012247060234225, + "tokens_seen": 1729822720 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038460982948846544, + "loss": 1.2476, + "theoretical_loss": 3.5012134208018457, + "tokens_seen": 1729888256 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038459378134403214, + "loss": 1.3194, + "theoretical_loss": 3.501202136127501, + "tokens_seen": 1729953792 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038457773319959884, + "loss": 1.3539, + "theoretical_loss": 3.5011908520003407, + "tokens_seen": 1730019328 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038456168505516554, + "loss": 1.3525, + "theoretical_loss": 3.501179568420318, + "tokens_seen": 1730084864 + }, + { + "epoch": 0.52, + "objective/train/docs_used": 641921, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1992993354797363, + "objective/train/theoretical_loss": 3.5011682853873856, + "objective/train/tokens_used": 1750610400, + "theoretical_loss": 3.5011682853873856, + "tokens_seen": 1730150400 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038454563691073224, + "loss": 1.1993, + "theoretical_loss": 3.5011682853873856, + "tokens_seen": 1730150400 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038452958876629894, + "loss": 1.3206, + "theoretical_loss": 3.501157002901495, + "tokens_seen": 1730215936 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038451354062186564, + "loss": 1.2603, + "theoretical_loss": 3.501145720962601, + "tokens_seen": 1730281472 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038449749247743234, + "loss": 1.3049, + "theoretical_loss": 3.5011344395706554, + "tokens_seen": 1730347008 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038448144433299904, + "loss": 1.2556, + "theoretical_loss": 3.501123158725611, + "tokens_seen": 1730412544 + }, + { + "epoch": 0.52, + "learning_rate": 0.00038446539618856575, + "loss": 1.3048, + "theoretical_loss": 3.5011118784274204, + "tokens_seen": 1730478080 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003844493480441324, + "loss": 1.3413, + "theoretical_loss": 3.501100598676036, + "tokens_seen": 1730543616 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003844332998996991, + "loss": 1.2065, + "theoretical_loss": 3.501089319471412, + "tokens_seen": 1730609152 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003844172517552658, + "loss": 1.2899, + "theoretical_loss": 3.5010780408135, + "tokens_seen": 1730674688 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003844012036108325, + "loss": 1.3113, + "theoretical_loss": 3.501066762702253, + "tokens_seen": 1730740224 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003843851554663992, + "loss": 1.4202, + "theoretical_loss": 3.501055485137624, + "tokens_seen": 1730805760 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003843691073219659, + "loss": 1.3048, + "theoretical_loss": 3.5010442081195654, + "tokens_seen": 1730871296 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003843530591775326, + "loss": 1.212, + "theoretical_loss": 3.5010329316480306, + "tokens_seen": 1730936832 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003843370110330993, + "loss": 1.2037, + "theoretical_loss": 3.501021655722972, + "tokens_seen": 1731002368 + }, + { + "epoch": 0.52, + "learning_rate": 0.000384320962888666, + "loss": 1.3061, + "theoretical_loss": 3.501010380344343, + "tokens_seen": 1731067904 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003843049147442327, + "loss": 1.324, + "theoretical_loss": 3.5009991055120957, + "tokens_seen": 1731133440 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003842888665997994, + "loss": 1.2311, + "theoretical_loss": 3.5009878312261833, + "tokens_seen": 1731198976 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003842728184553661, + "loss": 1.2152, + "theoretical_loss": 3.500976557486559, + "tokens_seen": 1731264512 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003842567703109328, + "loss": 1.3278, + "theoretical_loss": 3.5009652842931747, + "tokens_seen": 1731330048 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003842407221664995, + "loss": 1.3607, + "theoretical_loss": 3.5009540116459843, + "tokens_seen": 1731395584 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003842246740220662, + "loss": 1.3039, + "theoretical_loss": 3.5009427395449397, + "tokens_seen": 1731461120 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003842086258776329, + "loss": 1.2529, + "theoretical_loss": 3.5009314679899948, + "tokens_seen": 1731526656 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003841925777331996, + "loss": 1.3372, + "theoretical_loss": 3.5009201969811015, + "tokens_seen": 1731592192 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003841765295887663, + "loss": 1.347, + "theoretical_loss": 3.500908926518213, + "tokens_seen": 1731657728 + }, + { + "epoch": 0.52, + "learning_rate": 0.000384160481444333, + "loss": 1.3733, + "theoretical_loss": 3.500897656601283, + "tokens_seen": 1731723264 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003841444332998997, + "loss": 1.1952, + "theoretical_loss": 3.5008863872302634, + "tokens_seen": 1731788800 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003841283851554664, + "loss": 1.2982, + "theoretical_loss": 3.5008751184051072, + "tokens_seen": 1731854336 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003841123370110331, + "loss": 1.2475, + "theoretical_loss": 3.500863850125768, + "tokens_seen": 1731919872 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003840962888665998, + "loss": 1.3088, + "theoretical_loss": 3.5008525823921977, + "tokens_seen": 1731985408 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003840802407221665, + "loss": 1.293, + "theoretical_loss": 3.5008413152043496, + "tokens_seen": 1732050944 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003840641925777332, + "loss": 1.3779, + "theoretical_loss": 3.500830048562177, + "tokens_seen": 1732116480 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003840481444332999, + "loss": 1.2174, + "theoretical_loss": 3.5008187824656325, + "tokens_seen": 1732182016 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003840320962888666, + "loss": 1.3411, + "theoretical_loss": 3.5008075169146693, + "tokens_seen": 1732247552 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003840160481444333, + "loss": 1.3322, + "theoretical_loss": 3.50079625190924, + "tokens_seen": 1732313088 + }, + { + "epoch": 0.52, + "learning_rate": 0.000384, + "loss": 1.3654, + "theoretical_loss": 3.500784987449298, + "tokens_seen": 1732378624 + }, + { + "epoch": 0.52, + "learning_rate": 0.0003839839518555667, + "loss": 1.364, + "theoretical_loss": 3.5007737235347953, + "tokens_seen": 1732444160 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003839679037111334, + "loss": 1.3286, + "theoretical_loss": 3.500762460165686, + "tokens_seen": 1732509696 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003839518555667001, + "loss": 1.2703, + "theoretical_loss": 3.5007511973419225, + "tokens_seen": 1732575232 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003839358074222668, + "loss": 1.2505, + "theoretical_loss": 3.5007399350634576, + "tokens_seen": 1732640768 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003839197592778335, + "loss": 1.2592, + "theoretical_loss": 3.5007286733302445, + "tokens_seen": 1732706304 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003839037111334002, + "loss": 1.2797, + "theoretical_loss": 3.5007174121422366, + "tokens_seen": 1732771840 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003838876629889669, + "loss": 1.218, + "theoretical_loss": 3.5007061514993865, + "tokens_seen": 1732837376 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003838716148445336, + "loss": 1.1817, + "theoretical_loss": 3.500694891401647, + "tokens_seen": 1732902912 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003838555667001003, + "loss": 1.306, + "theoretical_loss": 3.500683631848971, + "tokens_seen": 1732968448 + }, + { + "epoch": 0.53, + "learning_rate": 0.000383839518555667, + "loss": 1.3168, + "theoretical_loss": 3.5006723728413123, + "tokens_seen": 1733033984 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003838234704112337, + "loss": 1.218, + "theoretical_loss": 3.500661114378623, + "tokens_seen": 1733099520 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003838074222668004, + "loss": 1.2776, + "theoretical_loss": 3.500649856460856, + "tokens_seen": 1733165056 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003837913741223671, + "loss": 1.2933, + "theoretical_loss": 3.5006385990879663, + "tokens_seen": 1733230592 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003837753259779338, + "loss": 1.3418, + "theoretical_loss": 3.500627342259904, + "tokens_seen": 1733296128 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003837592778335005, + "loss": 1.2755, + "theoretical_loss": 3.5006160859766244, + "tokens_seen": 1733361664 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003837432296890672, + "loss": 1.3763, + "theoretical_loss": 3.5006048302380792, + "tokens_seen": 1733427200 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003837271815446339, + "loss": 1.2265, + "theoretical_loss": 3.5005935750442223, + "tokens_seen": 1733492736 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003837111334002006, + "loss": 1.4039, + "theoretical_loss": 3.5005823203950066, + "tokens_seen": 1733558272 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003836950852557673, + "loss": 1.3288, + "theoretical_loss": 3.5005710662903846, + "tokens_seen": 1733623808 + }, + { + "epoch": 0.53, + "learning_rate": 0.000383679037111334, + "loss": 1.177, + "theoretical_loss": 3.50055981273031, + "tokens_seen": 1733689344 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003836629889669007, + "loss": 1.2796, + "theoretical_loss": 3.500548559714735, + "tokens_seen": 1733754880 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003836469408224674, + "loss": 1.3464, + "theoretical_loss": 3.500537307243614, + "tokens_seen": 1733820416 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003836308926780341, + "loss": 1.3586, + "theoretical_loss": 3.5005260553168993, + "tokens_seen": 1733885952 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003836148445336008, + "loss": 1.339, + "theoretical_loss": 3.5005148039345437, + "tokens_seen": 1733951488 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003835987963891675, + "loss": 1.3931, + "theoretical_loss": 3.500503553096501, + "tokens_seen": 1734017024 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003835827482447342, + "loss": 1.2968, + "theoretical_loss": 3.5004923028027237, + "tokens_seen": 1734082560 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003835667001003009, + "loss": 1.2789, + "theoretical_loss": 3.500481053053165, + "tokens_seen": 1734148096 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038355065195586763, + "loss": 1.3095, + "theoretical_loss": 3.5004698038477784, + "tokens_seen": 1734213632 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038353460381143433, + "loss": 1.3238, + "theoretical_loss": 3.500458555186517, + "tokens_seen": 1734279168 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038351855566700103, + "loss": 1.3055, + "theoretical_loss": 3.500447307069333, + "tokens_seen": 1734344704 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038350250752256773, + "loss": 1.3335, + "theoretical_loss": 3.500436059496181, + "tokens_seen": 1734410240 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038348645937813443, + "loss": 1.3311, + "theoretical_loss": 3.5004248124670125, + "tokens_seen": 1734475776 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038347041123370113, + "loss": 1.2849, + "theoretical_loss": 3.5004135659817823, + "tokens_seen": 1734541312 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038345436308926783, + "loss": 1.3201, + "theoretical_loss": 3.500402320040442, + "tokens_seen": 1734606848 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038343831494483453, + "loss": 1.3395, + "theoretical_loss": 3.5003910746429456, + "tokens_seen": 1734672384 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038342226680040123, + "loss": 1.3349, + "theoretical_loss": 3.5003798297892468, + "tokens_seen": 1734737920 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038340621865596793, + "loss": 1.2728, + "theoretical_loss": 3.500368585479297, + "tokens_seen": 1734803456 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038339017051153463, + "loss": 1.265, + "theoretical_loss": 3.500357341713051, + "tokens_seen": 1734868992 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038337412236710133, + "loss": 1.2503, + "theoretical_loss": 3.5003460984904615, + "tokens_seen": 1734934528 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038335807422266804, + "loss": 1.2114, + "theoretical_loss": 3.5003348558114817, + "tokens_seen": 1735000064 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038334202607823474, + "loss": 1.3465, + "theoretical_loss": 3.5003236136760645, + "tokens_seen": 1735065600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038332597793380144, + "loss": 1.3797, + "theoretical_loss": 3.5003123720841636, + "tokens_seen": 1735131136 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038330992978936814, + "loss": 1.2592, + "theoretical_loss": 3.500301131035731, + "tokens_seen": 1735196672 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038329388164493484, + "loss": 1.2673, + "theoretical_loss": 3.5002898905307216, + "tokens_seen": 1735262208 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038327783350050154, + "loss": 1.3435, + "theoretical_loss": 3.5002786505690873, + "tokens_seen": 1735327744 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038326178535606824, + "loss": 1.2983, + "theoretical_loss": 3.5002674111507823, + "tokens_seen": 1735393280 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038324573721163494, + "loss": 1.3299, + "theoretical_loss": 3.5002561722757592, + "tokens_seen": 1735458816 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038322968906720164, + "loss": 1.2967, + "theoretical_loss": 3.500244933943971, + "tokens_seen": 1735524352 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038321364092276834, + "loss": 1.2473, + "theoretical_loss": 3.5002336961553713, + "tokens_seen": 1735589888 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038319759277833504, + "loss": 1.3633, + "theoretical_loss": 3.5002224589099136, + "tokens_seen": 1735655424 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038318154463390174, + "loss": 1.2409, + "theoretical_loss": 3.5002112222075508, + "tokens_seen": 1735720960 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038316549648946844, + "loss": 1.2308, + "theoretical_loss": 3.5001999860482362, + "tokens_seen": 1735786496 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038314944834503514, + "loss": 1.292, + "theoretical_loss": 3.500188750431923, + "tokens_seen": 1735852032 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038313340020060184, + "loss": 1.2137, + "theoretical_loss": 3.5001775153585646, + "tokens_seen": 1735917568 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038311735205616854, + "loss": 1.2733, + "theoretical_loss": 3.5001662808281138, + "tokens_seen": 1735983104 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038310130391173525, + "loss": 1.3228, + "theoretical_loss": 3.5001550468405247, + "tokens_seen": 1736048640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038308525576730195, + "loss": 1.2218, + "theoretical_loss": 3.5001438133957494, + "tokens_seen": 1736114176 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038306920762286865, + "loss": 1.4227, + "theoretical_loss": 3.5001325804937427, + "tokens_seen": 1736179712 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003830531594784353, + "loss": 1.3137, + "theoretical_loss": 3.500121348134457, + "tokens_seen": 1736245248 + }, + { + "epoch": 0.53, + "learning_rate": 0.000383037111334002, + "loss": 1.2323, + "theoretical_loss": 3.5001101163178454, + "tokens_seen": 1736310784 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003830210631895687, + "loss": 1.2439, + "theoretical_loss": 3.5000988850438617, + "tokens_seen": 1736376320 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003830050150451354, + "loss": 1.2675, + "theoretical_loss": 3.500087654312459, + "tokens_seen": 1736441856 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003829889669007021, + "loss": 1.2665, + "theoretical_loss": 3.5000764241235904, + "tokens_seen": 1736507392 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003829729187562688, + "loss": 1.3168, + "theoretical_loss": 3.500065194477209, + "tokens_seen": 1736572928 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003829568706118355, + "loss": 1.3625, + "theoretical_loss": 3.5000539653732696, + "tokens_seen": 1736638464 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 644326, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3050986528396606, + "objective/train/theoretical_loss": 3.5000427368117233, + "objective/train/tokens_used": 1757164000, + "theoretical_loss": 3.5000427368117233, + "tokens_seen": 1736704000 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003829408224674022, + "loss": 1.3051, + "theoretical_loss": 3.5000427368117233, + "tokens_seen": 1736704000 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003829247743229689, + "loss": 1.3614, + "theoretical_loss": 3.5000315087925253, + "tokens_seen": 1736769536 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003829087261785356, + "loss": 1.2119, + "theoretical_loss": 3.5000202813156287, + "tokens_seen": 1736835072 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003828926780341023, + "loss": 1.3486, + "theoretical_loss": 3.5000090543809854, + "tokens_seen": 1736900608 + }, + { + "epoch": 0.53, + "learning_rate": 0.000382876629889669, + "loss": 1.3688, + "theoretical_loss": 3.4999978279885506, + "tokens_seen": 1736966144 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003828605817452357, + "loss": 1.1158, + "theoretical_loss": 3.4999866021382764, + "tokens_seen": 1737031680 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003828445336008024, + "loss": 1.3529, + "theoretical_loss": 3.4999753768301165, + "tokens_seen": 1737097216 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003828284854563691, + "loss": 1.2289, + "theoretical_loss": 3.499964152064025, + "tokens_seen": 1737162752 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003828124373119358, + "loss": 1.3047, + "theoretical_loss": 3.4999529278399537, + "tokens_seen": 1737228288 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003827963891675025, + "loss": 1.2696, + "theoretical_loss": 3.499941704157857, + "tokens_seen": 1737293824 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003827803410230692, + "loss": 1.2929, + "theoretical_loss": 3.4999304810176888, + "tokens_seen": 1737359360 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003827642928786359, + "loss": 1.2616, + "theoretical_loss": 3.499919258419401, + "tokens_seen": 1737424896 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003827482447342026, + "loss": 1.3212, + "theoretical_loss": 3.4999080363629487, + "tokens_seen": 1737490432 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003827321965897693, + "loss": 1.2265, + "theoretical_loss": 3.4998968148482845, + "tokens_seen": 1737555968 + }, + { + "epoch": 0.53, + "learning_rate": 0.000382716148445336, + "loss": 1.288, + "theoretical_loss": 3.4998855938753612, + "tokens_seen": 1737621504 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003827001003009027, + "loss": 1.1653, + "theoretical_loss": 3.499874373444133, + "tokens_seen": 1737687040 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003826840521564694, + "loss": 1.2796, + "theoretical_loss": 3.4998631535545535, + "tokens_seen": 1737752576 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003826680040120361, + "loss": 1.2938, + "theoretical_loss": 3.4998519342065753, + "tokens_seen": 1737818112 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003826519558676028, + "loss": 1.196, + "theoretical_loss": 3.4998407154001523, + "tokens_seen": 1737883648 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003826359077231695, + "loss": 1.2315, + "theoretical_loss": 3.499829497135238, + "tokens_seen": 1737949184 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003826198595787362, + "loss": 1.2646, + "theoretical_loss": 3.4998182794117856, + "tokens_seen": 1738014720 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003826038114343029, + "loss": 1.2998, + "theoretical_loss": 3.499807062229749, + "tokens_seen": 1738080256 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003825877632898696, + "loss": 1.2215, + "theoretical_loss": 3.4997958455890816, + "tokens_seen": 1738145792 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038257171514543637, + "loss": 1.2443, + "theoretical_loss": 3.4997846294897363, + "tokens_seen": 1738211328 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038255566700100307, + "loss": 1.3032, + "theoretical_loss": 3.499773413931667, + "tokens_seen": 1738276864 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038253961885656977, + "loss": 1.388, + "theoretical_loss": 3.4997621989148273, + "tokens_seen": 1738342400 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038252357071213647, + "loss": 1.3589, + "theoretical_loss": 3.49975098443917, + "tokens_seen": 1738407936 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038250752256770317, + "loss": 1.3303, + "theoretical_loss": 3.4997397705046493, + "tokens_seen": 1738473472 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038249147442326987, + "loss": 1.2803, + "theoretical_loss": 3.4997285571112178, + "tokens_seen": 1738539008 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038247542627883657, + "loss": 1.2943, + "theoretical_loss": 3.4997173442588307, + "tokens_seen": 1738604544 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038245937813440327, + "loss": 1.3404, + "theoretical_loss": 3.4997061319474394, + "tokens_seen": 1738670080 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038244332998996997, + "loss": 1.3078, + "theoretical_loss": 3.4996949201769993, + "tokens_seen": 1738735616 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003824272818455366, + "loss": 1.2631, + "theoretical_loss": 3.4996837089474626, + "tokens_seen": 1738801152 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003824112337011033, + "loss": 1.294, + "theoretical_loss": 3.4996724982587835, + "tokens_seen": 1738866688 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038239518555667, + "loss": 1.1924, + "theoretical_loss": 3.4996612881109153, + "tokens_seen": 1738932224 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003823791374122367, + "loss": 1.2323, + "theoretical_loss": 3.4996500785038114, + "tokens_seen": 1738997760 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003823630892678034, + "loss": 1.271, + "theoretical_loss": 3.499638869437425, + "tokens_seen": 1739063296 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003823470411233701, + "loss": 1.2995, + "theoretical_loss": 3.4996276609117105, + "tokens_seen": 1739128832 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003823309929789368, + "loss": 1.2148, + "theoretical_loss": 3.499616452926621, + "tokens_seen": 1739194368 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003823149448345035, + "loss": 1.295, + "theoretical_loss": 3.4996052454821105, + "tokens_seen": 1739259904 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003822988966900702, + "loss": 1.2613, + "theoretical_loss": 3.499594038578132, + "tokens_seen": 1739325440 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003822828485456369, + "loss": 1.1137, + "theoretical_loss": 3.4995828322146387, + "tokens_seen": 1739390976 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003822668004012036, + "loss": 1.3237, + "theoretical_loss": 3.4995716263915853, + "tokens_seen": 1739456512 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003822507522567703, + "loss": 1.422, + "theoretical_loss": 3.4995604211089244, + "tokens_seen": 1739522048 + }, + { + "epoch": 0.53, + "learning_rate": 0.000382234704112337, + "loss": 1.3192, + "theoretical_loss": 3.49954921636661, + "tokens_seen": 1739587584 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003822186559679037, + "loss": 1.3722, + "theoretical_loss": 3.499538012164596, + "tokens_seen": 1739653120 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003822026078234704, + "loss": 1.2704, + "theoretical_loss": 3.499526808502835, + "tokens_seen": 1739718656 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038218655967903713, + "loss": 1.4163, + "theoretical_loss": 3.4995156053812817, + "tokens_seen": 1739784192 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038217051153460383, + "loss": 1.3083, + "theoretical_loss": 3.4995044027998894, + "tokens_seen": 1739849728 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038215446339017053, + "loss": 1.2734, + "theoretical_loss": 3.499493200758611, + "tokens_seen": 1739915264 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038213841524573723, + "loss": 1.2435, + "theoretical_loss": 3.499481999257401, + "tokens_seen": 1739980800 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038212236710130393, + "loss": 1.2857, + "theoretical_loss": 3.4994707982962128, + "tokens_seen": 1740046336 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038210631895687063, + "loss": 1.3319, + "theoretical_loss": 3.499459597875, + "tokens_seen": 1740111872 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038209027081243733, + "loss": 1.2472, + "theoretical_loss": 3.4994483979937154, + "tokens_seen": 1740177408 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038207422266800403, + "loss": 1.3279, + "theoretical_loss": 3.4994371986523145, + "tokens_seen": 1740242944 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038205817452357073, + "loss": 1.3224, + "theoretical_loss": 3.499425999850749, + "tokens_seen": 1740308480 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038204212637913743, + "loss": 1.3617, + "theoretical_loss": 3.4994148015889737, + "tokens_seen": 1740374016 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038202607823470413, + "loss": 1.2463, + "theoretical_loss": 3.499403603866942, + "tokens_seen": 1740439552 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038201003009027083, + "loss": 1.2609, + "theoretical_loss": 3.499392406684607, + "tokens_seen": 1740505088 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038199398194583753, + "loss": 1.3215, + "theoretical_loss": 3.4993812100419235, + "tokens_seen": 1740570624 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038197793380140424, + "loss": 1.3098, + "theoretical_loss": 3.4993700139388446, + "tokens_seen": 1740636160 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038196188565697094, + "loss": 1.3877, + "theoretical_loss": 3.4993588183753235, + "tokens_seen": 1740701696 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038194583751253764, + "loss": 1.2922, + "theoretical_loss": 3.4993476233513148, + "tokens_seen": 1740767232 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038192978936810434, + "loss": 1.3097, + "theoretical_loss": 3.499336428866771, + "tokens_seen": 1740832768 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038191374122367104, + "loss": 1.3387, + "theoretical_loss": 3.4993252349216473, + "tokens_seen": 1740898304 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038189769307923774, + "loss": 1.3451, + "theoretical_loss": 3.4993140415158956, + "tokens_seen": 1740963840 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038188164493480444, + "loss": 1.2767, + "theoretical_loss": 3.4993028486494717, + "tokens_seen": 1741029376 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038186559679037114, + "loss": 1.2351, + "theoretical_loss": 3.4992916563223275, + "tokens_seen": 1741094912 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038184954864593784, + "loss": 1.3486, + "theoretical_loss": 3.4992804645344178, + "tokens_seen": 1741160448 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038183350050150454, + "loss": 1.2577, + "theoretical_loss": 3.499269273285696, + "tokens_seen": 1741225984 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038181745235707124, + "loss": 1.305, + "theoretical_loss": 3.499258082576116, + "tokens_seen": 1741291520 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038180140421263794, + "loss": 1.2764, + "theoretical_loss": 3.4992468924056306, + "tokens_seen": 1741357056 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038178535606820464, + "loss": 1.2273, + "theoretical_loss": 3.499235702774195, + "tokens_seen": 1741422592 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038176930792377134, + "loss": 1.2974, + "theoretical_loss": 3.499224513681762, + "tokens_seen": 1741488128 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038175325977933804, + "loss": 1.2456, + "theoretical_loss": 3.4992133251282853, + "tokens_seen": 1741553664 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038173721163490474, + "loss": 1.3281, + "theoretical_loss": 3.499202137113719, + "tokens_seen": 1741619200 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038172116349047145, + "loss": 1.1665, + "theoretical_loss": 3.499190949638017, + "tokens_seen": 1741684736 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038170511534603815, + "loss": 1.3041, + "theoretical_loss": 3.499179762701133, + "tokens_seen": 1741750272 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038168906720160485, + "loss": 1.2679, + "theoretical_loss": 3.4991685763030205, + "tokens_seen": 1741815808 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038167301905717155, + "loss": 1.3483, + "theoretical_loss": 3.4991573904436333, + "tokens_seen": 1741881344 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003816569709127382, + "loss": 1.2548, + "theoretical_loss": 3.499146205122926, + "tokens_seen": 1741946880 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003816409227683049, + "loss": 1.3515, + "theoretical_loss": 3.4991350203408507, + "tokens_seen": 1742012416 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003816248746238716, + "loss": 1.2782, + "theoretical_loss": 3.499123836097363, + "tokens_seen": 1742077952 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003816088264794383, + "loss": 1.3137, + "theoretical_loss": 3.499112652392416, + "tokens_seen": 1742143488 + }, + { + "epoch": 0.53, + "learning_rate": 0.000381592778335005, + "loss": 1.2995, + "theoretical_loss": 3.499101469225963, + "tokens_seen": 1742209024 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003815767301905717, + "loss": 1.3279, + "theoretical_loss": 3.4990902865979585, + "tokens_seen": 1742274560 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003815606820461384, + "loss": 1.3609, + "theoretical_loss": 3.499079104508356, + "tokens_seen": 1742340096 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003815446339017051, + "loss": 1.3115, + "theoretical_loss": 3.499067922957109, + "tokens_seen": 1742405632 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003815285857572718, + "loss": 1.2548, + "theoretical_loss": 3.4990567419441723, + "tokens_seen": 1742471168 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003815125376128385, + "loss": 1.2113, + "theoretical_loss": 3.499045561469499, + "tokens_seen": 1742536704 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003814964894684052, + "loss": 1.3926, + "theoretical_loss": 3.4990343815330434, + "tokens_seen": 1742602240 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003814804413239719, + "loss": 1.3529, + "theoretical_loss": 3.4990232021347585, + "tokens_seen": 1742667776 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003814643931795386, + "loss": 1.2343, + "theoretical_loss": 3.4990120232745987, + "tokens_seen": 1742733312 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003814483450351053, + "loss": 1.3227, + "theoretical_loss": 3.4990008449525183, + "tokens_seen": 1742798848 + }, + { + "epoch": 0.53, + "learning_rate": 0.000381432296890672, + "loss": 1.251, + "theoretical_loss": 3.4989896671684706, + "tokens_seen": 1742864384 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003814162487462387, + "loss": 1.2112, + "theoretical_loss": 3.49897848992241, + "tokens_seen": 1742929920 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003814002006018054, + "loss": 1.313, + "theoretical_loss": 3.4989673132142896, + "tokens_seen": 1742995456 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003813841524573721, + "loss": 1.3168, + "theoretical_loss": 3.498956137044064, + "tokens_seen": 1743060992 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038136810431293886, + "loss": 1.2179, + "theoretical_loss": 3.498944961411687, + "tokens_seen": 1743126528 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038135205616850556, + "loss": 1.2664, + "theoretical_loss": 3.4989337863171115, + "tokens_seen": 1743192064 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 646613, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2985121011734009, + "objective/train/theoretical_loss": 3.4989226117602925, + "objective/train/tokens_used": 1763717600, + "theoretical_loss": 3.4989226117602925, + "tokens_seen": 1743257600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038133600802407226, + "loss": 1.2985, + "theoretical_loss": 3.4989226117602925, + "tokens_seen": 1743257600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038131995987963896, + "loss": 1.3088, + "theoretical_loss": 3.498911437741184, + "tokens_seen": 1743323136 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038130391173520566, + "loss": 1.3527, + "theoretical_loss": 3.4989002642597393, + "tokens_seen": 1743388672 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038128786359077236, + "loss": 1.2178, + "theoretical_loss": 3.4988890913159123, + "tokens_seen": 1743454208 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038127181544633906, + "loss": 1.2452, + "theoretical_loss": 3.498877918909657, + "tokens_seen": 1743519744 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038125576730190576, + "loss": 1.2787, + "theoretical_loss": 3.4988667470409283, + "tokens_seen": 1743585280 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038123971915747246, + "loss": 1.3797, + "theoretical_loss": 3.498855575709679, + "tokens_seen": 1743650816 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038122367101303917, + "loss": 1.2631, + "theoretical_loss": 3.4988444049158627, + "tokens_seen": 1743716352 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038120762286860587, + "loss": 1.3138, + "theoretical_loss": 3.4988332346594344, + "tokens_seen": 1743781888 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038119157472417257, + "loss": 1.2757, + "theoretical_loss": 3.498822064940348, + "tokens_seen": 1743847424 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038117552657973927, + "loss": 1.345, + "theoretical_loss": 3.498810895758557, + "tokens_seen": 1743912960 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038115947843530597, + "loss": 1.3725, + "theoretical_loss": 3.498799727114015, + "tokens_seen": 1743978496 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038114343029087267, + "loss": 1.2652, + "theoretical_loss": 3.4987885590066767, + "tokens_seen": 1744044032 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038112738214643937, + "loss": 1.2894, + "theoretical_loss": 3.498777391436496, + "tokens_seen": 1744109568 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038111133400200607, + "loss": 1.3479, + "theoretical_loss": 3.4987662244034268, + "tokens_seen": 1744175104 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038109528585757277, + "loss": 1.2743, + "theoretical_loss": 3.498755057907423, + "tokens_seen": 1744240640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038107923771313947, + "loss": 1.2642, + "theoretical_loss": 3.4987438919484384, + "tokens_seen": 1744306176 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038106318956870617, + "loss": 1.388, + "theoretical_loss": 3.498732726526427, + "tokens_seen": 1744371712 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038104714142427287, + "loss": 1.3815, + "theoretical_loss": 3.4987215616413434, + "tokens_seen": 1744437248 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003810310932798396, + "loss": 1.3179, + "theoretical_loss": 3.498710397293141, + "tokens_seen": 1744502784 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003810150451354062, + "loss": 1.3044, + "theoretical_loss": 3.498699233481774, + "tokens_seen": 1744568320 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003809989969909729, + "loss": 1.2562, + "theoretical_loss": 3.498688070207197, + "tokens_seen": 1744633856 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003809829488465396, + "loss": 1.3316, + "theoretical_loss": 3.4986769074693624, + "tokens_seen": 1744699392 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003809669007021063, + "loss": 1.2169, + "theoretical_loss": 3.498665745268226, + "tokens_seen": 1744764928 + }, + { + "epoch": 0.53, + "learning_rate": 0.000380950852557673, + "loss": 1.3177, + "theoretical_loss": 3.498654583603741, + "tokens_seen": 1744830464 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003809348044132397, + "loss": 1.2921, + "theoretical_loss": 3.498643422475862, + "tokens_seen": 1744896000 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003809187562688064, + "loss": 1.2022, + "theoretical_loss": 3.4986322618845422, + "tokens_seen": 1744961536 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003809027081243731, + "loss": 1.269, + "theoretical_loss": 3.4986211018297357, + "tokens_seen": 1745027072 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003808866599799398, + "loss": 1.2948, + "theoretical_loss": 3.498609942311398, + "tokens_seen": 1745092608 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003808706118355065, + "loss": 1.1759, + "theoretical_loss": 3.4985987833294816, + "tokens_seen": 1745158144 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003808545636910732, + "loss": 1.2834, + "theoretical_loss": 3.4985876248839407, + "tokens_seen": 1745223680 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003808385155466399, + "loss": 1.3018, + "theoretical_loss": 3.49857646697473, + "tokens_seen": 1745289216 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003808224674022066, + "loss": 1.3318, + "theoretical_loss": 3.498565309601804, + "tokens_seen": 1745354752 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038080641925777333, + "loss": 1.3099, + "theoretical_loss": 3.498554152765115, + "tokens_seen": 1745420288 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038079037111334003, + "loss": 1.2239, + "theoretical_loss": 3.498542996464619, + "tokens_seen": 1745485824 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038077432296890673, + "loss": 1.3059, + "theoretical_loss": 3.4985318407002692, + "tokens_seen": 1745551360 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038075827482447343, + "loss": 1.2364, + "theoretical_loss": 3.4985206854720197, + "tokens_seen": 1745616896 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038074222668004013, + "loss": 1.3056, + "theoretical_loss": 3.498509530779825, + "tokens_seen": 1745682432 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038072617853560683, + "loss": 1.2736, + "theoretical_loss": 3.4984983766236386, + "tokens_seen": 1745747968 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038071013039117353, + "loss": 1.293, + "theoretical_loss": 3.498487223003415, + "tokens_seen": 1745813504 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038069408224674023, + "loss": 1.195, + "theoretical_loss": 3.498476069919109, + "tokens_seen": 1745879040 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038067803410230693, + "loss": 1.3238, + "theoretical_loss": 3.498464917370673, + "tokens_seen": 1745944576 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038066198595787363, + "loss": 1.2852, + "theoretical_loss": 3.498453765358063, + "tokens_seen": 1746010112 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038064593781344033, + "loss": 1.3209, + "theoretical_loss": 3.498442613881232, + "tokens_seen": 1746075648 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038062988966900703, + "loss": 1.3372, + "theoretical_loss": 3.4984314629401347, + "tokens_seen": 1746141184 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038061384152457374, + "loss": 1.2945, + "theoretical_loss": 3.498420312534725, + "tokens_seen": 1746206720 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038059779338014044, + "loss": 1.2476, + "theoretical_loss": 3.498409162664957, + "tokens_seen": 1746272256 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038058174523570714, + "loss": 1.3303, + "theoretical_loss": 3.4983980133307853, + "tokens_seen": 1746337792 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038056569709127384, + "loss": 1.3244, + "theoretical_loss": 3.498386864532163, + "tokens_seen": 1746403328 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038054964894684054, + "loss": 1.2776, + "theoretical_loss": 3.498375716269046, + "tokens_seen": 1746468864 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038053360080240724, + "loss": 1.2843, + "theoretical_loss": 3.498364568541387, + "tokens_seen": 1746534400 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038051755265797394, + "loss": 1.2347, + "theoretical_loss": 3.4983534213491407, + "tokens_seen": 1746599936 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038050150451354064, + "loss": 1.2487, + "theoretical_loss": 3.4983422746922614, + "tokens_seen": 1746665472 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038048545636910734, + "loss": 1.2516, + "theoretical_loss": 3.4983311285707033, + "tokens_seen": 1746731008 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038046940822467404, + "loss": 1.2798, + "theoretical_loss": 3.4983199829844205, + "tokens_seen": 1746796544 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038045336008024074, + "loss": 1.3117, + "theoretical_loss": 3.498308837933367, + "tokens_seen": 1746862080 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038043731193580744, + "loss": 1.2966, + "theoretical_loss": 3.4982976934174976, + "tokens_seen": 1746927616 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038042126379137414, + "loss": 1.29, + "theoretical_loss": 3.4982865494367656, + "tokens_seen": 1746993152 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038040521564694084, + "loss": 1.3897, + "theoretical_loss": 3.4982754059911265, + "tokens_seen": 1747058688 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038038916750250754, + "loss": 1.2236, + "theoretical_loss": 3.498264263080533, + "tokens_seen": 1747124224 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038037311935807424, + "loss": 1.3357, + "theoretical_loss": 3.498253120704941, + "tokens_seen": 1747189760 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038035707121364095, + "loss": 1.2582, + "theoretical_loss": 3.4982419788643035, + "tokens_seen": 1747255296 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038034102306920765, + "loss": 1.3539, + "theoretical_loss": 3.498230837558576, + "tokens_seen": 1747320832 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038032497492477435, + "loss": 1.3823, + "theoretical_loss": 3.498219696787711, + "tokens_seen": 1747386368 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038030892678034105, + "loss": 1.1922, + "theoretical_loss": 3.498208556551664, + "tokens_seen": 1747451904 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038029287863590775, + "loss": 1.2467, + "theoretical_loss": 3.4981974168503887, + "tokens_seen": 1747517440 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038027683049147445, + "loss": 1.2981, + "theoretical_loss": 3.49818627768384, + "tokens_seen": 1747582976 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038026078234704115, + "loss": 1.2435, + "theoretical_loss": 3.4981751390519715, + "tokens_seen": 1747648512 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003802447342026078, + "loss": 1.2628, + "theoretical_loss": 3.498164000954738, + "tokens_seen": 1747714048 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003802286860581745, + "loss": 1.3757, + "theoretical_loss": 3.498152863392094, + "tokens_seen": 1747779584 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003802126379137412, + "loss": 1.1661, + "theoretical_loss": 3.498141726363993, + "tokens_seen": 1747845120 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003801965897693079, + "loss": 1.2722, + "theoretical_loss": 3.4981305898703896, + "tokens_seen": 1747910656 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003801805416248746, + "loss": 1.3291, + "theoretical_loss": 3.4981194539112384, + "tokens_seen": 1747976192 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038016449348044135, + "loss": 1.3244, + "theoretical_loss": 3.4981083184864934, + "tokens_seen": 1748041728 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038014844533600805, + "loss": 1.3281, + "theoretical_loss": 3.4980971835961094, + "tokens_seen": 1748107264 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038013239719157475, + "loss": 1.2992, + "theoretical_loss": 3.49808604924004, + "tokens_seen": 1748172800 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038011634904714145, + "loss": 1.4689, + "theoretical_loss": 3.49807491541824, + "tokens_seen": 1748238336 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038010030090270816, + "loss": 1.3753, + "theoretical_loss": 3.4980637821306635, + "tokens_seen": 1748303872 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038008425275827486, + "loss": 1.2962, + "theoretical_loss": 3.4980526493772652, + "tokens_seen": 1748369408 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038006820461384156, + "loss": 1.4326, + "theoretical_loss": 3.498041517157999, + "tokens_seen": 1748434944 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038005215646940826, + "loss": 1.172, + "theoretical_loss": 3.49803038547282, + "tokens_seen": 1748500480 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038003610832497496, + "loss": 1.3313, + "theoretical_loss": 3.4980192543216813, + "tokens_seen": 1748566016 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038002006018054166, + "loss": 1.2436, + "theoretical_loss": 3.4980081237045386, + "tokens_seen": 1748631552 + }, + { + "epoch": 0.53, + "learning_rate": 0.00038000401203610836, + "loss": 1.3189, + "theoretical_loss": 3.4979969936213458, + "tokens_seen": 1748697088 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037998796389167506, + "loss": 1.3856, + "theoretical_loss": 3.497985864072057, + "tokens_seen": 1748762624 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037997191574724176, + "loss": 1.3996, + "theoretical_loss": 3.4979747350566264, + "tokens_seen": 1748828160 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037995586760280846, + "loss": 1.3532, + "theoretical_loss": 3.4979636065750093, + "tokens_seen": 1748893696 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037993981945837516, + "loss": 1.3095, + "theoretical_loss": 3.497952478627159, + "tokens_seen": 1748959232 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037992377131394186, + "loss": 1.3328, + "theoretical_loss": 3.4979413512130306, + "tokens_seen": 1749024768 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037990772316950856, + "loss": 1.207, + "theoretical_loss": 3.4979302243325785, + "tokens_seen": 1749090304 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037989167502507526, + "loss": 1.3038, + "theoretical_loss": 3.497919097985757, + "tokens_seen": 1749155840 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037987562688064196, + "loss": 1.2623, + "theoretical_loss": 3.49790797217252, + "tokens_seen": 1749221376 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037985957873620867, + "loss": 1.1614, + "theoretical_loss": 3.4978968468928224, + "tokens_seen": 1749286912 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037984353059177537, + "loss": 1.2948, + "theoretical_loss": 3.497885722146619, + "tokens_seen": 1749352448 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037982748244734207, + "loss": 1.2457, + "theoretical_loss": 3.4978745979338637, + "tokens_seen": 1749417984 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037981143430290877, + "loss": 1.3396, + "theoretical_loss": 3.497863474254511, + "tokens_seen": 1749483520 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037979538615847547, + "loss": 1.3406, + "theoretical_loss": 3.4978523511085156, + "tokens_seen": 1749549056 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037977933801404217, + "loss": 1.2741, + "theoretical_loss": 3.4978412284958313, + "tokens_seen": 1749614592 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037976328986960887, + "loss": 1.2838, + "theoretical_loss": 3.497830106416414, + "tokens_seen": 1749680128 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037974724172517557, + "loss": 1.3718, + "theoretical_loss": 3.497818984870216, + "tokens_seen": 1749745664 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 649057, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3276128768920898, + "objective/train/theoretical_loss": 3.4978078638571937, + "objective/train/tokens_used": 1770271200, + "theoretical_loss": 3.4978078638571937, + "tokens_seen": 1749811200 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037973119358074227, + "loss": 1.3276, + "theoretical_loss": 3.4978078638571937, + "tokens_seen": 1749811200 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037971514543630897, + "loss": 1.4764, + "theoretical_loss": 3.4977967433773003, + "tokens_seen": 1749876736 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037969909729187567, + "loss": 1.3239, + "theoretical_loss": 3.497785623430491, + "tokens_seen": 1749942272 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037968304914744237, + "loss": 1.2341, + "theoretical_loss": 3.4977745040167205, + "tokens_seen": 1750007808 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037966700100300907, + "loss": 1.3156, + "theoretical_loss": 3.4977633851359426, + "tokens_seen": 1750073344 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003796509528585758, + "loss": 1.2308, + "theoretical_loss": 3.497752266788112, + "tokens_seen": 1750138880 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003796349047141425, + "loss": 1.3055, + "theoretical_loss": 3.497741148973183, + "tokens_seen": 1750204416 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003796188565697091, + "loss": 1.3614, + "theoretical_loss": 3.497730031691111, + "tokens_seen": 1750269952 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003796028084252758, + "loss": 1.2677, + "theoretical_loss": 3.4977189149418493, + "tokens_seen": 1750335488 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003795867602808425, + "loss": 1.2818, + "theoretical_loss": 3.497707798725353, + "tokens_seen": 1750401024 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003795707121364092, + "loss": 1.3184, + "theoretical_loss": 3.497696683041577, + "tokens_seen": 1750466560 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003795546639919759, + "loss": 1.345, + "theoretical_loss": 3.4976855678904752, + "tokens_seen": 1750532096 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003795386158475426, + "loss": 1.3333, + "theoretical_loss": 3.497674453272002, + "tokens_seen": 1750597632 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003795225677031093, + "loss": 1.3238, + "theoretical_loss": 3.497663339186113, + "tokens_seen": 1750663168 + }, + { + "epoch": 0.53, + "learning_rate": 0.000379506519558676, + "loss": 1.2506, + "theoretical_loss": 3.497652225632762, + "tokens_seen": 1750728704 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003794904714142427, + "loss": 1.3671, + "theoretical_loss": 3.4976411126119036, + "tokens_seen": 1750794240 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003794744232698094, + "loss": 1.3271, + "theoretical_loss": 3.497630000123492, + "tokens_seen": 1750859776 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003794583751253761, + "loss": 1.3246, + "theoretical_loss": 3.497618888167482, + "tokens_seen": 1750925312 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037944232698094283, + "loss": 1.3035, + "theoretical_loss": 3.4976077767438287, + "tokens_seen": 1750990848 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037942627883650953, + "loss": 1.3198, + "theoretical_loss": 3.497596665852486, + "tokens_seen": 1751056384 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037941023069207623, + "loss": 1.3391, + "theoretical_loss": 3.497585555493409, + "tokens_seen": 1751121920 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037939418254764293, + "loss": 1.3285, + "theoretical_loss": 3.497574445666552, + "tokens_seen": 1751187456 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037937813440320963, + "loss": 1.3397, + "theoretical_loss": 3.4975633363718694, + "tokens_seen": 1751252992 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037936208625877633, + "loss": 1.3309, + "theoretical_loss": 3.497552227609316, + "tokens_seen": 1751318528 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037934603811434303, + "loss": 1.3243, + "theoretical_loss": 3.4975411193788464, + "tokens_seen": 1751384064 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037932998996990973, + "loss": 1.2281, + "theoretical_loss": 3.4975300116804156, + "tokens_seen": 1751449600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037931394182547643, + "loss": 1.4824, + "theoretical_loss": 3.497518904513978, + "tokens_seen": 1751515136 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037929789368104313, + "loss": 1.3076, + "theoretical_loss": 3.4975077978794875, + "tokens_seen": 1751580672 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037928184553660983, + "loss": 1.2676, + "theoretical_loss": 3.497496691776899, + "tokens_seen": 1751646208 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037926579739217653, + "loss": 1.3324, + "theoretical_loss": 3.497485586206168, + "tokens_seen": 1751711744 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037924974924774323, + "loss": 1.3943, + "theoretical_loss": 3.497474481167248, + "tokens_seen": 1751777280 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037923370110330994, + "loss": 1.3558, + "theoretical_loss": 3.4974633766600944, + "tokens_seen": 1751842816 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037921765295887664, + "loss": 1.3164, + "theoretical_loss": 3.4974522726846615, + "tokens_seen": 1751908352 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037920160481444334, + "loss": 1.3572, + "theoretical_loss": 3.497441169240904, + "tokens_seen": 1751973888 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037918555667001004, + "loss": 1.2264, + "theoretical_loss": 3.497430066328777, + "tokens_seen": 1752039424 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037916950852557674, + "loss": 1.4053, + "theoretical_loss": 3.4974189639482347, + "tokens_seen": 1752104960 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037915346038114344, + "loss": 1.3656, + "theoretical_loss": 3.497407862099232, + "tokens_seen": 1752170496 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037913741223671014, + "loss": 1.2535, + "theoretical_loss": 3.497396760781723, + "tokens_seen": 1752236032 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037912136409227684, + "loss": 1.3538, + "theoretical_loss": 3.497385659995663, + "tokens_seen": 1752301568 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037910531594784354, + "loss": 1.2463, + "theoretical_loss": 3.497374559741006, + "tokens_seen": 1752367104 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037908926780341024, + "loss": 1.28, + "theoretical_loss": 3.4973634600177075, + "tokens_seen": 1752432640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037907321965897694, + "loss": 1.3509, + "theoretical_loss": 3.497352360825722, + "tokens_seen": 1752498176 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037905717151454364, + "loss": 1.4171, + "theoretical_loss": 3.497341262165004, + "tokens_seen": 1752563712 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037904112337011034, + "loss": 1.23, + "theoretical_loss": 3.4973301640355086, + "tokens_seen": 1752629248 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037902507522567704, + "loss": 1.4456, + "theoretical_loss": 3.497319066437189, + "tokens_seen": 1752694784 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037900902708124374, + "loss": 1.3736, + "theoretical_loss": 3.497307969370002, + "tokens_seen": 1752760320 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037899297893681045, + "loss": 1.2296, + "theoretical_loss": 3.4972968728339016, + "tokens_seen": 1752825856 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037897693079237715, + "loss": 1.2612, + "theoretical_loss": 3.497285776828842, + "tokens_seen": 1752891392 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037896088264794385, + "loss": 1.3097, + "theoretical_loss": 3.4972746813547784, + "tokens_seen": 1752956928 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037894483450351055, + "loss": 1.4164, + "theoretical_loss": 3.4972635864116652, + "tokens_seen": 1753022464 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037892878635907725, + "loss": 1.2744, + "theoretical_loss": 3.4972524919994576, + "tokens_seen": 1753088000 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037891273821464395, + "loss": 1.2846, + "theoretical_loss": 3.49724139811811, + "tokens_seen": 1753153536 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037889669007021065, + "loss": 1.2857, + "theoretical_loss": 3.4972303047675775, + "tokens_seen": 1753219072 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037888064192577735, + "loss": 1.3341, + "theoretical_loss": 3.4972192119478143, + "tokens_seen": 1753284608 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037886459378134405, + "loss": 1.3851, + "theoretical_loss": 3.497208119658775, + "tokens_seen": 1753350144 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037884854563691075, + "loss": 1.2984, + "theoretical_loss": 3.4971970279004156, + "tokens_seen": 1753415680 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037883249749247745, + "loss": 1.3024, + "theoretical_loss": 3.4971859366726905, + "tokens_seen": 1753481216 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037881644934804415, + "loss": 1.3434, + "theoretical_loss": 3.497174845975553, + "tokens_seen": 1753546752 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037880040120361085, + "loss": 1.2333, + "theoretical_loss": 3.49716375580896, + "tokens_seen": 1753612288 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037878435305917755, + "loss": 1.2877, + "theoretical_loss": 3.4971526661728647, + "tokens_seen": 1753677824 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037876830491474425, + "loss": 1.3083, + "theoretical_loss": 3.4971415770672225, + "tokens_seen": 1753743360 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037875225677031095, + "loss": 1.2868, + "theoretical_loss": 3.4971304884919885, + "tokens_seen": 1753808896 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037873620862587766, + "loss": 1.2702, + "theoretical_loss": 3.4971194004471173, + "tokens_seen": 1753874432 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037872016048144436, + "loss": 1.2014, + "theoretical_loss": 3.4971083129325633, + "tokens_seen": 1753939968 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037870411233701106, + "loss": 1.323, + "theoretical_loss": 3.4970972259482824, + "tokens_seen": 1754005504 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037868806419257776, + "loss": 1.3539, + "theoretical_loss": 3.497086139494228, + "tokens_seen": 1754071040 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037867201604814446, + "loss": 1.307, + "theoretical_loss": 3.497075053570356, + "tokens_seen": 1754136576 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037865596790371116, + "loss": 1.2837, + "theoretical_loss": 3.4970639681766205, + "tokens_seen": 1754202112 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037863991975927786, + "loss": 1.2194, + "theoretical_loss": 3.4970528833129766, + "tokens_seen": 1754267648 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037862387161484456, + "loss": 1.1884, + "theoretical_loss": 3.49704179897938, + "tokens_seen": 1754333184 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037860782347041126, + "loss": 1.2718, + "theoretical_loss": 3.497030715175784, + "tokens_seen": 1754398720 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037859177532597796, + "loss": 1.3769, + "theoretical_loss": 3.4970196319021447, + "tokens_seen": 1754464256 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037857572718154466, + "loss": 1.2419, + "theoretical_loss": 3.4970085491584166, + "tokens_seen": 1754529792 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037855967903711136, + "loss": 1.3102, + "theoretical_loss": 3.496997466944554, + "tokens_seen": 1754595328 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037854363089267806, + "loss": 1.2267, + "theoretical_loss": 3.496986385260513, + "tokens_seen": 1754660864 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037852758274824476, + "loss": 1.209, + "theoretical_loss": 3.496975304106247, + "tokens_seen": 1754726400 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037851153460381146, + "loss": 1.3342, + "theoretical_loss": 3.4969642234817124, + "tokens_seen": 1754791936 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037849548645937816, + "loss": 1.2473, + "theoretical_loss": 3.496953143386863, + "tokens_seen": 1754857472 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037847943831494487, + "loss": 1.3206, + "theoretical_loss": 3.4969420638216535, + "tokens_seen": 1754923008 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037846339017051157, + "loss": 1.2715, + "theoretical_loss": 3.49693098478604, + "tokens_seen": 1754988544 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037844734202607827, + "loss": 1.3869, + "theoretical_loss": 3.496919906279977, + "tokens_seen": 1755054080 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037843129388164497, + "loss": 1.3689, + "theoretical_loss": 3.496908828303419, + "tokens_seen": 1755119616 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037841524573721167, + "loss": 1.2984, + "theoretical_loss": 3.496897750856321, + "tokens_seen": 1755185152 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037839919759277837, + "loss": 1.2678, + "theoretical_loss": 3.496886673938638, + "tokens_seen": 1755250688 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037838314944834507, + "loss": 1.4212, + "theoretical_loss": 3.496875597550325, + "tokens_seen": 1755316224 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037836710130391177, + "loss": 1.3151, + "theoretical_loss": 3.4968645216913368, + "tokens_seen": 1755381760 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037835105315947847, + "loss": 1.3384, + "theoretical_loss": 3.4968534463616283, + "tokens_seen": 1755447296 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037833500501504517, + "loss": 1.3234, + "theoretical_loss": 3.496842371561155, + "tokens_seen": 1755512832 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037831895687061187, + "loss": 1.3049, + "theoretical_loss": 3.496831297289871, + "tokens_seen": 1755578368 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037830290872617857, + "loss": 1.3583, + "theoretical_loss": 3.4968202235477324, + "tokens_seen": 1755643904 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003782868605817453, + "loss": 1.3182, + "theoretical_loss": 3.496809150334693, + "tokens_seen": 1755709440 + }, + { + "epoch": 0.53, + "learning_rate": 0.000378270812437312, + "loss": 1.3544, + "theoretical_loss": 3.4967980776507086, + "tokens_seen": 1755774976 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003782547642928787, + "loss": 1.2465, + "theoretical_loss": 3.4967870054957335, + "tokens_seen": 1755840512 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003782387161484454, + "loss": 1.2814, + "theoretical_loss": 3.4967759338697233, + "tokens_seen": 1755906048 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003782226680040121, + "loss": 1.3023, + "theoretical_loss": 3.496764862772632, + "tokens_seen": 1755971584 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003782066198595787, + "loss": 1.3111, + "theoretical_loss": 3.496753792204416, + "tokens_seen": 1756037120 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003781905717151454, + "loss": 1.2231, + "theoretical_loss": 3.4967427221650293, + "tokens_seen": 1756102656 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003781745235707121, + "loss": 1.264, + "theoretical_loss": 3.4967316526544274, + "tokens_seen": 1756168192 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003781584754262788, + "loss": 1.3618, + "theoretical_loss": 3.496720583672565, + "tokens_seen": 1756233728 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003781424272818455, + "loss": 1.278, + "theoretical_loss": 3.496709515219397, + "tokens_seen": 1756299264 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 651801, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2797825336456299, + "objective/train/theoretical_loss": 3.4966984472948788, + "objective/train/tokens_used": 1776824800, + "theoretical_loss": 3.4966984472948788, + "tokens_seen": 1756364800 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003781263791374122, + "loss": 1.2798, + "theoretical_loss": 3.4966984472948788, + "tokens_seen": 1756364800 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003781103309929789, + "loss": 1.2315, + "theoretical_loss": 3.4966873798989653, + "tokens_seen": 1756430336 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003780942828485456, + "loss": 1.2191, + "theoretical_loss": 3.4966763130316116, + "tokens_seen": 1756495872 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003780782347041123, + "loss": 1.2519, + "theoretical_loss": 3.4966652466927726, + "tokens_seen": 1756561408 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037806218655967903, + "loss": 1.3357, + "theoretical_loss": 3.496654180882403, + "tokens_seen": 1756626944 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037804613841524573, + "loss": 1.2607, + "theoretical_loss": 3.4966431156004587, + "tokens_seen": 1756692480 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037803009027081243, + "loss": 1.2685, + "theoretical_loss": 3.496632050846894, + "tokens_seen": 1756758016 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037801404212637913, + "loss": 1.2434, + "theoretical_loss": 3.4966209866216644, + "tokens_seen": 1756823552 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037799799398194583, + "loss": 1.3465, + "theoretical_loss": 3.496609922924725, + "tokens_seen": 1756889088 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037798194583751253, + "loss": 1.3626, + "theoretical_loss": 3.4965988597560305, + "tokens_seen": 1756954624 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037796589769307923, + "loss": 1.4077, + "theoretical_loss": 3.496587797115536, + "tokens_seen": 1757020160 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037794984954864593, + "loss": 1.2519, + "theoretical_loss": 3.496576735003197, + "tokens_seen": 1757085696 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037793380140421263, + "loss": 1.2949, + "theoretical_loss": 3.496565673418968, + "tokens_seen": 1757151232 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037791775325977933, + "loss": 1.26, + "theoretical_loss": 3.4965546123628046, + "tokens_seen": 1757216768 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037790170511534603, + "loss": 1.299, + "theoretical_loss": 3.4965435518346615, + "tokens_seen": 1757282304 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037788565697091273, + "loss": 1.2714, + "theoretical_loss": 3.496532491834494, + "tokens_seen": 1757347840 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037786960882647944, + "loss": 1.2342, + "theoretical_loss": 3.4965214323622575, + "tokens_seen": 1757413376 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037785356068204614, + "loss": 1.4217, + "theoretical_loss": 3.496510373417907, + "tokens_seen": 1757478912 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037783751253761284, + "loss": 1.2714, + "theoretical_loss": 3.4964993150013974, + "tokens_seen": 1757544448 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037782146439317954, + "loss": 1.3605, + "theoretical_loss": 3.4964882571126834, + "tokens_seen": 1757609984 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037780541624874624, + "loss": 1.353, + "theoretical_loss": 3.496477199751721, + "tokens_seen": 1757675520 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037778936810431294, + "loss": 1.3367, + "theoretical_loss": 3.496466142918465, + "tokens_seen": 1757741056 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003777733199598797, + "loss": 1.3048, + "theoretical_loss": 3.4964550866128703, + "tokens_seen": 1757806592 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003777572718154464, + "loss": 1.3871, + "theoretical_loss": 3.4964440308348923, + "tokens_seen": 1757872128 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003777412236710131, + "loss": 1.2906, + "theoretical_loss": 3.4964329755844856, + "tokens_seen": 1757937664 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003777251755265798, + "loss": 1.431, + "theoretical_loss": 3.4964219208616067, + "tokens_seen": 1758003200 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003777091273821465, + "loss": 1.4062, + "theoretical_loss": 3.49641086666621, + "tokens_seen": 1758068736 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003776930792377132, + "loss": 1.2596, + "theoretical_loss": 3.4963998129982494, + "tokens_seen": 1758134272 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003776770310932799, + "loss": 1.1795, + "theoretical_loss": 3.496388759857682, + "tokens_seen": 1758199808 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003776609829488466, + "loss": 1.2705, + "theoretical_loss": 3.4963777072444624, + "tokens_seen": 1758265344 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003776449348044133, + "loss": 1.2101, + "theoretical_loss": 3.4963666551585453, + "tokens_seen": 1758330880 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037762888665998, + "loss": 1.3403, + "theoretical_loss": 3.496355603599887, + "tokens_seen": 1758396416 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003776128385155467, + "loss": 1.3518, + "theoretical_loss": 3.496344552568441, + "tokens_seen": 1758461952 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003775967903711134, + "loss": 1.2791, + "theoretical_loss": 3.4963335020641635, + "tokens_seen": 1758527488 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037758074222668005, + "loss": 1.3481, + "theoretical_loss": 3.4963224520870098, + "tokens_seen": 1758593024 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037756469408224675, + "loss": 1.1984, + "theoretical_loss": 3.496311402636935, + "tokens_seen": 1758658560 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037754864593781345, + "loss": 1.2815, + "theoretical_loss": 3.4963003537138944, + "tokens_seen": 1758724096 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037753259779338015, + "loss": 1.3507, + "theoretical_loss": 3.4962893053178425, + "tokens_seen": 1758789632 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037751654964894685, + "loss": 1.2083, + "theoretical_loss": 3.4962782574487354, + "tokens_seen": 1758855168 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037750050150451355, + "loss": 1.3454, + "theoretical_loss": 3.496267210106528, + "tokens_seen": 1758920704 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037748445336008025, + "loss": 1.2418, + "theoretical_loss": 3.496256163291176, + "tokens_seen": 1758986240 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037746840521564695, + "loss": 1.2514, + "theoretical_loss": 3.496245117002634, + "tokens_seen": 1759051776 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037745235707121365, + "loss": 1.3407, + "theoretical_loss": 3.496234071240857, + "tokens_seen": 1759117312 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037743630892678035, + "loss": 1.2648, + "theoretical_loss": 3.4962230260058016, + "tokens_seen": 1759182848 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037742026078234705, + "loss": 1.2837, + "theoretical_loss": 3.4962119812974217, + "tokens_seen": 1759248384 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037740421263791375, + "loss": 1.4213, + "theoretical_loss": 3.496200937115673, + "tokens_seen": 1759313920 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037738816449348045, + "loss": 1.27, + "theoretical_loss": 3.496189893460511, + "tokens_seen": 1759379456 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037737211634904715, + "loss": 1.3216, + "theoretical_loss": 3.4961788503318907, + "tokens_seen": 1759444992 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037735606820461386, + "loss": 1.2788, + "theoretical_loss": 3.4961678077297673, + "tokens_seen": 1759510528 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037734002006018056, + "loss": 1.3175, + "theoretical_loss": 3.4961567656540966, + "tokens_seen": 1759576064 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037732397191574726, + "loss": 1.3132, + "theoretical_loss": 3.4961457241048337, + "tokens_seen": 1759641600 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037730792377131396, + "loss": 1.2886, + "theoretical_loss": 3.4961346830819338, + "tokens_seen": 1759707136 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037729187562688066, + "loss": 1.2804, + "theoretical_loss": 3.496123642585352, + "tokens_seen": 1759772672 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037727582748244736, + "loss": 1.3371, + "theoretical_loss": 3.4961126026150433, + "tokens_seen": 1759838208 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037725977933801406, + "loss": 1.294, + "theoretical_loss": 3.4961015631709644, + "tokens_seen": 1759903744 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037724373119358076, + "loss": 1.2786, + "theoretical_loss": 3.496090524253069, + "tokens_seen": 1759969280 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037722768304914746, + "loss": 1.4194, + "theoretical_loss": 3.496079485861314, + "tokens_seen": 1760034816 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037721163490471416, + "loss": 1.3895, + "theoretical_loss": 3.496068447995653, + "tokens_seen": 1760100352 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037719558676028086, + "loss": 1.3548, + "theoretical_loss": 3.496057410656043, + "tokens_seen": 1760165888 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037717953861584756, + "loss": 1.3332, + "theoretical_loss": 3.4960463738424385, + "tokens_seen": 1760231424 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037716349047141426, + "loss": 1.467, + "theoretical_loss": 3.496035337554795, + "tokens_seen": 1760296960 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037714744232698096, + "loss": 1.2277, + "theoretical_loss": 3.4960243017930672, + "tokens_seen": 1760362496 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037713139418254766, + "loss": 1.3594, + "theoretical_loss": 3.4960132665572115, + "tokens_seen": 1760428032 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037711534603811437, + "loss": 1.2322, + "theoretical_loss": 3.496002231847182, + "tokens_seen": 1760493568 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037709929789368107, + "loss": 1.2952, + "theoretical_loss": 3.495991197662936, + "tokens_seen": 1760559104 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037708324974924777, + "loss": 1.2471, + "theoretical_loss": 3.4959801640044272, + "tokens_seen": 1760624640 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037706720160481447, + "loss": 1.2729, + "theoretical_loss": 3.495969130871612, + "tokens_seen": 1760690176 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037705115346038117, + "loss": 1.3408, + "theoretical_loss": 3.4959580982644445, + "tokens_seen": 1760755712 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037703510531594787, + "loss": 1.2913, + "theoretical_loss": 3.4959470661828815, + "tokens_seen": 1760821248 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037701905717151457, + "loss": 1.3064, + "theoretical_loss": 3.495936034626878, + "tokens_seen": 1760886784 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037700300902708127, + "loss": 1.344, + "theoretical_loss": 3.4959250035963887, + "tokens_seen": 1760952320 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037698696088264797, + "loss": 1.2255, + "theoretical_loss": 3.49591397309137, + "tokens_seen": 1761017856 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037697091273821467, + "loss": 1.3125, + "theoretical_loss": 3.495902943111777, + "tokens_seen": 1761083392 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037695486459378137, + "loss": 1.2892, + "theoretical_loss": 3.4958919136575646, + "tokens_seen": 1761148928 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037693881644934807, + "loss": 1.2298, + "theoretical_loss": 3.4958808847286886, + "tokens_seen": 1761214464 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037692276830491477, + "loss": 1.2775, + "theoretical_loss": 3.495869856325105, + "tokens_seen": 1761280000 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003769067201604815, + "loss": 1.2953, + "theoretical_loss": 3.4958588284467678, + "tokens_seen": 1761345536 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003768906720160482, + "loss": 1.3371, + "theoretical_loss": 3.4958478010936336, + "tokens_seen": 1761411072 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003768746238716149, + "loss": 1.1876, + "theoretical_loss": 3.4958367742656575, + "tokens_seen": 1761476608 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003768585757271816, + "loss": 1.2331, + "theoretical_loss": 3.4958257479627957, + "tokens_seen": 1761542144 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003768425275827483, + "loss": 1.2259, + "theoretical_loss": 3.495814722185002, + "tokens_seen": 1761607680 + }, + { + "epoch": 0.53, + "learning_rate": 0.000376826479438315, + "loss": 1.2879, + "theoretical_loss": 3.4958036969322337, + "tokens_seen": 1761673216 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003768104312938816, + "loss": 1.2418, + "theoretical_loss": 3.495792672204445, + "tokens_seen": 1761738752 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003767943831494483, + "loss": 1.3548, + "theoretical_loss": 3.495781648001592, + "tokens_seen": 1761804288 + }, + { + "epoch": 0.53, + "learning_rate": 0.000376778335005015, + "loss": 1.2231, + "theoretical_loss": 3.4957706243236295, + "tokens_seen": 1761869824 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003767622868605817, + "loss": 1.2438, + "theoretical_loss": 3.495759601170514, + "tokens_seen": 1761935360 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003767462387161484, + "loss": 1.3131, + "theoretical_loss": 3.4957485785422, + "tokens_seen": 1762000896 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003767301905717151, + "loss": 1.3385, + "theoretical_loss": 3.4957375564386437, + "tokens_seen": 1762066432 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003767141424272818, + "loss": 1.3064, + "theoretical_loss": 3.4957265348598, + "tokens_seen": 1762131968 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037669809428284853, + "loss": 1.2705, + "theoretical_loss": 3.495715513805625, + "tokens_seen": 1762197504 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037668204613841523, + "loss": 1.4267, + "theoretical_loss": 3.495704493276074, + "tokens_seen": 1762263040 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037666599799398193, + "loss": 1.1838, + "theoretical_loss": 3.4956934732711025, + "tokens_seen": 1762328576 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037664994984954863, + "loss": 1.264, + "theoretical_loss": 3.4956824537906654, + "tokens_seen": 1762394112 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037663390170511533, + "loss": 1.3502, + "theoretical_loss": 3.4956714348347195, + "tokens_seen": 1762459648 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037661785356068203, + "loss": 1.2807, + "theoretical_loss": 3.4956604164032195, + "tokens_seen": 1762525184 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037660180541624873, + "loss": 1.3935, + "theoretical_loss": 3.4956493984961208, + "tokens_seen": 1762590720 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037658575727181543, + "loss": 1.3137, + "theoretical_loss": 3.4956383811133795, + "tokens_seen": 1762656256 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003765697091273822, + "loss": 1.2815, + "theoretical_loss": 3.4956273642549514, + "tokens_seen": 1762721792 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003765536609829489, + "loss": 1.2849, + "theoretical_loss": 3.495616347920791, + "tokens_seen": 1762787328 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003765376128385156, + "loss": 1.3091, + "theoretical_loss": 3.4956053321108547, + "tokens_seen": 1762852864 + }, + { + "epoch": 0.53, + "objective/train/docs_used": 653938, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3410481214523315, + "objective/train/theoretical_loss": 3.4955943168250974, + "objective/train/tokens_used": 1783378400, + "theoretical_loss": 3.4955943168250974, + "tokens_seen": 1762918400 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003765215646940823, + "loss": 1.341, + "theoretical_loss": 3.4955943168250974, + "tokens_seen": 1762918400 + }, + { + "epoch": 0.53, + "learning_rate": 0.000376505516549649, + "loss": 1.2773, + "theoretical_loss": 3.4955833020634755, + "tokens_seen": 1762983936 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003764894684052157, + "loss": 1.2225, + "theoretical_loss": 3.4955722878259436, + "tokens_seen": 1763049472 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003764734202607824, + "loss": 1.4515, + "theoretical_loss": 3.495561274112458, + "tokens_seen": 1763115008 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003764573721163491, + "loss": 1.3162, + "theoretical_loss": 3.4955502609229745, + "tokens_seen": 1763180544 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003764413239719158, + "loss": 1.2025, + "theoretical_loss": 3.4955392482574474, + "tokens_seen": 1763246080 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003764252758274825, + "loss": 1.3026, + "theoretical_loss": 3.4955282361158337, + "tokens_seen": 1763311616 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003764092276830492, + "loss": 1.3182, + "theoretical_loss": 3.495517224498089, + "tokens_seen": 1763377152 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003763931795386159, + "loss": 1.3047, + "theoretical_loss": 3.495506213404168, + "tokens_seen": 1763442688 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003763771313941826, + "loss": 1.3162, + "theoretical_loss": 3.495495202834027, + "tokens_seen": 1763508224 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003763610832497493, + "loss": 1.2838, + "theoretical_loss": 3.495484192787621, + "tokens_seen": 1763573760 + }, + { + "epoch": 0.53, + "learning_rate": 0.000376345035105316, + "loss": 1.3695, + "theoretical_loss": 3.495473183264906, + "tokens_seen": 1763639296 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003763289869608827, + "loss": 1.277, + "theoretical_loss": 3.495462174265838, + "tokens_seen": 1763704832 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003763129388164494, + "loss": 1.3029, + "theoretical_loss": 3.4954511657903717, + "tokens_seen": 1763770368 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003762968906720161, + "loss": 1.229, + "theoretical_loss": 3.4954401578384635, + "tokens_seen": 1763835904 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003762808425275828, + "loss": 1.3564, + "theoretical_loss": 3.495429150410069, + "tokens_seen": 1763901440 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003762647943831495, + "loss": 1.242, + "theoretical_loss": 3.4954181435051437, + "tokens_seen": 1763966976 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003762487462387162, + "loss": 1.3038, + "theoretical_loss": 3.495407137123643, + "tokens_seen": 1764032512 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003762326980942829, + "loss": 1.4232, + "theoretical_loss": 3.495396131265523, + "tokens_seen": 1764098048 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003762166499498496, + "loss": 1.2513, + "theoretical_loss": 3.495385125930739, + "tokens_seen": 1764163584 + }, + { + "epoch": 0.53, + "learning_rate": 0.0003762006018054163, + "loss": 1.2524, + "theoretical_loss": 3.495374121119247, + "tokens_seen": 1764229120 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037618455366098295, + "loss": 1.274, + "theoretical_loss": 3.4953631168310024, + "tokens_seen": 1764294656 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037616850551654965, + "loss": 1.3017, + "theoretical_loss": 3.4953521130659615, + "tokens_seen": 1764360192 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037615245737211635, + "loss": 1.2971, + "theoretical_loss": 3.4953411098240785, + "tokens_seen": 1764425728 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037613640922768305, + "loss": 1.2973, + "theoretical_loss": 3.4953301071053113, + "tokens_seen": 1764491264 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037612036108324975, + "loss": 1.2967, + "theoretical_loss": 3.4953191049096137, + "tokens_seen": 1764556800 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037610431293881645, + "loss": 1.3349, + "theoretical_loss": 3.495308103236942, + "tokens_seen": 1764622336 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037608826479438315, + "loss": 1.3216, + "theoretical_loss": 3.4952971020872527, + "tokens_seen": 1764687872 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037607221664994985, + "loss": 1.2522, + "theoretical_loss": 3.4952861014605006, + "tokens_seen": 1764753408 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037605616850551655, + "loss": 1.3048, + "theoretical_loss": 3.4952751013566417, + "tokens_seen": 1764818944 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037604012036108325, + "loss": 1.4214, + "theoretical_loss": 3.4952641017756316, + "tokens_seen": 1764884480 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037602407221664995, + "loss": 1.3009, + "theoretical_loss": 3.495253102717426, + "tokens_seen": 1764950016 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037600802407221665, + "loss": 1.24, + "theoretical_loss": 3.495242104181981, + "tokens_seen": 1765015552 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037599197592778336, + "loss": 1.3131, + "theoretical_loss": 3.495231106169252, + "tokens_seen": 1765081088 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037597592778335006, + "loss": 1.1849, + "theoretical_loss": 3.495220108679195, + "tokens_seen": 1765146624 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037595987963891676, + "loss": 1.4106, + "theoretical_loss": 3.4952091117117656, + "tokens_seen": 1765212160 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037594383149448346, + "loss": 1.345, + "theoretical_loss": 3.49519811526692, + "tokens_seen": 1765277696 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037592778335005016, + "loss": 1.2871, + "theoretical_loss": 3.495187119344613, + "tokens_seen": 1765343232 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037591173520561686, + "loss": 1.2209, + "theoretical_loss": 3.495176123944801, + "tokens_seen": 1765408768 + }, + { + "epoch": 0.53, + "learning_rate": 0.00037589568706118356, + "loss": 1.2446, + "theoretical_loss": 3.4951651290674395, + "tokens_seen": 1765474304 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037587963891675026, + "loss": 1.2112, + "theoretical_loss": 3.495154134712485, + "tokens_seen": 1765539840 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037586359077231696, + "loss": 1.2841, + "theoretical_loss": 3.4951431408798928, + "tokens_seen": 1765605376 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037584754262788366, + "loss": 1.2833, + "theoretical_loss": 3.4951321475696187, + "tokens_seen": 1765670912 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037583149448345036, + "loss": 1.2843, + "theoretical_loss": 3.495121154781618, + "tokens_seen": 1765736448 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037581544633901706, + "loss": 1.3418, + "theoretical_loss": 3.4951101625158474, + "tokens_seen": 1765801984 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037579939819458376, + "loss": 1.3131, + "theoretical_loss": 3.495099170772262, + "tokens_seen": 1765867520 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037578335005015046, + "loss": 1.2469, + "theoretical_loss": 3.4950881795508177, + "tokens_seen": 1765933056 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037576730190571716, + "loss": 1.3526, + "theoretical_loss": 3.4950771888514716, + "tokens_seen": 1765998592 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037575125376128386, + "loss": 1.2437, + "theoretical_loss": 3.495066198674177, + "tokens_seen": 1766064128 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037573520561685057, + "loss": 1.2676, + "theoretical_loss": 3.495055209018892, + "tokens_seen": 1766129664 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037571915747241727, + "loss": 1.311, + "theoretical_loss": 3.495044219885572, + "tokens_seen": 1766195200 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037570310932798397, + "loss": 1.2575, + "theoretical_loss": 3.4950332312741716, + "tokens_seen": 1766260736 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037568706118355067, + "loss": 1.1756, + "theoretical_loss": 3.495022243184648, + "tokens_seen": 1766326272 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037567101303911737, + "loss": 1.2839, + "theoretical_loss": 3.495011255616957, + "tokens_seen": 1766391808 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037565496489468407, + "loss": 1.2457, + "theoretical_loss": 3.4950002685710535, + "tokens_seen": 1766457344 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037563891675025077, + "loss": 1.4325, + "theoretical_loss": 3.4949892820468937, + "tokens_seen": 1766522880 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037562286860581747, + "loss": 1.1932, + "theoretical_loss": 3.494978296044434, + "tokens_seen": 1766588416 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037560682046138417, + "loss": 1.313, + "theoretical_loss": 3.49496731056363, + "tokens_seen": 1766653952 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037559077231695087, + "loss": 1.2789, + "theoretical_loss": 3.494956325604437, + "tokens_seen": 1766719488 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037557472417251757, + "loss": 1.3075, + "theoretical_loss": 3.4949453411668117, + "tokens_seen": 1766785024 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037555867602808427, + "loss": 1.2391, + "theoretical_loss": 3.4949343572507097, + "tokens_seen": 1766850560 + }, + { + "epoch": 0.54, + "learning_rate": 0.000375542627883651, + "loss": 1.1837, + "theoretical_loss": 3.4949233738560874, + "tokens_seen": 1766916096 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003755265797392177, + "loss": 1.3241, + "theoretical_loss": 3.4949123909828996, + "tokens_seen": 1766981632 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003755105315947844, + "loss": 1.2379, + "theoretical_loss": 3.4949014086311028, + "tokens_seen": 1767047168 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003754944834503511, + "loss": 1.2503, + "theoretical_loss": 3.4948904268006533, + "tokens_seen": 1767112704 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003754784353059178, + "loss": 1.1863, + "theoretical_loss": 3.494879445491506, + "tokens_seen": 1767178240 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003754623871614845, + "loss": 1.3481, + "theoretical_loss": 3.494868464703618, + "tokens_seen": 1767243776 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003754463390170512, + "loss": 1.3621, + "theoretical_loss": 3.4948574844369444, + "tokens_seen": 1767309312 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003754302908726179, + "loss": 1.3782, + "theoretical_loss": 3.4948465046914414, + "tokens_seen": 1767374848 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003754142427281845, + "loss": 1.3183, + "theoretical_loss": 3.494835525467065, + "tokens_seen": 1767440384 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003753981945837512, + "loss": 1.3075, + "theoretical_loss": 3.494824546763771, + "tokens_seen": 1767505920 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003753821464393179, + "loss": 1.2815, + "theoretical_loss": 3.494813568581516, + "tokens_seen": 1767571456 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003753660982948847, + "loss": 1.3812, + "theoretical_loss": 3.494802590920255, + "tokens_seen": 1767636992 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003753500501504514, + "loss": 1.227, + "theoretical_loss": 3.494791613779944, + "tokens_seen": 1767702528 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003753340020060181, + "loss": 1.2932, + "theoretical_loss": 3.49478063716054, + "tokens_seen": 1767768064 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003753179538615848, + "loss": 1.3219, + "theoretical_loss": 3.494769661061998, + "tokens_seen": 1767833600 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003753019057171515, + "loss": 1.3256, + "theoretical_loss": 3.494758685484274, + "tokens_seen": 1767899136 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003752858575727182, + "loss": 1.3479, + "theoretical_loss": 3.4947477104273243, + "tokens_seen": 1767964672 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003752698094282849, + "loss": 1.2814, + "theoretical_loss": 3.494736735891105, + "tokens_seen": 1768030208 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003752537612838516, + "loss": 1.2547, + "theoretical_loss": 3.494725761875572, + "tokens_seen": 1768095744 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003752377131394183, + "loss": 1.3023, + "theoretical_loss": 3.494714788380681, + "tokens_seen": 1768161280 + }, + { + "epoch": 0.54, + "learning_rate": 0.000375221664994985, + "loss": 1.2832, + "theoretical_loss": 3.494703815406388, + "tokens_seen": 1768226816 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003752056168505517, + "loss": 1.3405, + "theoretical_loss": 3.4946928429526496, + "tokens_seen": 1768292352 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003751895687061184, + "loss": 1.2642, + "theoretical_loss": 3.494681871019422, + "tokens_seen": 1768357888 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003751735205616851, + "loss": 1.2455, + "theoretical_loss": 3.4946708996066596, + "tokens_seen": 1768423424 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003751574724172518, + "loss": 1.259, + "theoretical_loss": 3.49465992871432, + "tokens_seen": 1768488960 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003751414242728185, + "loss": 1.3318, + "theoretical_loss": 3.494648958342359, + "tokens_seen": 1768554496 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003751253761283852, + "loss": 1.3421, + "theoretical_loss": 3.494637988490732, + "tokens_seen": 1768620032 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003751093279839519, + "loss": 1.3969, + "theoretical_loss": 3.4946270191593958, + "tokens_seen": 1768685568 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003750932798395186, + "loss": 1.2822, + "theoretical_loss": 3.4946160503483057, + "tokens_seen": 1768751104 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003750772316950853, + "loss": 1.4315, + "theoretical_loss": 3.494605082057418, + "tokens_seen": 1768816640 + }, + { + "epoch": 0.54, + "learning_rate": 0.000375061183550652, + "loss": 1.3198, + "theoretical_loss": 3.4945941142866888, + "tokens_seen": 1768882176 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003750451354062187, + "loss": 1.301, + "theoretical_loss": 3.494583147036075, + "tokens_seen": 1768947712 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003750290872617854, + "loss": 1.3133, + "theoretical_loss": 3.494572180305531, + "tokens_seen": 1769013248 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003750130391173521, + "loss": 1.2688, + "theoretical_loss": 3.494561214095014, + "tokens_seen": 1769078784 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003749969909729188, + "loss": 1.3001, + "theoretical_loss": 3.4945502484044804, + "tokens_seen": 1769144320 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003749809428284855, + "loss": 1.3247, + "theoretical_loss": 3.494539283233885, + "tokens_seen": 1769209856 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003749648946840522, + "loss": 1.392, + "theoretical_loss": 3.4945283185831846, + "tokens_seen": 1769275392 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003749488465396189, + "loss": 1.2468, + "theoretical_loss": 3.4945173544523356, + "tokens_seen": 1769340928 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003749327983951856, + "loss": 1.3476, + "theoretical_loss": 3.494506390841294, + "tokens_seen": 1769406464 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 656308, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.251791000366211, + "objective/train/theoretical_loss": 3.4944954277500155, + "objective/train/tokens_used": 1789932000, + "theoretical_loss": 3.4944954277500155, + "tokens_seen": 1769472000 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003749167502507523, + "loss": 1.2518, + "theoretical_loss": 3.4944954277500155, + "tokens_seen": 1769472000 + }, + { + "epoch": 0.54, + "learning_rate": 0.000374900702106319, + "loss": 1.4073, + "theoretical_loss": 3.4944844651784566, + "tokens_seen": 1769537536 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003748846539618857, + "loss": 1.3567, + "theoretical_loss": 3.4944735031265726, + "tokens_seen": 1769603072 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003748686058174524, + "loss": 1.2866, + "theoretical_loss": 3.494462541594321, + "tokens_seen": 1769668608 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003748525576730191, + "loss": 1.2437, + "theoretical_loss": 3.4944515805816563, + "tokens_seen": 1769734144 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003748365095285858, + "loss": 1.2296, + "theoretical_loss": 3.494440620088536, + "tokens_seen": 1769799680 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003748204613841525, + "loss": 1.2199, + "theoretical_loss": 3.494429660114916, + "tokens_seen": 1769865216 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003748044132397192, + "loss": 1.4278, + "theoretical_loss": 3.494418700660752, + "tokens_seen": 1769930752 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003747883650952859, + "loss": 1.3424, + "theoretical_loss": 3.4944077417260004, + "tokens_seen": 1769996288 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037477231695085255, + "loss": 1.3061, + "theoretical_loss": 3.4943967833106173, + "tokens_seen": 1770061824 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037475626880641925, + "loss": 1.3135, + "theoretical_loss": 3.4943858254145583, + "tokens_seen": 1770127360 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037474022066198595, + "loss": 1.3944, + "theoretical_loss": 3.494374868037781, + "tokens_seen": 1770192896 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037472417251755265, + "loss": 1.2357, + "theoretical_loss": 3.49436391118024, + "tokens_seen": 1770258432 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037470812437311935, + "loss": 1.3846, + "theoretical_loss": 3.4943529548418923, + "tokens_seen": 1770323968 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037469207622868605, + "loss": 1.2643, + "theoretical_loss": 3.4943419990226943, + "tokens_seen": 1770389504 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037467602808425275, + "loss": 1.2761, + "theoretical_loss": 3.4943310437226014, + "tokens_seen": 1770455040 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037465997993981945, + "loss": 1.3058, + "theoretical_loss": 3.49432008894157, + "tokens_seen": 1770520576 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037464393179538615, + "loss": 1.2449, + "theoretical_loss": 3.4943091346795567, + "tokens_seen": 1770586112 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037462788365095286, + "loss": 1.3193, + "theoretical_loss": 3.494298180936518, + "tokens_seen": 1770651648 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037461183550651956, + "loss": 1.3395, + "theoretical_loss": 3.4942872277124093, + "tokens_seen": 1770717184 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037459578736208626, + "loss": 1.262, + "theoretical_loss": 3.494276275007187, + "tokens_seen": 1770782720 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037457973921765296, + "loss": 1.3924, + "theoretical_loss": 3.494265322820808, + "tokens_seen": 1770848256 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037456369107321966, + "loss": 1.242, + "theoretical_loss": 3.494254371153227, + "tokens_seen": 1770913792 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037454764292878636, + "loss": 1.3509, + "theoretical_loss": 3.4942434200044015, + "tokens_seen": 1770979328 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037453159478435306, + "loss": 1.253, + "theoretical_loss": 3.494232469374287, + "tokens_seen": 1771044864 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037451554663991976, + "loss": 1.3111, + "theoretical_loss": 3.494221519262841, + "tokens_seen": 1771110400 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037449949849548646, + "loss": 1.2814, + "theoretical_loss": 3.4942105696700185, + "tokens_seen": 1771175936 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037448345035105316, + "loss": 1.4261, + "theoretical_loss": 3.4941996205957766, + "tokens_seen": 1771241472 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037446740220661986, + "loss": 1.4178, + "theoretical_loss": 3.4941886720400706, + "tokens_seen": 1771307008 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037445135406218656, + "loss": 1.3685, + "theoretical_loss": 3.4941777240028573, + "tokens_seen": 1771372544 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037443530591775326, + "loss": 1.2077, + "theoretical_loss": 3.494166776484093, + "tokens_seen": 1771438080 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037441925777331996, + "loss": 1.2986, + "theoretical_loss": 3.494155829483734, + "tokens_seen": 1771503616 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037440320962888666, + "loss": 1.2789, + "theoretical_loss": 3.494144883001736, + "tokens_seen": 1771569152 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037438716148445336, + "loss": 1.2659, + "theoretical_loss": 3.4941339370380566, + "tokens_seen": 1771634688 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037437111334002007, + "loss": 1.254, + "theoretical_loss": 3.4941229915926506, + "tokens_seen": 1771700224 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037435506519558677, + "loss": 1.4098, + "theoretical_loss": 3.4941120466654745, + "tokens_seen": 1771765760 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037433901705115347, + "loss": 1.2354, + "theoretical_loss": 3.4941011022564856, + "tokens_seen": 1771831296 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037432296890672017, + "loss": 1.3195, + "theoretical_loss": 3.4940901583656396, + "tokens_seen": 1771896832 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037430692076228687, + "loss": 1.2883, + "theoretical_loss": 3.494079214992893, + "tokens_seen": 1771962368 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037429087261785357, + "loss": 1.3291, + "theoretical_loss": 3.4940682721382017, + "tokens_seen": 1772027904 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037427482447342027, + "loss": 1.3505, + "theoretical_loss": 3.4940573298015223, + "tokens_seen": 1772093440 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037425877632898697, + "loss": 1.2915, + "theoretical_loss": 3.494046387982811, + "tokens_seen": 1772158976 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037424272818455367, + "loss": 1.3769, + "theoretical_loss": 3.494035446682024, + "tokens_seen": 1772224512 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037422668004012037, + "loss": 1.2896, + "theoretical_loss": 3.494024505899118, + "tokens_seen": 1772290048 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037421063189568707, + "loss": 1.2876, + "theoretical_loss": 3.4940135656340496, + "tokens_seen": 1772355584 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037419458375125377, + "loss": 1.3338, + "theoretical_loss": 3.4940026258867745, + "tokens_seen": 1772421120 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037417853560682047, + "loss": 1.34, + "theoretical_loss": 3.4939916866572487, + "tokens_seen": 1772486656 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037416248746238723, + "loss": 1.3064, + "theoretical_loss": 3.4939807479454297, + "tokens_seen": 1772552192 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003741464393179539, + "loss": 1.3602, + "theoretical_loss": 3.493969809751273, + "tokens_seen": 1772617728 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003741303911735206, + "loss": 1.2882, + "theoretical_loss": 3.4939588720747357, + "tokens_seen": 1772683264 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003741143430290873, + "loss": 1.3278, + "theoretical_loss": 3.493947934915773, + "tokens_seen": 1772748800 + }, + { + "epoch": 0.54, + "learning_rate": 0.000374098294884654, + "loss": 1.3808, + "theoretical_loss": 3.493936998274343, + "tokens_seen": 1772814336 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003740822467402207, + "loss": 1.3096, + "theoretical_loss": 3.4939260621504005, + "tokens_seen": 1772879872 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003740661985957874, + "loss": 1.1807, + "theoretical_loss": 3.493915126543902, + "tokens_seen": 1772945408 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003740501504513541, + "loss": 1.2776, + "theoretical_loss": 3.493904191454805, + "tokens_seen": 1773010944 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003740341023069208, + "loss": 1.3458, + "theoretical_loss": 3.493893256883065, + "tokens_seen": 1773076480 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003740180541624875, + "loss": 1.3917, + "theoretical_loss": 3.4938823228286386, + "tokens_seen": 1773142016 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003740020060180542, + "loss": 1.234, + "theoretical_loss": 3.493871389291482, + "tokens_seen": 1773207552 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003739859578736209, + "loss": 1.2791, + "theoretical_loss": 3.4938604562715527, + "tokens_seen": 1773273088 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003739699097291876, + "loss": 1.3543, + "theoretical_loss": 3.4938495237688056, + "tokens_seen": 1773338624 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003739538615847543, + "loss": 1.3617, + "theoretical_loss": 3.4938385917831978, + "tokens_seen": 1773404160 + }, + { + "epoch": 0.54, + "learning_rate": 0.000373937813440321, + "loss": 1.3319, + "theoretical_loss": 3.493827660314686, + "tokens_seen": 1773469696 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003739217652958877, + "loss": 1.2857, + "theoretical_loss": 3.4938167293632265, + "tokens_seen": 1773535232 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003739057171514544, + "loss": 1.2831, + "theoretical_loss": 3.493805798928775, + "tokens_seen": 1773600768 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003738896690070211, + "loss": 1.275, + "theoretical_loss": 3.493794869011289, + "tokens_seen": 1773666304 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003738736208625878, + "loss": 1.2863, + "theoretical_loss": 3.493783939610725, + "tokens_seen": 1773731840 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003738575727181545, + "loss": 1.2208, + "theoretical_loss": 3.493773010727038, + "tokens_seen": 1773797376 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003738415245737212, + "loss": 1.2966, + "theoretical_loss": 3.493762082360186, + "tokens_seen": 1773862912 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003738254764292879, + "loss": 1.2457, + "theoretical_loss": 3.4937511545101243, + "tokens_seen": 1773928448 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003738094282848546, + "loss": 1.3816, + "theoretical_loss": 3.4937402271768105, + "tokens_seen": 1773993984 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003737933801404213, + "loss": 1.383, + "theoretical_loss": 3.493729300360201, + "tokens_seen": 1774059520 + }, + { + "epoch": 0.54, + "learning_rate": 0.000373777331995988, + "loss": 1.3613, + "theoretical_loss": 3.4937183740602507, + "tokens_seen": 1774125056 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003737612838515547, + "loss": 1.3548, + "theoretical_loss": 3.4937074482769175, + "tokens_seen": 1774190592 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003737452357071214, + "loss": 1.2953, + "theoretical_loss": 3.4936965230101578, + "tokens_seen": 1774256128 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003737291875626881, + "loss": 1.2836, + "theoretical_loss": 3.493685598259928, + "tokens_seen": 1774321664 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003737131394182548, + "loss": 1.2464, + "theoretical_loss": 3.4936746740261846, + "tokens_seen": 1774387200 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003736970912738215, + "loss": 1.3198, + "theoretical_loss": 3.4936637503088837, + "tokens_seen": 1774452736 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003736810431293882, + "loss": 1.2822, + "theoretical_loss": 3.493652827107982, + "tokens_seen": 1774518272 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003736649949849549, + "loss": 1.2365, + "theoretical_loss": 3.493641904423437, + "tokens_seen": 1774583808 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003736489468405216, + "loss": 1.2811, + "theoretical_loss": 3.493630982255203, + "tokens_seen": 1774649344 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003736328986960883, + "loss": 1.2349, + "theoretical_loss": 3.493620060603239, + "tokens_seen": 1774714880 + }, + { + "epoch": 0.54, + "learning_rate": 0.000373616850551655, + "loss": 1.2782, + "theoretical_loss": 3.4936091394674995, + "tokens_seen": 1774780416 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003736008024072217, + "loss": 1.311, + "theoretical_loss": 3.4935982188479424, + "tokens_seen": 1774845952 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003735847542627884, + "loss": 1.3148, + "theoretical_loss": 3.493587298744524, + "tokens_seen": 1774911488 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003735687061183551, + "loss": 1.335, + "theoretical_loss": 3.4935763791572003, + "tokens_seen": 1774977024 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003735526579739218, + "loss": 1.3494, + "theoretical_loss": 3.4935654600859283, + "tokens_seen": 1775042560 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003735366098294885, + "loss": 1.3866, + "theoretical_loss": 3.4935545415306644, + "tokens_seen": 1775108096 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003735205616850552, + "loss": 1.262, + "theoretical_loss": 3.4935436234913655, + "tokens_seen": 1775173632 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003735045135406219, + "loss": 1.2552, + "theoretical_loss": 3.4935327059679873, + "tokens_seen": 1775239168 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003734884653961886, + "loss": 1.546, + "theoretical_loss": 3.4935217889604875, + "tokens_seen": 1775304704 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003734724172517553, + "loss": 1.1364, + "theoretical_loss": 3.493510872468822, + "tokens_seen": 1775370240 + }, + { + "epoch": 0.54, + "learning_rate": 0.000373456369107322, + "loss": 1.3052, + "theoretical_loss": 3.493499956492948, + "tokens_seen": 1775435776 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003734403209628887, + "loss": 1.2639, + "theoretical_loss": 3.493489041032821, + "tokens_seen": 1775501312 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003734242728184554, + "loss": 1.316, + "theoretical_loss": 3.493478126088398, + "tokens_seen": 1775566848 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003734082246740221, + "loss": 1.2847, + "theoretical_loss": 3.4934672116596364, + "tokens_seen": 1775632384 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003733921765295888, + "loss": 1.2441, + "theoretical_loss": 3.4934562977464925, + "tokens_seen": 1775697920 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037337612838515545, + "loss": 1.3006, + "theoretical_loss": 3.493445384348922, + "tokens_seen": 1775763456 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037336008024072215, + "loss": 1.3011, + "theoretical_loss": 3.493434471466882, + "tokens_seen": 1775828992 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037334403209628885, + "loss": 1.2988, + "theoretical_loss": 3.49342355910033, + "tokens_seen": 1775894528 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037332798395185555, + "loss": 1.2691, + "theoretical_loss": 3.4934126472492215, + "tokens_seen": 1775960064 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 658604, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3022226095199585, + "objective/train/theoretical_loss": 3.4934017359135137, + "objective/train/tokens_used": 1796485600, + "theoretical_loss": 3.4934017359135137, + "tokens_seen": 1776025600 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037331193580742225, + "loss": 1.3022, + "theoretical_loss": 3.4934017359135137, + "tokens_seen": 1776025600 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037329588766298895, + "loss": 1.2731, + "theoretical_loss": 3.4933908250931633, + "tokens_seen": 1776091136 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037327983951855565, + "loss": 1.3107, + "theoretical_loss": 3.493379914788126, + "tokens_seen": 1776156672 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037326379137412235, + "loss": 1.2608, + "theoretical_loss": 3.49336900499836, + "tokens_seen": 1776222208 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037324774322968906, + "loss": 1.369, + "theoretical_loss": 3.4933580957238206, + "tokens_seen": 1776287744 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037323169508525576, + "loss": 1.2272, + "theoretical_loss": 3.4933471869644652, + "tokens_seen": 1776353280 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037321564694082246, + "loss": 1.3714, + "theoretical_loss": 3.4933362787202507, + "tokens_seen": 1776418816 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037319959879638916, + "loss": 1.3204, + "theoretical_loss": 3.4933253709911325, + "tokens_seen": 1776484352 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037318355065195586, + "loss": 1.2744, + "theoretical_loss": 3.493314463777069, + "tokens_seen": 1776549888 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037316750250752256, + "loss": 1.2817, + "theoretical_loss": 3.493303557078015, + "tokens_seen": 1776615424 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037315145436308926, + "loss": 1.2684, + "theoretical_loss": 3.493292650893929, + "tokens_seen": 1776680960 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037313540621865596, + "loss": 1.2321, + "theoretical_loss": 3.4932817452247664, + "tokens_seen": 1776746496 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037311935807422266, + "loss": 1.3869, + "theoretical_loss": 3.493270840070484, + "tokens_seen": 1776812032 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037310330992978936, + "loss": 1.343, + "theoretical_loss": 3.4932599354310403, + "tokens_seen": 1776877568 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037308726178535606, + "loss": 1.2109, + "theoretical_loss": 3.493249031306389, + "tokens_seen": 1776943104 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037307121364092276, + "loss": 1.243, + "theoretical_loss": 3.4932381276964897, + "tokens_seen": 1777008640 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037305516549648946, + "loss": 1.2812, + "theoretical_loss": 3.4932272246012968, + "tokens_seen": 1777074176 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037303911735205616, + "loss": 1.1808, + "theoretical_loss": 3.4932163220207686, + "tokens_seen": 1777139712 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037302306920762286, + "loss": 1.3042, + "theoretical_loss": 3.493205419954861, + "tokens_seen": 1777205248 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037300702106318956, + "loss": 1.2571, + "theoretical_loss": 3.4931945184035307, + "tokens_seen": 1777270784 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037299097291875627, + "loss": 1.3308, + "theoretical_loss": 3.493183617366735, + "tokens_seen": 1777336320 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037297492477432297, + "loss": 1.3077, + "theoretical_loss": 3.4931727168444304, + "tokens_seen": 1777401856 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003729588766298897, + "loss": 1.3282, + "theoretical_loss": 3.4931618168365737, + "tokens_seen": 1777467392 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003729428284854564, + "loss": 1.2921, + "theoretical_loss": 3.4931509173431214, + "tokens_seen": 1777532928 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003729267803410231, + "loss": 1.2497, + "theoretical_loss": 3.4931400183640307, + "tokens_seen": 1777598464 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003729107321965898, + "loss": 1.2586, + "theoretical_loss": 3.4931291198992582, + "tokens_seen": 1777664000 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003728946840521565, + "loss": 1.2318, + "theoretical_loss": 3.49311822194876, + "tokens_seen": 1777729536 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003728786359077232, + "loss": 1.2395, + "theoretical_loss": 3.493107324512494, + "tokens_seen": 1777795072 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003728625877632899, + "loss": 1.3368, + "theoretical_loss": 3.493096427590416, + "tokens_seen": 1777860608 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003728465396188566, + "loss": 1.2651, + "theoretical_loss": 3.4930855311824835, + "tokens_seen": 1777926144 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003728304914744233, + "loss": 1.3089, + "theoretical_loss": 3.493074635288653, + "tokens_seen": 1777991680 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037281444332999, + "loss": 1.2784, + "theoretical_loss": 3.493063739908881, + "tokens_seen": 1778057216 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037279839518555673, + "loss": 1.2582, + "theoretical_loss": 3.493052845043125, + "tokens_seen": 1778122752 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037278234704112343, + "loss": 1.289, + "theoretical_loss": 3.493041950691341, + "tokens_seen": 1778188288 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037276629889669013, + "loss": 1.3668, + "theoretical_loss": 3.4930310568534866, + "tokens_seen": 1778253824 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003727502507522568, + "loss": 1.246, + "theoretical_loss": 3.4930201635295175, + "tokens_seen": 1778319360 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003727342026078235, + "loss": 1.3166, + "theoretical_loss": 3.493009270719392, + "tokens_seen": 1778384896 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003727181544633902, + "loss": 1.2615, + "theoretical_loss": 3.4929983784230663, + "tokens_seen": 1778450432 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003727021063189569, + "loss": 1.2992, + "theoretical_loss": 3.492987486640496, + "tokens_seen": 1778515968 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003726860581745236, + "loss": 1.311, + "theoretical_loss": 3.4929765953716405, + "tokens_seen": 1778581504 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003726700100300903, + "loss": 1.3501, + "theoretical_loss": 3.4929657046164544, + "tokens_seen": 1778647040 + }, + { + "epoch": 0.54, + "learning_rate": 0.000372653961885657, + "loss": 1.3801, + "theoretical_loss": 3.4929548143748956, + "tokens_seen": 1778712576 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003726379137412237, + "loss": 1.2638, + "theoretical_loss": 3.4929439246469203, + "tokens_seen": 1778778112 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003726218655967904, + "loss": 1.3304, + "theoretical_loss": 3.492933035432486, + "tokens_seen": 1778843648 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003726058174523571, + "loss": 1.385, + "theoretical_loss": 3.492922146731549, + "tokens_seen": 1778909184 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003725897693079238, + "loss": 1.2565, + "theoretical_loss": 3.4929112585440674, + "tokens_seen": 1778974720 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003725737211634905, + "loss": 1.3054, + "theoretical_loss": 3.492900370869996, + "tokens_seen": 1779040256 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003725576730190572, + "loss": 1.3733, + "theoretical_loss": 3.492889483709294, + "tokens_seen": 1779105792 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003725416248746239, + "loss": 1.3825, + "theoretical_loss": 3.4928785970619165, + "tokens_seen": 1779171328 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003725255767301906, + "loss": 1.3172, + "theoretical_loss": 3.492867710927821, + "tokens_seen": 1779236864 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003725095285857573, + "loss": 1.2998, + "theoretical_loss": 3.4928568253069643, + "tokens_seen": 1779302400 + }, + { + "epoch": 0.54, + "learning_rate": 0.000372493480441324, + "loss": 1.3369, + "theoretical_loss": 3.4928459401993033, + "tokens_seen": 1779367936 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003724774322968907, + "loss": 1.286, + "theoretical_loss": 3.492835055604796, + "tokens_seen": 1779433472 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003724613841524574, + "loss": 1.456, + "theoretical_loss": 3.492824171523397, + "tokens_seen": 1779499008 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003724453360080241, + "loss": 1.3148, + "theoretical_loss": 3.492813287955065, + "tokens_seen": 1779564544 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003724292878635908, + "loss": 1.2406, + "theoretical_loss": 3.4928024048997566, + "tokens_seen": 1779630080 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003724132397191575, + "loss": 1.236, + "theoretical_loss": 3.492791522357429, + "tokens_seen": 1779695616 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003723971915747242, + "loss": 1.3426, + "theoretical_loss": 3.4927806403280384, + "tokens_seen": 1779761152 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003723811434302909, + "loss": 1.3456, + "theoretical_loss": 3.4927697588115416, + "tokens_seen": 1779826688 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003723650952858576, + "loss": 1.3154, + "theoretical_loss": 3.492758877807897, + "tokens_seen": 1779892224 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003723490471414243, + "loss": 1.3, + "theoretical_loss": 3.4927479973170596, + "tokens_seen": 1779957760 + }, + { + "epoch": 0.54, + "learning_rate": 0.000372332998996991, + "loss": 1.2622, + "theoretical_loss": 3.4927371173389874, + "tokens_seen": 1780023296 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003723169508525577, + "loss": 1.4461, + "theoretical_loss": 3.4927262378736375, + "tokens_seen": 1780088832 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003723009027081244, + "loss": 1.2829, + "theoretical_loss": 3.4927153589209667, + "tokens_seen": 1780154368 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003722848545636911, + "loss": 1.3025, + "theoretical_loss": 3.492704480480932, + "tokens_seen": 1780219904 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003722688064192578, + "loss": 1.3253, + "theoretical_loss": 3.49269360255349, + "tokens_seen": 1780285440 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003722527582748245, + "loss": 1.2941, + "theoretical_loss": 3.492682725138598, + "tokens_seen": 1780350976 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003722367101303912, + "loss": 1.3153, + "theoretical_loss": 3.492671848236213, + "tokens_seen": 1780416512 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003722206619859579, + "loss": 1.3559, + "theoretical_loss": 3.4926609718462918, + "tokens_seen": 1780482048 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003722046138415246, + "loss": 1.2704, + "theoretical_loss": 3.4926500959687914, + "tokens_seen": 1780547584 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003721885656970913, + "loss": 1.2936, + "theoretical_loss": 3.492639220603669, + "tokens_seen": 1780613120 + }, + { + "epoch": 0.54, + "learning_rate": 0.000372172517552658, + "loss": 1.2354, + "theoretical_loss": 3.4926283457508815, + "tokens_seen": 1780678656 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003721564694082247, + "loss": 1.1937, + "theoretical_loss": 3.4926174714103864, + "tokens_seen": 1780744192 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003721404212637914, + "loss": 1.2464, + "theoretical_loss": 3.4926065975821396, + "tokens_seen": 1780809728 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003721243731193581, + "loss": 1.3382, + "theoretical_loss": 3.4925957242660988, + "tokens_seen": 1780875264 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003721083249749248, + "loss": 1.3577, + "theoretical_loss": 3.4925848514622215, + "tokens_seen": 1780940800 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003720922768304915, + "loss": 1.3563, + "theoretical_loss": 3.4925739791704635, + "tokens_seen": 1781006336 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003720762286860582, + "loss": 1.2716, + "theoretical_loss": 3.492563107390783, + "tokens_seen": 1781071872 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003720601805416249, + "loss": 1.3027, + "theoretical_loss": 3.4925522361231365, + "tokens_seen": 1781137408 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003720441323971916, + "loss": 1.2959, + "theoretical_loss": 3.4925413653674813, + "tokens_seen": 1781202944 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003720280842527583, + "loss": 1.3184, + "theoretical_loss": 3.4925304951237734, + "tokens_seen": 1781268480 + }, + { + "epoch": 0.54, + "learning_rate": 0.000372012036108325, + "loss": 1.4012, + "theoretical_loss": 3.4925196253919717, + "tokens_seen": 1781334016 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003719959879638917, + "loss": 1.3409, + "theoretical_loss": 3.4925087561720325, + "tokens_seen": 1781399552 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003719799398194584, + "loss": 1.3597, + "theoretical_loss": 3.4924978874639114, + "tokens_seen": 1781465088 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037196389167502505, + "loss": 1.2151, + "theoretical_loss": 3.4924870192675677, + "tokens_seen": 1781530624 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037194784353059175, + "loss": 1.2593, + "theoretical_loss": 3.4924761515829577, + "tokens_seen": 1781596160 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037193179538615845, + "loss": 1.2752, + "theoretical_loss": 3.492465284410038, + "tokens_seen": 1781661696 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037191574724172515, + "loss": 1.3261, + "theoretical_loss": 3.4924544177487657, + "tokens_seen": 1781727232 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037189969909729185, + "loss": 1.3055, + "theoretical_loss": 3.4924435515990986, + "tokens_seen": 1781792768 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037188365095285856, + "loss": 1.3562, + "theoretical_loss": 3.492432685960993, + "tokens_seen": 1781858304 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037186760280842526, + "loss": 1.2734, + "theoretical_loss": 3.492421820834406, + "tokens_seen": 1781923840 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037185155466399196, + "loss": 1.3165, + "theoretical_loss": 3.492410956219296, + "tokens_seen": 1781989376 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037183550651955866, + "loss": 1.313, + "theoretical_loss": 3.492400092115619, + "tokens_seen": 1782054912 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037181945837512536, + "loss": 1.2712, + "theoretical_loss": 3.492389228523332, + "tokens_seen": 1782120448 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037180341023069206, + "loss": 1.2069, + "theoretical_loss": 3.4923783654423923, + "tokens_seen": 1782185984 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037178736208625876, + "loss": 1.3187, + "theoretical_loss": 3.4923675028727574, + "tokens_seen": 1782251520 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003717713139418255, + "loss": 1.2561, + "theoretical_loss": 3.4923566408143842, + "tokens_seen": 1782317056 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003717552657973922, + "loss": 1.2024, + "theoretical_loss": 3.49234577926723, + "tokens_seen": 1782382592 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003717392176529589, + "loss": 1.177, + "theoretical_loss": 3.4923349182312515, + "tokens_seen": 1782448128 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003717231695085256, + "loss": 1.3196, + "theoretical_loss": 3.4923240577064067, + "tokens_seen": 1782513664 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 660818, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.347831130027771, + "objective/train/theoretical_loss": 3.4923131976926514, + "objective/train/tokens_used": 1803039200, + "theoretical_loss": 3.4923131976926514, + "tokens_seen": 1782579200 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003717071213640923, + "loss": 1.3478, + "theoretical_loss": 3.4923131976926514, + "tokens_seen": 1782579200 + }, + { + "epoch": 0.54, + "learning_rate": 0.000371691073219659, + "loss": 1.3183, + "theoretical_loss": 3.492302338189944, + "tokens_seen": 1782644736 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003716750250752257, + "loss": 1.2595, + "theoretical_loss": 3.492291479198241, + "tokens_seen": 1782710272 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003716589769307924, + "loss": 1.3671, + "theoretical_loss": 3.4922806207174997, + "tokens_seen": 1782775808 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003716429287863591, + "loss": 1.2999, + "theoretical_loss": 3.4922697627476778, + "tokens_seen": 1782841344 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003716268806419258, + "loss": 1.2379, + "theoretical_loss": 3.4922589052887316, + "tokens_seen": 1782906880 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003716108324974925, + "loss": 1.2423, + "theoretical_loss": 3.492248048340619, + "tokens_seen": 1782972416 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003715947843530592, + "loss": 1.3165, + "theoretical_loss": 3.492237191903297, + "tokens_seen": 1783037952 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003715787362086259, + "loss": 1.4025, + "theoretical_loss": 3.4922263359767225, + "tokens_seen": 1783103488 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003715626880641926, + "loss": 1.1711, + "theoretical_loss": 3.492215480560853, + "tokens_seen": 1783169024 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003715466399197593, + "loss": 1.2091, + "theoretical_loss": 3.4922046256556456, + "tokens_seen": 1783234560 + }, + { + "epoch": 0.54, + "learning_rate": 0.000371530591775326, + "loss": 1.3337, + "theoretical_loss": 3.4921937712610576, + "tokens_seen": 1783300096 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003715145436308927, + "loss": 1.2285, + "theoretical_loss": 3.492182917377046, + "tokens_seen": 1783365632 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003714984954864594, + "loss": 1.3318, + "theoretical_loss": 3.492172064003568, + "tokens_seen": 1783431168 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003714824473420261, + "loss": 1.2709, + "theoretical_loss": 3.492161211140581, + "tokens_seen": 1783496704 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003714663991975928, + "loss": 1.3159, + "theoretical_loss": 3.4921503587880425, + "tokens_seen": 1783562240 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003714503510531595, + "loss": 1.2836, + "theoretical_loss": 3.4921395069459096, + "tokens_seen": 1783627776 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037143430290872623, + "loss": 1.2065, + "theoretical_loss": 3.492128655614139, + "tokens_seen": 1783693312 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037141825476429293, + "loss": 1.2481, + "theoretical_loss": 3.492117804792689, + "tokens_seen": 1783758848 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037140220661985963, + "loss": 1.1959, + "theoretical_loss": 3.4921069544815153, + "tokens_seen": 1783824384 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037138615847542633, + "loss": 1.327, + "theoretical_loss": 3.4920961046805763, + "tokens_seen": 1783889920 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037137011033099303, + "loss": 1.2133, + "theoretical_loss": 3.4920852553898296, + "tokens_seen": 1783955456 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037135406218655973, + "loss": 1.2372, + "theoretical_loss": 3.4920744066092313, + "tokens_seen": 1784020992 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003713380140421264, + "loss": 1.2869, + "theoretical_loss": 3.4920635583387396, + "tokens_seen": 1784086528 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003713219658976931, + "loss": 1.2881, + "theoretical_loss": 3.492052710578311, + "tokens_seen": 1784152064 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003713059177532598, + "loss": 1.288, + "theoretical_loss": 3.492041863327904, + "tokens_seen": 1784217600 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003712898696088265, + "loss": 1.2846, + "theoretical_loss": 3.4920310165874744, + "tokens_seen": 1784283136 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003712738214643932, + "loss": 1.301, + "theoretical_loss": 3.49202017035698, + "tokens_seen": 1784348672 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003712577733199599, + "loss": 1.2329, + "theoretical_loss": 3.492009324636379, + "tokens_seen": 1784414208 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003712417251755266, + "loss": 1.3644, + "theoretical_loss": 3.491998479425628, + "tokens_seen": 1784479744 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003712256770310933, + "loss": 1.252, + "theoretical_loss": 3.491987634724684, + "tokens_seen": 1784545280 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037120962888666, + "loss": 1.2651, + "theoretical_loss": 3.4919767905335046, + "tokens_seen": 1784610816 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003711935807422267, + "loss": 1.225, + "theoretical_loss": 3.491965946852047, + "tokens_seen": 1784676352 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003711775325977934, + "loss": 1.3224, + "theoretical_loss": 3.491955103680269, + "tokens_seen": 1784741888 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003711614844533601, + "loss": 1.21, + "theoretical_loss": 3.4919442610181277, + "tokens_seen": 1784807424 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003711454363089268, + "loss": 1.1885, + "theoretical_loss": 3.49193341886558, + "tokens_seen": 1784872960 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003711293881644935, + "loss": 1.2025, + "theoretical_loss": 3.491922577222584, + "tokens_seen": 1784938496 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003711133400200602, + "loss": 1.3729, + "theoretical_loss": 3.4919117360890963, + "tokens_seen": 1785004032 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003710972918756269, + "loss": 1.2907, + "theoretical_loss": 3.4919008954650748, + "tokens_seen": 1785069568 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003710812437311936, + "loss": 1.4341, + "theoretical_loss": 3.491890055350477, + "tokens_seen": 1785135104 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003710651955867603, + "loss": 1.3643, + "theoretical_loss": 3.491879215745259, + "tokens_seen": 1785200640 + }, + { + "epoch": 0.54, + "learning_rate": 0.000371049147442327, + "loss": 1.3265, + "theoretical_loss": 3.4918683766493794, + "tokens_seen": 1785266176 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003710330992978937, + "loss": 1.4585, + "theoretical_loss": 3.491857538062795, + "tokens_seen": 1785331712 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003710170511534604, + "loss": 1.3621, + "theoretical_loss": 3.491846699985464, + "tokens_seen": 1785397248 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003710010030090271, + "loss": 1.2263, + "theoretical_loss": 3.491835862417343, + "tokens_seen": 1785462784 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003709849548645938, + "loss": 1.2819, + "theoretical_loss": 3.491825025358389, + "tokens_seen": 1785528320 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003709689067201605, + "loss": 1.2953, + "theoretical_loss": 3.4918141888085605, + "tokens_seen": 1785593856 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003709528585757272, + "loss": 1.2274, + "theoretical_loss": 3.4918033527678145, + "tokens_seen": 1785659392 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003709368104312939, + "loss": 1.3808, + "theoretical_loss": 3.491792517236108, + "tokens_seen": 1785724928 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003709207622868606, + "loss": 1.2612, + "theoretical_loss": 3.4917816822133982, + "tokens_seen": 1785790464 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003709047141424273, + "loss": 1.2991, + "theoretical_loss": 3.4917708476996436, + "tokens_seen": 1785856000 + }, + { + "epoch": 0.54, + "learning_rate": 0.000370888665997994, + "loss": 1.4606, + "theoretical_loss": 3.491760013694801, + "tokens_seen": 1785921536 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003708726178535607, + "loss": 1.2648, + "theoretical_loss": 3.491749180198828, + "tokens_seen": 1785987072 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003708565697091274, + "loss": 1.3119, + "theoretical_loss": 3.4917383472116814, + "tokens_seen": 1786052608 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003708405215646941, + "loss": 1.2952, + "theoretical_loss": 3.491727514733319, + "tokens_seen": 1786118144 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003708244734202608, + "loss": 1.3323, + "theoretical_loss": 3.4917166827636987, + "tokens_seen": 1786183680 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003708084252758275, + "loss": 1.2391, + "theoretical_loss": 3.491705851302777, + "tokens_seen": 1786249216 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003707923771313942, + "loss": 1.1623, + "theoretical_loss": 3.4916950203505124, + "tokens_seen": 1786314752 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003707763289869609, + "loss": 1.3543, + "theoretical_loss": 3.491684189906862, + "tokens_seen": 1786380288 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003707602808425276, + "loss": 1.1578, + "theoretical_loss": 3.4916733599717826, + "tokens_seen": 1786445824 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003707442326980943, + "loss": 1.241, + "theoretical_loss": 3.4916625305452325, + "tokens_seen": 1786511360 + }, + { + "epoch": 0.54, + "learning_rate": 0.000370728184553661, + "loss": 1.3273, + "theoretical_loss": 3.491651701627169, + "tokens_seen": 1786576896 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003707121364092277, + "loss": 1.2918, + "theoretical_loss": 3.4916408732175492, + "tokens_seen": 1786642432 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003706960882647944, + "loss": 1.3159, + "theoretical_loss": 3.491630045316331, + "tokens_seen": 1786707968 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003706800401203611, + "loss": 1.2811, + "theoretical_loss": 3.4916192179234713, + "tokens_seen": 1786773504 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003706639919759278, + "loss": 1.38, + "theoretical_loss": 3.491608391038928, + "tokens_seen": 1786839040 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003706479438314945, + "loss": 1.3665, + "theoretical_loss": 3.4915975646626594, + "tokens_seen": 1786904576 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003706318956870612, + "loss": 1.1962, + "theoretical_loss": 3.491586738794621, + "tokens_seen": 1786970112 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003706158475426279, + "loss": 1.2381, + "theoretical_loss": 3.491575913434772, + "tokens_seen": 1787035648 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003705997993981946, + "loss": 1.3481, + "theoretical_loss": 3.4915650885830694, + "tokens_seen": 1787101184 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003705837512537613, + "loss": 1.2411, + "theoretical_loss": 3.4915542642394706, + "tokens_seen": 1787166720 + }, + { + "epoch": 0.54, + "learning_rate": 0.000370567703109328, + "loss": 1.3061, + "theoretical_loss": 3.4915434404039334, + "tokens_seen": 1787232256 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003705516549648947, + "loss": 1.2662, + "theoretical_loss": 3.491532617076415, + "tokens_seen": 1787297792 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003705356068204614, + "loss": 1.2532, + "theoretical_loss": 3.491521794256873, + "tokens_seen": 1787363328 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003705195586760281, + "loss": 1.2579, + "theoretical_loss": 3.4915109719452646, + "tokens_seen": 1787428864 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003705035105315948, + "loss": 1.2176, + "theoretical_loss": 3.4915001501415484, + "tokens_seen": 1787494400 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003704874623871615, + "loss": 1.3487, + "theoretical_loss": 3.491489328845681, + "tokens_seen": 1787559936 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003704714142427282, + "loss": 1.192, + "theoretical_loss": 3.4914785080576207, + "tokens_seen": 1787625472 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003704553660982949, + "loss": 1.2312, + "theoretical_loss": 3.491467687777324, + "tokens_seen": 1787691008 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003704393179538616, + "loss": 1.2987, + "theoretical_loss": 3.4914568680047493, + "tokens_seen": 1787756544 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003704232698094283, + "loss": 1.257, + "theoretical_loss": 3.4914460487398538, + "tokens_seen": 1787822080 + }, + { + "epoch": 0.54, + "learning_rate": 0.000370407221664995, + "loss": 1.1996, + "theoretical_loss": 3.491435229982595, + "tokens_seen": 1787887616 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003703911735205617, + "loss": 1.2332, + "theoretical_loss": 3.4914244117329307, + "tokens_seen": 1787953152 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003703751253761284, + "loss": 1.3671, + "theoretical_loss": 3.491413593990819, + "tokens_seen": 1788018688 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003703590772316951, + "loss": 1.2866, + "theoretical_loss": 3.4914027767562166, + "tokens_seen": 1788084224 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003703430290872618, + "loss": 1.1812, + "theoretical_loss": 3.4913919600290813, + "tokens_seen": 1788149760 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003703269809428285, + "loss": 1.3575, + "theoretical_loss": 3.491381143809371, + "tokens_seen": 1788215296 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003703109327983952, + "loss": 1.2746, + "theoretical_loss": 3.4913703280970427, + "tokens_seen": 1788280832 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003702948846539619, + "loss": 1.4495, + "theoretical_loss": 3.4913595128920547, + "tokens_seen": 1788346368 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003702788365095286, + "loss": 1.3679, + "theoretical_loss": 3.4913486981943644, + "tokens_seen": 1788411904 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003702627883650953, + "loss": 1.3308, + "theoretical_loss": 3.4913378840039293, + "tokens_seen": 1788477440 + }, + { + "epoch": 0.54, + "learning_rate": 0.000370246740220662, + "loss": 1.2383, + "theoretical_loss": 3.491327070320707, + "tokens_seen": 1788542976 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003702306920762287, + "loss": 1.2594, + "theoretical_loss": 3.491316257144655, + "tokens_seen": 1788608512 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003702146439317954, + "loss": 1.2863, + "theoretical_loss": 3.4913054444757314, + "tokens_seen": 1788674048 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003701985957873621, + "loss": 1.3374, + "theoretical_loss": 3.4912946323138936, + "tokens_seen": 1788739584 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003701825476429288, + "loss": 1.424, + "theoretical_loss": 3.491283820659099, + "tokens_seen": 1788805120 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003701664994984955, + "loss": 1.2867, + "theoretical_loss": 3.491273009511306, + "tokens_seen": 1788870656 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003701504513540622, + "loss": 1.4129, + "theoretical_loss": 3.491262198870471, + "tokens_seen": 1788936192 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003701344032096289, + "loss": 1.2612, + "theoretical_loss": 3.491251388736553, + "tokens_seen": 1789001728 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003701183550651956, + "loss": 1.2609, + "theoretical_loss": 3.4912405791095082, + "tokens_seen": 1789067264 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 663117, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3809022903442383, + "objective/train/theoretical_loss": 3.491229769989296, + "objective/train/tokens_used": 1809592800, + "theoretical_loss": 3.491229769989296, + "tokens_seen": 1789132800 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003701023069207623, + "loss": 1.3809, + "theoretical_loss": 3.491229769989296, + "tokens_seen": 1789132800 + }, + { + "epoch": 0.54, + "learning_rate": 0.000370086258776329, + "loss": 1.3233, + "theoretical_loss": 3.4912189613758726, + "tokens_seen": 1789198336 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003700702106318957, + "loss": 1.4455, + "theoretical_loss": 3.4912081532691963, + "tokens_seen": 1789263872 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037005416248746243, + "loss": 1.287, + "theoretical_loss": 3.491197345669225, + "tokens_seen": 1789329408 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037003811434302913, + "loss": 1.2059, + "theoretical_loss": 3.4911865385759158, + "tokens_seen": 1789394944 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037002206619859583, + "loss": 1.3375, + "theoretical_loss": 3.4911757319892267, + "tokens_seen": 1789460480 + }, + { + "epoch": 0.54, + "learning_rate": 0.00037000601805416253, + "loss": 1.1849, + "theoretical_loss": 3.4911649259091155, + "tokens_seen": 1789526016 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036998996990972923, + "loss": 1.4352, + "theoretical_loss": 3.4911541203355396, + "tokens_seen": 1789591552 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036997392176529593, + "loss": 1.278, + "theoretical_loss": 3.4911433152684572, + "tokens_seen": 1789657088 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036995787362086263, + "loss": 1.3877, + "theoretical_loss": 3.4911325107078257, + "tokens_seen": 1789722624 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003699418254764293, + "loss": 1.2787, + "theoretical_loss": 3.491121706653603, + "tokens_seen": 1789788160 + }, + { + "epoch": 0.54, + "learning_rate": 0.000369925777331996, + "loss": 1.2788, + "theoretical_loss": 3.4911109031057457, + "tokens_seen": 1789853696 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003699097291875627, + "loss": 1.3872, + "theoretical_loss": 3.4911001000642132, + "tokens_seen": 1789919232 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003698936810431294, + "loss": 1.2819, + "theoretical_loss": 3.491089297528963, + "tokens_seen": 1789984768 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003698776328986961, + "loss": 1.2484, + "theoretical_loss": 3.4910784954999516, + "tokens_seen": 1790050304 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003698615847542628, + "loss": 1.2361, + "theoretical_loss": 3.491067693977138, + "tokens_seen": 1790115840 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003698455366098295, + "loss": 1.2961, + "theoretical_loss": 3.4910568929604793, + "tokens_seen": 1790181376 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003698294884653962, + "loss": 1.3272, + "theoretical_loss": 3.491046092449933, + "tokens_seen": 1790246912 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003698134403209629, + "loss": 1.3912, + "theoretical_loss": 3.4910352924454577, + "tokens_seen": 1790312448 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003697973921765296, + "loss": 1.359, + "theoretical_loss": 3.4910244929470107, + "tokens_seen": 1790377984 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003697813440320963, + "loss": 1.2548, + "theoretical_loss": 3.4910136939545495, + "tokens_seen": 1790443520 + }, + { + "epoch": 0.54, + "learning_rate": 0.000369765295887663, + "loss": 1.2315, + "theoretical_loss": 3.4910028954680326, + "tokens_seen": 1790509056 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003697492477432297, + "loss": 1.233, + "theoretical_loss": 3.490992097487417, + "tokens_seen": 1790574592 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003697331995987964, + "loss": 1.2235, + "theoretical_loss": 3.490981300012661, + "tokens_seen": 1790640128 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003697171514543631, + "loss": 1.3814, + "theoretical_loss": 3.490970503043722, + "tokens_seen": 1790705664 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003697011033099298, + "loss": 1.3567, + "theoretical_loss": 3.4909597065805587, + "tokens_seen": 1790771200 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003696850551654965, + "loss": 1.2018, + "theoretical_loss": 3.4909489106231275, + "tokens_seen": 1790836736 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003696690070210632, + "loss": 1.3169, + "theoretical_loss": 3.490938115171387, + "tokens_seen": 1790902272 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003696529588766299, + "loss": 1.2957, + "theoretical_loss": 3.4909273202252953, + "tokens_seen": 1790967808 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003696369107321966, + "loss": 1.2328, + "theoretical_loss": 3.49091652578481, + "tokens_seen": 1791033344 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003696208625877633, + "loss": 1.311, + "theoretical_loss": 3.490905731849888, + "tokens_seen": 1791098880 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036960481444333, + "loss": 1.2407, + "theoretical_loss": 3.4908949384204884, + "tokens_seen": 1791164416 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003695887662988967, + "loss": 1.3243, + "theoretical_loss": 3.490884145496568, + "tokens_seen": 1791229952 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003695727181544634, + "loss": 1.35, + "theoretical_loss": 3.4908733530780856, + "tokens_seen": 1791295488 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003695566700100301, + "loss": 1.1884, + "theoretical_loss": 3.490862561164999, + "tokens_seen": 1791361024 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003695406218655968, + "loss": 1.3656, + "theoretical_loss": 3.490851769757265, + "tokens_seen": 1791426560 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003695245737211635, + "loss": 1.2884, + "theoretical_loss": 3.490840978854842, + "tokens_seen": 1791492096 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003695085255767302, + "loss": 1.2381, + "theoretical_loss": 3.4908301884576884, + "tokens_seen": 1791557632 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003694924774322969, + "loss": 1.371, + "theoretical_loss": 3.4908193985657614, + "tokens_seen": 1791623168 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003694764292878636, + "loss": 1.2292, + "theoretical_loss": 3.490808609179019, + "tokens_seen": 1791688704 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003694603811434303, + "loss": 1.2385, + "theoretical_loss": 3.490797820297419, + "tokens_seen": 1791754240 + }, + { + "epoch": 0.54, + "learning_rate": 0.000369444332998997, + "loss": 1.2468, + "theoretical_loss": 3.4907870319209198, + "tokens_seen": 1791819776 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003694282848545637, + "loss": 1.3219, + "theoretical_loss": 3.4907762440494787, + "tokens_seen": 1791885312 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003694122367101304, + "loss": 1.233, + "theoretical_loss": 3.4907654566830537, + "tokens_seen": 1791950848 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003693961885656971, + "loss": 1.3471, + "theoretical_loss": 3.4907546698216025, + "tokens_seen": 1792016384 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003693801404212638, + "loss": 1.3657, + "theoretical_loss": 3.4907438834650844, + "tokens_seen": 1792081920 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036936409227683055, + "loss": 1.4085, + "theoretical_loss": 3.4907330976134547, + "tokens_seen": 1792147456 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036934804413239726, + "loss": 1.2566, + "theoretical_loss": 3.4907223122666737, + "tokens_seen": 1792212992 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036933199598796396, + "loss": 1.2379, + "theoretical_loss": 3.490711527424698, + "tokens_seen": 1792278528 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036931594784353066, + "loss": 1.2437, + "theoretical_loss": 3.490700743087486, + "tokens_seen": 1792344064 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003692998996990973, + "loss": 1.1985, + "theoretical_loss": 3.4906899592549956, + "tokens_seen": 1792409600 + }, + { + "epoch": 0.54, + "learning_rate": 0.000369283851554664, + "loss": 1.3272, + "theoretical_loss": 3.4906791759271845, + "tokens_seen": 1792475136 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003692678034102307, + "loss": 1.2838, + "theoretical_loss": 3.490668393104011, + "tokens_seen": 1792540672 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003692517552657974, + "loss": 1.25, + "theoretical_loss": 3.4906576107854326, + "tokens_seen": 1792606208 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003692357071213641, + "loss": 1.2745, + "theoretical_loss": 3.490646828971407, + "tokens_seen": 1792671744 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003692196589769308, + "loss": 1.2278, + "theoretical_loss": 3.4906360476618934, + "tokens_seen": 1792737280 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003692036108324975, + "loss": 1.3349, + "theoretical_loss": 3.4906252668568483, + "tokens_seen": 1792802816 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003691875626880642, + "loss": 1.2057, + "theoretical_loss": 3.4906144865562307, + "tokens_seen": 1792868352 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003691715145436309, + "loss": 1.2614, + "theoretical_loss": 3.490603706759998, + "tokens_seen": 1792933888 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003691554663991976, + "loss": 1.3351, + "theoretical_loss": 3.490592927468108, + "tokens_seen": 1792999424 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003691394182547643, + "loss": 1.2624, + "theoretical_loss": 3.4905821486805197, + "tokens_seen": 1793064960 + }, + { + "epoch": 0.54, + "learning_rate": 0.000369123370110331, + "loss": 1.3769, + "theoretical_loss": 3.49057137039719, + "tokens_seen": 1793130496 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003691073219658977, + "loss": 1.279, + "theoretical_loss": 3.490560592618077, + "tokens_seen": 1793196032 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003690912738214644, + "loss": 1.2559, + "theoretical_loss": 3.4905498153431394, + "tokens_seen": 1793261568 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003690752256770311, + "loss": 1.3415, + "theoretical_loss": 3.4905390385723347, + "tokens_seen": 1793327104 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003690591775325978, + "loss": 1.1491, + "theoretical_loss": 3.49052826230562, + "tokens_seen": 1793392640 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003690431293881645, + "loss": 1.3898, + "theoretical_loss": 3.490517486542956, + "tokens_seen": 1793458176 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003690270812437312, + "loss": 1.2731, + "theoretical_loss": 3.490506711284297, + "tokens_seen": 1793523712 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003690110330992979, + "loss": 1.2316, + "theoretical_loss": 3.490495936529604, + "tokens_seen": 1793589248 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003689949849548646, + "loss": 1.3368, + "theoretical_loss": 3.490485162278834, + "tokens_seen": 1793654784 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003689789368104313, + "loss": 1.2938, + "theoretical_loss": 3.490474388531945, + "tokens_seen": 1793720320 + }, + { + "epoch": 0.54, + "learning_rate": 0.000368962888665998, + "loss": 1.3329, + "theoretical_loss": 3.4904636152888946, + "tokens_seen": 1793785856 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003689468405215647, + "loss": 1.2457, + "theoretical_loss": 3.4904528425496415, + "tokens_seen": 1793851392 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003689307923771314, + "loss": 1.2522, + "theoretical_loss": 3.490442070314143, + "tokens_seen": 1793916928 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003689147442326981, + "loss": 1.2524, + "theoretical_loss": 3.490431298582358, + "tokens_seen": 1793982464 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003688986960882648, + "loss": 1.4166, + "theoretical_loss": 3.490420527354244, + "tokens_seen": 1794048000 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003688826479438315, + "loss": 1.4053, + "theoretical_loss": 3.49040975662976, + "tokens_seen": 1794113536 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003688665997993982, + "loss": 1.3083, + "theoretical_loss": 3.4903989864088625, + "tokens_seen": 1794179072 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003688505516549649, + "loss": 1.3791, + "theoretical_loss": 3.4903882166915103, + "tokens_seen": 1794244608 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003688345035105316, + "loss": 1.2407, + "theoretical_loss": 3.4903774474776617, + "tokens_seen": 1794310144 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003688184553660983, + "loss": 1.3954, + "theoretical_loss": 3.4903666787672747, + "tokens_seen": 1794375680 + }, + { + "epoch": 0.54, + "learning_rate": 0.000368802407221665, + "loss": 1.4862, + "theoretical_loss": 3.4903559105603073, + "tokens_seen": 1794441216 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003687863590772317, + "loss": 1.2098, + "theoretical_loss": 3.490345142856717, + "tokens_seen": 1794506752 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003687703109327984, + "loss": 1.3053, + "theoretical_loss": 3.4903343756564627, + "tokens_seen": 1794572288 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003687542627883651, + "loss": 1.3729, + "theoretical_loss": 3.4903236089595024, + "tokens_seen": 1794637824 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003687382146439318, + "loss": 1.3212, + "theoretical_loss": 3.4903128427657935, + "tokens_seen": 1794703360 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003687221664994985, + "loss": 1.3474, + "theoretical_loss": 3.4903020770752953, + "tokens_seen": 1794768896 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003687061183550652, + "loss": 1.2019, + "theoretical_loss": 3.4902913118879644, + "tokens_seen": 1794834432 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036869007021063193, + "loss": 1.2913, + "theoretical_loss": 3.4902805472037604, + "tokens_seen": 1794899968 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036867402206619863, + "loss": 1.2855, + "theoretical_loss": 3.4902697830226406, + "tokens_seen": 1794965504 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036865797392176533, + "loss": 1.2733, + "theoretical_loss": 3.490259019344563, + "tokens_seen": 1795031040 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036864192577733203, + "loss": 1.3147, + "theoretical_loss": 3.490248256169486, + "tokens_seen": 1795096576 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036862587763289873, + "loss": 1.3376, + "theoretical_loss": 3.490237493497368, + "tokens_seen": 1795162112 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036860982948846543, + "loss": 1.3253, + "theoretical_loss": 3.4902267313281667, + "tokens_seen": 1795227648 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036859378134403213, + "loss": 1.2435, + "theoretical_loss": 3.4902159696618402, + "tokens_seen": 1795293184 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036857773319959883, + "loss": 1.3031, + "theoretical_loss": 3.490205208498347, + "tokens_seen": 1795358720 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036856168505516553, + "loss": 1.2845, + "theoretical_loss": 3.490194447837645, + "tokens_seen": 1795424256 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036854563691073223, + "loss": 1.3905, + "theoretical_loss": 3.490183687679693, + "tokens_seen": 1795489792 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003685295887662989, + "loss": 1.1638, + "theoretical_loss": 3.490172928024448, + "tokens_seen": 1795555328 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003685135406218656, + "loss": 1.3908, + "theoretical_loss": 3.490162168871869, + "tokens_seen": 1795620864 + }, + { + "epoch": 0.54, + "objective/train/docs_used": 665936, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2389717102050781, + "objective/train/theoretical_loss": 3.4901514102219138, + "objective/train/tokens_used": 1816146400, + "theoretical_loss": 3.4901514102219138, + "tokens_seen": 1795686400 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003684974924774323, + "loss": 1.239, + "theoretical_loss": 3.4901514102219138, + "tokens_seen": 1795686400 + }, + { + "epoch": 0.54, + "learning_rate": 0.000368481444332999, + "loss": 1.4081, + "theoretical_loss": 3.490140652074541, + "tokens_seen": 1795751936 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003684653961885657, + "loss": 1.2562, + "theoretical_loss": 3.4901298944297086, + "tokens_seen": 1795817472 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003684493480441324, + "loss": 1.3515, + "theoretical_loss": 3.4901191372873743, + "tokens_seen": 1795883008 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003684332998996991, + "loss": 1.4063, + "theoretical_loss": 3.490108380647497, + "tokens_seen": 1795948544 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003684172517552658, + "loss": 1.2753, + "theoretical_loss": 3.4900976245100344, + "tokens_seen": 1796014080 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003684012036108325, + "loss": 1.2901, + "theoretical_loss": 3.4900868688749442, + "tokens_seen": 1796079616 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003683851554663992, + "loss": 1.259, + "theoretical_loss": 3.4900761137421865, + "tokens_seen": 1796145152 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003683691073219659, + "loss": 1.1931, + "theoretical_loss": 3.4900653591117177, + "tokens_seen": 1796210688 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003683530591775326, + "loss": 1.2711, + "theoretical_loss": 3.490054604983497, + "tokens_seen": 1796276224 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003683370110330993, + "loss": 1.2071, + "theoretical_loss": 3.4900438513574814, + "tokens_seen": 1796341760 + }, + { + "epoch": 0.54, + "learning_rate": 0.000368320962888666, + "loss": 1.3694, + "theoretical_loss": 3.490033098233631, + "tokens_seen": 1796407296 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003683049147442327, + "loss": 1.3474, + "theoretical_loss": 3.4900223456119024, + "tokens_seen": 1796472832 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003682888665997994, + "loss": 1.2065, + "theoretical_loss": 3.4900115934922544, + "tokens_seen": 1796538368 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003682728184553661, + "loss": 1.2808, + "theoretical_loss": 3.4900008418746458, + "tokens_seen": 1796603904 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003682567703109328, + "loss": 1.285, + "theoretical_loss": 3.4899900907590338, + "tokens_seen": 1796669440 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003682407221664995, + "loss": 1.3141, + "theoretical_loss": 3.4899793401453776, + "tokens_seen": 1796734976 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003682246740220662, + "loss": 1.3117, + "theoretical_loss": 3.4899685900336346, + "tokens_seen": 1796800512 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003682086258776329, + "loss": 1.2707, + "theoretical_loss": 3.489957840423764, + "tokens_seen": 1796866048 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003681925777331996, + "loss": 1.2678, + "theoretical_loss": 3.489947091315723, + "tokens_seen": 1796931584 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003681765295887663, + "loss": 1.2385, + "theoretical_loss": 3.4899363427094707, + "tokens_seen": 1796997120 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036816048144433305, + "loss": 1.3248, + "theoretical_loss": 3.489925594604965, + "tokens_seen": 1797062656 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036814443329989975, + "loss": 1.3171, + "theoretical_loss": 3.4899148470021646, + "tokens_seen": 1797128192 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036812838515546645, + "loss": 1.2135, + "theoretical_loss": 3.489904099901027, + "tokens_seen": 1797193728 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036811233701103315, + "loss": 1.389, + "theoretical_loss": 3.4898933533015115, + "tokens_seen": 1797259264 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036809628886659985, + "loss": 1.1908, + "theoretical_loss": 3.489882607203576, + "tokens_seen": 1797324800 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036808024072216655, + "loss": 1.3199, + "theoretical_loss": 3.489871861607178, + "tokens_seen": 1797390336 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036806419257773325, + "loss": 1.3055, + "theoretical_loss": 3.4898611165122766, + "tokens_seen": 1797455872 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036804814443329995, + "loss": 1.4186, + "theoretical_loss": 3.4898503719188305, + "tokens_seen": 1797521408 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036803209628886665, + "loss": 1.3508, + "theoretical_loss": 3.489839627826797, + "tokens_seen": 1797586944 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036801604814443335, + "loss": 1.3014, + "theoretical_loss": 3.489828884236135, + "tokens_seen": 1797652480 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036800000000000005, + "loss": 1.3022, + "theoretical_loss": 3.489818141146803, + "tokens_seen": 1797718016 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036798395185556676, + "loss": 1.3796, + "theoretical_loss": 3.489807398558759, + "tokens_seen": 1797783552 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036796790371113346, + "loss": 1.2867, + "theoretical_loss": 3.4897966564719614, + "tokens_seen": 1797849088 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036795185556670016, + "loss": 1.2297, + "theoretical_loss": 3.489785914886369, + "tokens_seen": 1797914624 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036793580742226686, + "loss": 1.3133, + "theoretical_loss": 3.489775173801939, + "tokens_seen": 1797980160 + }, + { + "epoch": 0.54, + "learning_rate": 0.00036791975927783356, + "loss": 1.4143, + "theoretical_loss": 3.489764433218631, + "tokens_seen": 1798045696 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003679037111334002, + "loss": 1.3326, + "theoretical_loss": 3.489753693136402, + "tokens_seen": 1798111232 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003678876629889669, + "loss": 1.3359, + "theoretical_loss": 3.4897429535552122, + "tokens_seen": 1798176768 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003678716148445336, + "loss": 1.2186, + "theoretical_loss": 3.4897322144750182, + "tokens_seen": 1798242304 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003678555667001003, + "loss": 1.2614, + "theoretical_loss": 3.4897214758957795, + "tokens_seen": 1798307840 + }, + { + "epoch": 0.54, + "learning_rate": 0.000367839518555667, + "loss": 1.3278, + "theoretical_loss": 3.489710737817454, + "tokens_seen": 1798373376 + }, + { + "epoch": 0.54, + "learning_rate": 0.0003678234704112337, + "loss": 1.1945, + "theoretical_loss": 3.48970000024, + "tokens_seen": 1798438912 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003678074222668004, + "loss": 1.3535, + "theoretical_loss": 3.4896892631633762, + "tokens_seen": 1798504448 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003677913741223671, + "loss": 1.2458, + "theoretical_loss": 3.489678526587541, + "tokens_seen": 1798569984 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003677753259779338, + "loss": 1.1784, + "theoretical_loss": 3.4896677905124527, + "tokens_seen": 1798635520 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003677592778335005, + "loss": 1.2622, + "theoretical_loss": 3.4896570549380694, + "tokens_seen": 1798701056 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003677432296890672, + "loss": 1.3325, + "theoretical_loss": 3.4896463198643497, + "tokens_seen": 1798766592 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003677271815446339, + "loss": 1.3123, + "theoretical_loss": 3.4896355852912526, + "tokens_seen": 1798832128 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003677111334002006, + "loss": 1.2087, + "theoretical_loss": 3.4896248512187356, + "tokens_seen": 1798897664 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003676950852557673, + "loss": 1.3013, + "theoretical_loss": 3.4896141176467577, + "tokens_seen": 1798963200 + }, + { + "epoch": 0.55, + "learning_rate": 0.000367679037111334, + "loss": 1.2104, + "theoretical_loss": 3.4896033845752763, + "tokens_seen": 1799028736 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003676629889669007, + "loss": 1.3489, + "theoretical_loss": 3.4895926520042515, + "tokens_seen": 1799094272 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003676469408224674, + "loss": 1.2453, + "theoretical_loss": 3.4895819199336406, + "tokens_seen": 1799159808 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003676308926780341, + "loss": 1.1726, + "theoretical_loss": 3.489571188363403, + "tokens_seen": 1799225344 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003676148445336008, + "loss": 1.3137, + "theoretical_loss": 3.4895604572934955, + "tokens_seen": 1799290880 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003675987963891675, + "loss": 1.3139, + "theoretical_loss": 3.489549726723878, + "tokens_seen": 1799356416 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003675827482447342, + "loss": 1.3578, + "theoretical_loss": 3.4895389966545087, + "tokens_seen": 1799421952 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003675667001003009, + "loss": 1.3309, + "theoretical_loss": 3.489528267085346, + "tokens_seen": 1799487488 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003675506519558676, + "loss": 1.2943, + "theoretical_loss": 3.4895175380163472, + "tokens_seen": 1799553024 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003675346038114343, + "loss": 1.2816, + "theoretical_loss": 3.4895068094474726, + "tokens_seen": 1799618560 + }, + { + "epoch": 0.55, + "learning_rate": 0.000367518555667001, + "loss": 1.3235, + "theoretical_loss": 3.4894960813786797, + "tokens_seen": 1799684096 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003675025075225677, + "loss": 1.3456, + "theoretical_loss": 3.4894853538099273, + "tokens_seen": 1799749632 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003674864593781344, + "loss": 1.2696, + "theoretical_loss": 3.489474626741174, + "tokens_seen": 1799815168 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003674704112337011, + "loss": 1.416, + "theoretical_loss": 3.4894639001723773, + "tokens_seen": 1799880704 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003674543630892678, + "loss": 1.3812, + "theoretical_loss": 3.4894531741034966, + "tokens_seen": 1799946240 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003674383149448345, + "loss": 1.4034, + "theoretical_loss": 3.489442448534491, + "tokens_seen": 1800011776 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003674222668004012, + "loss": 1.4114, + "theoretical_loss": 3.489431723465317, + "tokens_seen": 1800077312 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003674062186559679, + "loss": 1.3485, + "theoretical_loss": 3.489420998895935, + "tokens_seen": 1800142848 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003673901705115346, + "loss": 1.4037, + "theoretical_loss": 3.489410274826303, + "tokens_seen": 1800208384 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003673741223671013, + "loss": 1.3203, + "theoretical_loss": 3.489399551256379, + "tokens_seen": 1800273920 + }, + { + "epoch": 0.55, + "learning_rate": 0.000367358074222668, + "loss": 1.4203, + "theoretical_loss": 3.4893888281861223, + "tokens_seen": 1800339456 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003673420260782347, + "loss": 1.2834, + "theoretical_loss": 3.489378105615491, + "tokens_seen": 1800404992 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003673259779338014, + "loss": 1.3507, + "theoretical_loss": 3.489367383544443, + "tokens_seen": 1800470528 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036730992978936813, + "loss": 1.2503, + "theoretical_loss": 3.489356661972938, + "tokens_seen": 1800536064 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036729388164493483, + "loss": 1.3611, + "theoretical_loss": 3.489345940900934, + "tokens_seen": 1800601600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036727783350050153, + "loss": 1.3426, + "theoretical_loss": 3.48933522032839, + "tokens_seen": 1800667136 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036726178535606823, + "loss": 1.2965, + "theoretical_loss": 3.4893245002552638, + "tokens_seen": 1800732672 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036724573721163493, + "loss": 1.4003, + "theoretical_loss": 3.489313780681514, + "tokens_seen": 1800798208 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036722968906720163, + "loss": 1.3827, + "theoretical_loss": 3.4893030616070995, + "tokens_seen": 1800863744 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036721364092276833, + "loss": 1.2578, + "theoretical_loss": 3.4892923430319795, + "tokens_seen": 1800929280 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036719759277833503, + "loss": 1.1796, + "theoretical_loss": 3.4892816249561114, + "tokens_seen": 1800994816 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036718154463390173, + "loss": 1.368, + "theoretical_loss": 3.4892709073794546, + "tokens_seen": 1801060352 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036716549648946843, + "loss": 1.2741, + "theoretical_loss": 3.489260190301967, + "tokens_seen": 1801125888 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036714944834503513, + "loss": 1.3274, + "theoretical_loss": 3.489249473723608, + "tokens_seen": 1801191424 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003671334002006018, + "loss": 1.2026, + "theoretical_loss": 3.4892387576443356, + "tokens_seen": 1801256960 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003671173520561685, + "loss": 1.2063, + "theoretical_loss": 3.4892280420641084, + "tokens_seen": 1801322496 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003671013039117352, + "loss": 1.3498, + "theoretical_loss": 3.4892173269828852, + "tokens_seen": 1801388032 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003670852557673019, + "loss": 1.3878, + "theoretical_loss": 3.489206612400625, + "tokens_seen": 1801453568 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003670692076228686, + "loss": 1.2845, + "theoretical_loss": 3.489195898317286, + "tokens_seen": 1801519104 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003670531594784353, + "loss": 1.3333, + "theoretical_loss": 3.4891851847328264, + "tokens_seen": 1801584640 + }, + { + "epoch": 0.55, + "learning_rate": 0.000367037111334002, + "loss": 1.2437, + "theoretical_loss": 3.489174471647205, + "tokens_seen": 1801650176 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003670210631895687, + "loss": 1.2544, + "theoretical_loss": 3.4891637590603812, + "tokens_seen": 1801715712 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003670050150451354, + "loss": 1.2525, + "theoretical_loss": 3.489153046972313, + "tokens_seen": 1801781248 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003669889669007021, + "loss": 1.2768, + "theoretical_loss": 3.4891423353829594, + "tokens_seen": 1801846784 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003669729187562688, + "loss": 1.2543, + "theoretical_loss": 3.4891316242922787, + "tokens_seen": 1801912320 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036695687061183554, + "loss": 1.301, + "theoretical_loss": 3.489120913700229, + "tokens_seen": 1801977856 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036694082246740224, + "loss": 1.2873, + "theoretical_loss": 3.4891102036067703, + "tokens_seen": 1802043392 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036692477432296894, + "loss": 1.228, + "theoretical_loss": 3.4890994940118603, + "tokens_seen": 1802108928 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036690872617853564, + "loss": 1.1926, + "theoretical_loss": 3.4890887849154577, + "tokens_seen": 1802174464 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 668118, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2964489459991455, + "objective/train/theoretical_loss": 3.4890780763175218, + "objective/train/tokens_used": 1822700000, + "theoretical_loss": 3.4890780763175218, + "tokens_seen": 1802240000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036689267803410234, + "loss": 1.2964, + "theoretical_loss": 3.4890780763175218, + "tokens_seen": 1802240000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036687662988966904, + "loss": 1.3281, + "theoretical_loss": 3.4890673682180107, + "tokens_seen": 1802305536 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036686058174523575, + "loss": 1.3672, + "theoretical_loss": 3.489056660616883, + "tokens_seen": 1802371072 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036684453360080245, + "loss": 1.2708, + "theoretical_loss": 3.489045953514098, + "tokens_seen": 1802436608 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036682848545636915, + "loss": 1.3256, + "theoretical_loss": 3.489035246909614, + "tokens_seen": 1802502144 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036681243731193585, + "loss": 1.2375, + "theoretical_loss": 3.4890245408033893, + "tokens_seen": 1802567680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036679638916750255, + "loss": 1.3493, + "theoretical_loss": 3.489013835195383, + "tokens_seen": 1802633216 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036678034102306925, + "loss": 1.2835, + "theoretical_loss": 3.489003130085554, + "tokens_seen": 1802698752 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036676429287863595, + "loss": 1.238, + "theoretical_loss": 3.488992425473861, + "tokens_seen": 1802764288 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036674824473420265, + "loss": 1.3071, + "theoretical_loss": 3.4889817213602625, + "tokens_seen": 1802829824 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036673219658976935, + "loss": 1.357, + "theoretical_loss": 3.488971017744717, + "tokens_seen": 1802895360 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036671614844533605, + "loss": 1.289, + "theoretical_loss": 3.4889603146271835, + "tokens_seen": 1802960896 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036670010030090275, + "loss": 1.3182, + "theoretical_loss": 3.488949612007621, + "tokens_seen": 1803026432 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036668405215646945, + "loss": 1.2912, + "theoretical_loss": 3.488938909885988, + "tokens_seen": 1803091968 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036666800401203615, + "loss": 1.3946, + "theoretical_loss": 3.4889282082622426, + "tokens_seen": 1803157504 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036665195586760285, + "loss": 1.3705, + "theoretical_loss": 3.4889175071363443, + "tokens_seen": 1803223040 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036663590772316955, + "loss": 1.2332, + "theoretical_loss": 3.4889068065082522, + "tokens_seen": 1803288576 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036661985957873625, + "loss": 1.2533, + "theoretical_loss": 3.488896106377924, + "tokens_seen": 1803354112 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036660381143430296, + "loss": 1.2289, + "theoretical_loss": 3.488885406745319, + "tokens_seen": 1803419648 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036658776328986966, + "loss": 1.3069, + "theoretical_loss": 3.4888747076103956, + "tokens_seen": 1803485184 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036657171514543636, + "loss": 1.2168, + "theoretical_loss": 3.4888640089731133, + "tokens_seen": 1803550720 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036655566700100306, + "loss": 1.2705, + "theoretical_loss": 3.4888533108334303, + "tokens_seen": 1803616256 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036653961885656976, + "loss": 1.2525, + "theoretical_loss": 3.4888426131913057, + "tokens_seen": 1803681792 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036652357071213646, + "loss": 1.3686, + "theoretical_loss": 3.4888319160466983, + "tokens_seen": 1803747328 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003665075225677031, + "loss": 1.3212, + "theoretical_loss": 3.488821219399566, + "tokens_seen": 1803812864 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003664914744232698, + "loss": 1.3219, + "theoretical_loss": 3.488810523249869, + "tokens_seen": 1803878400 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003664754262788365, + "loss": 1.2772, + "theoretical_loss": 3.488799827597565, + "tokens_seen": 1803943936 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003664593781344032, + "loss": 1.2365, + "theoretical_loss": 3.4887891324426135, + "tokens_seen": 1804009472 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003664433299899699, + "loss": 1.3054, + "theoretical_loss": 3.4887784377849727, + "tokens_seen": 1804075008 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003664272818455366, + "loss": 1.2577, + "theoretical_loss": 3.488767743624602, + "tokens_seen": 1804140544 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003664112337011033, + "loss": 1.3047, + "theoretical_loss": 3.4887570499614595, + "tokens_seen": 1804206080 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036639518555667, + "loss": 1.2722, + "theoretical_loss": 3.488746356795505, + "tokens_seen": 1804271616 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003663791374122367, + "loss": 1.3224, + "theoretical_loss": 3.488735664126697, + "tokens_seen": 1804337152 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003663630892678034, + "loss": 1.4133, + "theoretical_loss": 3.4887249719549933, + "tokens_seen": 1804402688 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003663470411233701, + "loss": 1.2678, + "theoretical_loss": 3.488714280280354, + "tokens_seen": 1804468224 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003663309929789368, + "loss": 1.3916, + "theoretical_loss": 3.4887035891027374, + "tokens_seen": 1804533760 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003663149448345035, + "loss": 1.3092, + "theoretical_loss": 3.4886928984221024, + "tokens_seen": 1804599296 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003662988966900702, + "loss": 1.2787, + "theoretical_loss": 3.4886822082384077, + "tokens_seen": 1804664832 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003662828485456369, + "loss": 1.2304, + "theoretical_loss": 3.4886715185516124, + "tokens_seen": 1804730368 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003662668004012036, + "loss": 1.2904, + "theoretical_loss": 3.4886608293616757, + "tokens_seen": 1804795904 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003662507522567703, + "loss": 1.3689, + "theoretical_loss": 3.4886501406685557, + "tokens_seen": 1804861440 + }, + { + "epoch": 0.55, + "learning_rate": 0.000366234704112337, + "loss": 1.1734, + "theoretical_loss": 3.4886394524722117, + "tokens_seen": 1804926976 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003662186559679037, + "loss": 1.2281, + "theoretical_loss": 3.488628764772603, + "tokens_seen": 1804992512 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003662026078234704, + "loss": 1.3037, + "theoretical_loss": 3.4886180775696873, + "tokens_seen": 1805058048 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003661865596790371, + "loss": 1.3697, + "theoretical_loss": 3.4886073908634243, + "tokens_seen": 1805123584 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003661705115346038, + "loss": 1.3233, + "theoretical_loss": 3.488596704653773, + "tokens_seen": 1805189120 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003661544633901705, + "loss": 1.2426, + "theoretical_loss": 3.488586018940692, + "tokens_seen": 1805254656 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003661384152457372, + "loss": 1.2545, + "theoretical_loss": 3.4885753337241403, + "tokens_seen": 1805320192 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003661223671013039, + "loss": 1.3232, + "theoretical_loss": 3.4885646490040765, + "tokens_seen": 1805385728 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003661063189568706, + "loss": 1.2528, + "theoretical_loss": 3.48855396478046, + "tokens_seen": 1805451264 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003660902708124373, + "loss": 1.3276, + "theoretical_loss": 3.4885432810532495, + "tokens_seen": 1805516800 + }, + { + "epoch": 0.55, + "learning_rate": 0.000366074222668004, + "loss": 1.415, + "theoretical_loss": 3.4885325978224038, + "tokens_seen": 1805582336 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003660581745235707, + "loss": 1.3813, + "theoretical_loss": 3.488521915087882, + "tokens_seen": 1805647872 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003660421263791374, + "loss": 1.3567, + "theoretical_loss": 3.4885112328496426, + "tokens_seen": 1805713408 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003660260782347041, + "loss": 1.3208, + "theoretical_loss": 3.4885005511076455, + "tokens_seen": 1805778944 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003660100300902708, + "loss": 1.1693, + "theoretical_loss": 3.4884898698618487, + "tokens_seen": 1805844480 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003659939819458375, + "loss": 1.3427, + "theoretical_loss": 3.4884791891122116, + "tokens_seen": 1805910016 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003659779338014042, + "loss": 1.2846, + "theoretical_loss": 3.488468508858693, + "tokens_seen": 1805975552 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003659618856569709, + "loss": 1.3282, + "theoretical_loss": 3.488457829101252, + "tokens_seen": 1806041088 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036594583751253763, + "loss": 1.1916, + "theoretical_loss": 3.488447149839847, + "tokens_seen": 1806106624 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036592978936810433, + "loss": 1.3181, + "theoretical_loss": 3.4884364710744378, + "tokens_seen": 1806172160 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036591374122367103, + "loss": 1.2587, + "theoretical_loss": 3.488425792804983, + "tokens_seen": 1806237696 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036589769307923773, + "loss": 1.372, + "theoretical_loss": 3.488415115031441, + "tokens_seen": 1806303232 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036588164493480443, + "loss": 1.3021, + "theoretical_loss": 3.488404437753772, + "tokens_seen": 1806368768 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036586559679037113, + "loss": 1.2783, + "theoretical_loss": 3.4883937609719338, + "tokens_seen": 1806434304 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036584954864593783, + "loss": 1.3012, + "theoretical_loss": 3.4883830846858856, + "tokens_seen": 1806499840 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036583350050150453, + "loss": 1.3005, + "theoretical_loss": 3.4883724088955876, + "tokens_seen": 1806565376 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036581745235707123, + "loss": 1.3229, + "theoretical_loss": 3.488361733600997, + "tokens_seen": 1806630912 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036580140421263793, + "loss": 1.2381, + "theoretical_loss": 3.488351058802074, + "tokens_seen": 1806696448 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036578535606820463, + "loss": 1.3605, + "theoretical_loss": 3.4883403844987777, + "tokens_seen": 1806761984 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036576930792377133, + "loss": 1.3745, + "theoretical_loss": 3.4883297106910662, + "tokens_seen": 1806827520 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036575325977933803, + "loss": 1.3036, + "theoretical_loss": 3.4883190373788997, + "tokens_seen": 1806893056 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036573721163490474, + "loss": 1.2695, + "theoretical_loss": 3.4883083645622355, + "tokens_seen": 1806958592 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036572116349047144, + "loss": 1.2547, + "theoretical_loss": 3.4882976922410345, + "tokens_seen": 1807024128 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036570511534603814, + "loss": 1.3892, + "theoretical_loss": 3.4882870204152545, + "tokens_seen": 1807089664 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036568906720160484, + "loss": 1.2595, + "theoretical_loss": 3.488276349084855, + "tokens_seen": 1807155200 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036567301905717154, + "loss": 1.2439, + "theoretical_loss": 3.488265678249795, + "tokens_seen": 1807220736 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036565697091273824, + "loss": 1.2159, + "theoretical_loss": 3.4882550079100336, + "tokens_seen": 1807286272 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036564092276830494, + "loss": 1.2528, + "theoretical_loss": 3.4882443380655297, + "tokens_seen": 1807351808 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036562487462387164, + "loss": 1.2708, + "theoretical_loss": 3.4882336687162425, + "tokens_seen": 1807417344 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036560882647943834, + "loss": 1.24, + "theoretical_loss": 3.4882229998621304, + "tokens_seen": 1807482880 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036559277833500504, + "loss": 1.3216, + "theoretical_loss": 3.4882123315031537, + "tokens_seen": 1807548416 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036557673019057174, + "loss": 1.2807, + "theoretical_loss": 3.4882016636392708, + "tokens_seen": 1807613952 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036556068204613844, + "loss": 1.3384, + "theoretical_loss": 3.4881909962704407, + "tokens_seen": 1807679488 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036554463390170514, + "loss": 1.298, + "theoretical_loss": 3.4881803293966227, + "tokens_seen": 1807745024 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036552858575727184, + "loss": 1.3619, + "theoretical_loss": 3.488169663017775, + "tokens_seen": 1807810560 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036551253761283854, + "loss": 1.301, + "theoretical_loss": 3.4881589971338585, + "tokens_seen": 1807876096 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036549648946840525, + "loss": 1.2928, + "theoretical_loss": 3.4881483317448305, + "tokens_seen": 1807941632 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036548044132397195, + "loss": 1.287, + "theoretical_loss": 3.488137666850651, + "tokens_seen": 1808007168 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036546439317953865, + "loss": 1.2864, + "theoretical_loss": 3.488127002451279, + "tokens_seen": 1808072704 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036544834503510535, + "loss": 1.3589, + "theoretical_loss": 3.488116338546674, + "tokens_seen": 1808138240 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036543229689067205, + "loss": 1.2781, + "theoretical_loss": 3.488105675136794, + "tokens_seen": 1808203776 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036541624874623875, + "loss": 1.3131, + "theoretical_loss": 3.4880950122215992, + "tokens_seen": 1808269312 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036540020060180545, + "loss": 1.3087, + "theoretical_loss": 3.4880843498010483, + "tokens_seen": 1808334848 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036538415245737215, + "loss": 1.3353, + "theoretical_loss": 3.4880736878751, + "tokens_seen": 1808400384 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036536810431293885, + "loss": 1.258, + "theoretical_loss": 3.4880630264437142, + "tokens_seen": 1808465920 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036535205616850555, + "loss": 1.3441, + "theoretical_loss": 3.48805236550685, + "tokens_seen": 1808531456 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036533600802407225, + "loss": 1.2953, + "theoretical_loss": 3.488041705064466, + "tokens_seen": 1808596992 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036531995987963895, + "loss": 1.2533, + "theoretical_loss": 3.4880310451165215, + "tokens_seen": 1808662528 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036530391173520565, + "loss": 1.3351, + "theoretical_loss": 3.488020385662976, + "tokens_seen": 1808728064 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 670351, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2942354679107666, + "objective/train/theoretical_loss": 3.4880097267037877, + "objective/train/tokens_used": 1829253600, + "theoretical_loss": 3.4880097267037877, + "tokens_seen": 1808793600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036528786359077235, + "loss": 1.2942, + "theoretical_loss": 3.4880097267037877, + "tokens_seen": 1808793600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036527181544633905, + "loss": 1.2962, + "theoretical_loss": 3.4879990682389175, + "tokens_seen": 1808859136 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036525576730190575, + "loss": 1.4249, + "theoretical_loss": 3.487988410268323, + "tokens_seen": 1808924672 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036523971915747246, + "loss": 1.3076, + "theoretical_loss": 3.4879777527919638, + "tokens_seen": 1808990208 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036522367101303916, + "loss": 1.2883, + "theoretical_loss": 3.4879670958097995, + "tokens_seen": 1809055744 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036520762286860586, + "loss": 1.4295, + "theoretical_loss": 3.4879564393217892, + "tokens_seen": 1809121280 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036519157472417256, + "loss": 1.2959, + "theoretical_loss": 3.4879457833278913, + "tokens_seen": 1809186816 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036517552657973926, + "loss": 1.3216, + "theoretical_loss": 3.4879351278280657, + "tokens_seen": 1809252352 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036515947843530596, + "loss": 1.3, + "theoretical_loss": 3.4879244728222716, + "tokens_seen": 1809317888 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036514343029087266, + "loss": 1.2188, + "theoretical_loss": 3.487913818310468, + "tokens_seen": 1809383424 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036512738214643936, + "loss": 1.2539, + "theoretical_loss": 3.4879031642926144, + "tokens_seen": 1809448960 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036511133400200606, + "loss": 1.3825, + "theoretical_loss": 3.4878925107686696, + "tokens_seen": 1809514496 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003650952858575727, + "loss": 1.3235, + "theoretical_loss": 3.487881857738593, + "tokens_seen": 1809580032 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003650792377131394, + "loss": 1.4282, + "theoretical_loss": 3.4878712052023433, + "tokens_seen": 1809645568 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003650631895687061, + "loss": 1.2714, + "theoretical_loss": 3.487860553159881, + "tokens_seen": 1809711104 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003650471414242728, + "loss": 1.284, + "theoretical_loss": 3.487849901611164, + "tokens_seen": 1809776640 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003650310932798395, + "loss": 1.2127, + "theoretical_loss": 3.487839250556153, + "tokens_seen": 1809842176 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003650150451354062, + "loss": 1.2076, + "theoretical_loss": 3.4878285999948053, + "tokens_seen": 1809907712 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003649989969909729, + "loss": 1.4587, + "theoretical_loss": 3.4878179499270816, + "tokens_seen": 1809973248 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003649829488465396, + "loss": 1.3039, + "theoretical_loss": 3.487807300352941, + "tokens_seen": 1810038784 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003649669007021063, + "loss": 1.2669, + "theoretical_loss": 3.487796651272342, + "tokens_seen": 1810104320 + }, + { + "epoch": 0.55, + "learning_rate": 0.000364950852557673, + "loss": 1.2684, + "theoretical_loss": 3.4877860026852443, + "tokens_seen": 1810169856 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003649348044132397, + "loss": 1.2345, + "theoretical_loss": 3.4877753545916077, + "tokens_seen": 1810235392 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003649187562688064, + "loss": 1.4486, + "theoretical_loss": 3.487764706991391, + "tokens_seen": 1810300928 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003649027081243731, + "loss": 1.3241, + "theoretical_loss": 3.487754059884553, + "tokens_seen": 1810366464 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003648866599799398, + "loss": 1.3389, + "theoretical_loss": 3.487743413271054, + "tokens_seen": 1810432000 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003648706118355065, + "loss": 1.3809, + "theoretical_loss": 3.487732767150852, + "tokens_seen": 1810497536 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003648545636910732, + "loss": 1.289, + "theoretical_loss": 3.487722121523907, + "tokens_seen": 1810563072 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003648385155466399, + "loss": 1.2756, + "theoretical_loss": 3.487711476390179, + "tokens_seen": 1810628608 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003648224674022066, + "loss": 1.2647, + "theoretical_loss": 3.4877008317496268, + "tokens_seen": 1810694144 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003648064192577733, + "loss": 1.2127, + "theoretical_loss": 3.4876901876022086, + "tokens_seen": 1810759680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036479037111334, + "loss": 1.3807, + "theoretical_loss": 3.487679543947885, + "tokens_seen": 1810825216 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003647743229689067, + "loss": 1.2578, + "theoretical_loss": 3.4876689007866153, + "tokens_seen": 1810890752 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003647582748244734, + "loss": 1.376, + "theoretical_loss": 3.4876582581183575, + "tokens_seen": 1810956288 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003647422266800401, + "loss": 1.3691, + "theoretical_loss": 3.4876476159430725, + "tokens_seen": 1811021824 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003647261785356068, + "loss": 1.2786, + "theoretical_loss": 3.4876369742607194, + "tokens_seen": 1811087360 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003647101303911735, + "loss": 1.2784, + "theoretical_loss": 3.4876263330712565, + "tokens_seen": 1811152896 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003646940822467402, + "loss": 1.3162, + "theoretical_loss": 3.4876156923746437, + "tokens_seen": 1811218432 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003646780341023069, + "loss": 1.3026, + "theoretical_loss": 3.487605052170841, + "tokens_seen": 1811283968 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003646619859578736, + "loss": 1.2396, + "theoretical_loss": 3.4875944124598064, + "tokens_seen": 1811349504 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003646459378134403, + "loss": 1.3948, + "theoretical_loss": 3.487583773241501, + "tokens_seen": 1811415040 + }, + { + "epoch": 0.55, + "learning_rate": 0.000364629889669007, + "loss": 1.2872, + "theoretical_loss": 3.4875731345158822, + "tokens_seen": 1811480576 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003646138415245737, + "loss": 1.3545, + "theoretical_loss": 3.4875624962829104, + "tokens_seen": 1811546112 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003645977933801404, + "loss": 1.1948, + "theoretical_loss": 3.4875518585425453, + "tokens_seen": 1811611648 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003645817452357071, + "loss": 1.3304, + "theoretical_loss": 3.487541221294746, + "tokens_seen": 1811677184 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036456569709127383, + "loss": 1.3806, + "theoretical_loss": 3.4875305845394715, + "tokens_seen": 1811742720 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003645496489468406, + "loss": 1.3106, + "theoretical_loss": 3.487519948276681, + "tokens_seen": 1811808256 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003645336008024073, + "loss": 1.3466, + "theoretical_loss": 3.487509312506335, + "tokens_seen": 1811873792 + }, + { + "epoch": 0.55, + "learning_rate": 0.000364517552657974, + "loss": 1.2387, + "theoretical_loss": 3.4874986772283916, + "tokens_seen": 1811939328 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003645015045135407, + "loss": 1.2026, + "theoretical_loss": 3.487488042442811, + "tokens_seen": 1812004864 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003644854563691074, + "loss": 1.2548, + "theoretical_loss": 3.4874774081495525, + "tokens_seen": 1812070400 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036446940822467403, + "loss": 1.4227, + "theoretical_loss": 3.487466774348575, + "tokens_seen": 1812135936 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036445336008024073, + "loss": 1.327, + "theoretical_loss": 3.4874561410398393, + "tokens_seen": 1812201472 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036443731193580743, + "loss": 1.2831, + "theoretical_loss": 3.4874455082233027, + "tokens_seen": 1812267008 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036442126379137413, + "loss": 1.2884, + "theoretical_loss": 3.487434875898926, + "tokens_seen": 1812332544 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036440521564694083, + "loss": 1.3583, + "theoretical_loss": 3.4874242440666685, + "tokens_seen": 1812398080 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036438916750250753, + "loss": 1.3551, + "theoretical_loss": 3.487413612726489, + "tokens_seen": 1812463616 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036437311935807424, + "loss": 1.236, + "theoretical_loss": 3.4874029818783487, + "tokens_seen": 1812529152 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036435707121364094, + "loss": 1.2949, + "theoretical_loss": 3.4873923515222045, + "tokens_seen": 1812594688 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036434102306920764, + "loss": 1.2706, + "theoretical_loss": 3.487381721658018, + "tokens_seen": 1812660224 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036432497492477434, + "loss": 1.4093, + "theoretical_loss": 3.487371092285747, + "tokens_seen": 1812725760 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036430892678034104, + "loss": 1.2057, + "theoretical_loss": 3.487360463405352, + "tokens_seen": 1812791296 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036429287863590774, + "loss": 1.3148, + "theoretical_loss": 3.4873498350167917, + "tokens_seen": 1812856832 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036427683049147444, + "loss": 1.2817, + "theoretical_loss": 3.4873392071200264, + "tokens_seen": 1812922368 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036426078234704114, + "loss": 1.3325, + "theoretical_loss": 3.4873285797150153, + "tokens_seen": 1812987904 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036424473420260784, + "loss": 1.2395, + "theoretical_loss": 3.4873179528017175, + "tokens_seen": 1813053440 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036422868605817454, + "loss": 1.2677, + "theoretical_loss": 3.487307326380093, + "tokens_seen": 1813118976 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036421263791374124, + "loss": 1.2872, + "theoretical_loss": 3.487296700450101, + "tokens_seen": 1813184512 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036419658976930794, + "loss": 1.3708, + "theoretical_loss": 3.487286075011701, + "tokens_seen": 1813250048 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036418054162487464, + "loss": 1.3452, + "theoretical_loss": 3.4872754500648524, + "tokens_seen": 1813315584 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036416449348044134, + "loss": 1.2533, + "theoretical_loss": 3.4872648256095147, + "tokens_seen": 1813381120 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036414844533600804, + "loss": 1.3935, + "theoretical_loss": 3.487254201645648, + "tokens_seen": 1813446656 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036413239719157474, + "loss": 1.237, + "theoretical_loss": 3.4872435781732105, + "tokens_seen": 1813512192 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036411634904714145, + "loss": 1.2853, + "theoretical_loss": 3.487232955192163, + "tokens_seen": 1813577728 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036410030090270815, + "loss": 1.2878, + "theoretical_loss": 3.487222332702464, + "tokens_seen": 1813643264 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036408425275827485, + "loss": 1.2768, + "theoretical_loss": 3.487211710704074, + "tokens_seen": 1813708800 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036406820461384155, + "loss": 1.3713, + "theoretical_loss": 3.487201089196952, + "tokens_seen": 1813774336 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036405215646940825, + "loss": 1.2252, + "theoretical_loss": 3.4871904681810575, + "tokens_seen": 1813839872 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036403610832497495, + "loss": 1.3003, + "theoretical_loss": 3.48717984765635, + "tokens_seen": 1813905408 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036402006018054165, + "loss": 1.4037, + "theoretical_loss": 3.4871692276227897, + "tokens_seen": 1813970944 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036400401203610835, + "loss": 1.4759, + "theoretical_loss": 3.487158608080335, + "tokens_seen": 1814036480 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036398796389167505, + "loss": 1.2166, + "theoretical_loss": 3.487147989028946, + "tokens_seen": 1814102016 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036397191574724175, + "loss": 1.2011, + "theoretical_loss": 3.4871373704685826, + "tokens_seen": 1814167552 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036395586760280845, + "loss": 1.4108, + "theoretical_loss": 3.487126752399204, + "tokens_seen": 1814233088 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036393981945837515, + "loss": 1.3107, + "theoretical_loss": 3.48711613482077, + "tokens_seen": 1814298624 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036392377131394185, + "loss": 1.2533, + "theoretical_loss": 3.48710551773324, + "tokens_seen": 1814364160 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036390772316950855, + "loss": 1.2205, + "theoretical_loss": 3.4870949011365733, + "tokens_seen": 1814429696 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036389167502507525, + "loss": 1.4819, + "theoretical_loss": 3.48708428503073, + "tokens_seen": 1814495232 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036387562688064195, + "loss": 1.2047, + "theoretical_loss": 3.487073669415669, + "tokens_seen": 1814560768 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036385957873620866, + "loss": 1.2598, + "theoretical_loss": 3.487063054291351, + "tokens_seen": 1814626304 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036384353059177536, + "loss": 1.2209, + "theoretical_loss": 3.4870524396577345, + "tokens_seen": 1814691840 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036382748244734206, + "loss": 1.2011, + "theoretical_loss": 3.4870418255147793, + "tokens_seen": 1814757376 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036381143430290876, + "loss": 1.3592, + "theoretical_loss": 3.487031211862446, + "tokens_seen": 1814822912 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036379538615847546, + "loss": 1.2237, + "theoretical_loss": 3.4870205987006924, + "tokens_seen": 1814888448 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036377933801404216, + "loss": 1.3846, + "theoretical_loss": 3.48700998602948, + "tokens_seen": 1814953984 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036376328986960886, + "loss": 1.4417, + "theoretical_loss": 3.486999373848767, + "tokens_seen": 1815019520 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036374724172517556, + "loss": 1.298, + "theoretical_loss": 3.4869887621585134, + "tokens_seen": 1815085056 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036373119358074226, + "loss": 1.2614, + "theoretical_loss": 3.486978150958679, + "tokens_seen": 1815150592 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036371514543630896, + "loss": 1.3218, + "theoretical_loss": 3.486967540249224, + "tokens_seen": 1815216128 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003636990972918756, + "loss": 1.3916, + "theoretical_loss": 3.486956930030107, + "tokens_seen": 1815281664 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 672501, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3391975164413452, + "objective/train/theoretical_loss": 3.4869463203012883, + "objective/train/tokens_used": 1835807200, + "theoretical_loss": 3.4869463203012883, + "tokens_seen": 1815347200 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003636830491474423, + "loss": 1.3392, + "theoretical_loss": 3.4869463203012883, + "tokens_seen": 1815347200 + }, + { + "epoch": 0.55, + "learning_rate": 0.000363667001003009, + "loss": 1.4527, + "theoretical_loss": 3.486935711062727, + "tokens_seen": 1815412736 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003636509528585757, + "loss": 1.3423, + "theoretical_loss": 3.4869251023143835, + "tokens_seen": 1815478272 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003636349047141424, + "loss": 1.2005, + "theoretical_loss": 3.486914494056217, + "tokens_seen": 1815543808 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003636188565697091, + "loss": 1.2742, + "theoretical_loss": 3.4869038862881867, + "tokens_seen": 1815609344 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003636028084252758, + "loss": 1.3306, + "theoretical_loss": 3.4868932790102534, + "tokens_seen": 1815674880 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003635867602808425, + "loss": 1.3567, + "theoretical_loss": 3.4868826722223756, + "tokens_seen": 1815740416 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003635707121364092, + "loss": 1.405, + "theoretical_loss": 3.486872065924514, + "tokens_seen": 1815805952 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003635546639919759, + "loss": 1.3109, + "theoretical_loss": 3.486861460116627, + "tokens_seen": 1815871488 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003635386158475426, + "loss": 1.262, + "theoretical_loss": 3.4868508547986754, + "tokens_seen": 1815937024 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003635225677031093, + "loss": 1.3309, + "theoretical_loss": 3.486840249970619, + "tokens_seen": 1816002560 + }, + { + "epoch": 0.55, + "learning_rate": 0.000363506519558676, + "loss": 1.3596, + "theoretical_loss": 3.4868296456324166, + "tokens_seen": 1816068096 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003634904714142427, + "loss": 1.3024, + "theoretical_loss": 3.4868190417840284, + "tokens_seen": 1816133632 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003634744232698094, + "loss": 1.2944, + "theoretical_loss": 3.486808438425414, + "tokens_seen": 1816199168 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003634583751253761, + "loss": 1.1889, + "theoretical_loss": 3.4867978355565334, + "tokens_seen": 1816264704 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003634423269809428, + "loss": 1.1839, + "theoretical_loss": 3.4867872331773455, + "tokens_seen": 1816330240 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003634262788365095, + "loss": 1.2992, + "theoretical_loss": 3.4867766312878112, + "tokens_seen": 1816395776 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003634102306920762, + "loss": 1.3387, + "theoretical_loss": 3.486766029887889, + "tokens_seen": 1816461312 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003633941825476429, + "loss": 1.2463, + "theoretical_loss": 3.4867554289775398, + "tokens_seen": 1816526848 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003633781344032096, + "loss": 1.3317, + "theoretical_loss": 3.4867448285567226, + "tokens_seen": 1816592384 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003633620862587764, + "loss": 1.3097, + "theoretical_loss": 3.486734228625397, + "tokens_seen": 1816657920 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003633460381143431, + "loss": 1.2383, + "theoretical_loss": 3.486723629183523, + "tokens_seen": 1816723456 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003633299899699098, + "loss": 1.2863, + "theoretical_loss": 3.4867130302310607, + "tokens_seen": 1816788992 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003633139418254765, + "loss": 1.2137, + "theoretical_loss": 3.486702431767969, + "tokens_seen": 1816854528 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003632978936810432, + "loss": 1.346, + "theoretical_loss": 3.4866918337942088, + "tokens_seen": 1816920064 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003632818455366099, + "loss": 1.292, + "theoretical_loss": 3.486681236309739, + "tokens_seen": 1816985600 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003632657973921766, + "loss": 1.3312, + "theoretical_loss": 3.4866706393145197, + "tokens_seen": 1817051136 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003632497492477433, + "loss": 1.3065, + "theoretical_loss": 3.48666004280851, + "tokens_seen": 1817116672 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036323370110331, + "loss": 1.3929, + "theoretical_loss": 3.4866494467916707, + "tokens_seen": 1817182208 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003632176529588767, + "loss": 1.2098, + "theoretical_loss": 3.486638851263961, + "tokens_seen": 1817247744 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003632016048144434, + "loss": 1.2935, + "theoretical_loss": 3.486628256225341, + "tokens_seen": 1817313280 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003631855566700101, + "loss": 1.2898, + "theoretical_loss": 3.48661766167577, + "tokens_seen": 1817378816 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003631695085255768, + "loss": 1.3324, + "theoretical_loss": 3.486607067615208, + "tokens_seen": 1817444352 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003631534603811435, + "loss": 1.3072, + "theoretical_loss": 3.486596474043615, + "tokens_seen": 1817509888 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003631374122367102, + "loss": 1.2845, + "theoretical_loss": 3.4865858809609507, + "tokens_seen": 1817575424 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003631213640922769, + "loss": 1.4074, + "theoretical_loss": 3.486575288367175, + "tokens_seen": 1817640960 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003631053159478436, + "loss": 1.3057, + "theoretical_loss": 3.4865646962622474, + "tokens_seen": 1817706496 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003630892678034103, + "loss": 1.322, + "theoretical_loss": 3.486554104646128, + "tokens_seen": 1817772032 + }, + { + "epoch": 0.55, + "learning_rate": 0.000363073219658977, + "loss": 1.2198, + "theoretical_loss": 3.486543513518776, + "tokens_seen": 1817837568 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036305717151454363, + "loss": 1.3305, + "theoretical_loss": 3.4865329228801523, + "tokens_seen": 1817903104 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036304112337011033, + "loss": 1.2235, + "theoretical_loss": 3.486522332730216, + "tokens_seen": 1817968640 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036302507522567703, + "loss": 1.2335, + "theoretical_loss": 3.4865117430689274, + "tokens_seen": 1818034176 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036300902708124374, + "loss": 1.3028, + "theoretical_loss": 3.4865011538962456, + "tokens_seen": 1818099712 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036299297893681044, + "loss": 1.2288, + "theoretical_loss": 3.486490565212131, + "tokens_seen": 1818165248 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036297693079237714, + "loss": 1.2372, + "theoretical_loss": 3.486479977016544, + "tokens_seen": 1818230784 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036296088264794384, + "loss": 1.2498, + "theoretical_loss": 3.486469389309443, + "tokens_seen": 1818296320 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036294483450351054, + "loss": 1.1995, + "theoretical_loss": 3.486458802090789, + "tokens_seen": 1818361856 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036292878635907724, + "loss": 1.2022, + "theoretical_loss": 3.4864482153605416, + "tokens_seen": 1818427392 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036291273821464394, + "loss": 1.2982, + "theoretical_loss": 3.4864376291186607, + "tokens_seen": 1818492928 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036289669007021064, + "loss": 1.3618, + "theoretical_loss": 3.4864270433651052, + "tokens_seen": 1818558464 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036288064192577734, + "loss": 1.2385, + "theoretical_loss": 3.486416458099837, + "tokens_seen": 1818624000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036286459378134404, + "loss": 1.2754, + "theoretical_loss": 3.486405873322814, + "tokens_seen": 1818689536 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036284854563691074, + "loss": 1.3664, + "theoretical_loss": 3.4863952890339975, + "tokens_seen": 1818755072 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036283249749247744, + "loss": 1.2451, + "theoretical_loss": 3.486384705233346, + "tokens_seen": 1818820608 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036281644934804414, + "loss": 1.3629, + "theoretical_loss": 3.4863741219208206, + "tokens_seen": 1818886144 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036280040120361084, + "loss": 1.3382, + "theoretical_loss": 3.486363539096381, + "tokens_seen": 1818951680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036278435305917754, + "loss": 1.3198, + "theoretical_loss": 3.486352956759987, + "tokens_seen": 1819017216 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036276830491474424, + "loss": 1.1998, + "theoretical_loss": 3.4863423749115983, + "tokens_seen": 1819082752 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036275225677031095, + "loss": 1.4538, + "theoretical_loss": 3.486331793551175, + "tokens_seen": 1819148288 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036273620862587765, + "loss": 1.2595, + "theoretical_loss": 3.4863212126786762, + "tokens_seen": 1819213824 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036272016048144435, + "loss": 1.312, + "theoretical_loss": 3.4863106322940633, + "tokens_seen": 1819279360 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036270411233701105, + "loss": 1.2293, + "theoretical_loss": 3.4863000523972953, + "tokens_seen": 1819344896 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036268806419257775, + "loss": 1.3261, + "theoretical_loss": 3.4862894729883322, + "tokens_seen": 1819410432 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036267201604814445, + "loss": 1.2724, + "theoretical_loss": 3.4862788940671345, + "tokens_seen": 1819475968 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036265596790371115, + "loss": 1.3279, + "theoretical_loss": 3.4862683156336614, + "tokens_seen": 1819541504 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036263991975927785, + "loss": 1.1928, + "theoretical_loss": 3.486257737687873, + "tokens_seen": 1819607040 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036262387161484455, + "loss": 1.2439, + "theoretical_loss": 3.4862471602297296, + "tokens_seen": 1819672576 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036260782347041125, + "loss": 1.2307, + "theoretical_loss": 3.486236583259191, + "tokens_seen": 1819738112 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036259177532597795, + "loss": 1.2573, + "theoretical_loss": 3.4862260067762176, + "tokens_seen": 1819803648 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036257572718154465, + "loss": 1.2904, + "theoretical_loss": 3.486215430780769, + "tokens_seen": 1819869184 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036255967903711135, + "loss": 1.2405, + "theoretical_loss": 3.486204855272804, + "tokens_seen": 1819934720 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036254363089267805, + "loss": 1.2037, + "theoretical_loss": 3.4861942802522843, + "tokens_seen": 1820000256 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036252758274824475, + "loss": 1.3141, + "theoretical_loss": 3.486183705719169, + "tokens_seen": 1820065792 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036251153460381145, + "loss": 1.3552, + "theoretical_loss": 3.486173131673419, + "tokens_seen": 1820131328 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036249548645937816, + "loss": 1.3386, + "theoretical_loss": 3.4861625581149926, + "tokens_seen": 1820196864 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036247943831494486, + "loss": 1.2109, + "theoretical_loss": 3.4861519850438514, + "tokens_seen": 1820262400 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036246339017051156, + "loss": 1.2605, + "theoretical_loss": 3.486141412459955, + "tokens_seen": 1820327936 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036244734202607826, + "loss": 1.2905, + "theoretical_loss": 3.486130840363263, + "tokens_seen": 1820393472 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036243129388164496, + "loss": 1.253, + "theoretical_loss": 3.4861202687537354, + "tokens_seen": 1820459008 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036241524573721166, + "loss": 1.2083, + "theoretical_loss": 3.486109697631333, + "tokens_seen": 1820524544 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036239919759277836, + "loss": 1.3264, + "theoretical_loss": 3.486099126996015, + "tokens_seen": 1820590080 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036238314944834506, + "loss": 1.3942, + "theoretical_loss": 3.4860885568477413, + "tokens_seen": 1820655616 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036236710130391176, + "loss": 1.246, + "theoretical_loss": 3.486077987186473, + "tokens_seen": 1820721152 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036235105315947846, + "loss": 1.2363, + "theoretical_loss": 3.486067418012169, + "tokens_seen": 1820786688 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036233500501504516, + "loss": 1.2962, + "theoretical_loss": 3.4860568493247897, + "tokens_seen": 1820852224 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036231895687061186, + "loss": 1.1614, + "theoretical_loss": 3.4860462811242954, + "tokens_seen": 1820917760 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036230290872617856, + "loss": 1.3428, + "theoretical_loss": 3.4860357134106463, + "tokens_seen": 1820983296 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003622868605817452, + "loss": 1.3023, + "theoretical_loss": 3.4860251461838017, + "tokens_seen": 1821048832 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003622708124373119, + "loss": 1.3458, + "theoretical_loss": 3.4860145794437223, + "tokens_seen": 1821114368 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003622547642928786, + "loss": 1.3082, + "theoretical_loss": 3.486004013190368, + "tokens_seen": 1821179904 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003622387161484453, + "loss": 1.1885, + "theoretical_loss": 3.4859934474236987, + "tokens_seen": 1821245440 + }, + { + "epoch": 0.55, + "learning_rate": 0.000362222668004012, + "loss": 1.3075, + "theoretical_loss": 3.4859828821436745, + "tokens_seen": 1821310976 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003622066198595787, + "loss": 1.1836, + "theoretical_loss": 3.4859723173502557, + "tokens_seen": 1821376512 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003621905717151454, + "loss": 1.298, + "theoretical_loss": 3.4859617530434024, + "tokens_seen": 1821442048 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003621745235707121, + "loss": 1.3124, + "theoretical_loss": 3.485951189223074, + "tokens_seen": 1821507584 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036215847542627887, + "loss": 1.3697, + "theoretical_loss": 3.4859406258892314, + "tokens_seen": 1821573120 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036214242728184557, + "loss": 1.2871, + "theoretical_loss": 3.4859300630418346, + "tokens_seen": 1821638656 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036212637913741227, + "loss": 1.301, + "theoretical_loss": 3.4859195006808434, + "tokens_seen": 1821704192 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036211033099297897, + "loss": 1.353, + "theoretical_loss": 3.485908938806218, + "tokens_seen": 1821769728 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036209428284854567, + "loss": 1.2603, + "theoretical_loss": 3.485898377417919, + "tokens_seen": 1821835264 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 674808, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2721842527389526, + "objective/train/theoretical_loss": 3.4858878165159055, + "objective/train/tokens_used": 1842360800, + "theoretical_loss": 3.4858878165159055, + "tokens_seen": 1821900800 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036207823470411237, + "loss": 1.2722, + "theoretical_loss": 3.4858878165159055, + "tokens_seen": 1821900800 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036206218655967907, + "loss": 1.3032, + "theoretical_loss": 3.485877256100138, + "tokens_seen": 1821966336 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003620461384152458, + "loss": 1.2667, + "theoretical_loss": 3.4858666961705773, + "tokens_seen": 1822031872 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003620300902708125, + "loss": 1.2581, + "theoretical_loss": 3.4858561367271825, + "tokens_seen": 1822097408 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003620140421263792, + "loss": 1.3354, + "theoretical_loss": 3.485845577769915, + "tokens_seen": 1822162944 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003619979939819459, + "loss": 1.2404, + "theoretical_loss": 3.4858350192987335, + "tokens_seen": 1822228480 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003619819458375126, + "loss": 1.0984, + "theoretical_loss": 3.485824461313599, + "tokens_seen": 1822294016 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003619658976930793, + "loss": 1.215, + "theoretical_loss": 3.4858139038144715, + "tokens_seen": 1822359552 + }, + { + "epoch": 0.55, + "learning_rate": 0.000361949849548646, + "loss": 1.206, + "theoretical_loss": 3.485803346801311, + "tokens_seen": 1822425088 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003619338014042127, + "loss": 1.3283, + "theoretical_loss": 3.485792790274078, + "tokens_seen": 1822490624 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003619177532597794, + "loss": 1.1211, + "theoretical_loss": 3.4857822342327323, + "tokens_seen": 1822556160 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003619017051153461, + "loss": 1.3168, + "theoretical_loss": 3.4857716786772346, + "tokens_seen": 1822621696 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003618856569709128, + "loss": 1.2378, + "theoretical_loss": 3.4857611236075443, + "tokens_seen": 1822687232 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003618696088264795, + "loss": 1.3078, + "theoretical_loss": 3.4857505690236223, + "tokens_seen": 1822752768 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003618535606820462, + "loss": 1.3185, + "theoretical_loss": 3.485740014925428, + "tokens_seen": 1822818304 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003618375125376129, + "loss": 1.1433, + "theoretical_loss": 3.485729461312922, + "tokens_seen": 1822883840 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003618214643931796, + "loss": 1.1671, + "theoretical_loss": 3.4857189081860644, + "tokens_seen": 1822949376 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003618054162487463, + "loss": 1.3716, + "theoretical_loss": 3.485708355544816, + "tokens_seen": 1823014912 + }, + { + "epoch": 0.55, + "learning_rate": 0.000361789368104313, + "loss": 1.2111, + "theoretical_loss": 3.4856978033891366, + "tokens_seen": 1823080448 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003617733199598797, + "loss": 1.1952, + "theoretical_loss": 3.4856872517189856, + "tokens_seen": 1823145984 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003617572718154464, + "loss": 1.2927, + "theoretical_loss": 3.4856767005343245, + "tokens_seen": 1823211520 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003617412236710131, + "loss": 1.4192, + "theoretical_loss": 3.4856661498351125, + "tokens_seen": 1823277056 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003617251755265798, + "loss": 1.3046, + "theoretical_loss": 3.4856555996213103, + "tokens_seen": 1823342592 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003617091273821465, + "loss": 1.2615, + "theoretical_loss": 3.485645049892878, + "tokens_seen": 1823408128 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003616930792377132, + "loss": 1.1707, + "theoretical_loss": 3.485634500649777, + "tokens_seen": 1823473664 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003616770310932799, + "loss": 1.2532, + "theoretical_loss": 3.4856239518919647, + "tokens_seen": 1823539200 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036166098294884653, + "loss": 1.2748, + "theoretical_loss": 3.4856134036194044, + "tokens_seen": 1823604736 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036164493480441323, + "loss": 1.3359, + "theoretical_loss": 3.485602855832054, + "tokens_seen": 1823670272 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036162888665997994, + "loss": 1.3597, + "theoretical_loss": 3.4855923085298754, + "tokens_seen": 1823735808 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036161283851554664, + "loss": 1.3289, + "theoretical_loss": 3.4855817617128277, + "tokens_seen": 1823801344 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036159679037111334, + "loss": 1.3028, + "theoretical_loss": 3.4855712153808724, + "tokens_seen": 1823866880 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036158074222668004, + "loss": 1.1622, + "theoretical_loss": 3.4855606695339683, + "tokens_seen": 1823932416 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036156469408224674, + "loss": 1.2859, + "theoretical_loss": 3.485550124172077, + "tokens_seen": 1823997952 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036154864593781344, + "loss": 1.3026, + "theoretical_loss": 3.4855395792951573, + "tokens_seen": 1824063488 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036153259779338014, + "loss": 1.279, + "theoretical_loss": 3.4855290349031707, + "tokens_seen": 1824129024 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036151654964894684, + "loss": 1.2973, + "theoretical_loss": 3.4855184909960775, + "tokens_seen": 1824194560 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036150050150451354, + "loss": 1.2321, + "theoretical_loss": 3.4855079475738373, + "tokens_seen": 1824260096 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036148445336008024, + "loss": 1.2607, + "theoretical_loss": 3.4854974046364102, + "tokens_seen": 1824325632 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036146840521564694, + "loss": 1.3147, + "theoretical_loss": 3.485486862183757, + "tokens_seen": 1824391168 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036145235707121364, + "loss": 1.223, + "theoretical_loss": 3.4854763202158385, + "tokens_seen": 1824456704 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036143630892678034, + "loss": 1.2575, + "theoretical_loss": 3.485465778732614, + "tokens_seen": 1824522240 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036142026078234704, + "loss": 1.2681, + "theoretical_loss": 3.485455237734045, + "tokens_seen": 1824587776 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036140421263791374, + "loss": 1.2711, + "theoretical_loss": 3.4854446972200903, + "tokens_seen": 1824653312 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036138816449348044, + "loss": 1.4449, + "theoretical_loss": 3.485434157190711, + "tokens_seen": 1824718848 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036137211634904715, + "loss": 1.2158, + "theoretical_loss": 3.4854236176458677, + "tokens_seen": 1824784384 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036135606820461385, + "loss": 1.2879, + "theoretical_loss": 3.4854130785855206, + "tokens_seen": 1824849920 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036134002006018055, + "loss": 1.2785, + "theoretical_loss": 3.4854025400096296, + "tokens_seen": 1824915456 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036132397191574725, + "loss": 1.2894, + "theoretical_loss": 3.485392001918155, + "tokens_seen": 1824980992 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036130792377131395, + "loss": 1.1819, + "theoretical_loss": 3.485381464311058, + "tokens_seen": 1825046528 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036129187562688065, + "loss": 1.171, + "theoretical_loss": 3.485370927188298, + "tokens_seen": 1825112064 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036127582748244735, + "loss": 1.2869, + "theoretical_loss": 3.4853603905498356, + "tokens_seen": 1825177600 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036125977933801405, + "loss": 1.2825, + "theoretical_loss": 3.485349854395632, + "tokens_seen": 1825243136 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036124373119358075, + "loss": 1.2787, + "theoretical_loss": 3.4853393187256456, + "tokens_seen": 1825308672 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036122768304914745, + "loss": 1.3809, + "theoretical_loss": 3.4853287835398388, + "tokens_seen": 1825374208 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036121163490471415, + "loss": 1.2724, + "theoretical_loss": 3.485318248838171, + "tokens_seen": 1825439744 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036119558676028085, + "loss": 1.1571, + "theoretical_loss": 3.485307714620603, + "tokens_seen": 1825505280 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036117953861584755, + "loss": 1.2738, + "theoretical_loss": 3.4852971808870947, + "tokens_seen": 1825570816 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036116349047141425, + "loss": 1.2857, + "theoretical_loss": 3.4852866476376065, + "tokens_seen": 1825636352 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036114744232698095, + "loss": 1.2802, + "theoretical_loss": 3.485276114872099, + "tokens_seen": 1825701888 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036113139418254766, + "loss": 1.2804, + "theoretical_loss": 3.4852655825905328, + "tokens_seen": 1825767424 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036111534603811436, + "loss": 1.3049, + "theoretical_loss": 3.485255050792868, + "tokens_seen": 1825832960 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036109929789368106, + "loss": 1.2965, + "theoretical_loss": 3.485244519479065, + "tokens_seen": 1825898496 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036108324974924776, + "loss": 1.301, + "theoretical_loss": 3.4852339886490844, + "tokens_seen": 1825964032 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036106720160481446, + "loss": 1.2356, + "theoretical_loss": 3.4852234583028863, + "tokens_seen": 1826029568 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036105115346038116, + "loss": 1.2331, + "theoretical_loss": 3.485212928440431, + "tokens_seen": 1826095104 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036103510531594786, + "loss": 1.1888, + "theoretical_loss": 3.48520239906168, + "tokens_seen": 1826160640 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036101905717151456, + "loss": 1.2205, + "theoretical_loss": 3.4851918701665925, + "tokens_seen": 1826226176 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036100300902708126, + "loss": 1.2495, + "theoretical_loss": 3.485181341755129, + "tokens_seen": 1826291712 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036098696088264796, + "loss": 1.2872, + "theoretical_loss": 3.4851708138272506, + "tokens_seen": 1826357248 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036097091273821466, + "loss": 1.2295, + "theoretical_loss": 3.4851602863829174, + "tokens_seen": 1826422784 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036095486459378136, + "loss": 1.3657, + "theoretical_loss": 3.48514975942209, + "tokens_seen": 1826488320 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036093881644934806, + "loss": 1.4513, + "theoretical_loss": 3.4851392329447286, + "tokens_seen": 1826553856 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036092276830491476, + "loss": 1.334, + "theoretical_loss": 3.4851287069507935, + "tokens_seen": 1826619392 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036090672016048146, + "loss": 1.2897, + "theoretical_loss": 3.485118181440246, + "tokens_seen": 1826684928 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036089067201604816, + "loss": 1.337, + "theoretical_loss": 3.4851076564130454, + "tokens_seen": 1826750464 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036087462387161487, + "loss": 1.26, + "theoretical_loss": 3.485097131869153, + "tokens_seen": 1826816000 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036085857572718157, + "loss": 1.3004, + "theoretical_loss": 3.4850866078085287, + "tokens_seen": 1826881536 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036084252758274827, + "loss": 1.27, + "theoretical_loss": 3.485076084231134, + "tokens_seen": 1826947072 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036082647943831497, + "loss": 1.2081, + "theoretical_loss": 3.485065561136928, + "tokens_seen": 1827012608 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036081043129388167, + "loss": 1.271, + "theoretical_loss": 3.4850550385258723, + "tokens_seen": 1827078144 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036079438314944837, + "loss": 1.2778, + "theoretical_loss": 3.4850445163979265, + "tokens_seen": 1827143680 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036077833500501507, + "loss": 1.3709, + "theoretical_loss": 3.485033994753052, + "tokens_seen": 1827209216 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036076228686058177, + "loss": 1.2897, + "theoretical_loss": 3.4850234735912085, + "tokens_seen": 1827274752 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036074623871614847, + "loss": 1.1365, + "theoretical_loss": 3.4850129529123564, + "tokens_seen": 1827340288 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036073019057171517, + "loss": 1.433, + "theoretical_loss": 3.4850024327164575, + "tokens_seen": 1827405824 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036071414242728187, + "loss": 1.1952, + "theoretical_loss": 3.484991913003471, + "tokens_seen": 1827471360 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036069809428284857, + "loss": 1.3011, + "theoretical_loss": 3.484981393773358, + "tokens_seen": 1827536896 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036068204613841527, + "loss": 1.253, + "theoretical_loss": 3.484970875026079, + "tokens_seen": 1827602432 + }, + { + "epoch": 0.55, + "learning_rate": 0.000360665997993982, + "loss": 1.3398, + "theoretical_loss": 3.4849603567615937, + "tokens_seen": 1827667968 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003606499498495487, + "loss": 1.3193, + "theoretical_loss": 3.4849498389798637, + "tokens_seen": 1827733504 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003606339017051154, + "loss": 1.2925, + "theoretical_loss": 3.4849393216808497, + "tokens_seen": 1827799040 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003606178535606821, + "loss": 1.2159, + "theoretical_loss": 3.484928804864511, + "tokens_seen": 1827864576 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003606018054162488, + "loss": 1.2627, + "theoretical_loss": 3.4849182885308094, + "tokens_seen": 1827930112 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003605857572718155, + "loss": 1.2849, + "theoretical_loss": 3.484907772679705, + "tokens_seen": 1827995648 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003605697091273822, + "loss": 1.355, + "theoretical_loss": 3.4848972573111574, + "tokens_seen": 1828061184 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003605536609829489, + "loss": 1.2754, + "theoretical_loss": 3.484886742425129, + "tokens_seen": 1828126720 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003605376128385156, + "loss": 1.1823, + "theoretical_loss": 3.4848762280215784, + "tokens_seen": 1828192256 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003605215646940823, + "loss": 1.1458, + "theoretical_loss": 3.4848657141004677, + "tokens_seen": 1828257792 + }, + { + "epoch": 0.55, + "learning_rate": 0.000360505516549649, + "loss": 1.2216, + "theoretical_loss": 3.484855200661757, + "tokens_seen": 1828323328 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003604894684052157, + "loss": 1.3, + "theoretical_loss": 3.4848446877054062, + "tokens_seen": 1828388864 + }, + { + "epoch": 0.55, + "objective/train/docs_used": 677204, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2367684841156006, + "objective/train/theoretical_loss": 3.484834175231377, + "objective/train/tokens_used": 1848914400, + "theoretical_loss": 3.484834175231377, + "tokens_seen": 1828454400 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003604734202607824, + "loss": 1.2368, + "theoretical_loss": 3.484834175231377, + "tokens_seen": 1828454400 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003604573721163491, + "loss": 1.3096, + "theoretical_loss": 3.4848236632396294, + "tokens_seen": 1828519936 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003604413239719158, + "loss": 1.2633, + "theoretical_loss": 3.484813151730124, + "tokens_seen": 1828585472 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003604252758274825, + "loss": 1.3248, + "theoretical_loss": 3.484802640702821, + "tokens_seen": 1828651008 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003604092276830492, + "loss": 1.2774, + "theoretical_loss": 3.484792130157682, + "tokens_seen": 1828716544 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003603931795386159, + "loss": 1.2364, + "theoretical_loss": 3.484781620094667, + "tokens_seen": 1828782080 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003603771313941826, + "loss": 1.2121, + "theoretical_loss": 3.4847711105137367, + "tokens_seen": 1828847616 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003603610832497493, + "loss": 1.3228, + "theoretical_loss": 3.4847606014148518, + "tokens_seen": 1828913152 + }, + { + "epoch": 0.55, + "learning_rate": 0.000360345035105316, + "loss": 1.2325, + "theoretical_loss": 3.484750092797972, + "tokens_seen": 1828978688 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003603289869608827, + "loss": 1.2337, + "theoretical_loss": 3.4847395846630596, + "tokens_seen": 1829044224 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003603129388164494, + "loss": 1.2489, + "theoretical_loss": 3.4847290770100736, + "tokens_seen": 1829109760 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003602968906720161, + "loss": 1.2183, + "theoretical_loss": 3.484718569838976, + "tokens_seen": 1829175296 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003602808425275828, + "loss": 1.3433, + "theoretical_loss": 3.484708063149726, + "tokens_seen": 1829240832 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036026479438314944, + "loss": 1.4086, + "theoretical_loss": 3.484697556942286, + "tokens_seen": 1829306368 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036024874623871614, + "loss": 1.2972, + "theoretical_loss": 3.484687051216615, + "tokens_seen": 1829371904 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036023269809428284, + "loss": 1.3328, + "theoretical_loss": 3.4846765459726745, + "tokens_seen": 1829437440 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036021664994984954, + "loss": 1.2857, + "theoretical_loss": 3.484666041210425, + "tokens_seen": 1829502976 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036020060180541624, + "loss": 1.3578, + "theoretical_loss": 3.484655536929827, + "tokens_seen": 1829568512 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036018455366098294, + "loss": 1.1524, + "theoretical_loss": 3.4846450331308416, + "tokens_seen": 1829634048 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036016850551654964, + "loss": 1.1948, + "theoretical_loss": 3.484634529813429, + "tokens_seen": 1829699584 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036015245737211634, + "loss": 1.2525, + "theoretical_loss": 3.48462402697755, + "tokens_seen": 1829765120 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036013640922768304, + "loss": 1.2323, + "theoretical_loss": 3.4846135246231658, + "tokens_seen": 1829830656 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036012036108324974, + "loss": 1.2172, + "theoretical_loss": 3.4846030227502363, + "tokens_seen": 1829896192 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036010431293881644, + "loss": 1.3382, + "theoretical_loss": 3.484592521358722, + "tokens_seen": 1829961728 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036008826479438314, + "loss": 1.2711, + "theoretical_loss": 3.4845820204485847, + "tokens_seen": 1830027264 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036007221664994984, + "loss": 1.2805, + "theoretical_loss": 3.4845715200197844, + "tokens_seen": 1830092800 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036005616850551654, + "loss": 1.196, + "theoretical_loss": 3.484561020072282, + "tokens_seen": 1830158336 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036004012036108324, + "loss": 1.2159, + "theoretical_loss": 3.484550520606038, + "tokens_seen": 1830223872 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036002407221664994, + "loss": 1.2029, + "theoretical_loss": 3.484540021621013, + "tokens_seen": 1830289408 + }, + { + "epoch": 0.55, + "learning_rate": 0.00036000802407221665, + "loss": 1.2573, + "theoretical_loss": 3.484529523117168, + "tokens_seen": 1830354944 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035999197592778335, + "loss": 1.3204, + "theoretical_loss": 3.4845190250944635, + "tokens_seen": 1830420480 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035997592778335005, + "loss": 1.1455, + "theoretical_loss": 3.484508527552861, + "tokens_seen": 1830486016 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035995987963891675, + "loss": 1.2278, + "theoretical_loss": 3.48449803049232, + "tokens_seen": 1830551552 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035994383149448345, + "loss": 1.2978, + "theoretical_loss": 3.4844875339128016, + "tokens_seen": 1830617088 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035992778335005015, + "loss": 1.485, + "theoretical_loss": 3.4844770378142673, + "tokens_seen": 1830682624 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035991173520561685, + "loss": 1.2834, + "theoretical_loss": 3.484466542196677, + "tokens_seen": 1830748160 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035989568706118355, + "loss": 1.2538, + "theoretical_loss": 3.484456047059992, + "tokens_seen": 1830813696 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035987963891675025, + "loss": 1.3566, + "theoretical_loss": 3.484445552404173, + "tokens_seen": 1830879232 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035986359077231695, + "loss": 1.2636, + "theoretical_loss": 3.48443505822918, + "tokens_seen": 1830944768 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035984754262788365, + "loss": 1.4141, + "theoretical_loss": 3.484424564534975, + "tokens_seen": 1831010304 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035983149448345035, + "loss": 1.3119, + "theoretical_loss": 3.4844140713215177, + "tokens_seen": 1831075840 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035981544633901705, + "loss": 1.3809, + "theoretical_loss": 3.4844035785887693, + "tokens_seen": 1831141376 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035979939819458375, + "loss": 1.2798, + "theoretical_loss": 3.4843930863366905, + "tokens_seen": 1831206912 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035978335005015045, + "loss": 1.221, + "theoretical_loss": 3.4843825945652425, + "tokens_seen": 1831272448 + }, + { + "epoch": 0.55, + "learning_rate": 0.00035976730190571715, + "loss": 1.3016, + "theoretical_loss": 3.484372103274385, + "tokens_seen": 1831337984 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003597512537612839, + "loss": 1.3521, + "theoretical_loss": 3.4843616124640797, + "tokens_seen": 1831403520 + }, + { + "epoch": 0.55, + "learning_rate": 0.0003597352056168506, + "loss": 1.2447, + "theoretical_loss": 3.4843511221342873, + "tokens_seen": 1831469056 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003597191574724173, + "loss": 1.3746, + "theoretical_loss": 3.484340632284969, + "tokens_seen": 1831534592 + }, + { + "epoch": 0.56, + "learning_rate": 0.000359703109327984, + "loss": 1.4809, + "theoretical_loss": 3.4843301429160842, + "tokens_seen": 1831600128 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003596870611835507, + "loss": 1.2925, + "theoretical_loss": 3.484319654027595, + "tokens_seen": 1831665664 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003596710130391174, + "loss": 1.2497, + "theoretical_loss": 3.484309165619462, + "tokens_seen": 1831731200 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003596549648946841, + "loss": 1.3549, + "theoretical_loss": 3.484298677691645, + "tokens_seen": 1831796736 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003596389167502508, + "loss": 1.2066, + "theoretical_loss": 3.484288190244106, + "tokens_seen": 1831862272 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035962286860581746, + "loss": 1.3559, + "theoretical_loss": 3.484277703276806, + "tokens_seen": 1831927808 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035960682046138416, + "loss": 1.4067, + "theoretical_loss": 3.484267216789705, + "tokens_seen": 1831993344 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035959077231695086, + "loss": 1.3232, + "theoretical_loss": 3.4842567307827643, + "tokens_seen": 1832058880 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035957472417251756, + "loss": 1.419, + "theoretical_loss": 3.484246245255944, + "tokens_seen": 1832124416 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035955867602808426, + "loss": 1.3281, + "theoretical_loss": 3.484235760209206, + "tokens_seen": 1832189952 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035954262788365096, + "loss": 1.3293, + "theoretical_loss": 3.4842252756425105, + "tokens_seen": 1832255488 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035952657973921766, + "loss": 1.3247, + "theoretical_loss": 3.4842147915558184, + "tokens_seen": 1832321024 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035951053159478436, + "loss": 1.4027, + "theoretical_loss": 3.4842043079490908, + "tokens_seen": 1832386560 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035949448345035107, + "loss": 1.3235, + "theoretical_loss": 3.4841938248222886, + "tokens_seen": 1832452096 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035947843530591777, + "loss": 1.2204, + "theoretical_loss": 3.4841833421753723, + "tokens_seen": 1832517632 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035946238716148447, + "loss": 1.2271, + "theoretical_loss": 3.484172860008303, + "tokens_seen": 1832583168 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035944633901705117, + "loss": 1.4005, + "theoretical_loss": 3.4841623783210416, + "tokens_seen": 1832648704 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035943029087261787, + "loss": 1.1811, + "theoretical_loss": 3.4841518971135486, + "tokens_seen": 1832714240 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035941424272818457, + "loss": 1.3683, + "theoretical_loss": 3.4841414163857856, + "tokens_seen": 1832779776 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035939819458375127, + "loss": 1.2616, + "theoretical_loss": 3.484130936137713, + "tokens_seen": 1832845312 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035938214643931797, + "loss": 1.3673, + "theoretical_loss": 3.4841204563692916, + "tokens_seen": 1832910848 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035936609829488467, + "loss": 1.2529, + "theoretical_loss": 3.484109977080483, + "tokens_seen": 1832976384 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035935005015045137, + "loss": 1.2394, + "theoretical_loss": 3.484099498271247, + "tokens_seen": 1833041920 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035933400200601807, + "loss": 1.2712, + "theoretical_loss": 3.4840890199415453, + "tokens_seen": 1833107456 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035931795386158477, + "loss": 1.2454, + "theoretical_loss": 3.4840785420913387, + "tokens_seen": 1833172992 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003593019057171515, + "loss": 1.3403, + "theoretical_loss": 3.484068064720588, + "tokens_seen": 1833238528 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003592858575727182, + "loss": 1.2649, + "theoretical_loss": 3.4840575878292546, + "tokens_seen": 1833304064 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003592698094282849, + "loss": 1.2573, + "theoretical_loss": 3.484047111417298, + "tokens_seen": 1833369600 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003592537612838516, + "loss": 1.343, + "theoretical_loss": 3.484036635484681, + "tokens_seen": 1833435136 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003592377131394183, + "loss": 1.2579, + "theoretical_loss": 3.4840261600313633, + "tokens_seen": 1833500672 + }, + { + "epoch": 0.56, + "learning_rate": 0.000359221664994985, + "loss": 1.3222, + "theoretical_loss": 3.4840156850573063, + "tokens_seen": 1833566208 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003592056168505517, + "loss": 1.3827, + "theoretical_loss": 3.4840052105624704, + "tokens_seen": 1833631744 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003591895687061184, + "loss": 1.3006, + "theoretical_loss": 3.4839947365468174, + "tokens_seen": 1833697280 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003591735205616851, + "loss": 1.1579, + "theoretical_loss": 3.483984263010308, + "tokens_seen": 1833762816 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003591574724172518, + "loss": 1.2496, + "theoretical_loss": 3.483973789952903, + "tokens_seen": 1833828352 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003591414242728185, + "loss": 1.458, + "theoretical_loss": 3.4839633173745628, + "tokens_seen": 1833893888 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003591253761283852, + "loss": 1.3127, + "theoretical_loss": 3.4839528452752493, + "tokens_seen": 1833959424 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003591093279839519, + "loss": 1.3194, + "theoretical_loss": 3.483942373654923, + "tokens_seen": 1834024960 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003590932798395186, + "loss": 1.2295, + "theoretical_loss": 3.483931902513545, + "tokens_seen": 1834090496 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003590772316950853, + "loss": 1.3001, + "theoretical_loss": 3.4839214318510763, + "tokens_seen": 1834156032 + }, + { + "epoch": 0.56, + "learning_rate": 0.000359061183550652, + "loss": 1.2751, + "theoretical_loss": 3.483910961667478, + "tokens_seen": 1834221568 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003590451354062187, + "loss": 1.2203, + "theoretical_loss": 3.483900491962711, + "tokens_seen": 1834287104 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003590290872617854, + "loss": 1.2461, + "theoretical_loss": 3.4838900227367358, + "tokens_seen": 1834352640 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003590130391173521, + "loss": 1.5078, + "theoretical_loss": 3.4838795539895138, + "tokens_seen": 1834418176 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003589969909729188, + "loss": 1.3334, + "theoretical_loss": 3.483869085721006, + "tokens_seen": 1834483712 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003589809428284855, + "loss": 1.2932, + "theoretical_loss": 3.483858617931174, + "tokens_seen": 1834549248 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003589648946840522, + "loss": 1.3352, + "theoretical_loss": 3.483848150619978, + "tokens_seen": 1834614784 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003589488465396189, + "loss": 1.306, + "theoretical_loss": 3.483837683787379, + "tokens_seen": 1834680320 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003589327983951856, + "loss": 1.2541, + "theoretical_loss": 3.4838272174333387, + "tokens_seen": 1834745856 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003589167502507523, + "loss": 1.3995, + "theoretical_loss": 3.4838167515578173, + "tokens_seen": 1834811392 + }, + { + "epoch": 0.56, + "learning_rate": 0.000358900702106319, + "loss": 1.2525, + "theoretical_loss": 3.4838062861607764, + "tokens_seen": 1834876928 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003588846539618857, + "loss": 1.309, + "theoretical_loss": 3.483795821242177, + "tokens_seen": 1834942464 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 679738, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2952748537063599, + "objective/train/theoretical_loss": 3.48378535680198, + "objective/train/tokens_used": 1855468000, + "theoretical_loss": 3.48378535680198, + "tokens_seen": 1835008000 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003588686058174524, + "loss": 1.2953, + "theoretical_loss": 3.48378535680198, + "tokens_seen": 1835008000 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035885255767301904, + "loss": 1.401, + "theoretical_loss": 3.4837748928401466, + "tokens_seen": 1835073536 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035883650952858574, + "loss": 1.2941, + "theoretical_loss": 3.4837644293566377, + "tokens_seen": 1835139072 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035882046138415244, + "loss": 1.2303, + "theoretical_loss": 3.483753966351414, + "tokens_seen": 1835204608 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035880441323971914, + "loss": 1.2917, + "theoretical_loss": 3.4837435038244378, + "tokens_seen": 1835270144 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035878836509528584, + "loss": 1.2749, + "theoretical_loss": 3.4837330417756682, + "tokens_seen": 1835335680 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035877231695085254, + "loss": 1.3176, + "theoretical_loss": 3.483722580205068, + "tokens_seen": 1835401216 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035875626880641924, + "loss": 1.323, + "theoretical_loss": 3.4837121191125977, + "tokens_seen": 1835466752 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035874022066198594, + "loss": 1.2741, + "theoretical_loss": 3.483701658498218, + "tokens_seen": 1835532288 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035872417251755264, + "loss": 1.3139, + "theoretical_loss": 3.483691198361891, + "tokens_seen": 1835597824 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035870812437311934, + "loss": 1.3271, + "theoretical_loss": 3.483680738703576, + "tokens_seen": 1835663360 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035869207622868604, + "loss": 1.1153, + "theoretical_loss": 3.4836702795232357, + "tokens_seen": 1835728896 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035867602808425274, + "loss": 1.3463, + "theoretical_loss": 3.4836598208208307, + "tokens_seen": 1835794432 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035865997993981944, + "loss": 1.2284, + "theoretical_loss": 3.483649362596322, + "tokens_seen": 1835859968 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035864393179538614, + "loss": 1.2052, + "theoretical_loss": 3.4836389048496708, + "tokens_seen": 1835925504 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035862788365095285, + "loss": 1.3464, + "theoretical_loss": 3.4836284475808386, + "tokens_seen": 1835991040 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035861183550651955, + "loss": 1.2312, + "theoretical_loss": 3.4836179907897855, + "tokens_seen": 1836056576 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035859578736208625, + "loss": 1.2213, + "theoretical_loss": 3.4836075344764734, + "tokens_seen": 1836122112 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035857973921765295, + "loss": 1.2676, + "theoretical_loss": 3.483597078640863, + "tokens_seen": 1836187648 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035856369107321965, + "loss": 1.287, + "theoretical_loss": 3.4835866232829167, + "tokens_seen": 1836253184 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003585476429287864, + "loss": 1.3312, + "theoretical_loss": 3.4835761684025934, + "tokens_seen": 1836318720 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003585315947843531, + "loss": 1.2455, + "theoretical_loss": 3.483565713999856, + "tokens_seen": 1836384256 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003585155466399198, + "loss": 1.2484, + "theoretical_loss": 3.483555260074665, + "tokens_seen": 1836449792 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003584994984954865, + "loss": 1.1925, + "theoretical_loss": 3.4835448066269814, + "tokens_seen": 1836515328 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003584834503510532, + "loss": 1.2854, + "theoretical_loss": 3.483534353656767, + "tokens_seen": 1836580864 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003584674022066199, + "loss": 1.3884, + "theoretical_loss": 3.4835239011639825, + "tokens_seen": 1836646400 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003584513540621866, + "loss": 1.3074, + "theoretical_loss": 3.483513449148589, + "tokens_seen": 1836711936 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003584353059177533, + "loss": 1.2846, + "theoretical_loss": 3.4835029976105476, + "tokens_seen": 1836777472 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035841925777332, + "loss": 1.2531, + "theoretical_loss": 3.4834925465498197, + "tokens_seen": 1836843008 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003584032096288867, + "loss": 1.275, + "theoretical_loss": 3.483482095966366, + "tokens_seen": 1836908544 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003583871614844534, + "loss": 1.2522, + "theoretical_loss": 3.4834716458601482, + "tokens_seen": 1836974080 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003583711133400201, + "loss": 1.3039, + "theoretical_loss": 3.483461196231128, + "tokens_seen": 1837039616 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003583550651955868, + "loss": 1.3846, + "theoretical_loss": 3.4834507470792655, + "tokens_seen": 1837105152 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003583390170511535, + "loss": 1.2717, + "theoretical_loss": 3.4834402984045223, + "tokens_seen": 1837170688 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003583229689067202, + "loss": 1.2554, + "theoretical_loss": 3.4834298502068597, + "tokens_seen": 1837236224 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003583069207622869, + "loss": 1.2555, + "theoretical_loss": 3.4834194024862386, + "tokens_seen": 1837301760 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003582908726178536, + "loss": 1.3575, + "theoretical_loss": 3.483408955242621, + "tokens_seen": 1837367296 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003582748244734203, + "loss": 1.198, + "theoretical_loss": 3.483398508475967, + "tokens_seen": 1837432832 + }, + { + "epoch": 0.56, + "learning_rate": 0.000358258776328987, + "loss": 1.2773, + "theoretical_loss": 3.4833880621862385, + "tokens_seen": 1837498368 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003582427281845537, + "loss": 1.2469, + "theoretical_loss": 3.4833776163733967, + "tokens_seen": 1837563904 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035822668004012036, + "loss": 1.3568, + "theoretical_loss": 3.4833671710374023, + "tokens_seen": 1837629440 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035821063189568706, + "loss": 1.2855, + "theoretical_loss": 3.4833567261782177, + "tokens_seen": 1837694976 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035819458375125376, + "loss": 1.4882, + "theoretical_loss": 3.4833462817958027, + "tokens_seen": 1837760512 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035817853560682046, + "loss": 1.3101, + "theoretical_loss": 3.4833358378901194, + "tokens_seen": 1837826048 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035816248746238716, + "loss": 1.1893, + "theoretical_loss": 3.4833253944611284, + "tokens_seen": 1837891584 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035814643931795386, + "loss": 1.4205, + "theoretical_loss": 3.483314951508792, + "tokens_seen": 1837957120 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035813039117352057, + "loss": 1.249, + "theoretical_loss": 3.4833045090330703, + "tokens_seen": 1838022656 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035811434302908727, + "loss": 1.2554, + "theoretical_loss": 3.483294067033926, + "tokens_seen": 1838088192 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035809829488465397, + "loss": 1.2927, + "theoretical_loss": 3.4832836255113184, + "tokens_seen": 1838153728 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035808224674022067, + "loss": 1.2619, + "theoretical_loss": 3.48327318446521, + "tokens_seen": 1838219264 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035806619859578737, + "loss": 1.3203, + "theoretical_loss": 3.483262743895562, + "tokens_seen": 1838284800 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035805015045135407, + "loss": 1.2844, + "theoretical_loss": 3.4832523038023355, + "tokens_seen": 1838350336 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035803410230692077, + "loss": 1.2744, + "theoretical_loss": 3.483241864185492, + "tokens_seen": 1838415872 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035801805416248747, + "loss": 1.3032, + "theoretical_loss": 3.483231425044993, + "tokens_seen": 1838481408 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035800200601805417, + "loss": 1.3436, + "theoretical_loss": 3.4832209863807986, + "tokens_seen": 1838546944 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035798595787362087, + "loss": 1.3063, + "theoretical_loss": 3.4832105481928712, + "tokens_seen": 1838612480 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035796990972918757, + "loss": 1.2119, + "theoretical_loss": 3.483200110481172, + "tokens_seen": 1838678016 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035795386158475427, + "loss": 1.3683, + "theoretical_loss": 3.4831896732456613, + "tokens_seen": 1838743552 + }, + { + "epoch": 0.56, + "learning_rate": 0.000357937813440321, + "loss": 1.3015, + "theoretical_loss": 3.483179236486302, + "tokens_seen": 1838809088 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003579217652958877, + "loss": 1.3264, + "theoretical_loss": 3.483168800203054, + "tokens_seen": 1838874624 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003579057171514544, + "loss": 1.2832, + "theoretical_loss": 3.4831583643958797, + "tokens_seen": 1838940160 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003578896690070211, + "loss": 1.1828, + "theoretical_loss": 3.4831479290647396, + "tokens_seen": 1839005696 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003578736208625878, + "loss": 1.2759, + "theoretical_loss": 3.483137494209595, + "tokens_seen": 1839071232 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003578575727181545, + "loss": 1.2862, + "theoretical_loss": 3.4831270598304087, + "tokens_seen": 1839136768 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003578415245737212, + "loss": 1.2731, + "theoretical_loss": 3.48311662592714, + "tokens_seen": 1839202304 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003578254764292879, + "loss": 1.2257, + "theoretical_loss": 3.4831061924997515, + "tokens_seen": 1839267840 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003578094282848546, + "loss": 1.4264, + "theoretical_loss": 3.4830957595482035, + "tokens_seen": 1839333376 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003577933801404213, + "loss": 1.2239, + "theoretical_loss": 3.483085327072459, + "tokens_seen": 1839398912 + }, + { + "epoch": 0.56, + "learning_rate": 0.000357777331995988, + "loss": 1.3255, + "theoretical_loss": 3.4830748950724777, + "tokens_seen": 1839464448 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003577612838515547, + "loss": 1.343, + "theoretical_loss": 3.4830644635482217, + "tokens_seen": 1839529984 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003577452357071214, + "loss": 1.3788, + "theoretical_loss": 3.483054032499653, + "tokens_seen": 1839595520 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003577291875626881, + "loss": 1.2454, + "theoretical_loss": 3.483043601926731, + "tokens_seen": 1839661056 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003577131394182548, + "loss": 1.224, + "theoretical_loss": 3.483033171829419, + "tokens_seen": 1839726592 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003576970912738215, + "loss": 1.2418, + "theoretical_loss": 3.4830227422076776, + "tokens_seen": 1839792128 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003576810431293882, + "loss": 1.2135, + "theoretical_loss": 3.4830123130614687, + "tokens_seen": 1839857664 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003576649949849549, + "loss": 1.2745, + "theoretical_loss": 3.4830018843907524, + "tokens_seen": 1839923200 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003576489468405216, + "loss": 1.4208, + "theoretical_loss": 3.4829914561954913, + "tokens_seen": 1839988736 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003576328986960883, + "loss": 1.3332, + "theoretical_loss": 3.4829810284756464, + "tokens_seen": 1840054272 + }, + { + "epoch": 0.56, + "learning_rate": 0.000357616850551655, + "loss": 1.3302, + "theoretical_loss": 3.4829706012311794, + "tokens_seen": 1840119808 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003576008024072217, + "loss": 1.413, + "theoretical_loss": 3.4829601744620513, + "tokens_seen": 1840185344 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003575847542627884, + "loss": 1.3436, + "theoretical_loss": 3.4829497481682234, + "tokens_seen": 1840250880 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003575687061183551, + "loss": 1.2637, + "theoretical_loss": 3.482939322349657, + "tokens_seen": 1840316416 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003575526579739218, + "loss": 1.3617, + "theoretical_loss": 3.482928897006315, + "tokens_seen": 1840381952 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003575366098294885, + "loss": 1.3367, + "theoretical_loss": 3.4829184721381568, + "tokens_seen": 1840447488 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003575205616850552, + "loss": 1.2895, + "theoretical_loss": 3.482908047745145, + "tokens_seen": 1840513024 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003575045135406219, + "loss": 1.286, + "theoretical_loss": 3.48289762382724, + "tokens_seen": 1840578560 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003574884653961886, + "loss": 1.3803, + "theoretical_loss": 3.4828872003844045, + "tokens_seen": 1840644096 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003574724172517553, + "loss": 1.2066, + "theoretical_loss": 3.482876777416599, + "tokens_seen": 1840709632 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035745636910732194, + "loss": 1.3016, + "theoretical_loss": 3.482866354923786, + "tokens_seen": 1840775168 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035744032096288864, + "loss": 1.2025, + "theoretical_loss": 3.4828559329059257, + "tokens_seen": 1840840704 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035742427281845534, + "loss": 1.3326, + "theoretical_loss": 3.48284551136298, + "tokens_seen": 1840906240 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035740822467402204, + "loss": 1.3821, + "theoretical_loss": 3.482835090294911, + "tokens_seen": 1840971776 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035739217652958874, + "loss": 1.2534, + "theoretical_loss": 3.482824669701679, + "tokens_seen": 1841037312 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035737612838515544, + "loss": 1.2105, + "theoretical_loss": 3.4828142495832464, + "tokens_seen": 1841102848 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035736008024072214, + "loss": 1.3183, + "theoretical_loss": 3.4828038299395745, + "tokens_seen": 1841168384 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003573440320962889, + "loss": 1.384, + "theoretical_loss": 3.482793410770624, + "tokens_seen": 1841233920 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003573279839518556, + "loss": 1.2149, + "theoretical_loss": 3.4827829920763573, + "tokens_seen": 1841299456 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003573119358074223, + "loss": 1.2485, + "theoretical_loss": 3.482772573856736, + "tokens_seen": 1841364992 + }, + { + "epoch": 0.56, + "learning_rate": 0.000357295887662989, + "loss": 1.3124, + "theoretical_loss": 3.4827621561117206, + "tokens_seen": 1841430528 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003572798395185557, + "loss": 1.2336, + "theoretical_loss": 3.4827517388412734, + "tokens_seen": 1841496064 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 682020, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3616777658462524, + "objective/train/theoretical_loss": 3.4827413220453556, + "objective/train/tokens_used": 1862021600, + "theoretical_loss": 3.4827413220453556, + "tokens_seen": 1841561600 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003572637913741224, + "loss": 1.3617, + "theoretical_loss": 3.4827413220453556, + "tokens_seen": 1841561600 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003572477432296891, + "loss": 1.3909, + "theoretical_loss": 3.482730905723929, + "tokens_seen": 1841627136 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003572316950852558, + "loss": 1.3184, + "theoretical_loss": 3.482720489876954, + "tokens_seen": 1841692672 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003572156469408225, + "loss": 1.3754, + "theoretical_loss": 3.4827100745043937, + "tokens_seen": 1841758208 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003571995987963892, + "loss": 1.3295, + "theoretical_loss": 3.4826996596062085, + "tokens_seen": 1841823744 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003571835506519559, + "loss": 1.2672, + "theoretical_loss": 3.4826892451823603, + "tokens_seen": 1841889280 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003571675025075226, + "loss": 1.234, + "theoretical_loss": 3.4826788312328105, + "tokens_seen": 1841954816 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003571514543630893, + "loss": 1.3839, + "theoretical_loss": 3.4826684177575213, + "tokens_seen": 1842020352 + }, + { + "epoch": 0.56, + "learning_rate": 0.000357135406218656, + "loss": 1.3383, + "theoretical_loss": 3.482658004756453, + "tokens_seen": 1842085888 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003571193580742227, + "loss": 1.2972, + "theoretical_loss": 3.4826475922295677, + "tokens_seen": 1842151424 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003571033099297894, + "loss": 1.2778, + "theoretical_loss": 3.4826371801768277, + "tokens_seen": 1842216960 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003570872617853561, + "loss": 1.2536, + "theoretical_loss": 3.4826267685981933, + "tokens_seen": 1842282496 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003570712136409228, + "loss": 1.2848, + "theoretical_loss": 3.482616357493627, + "tokens_seen": 1842348032 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003570551654964895, + "loss": 1.2966, + "theoretical_loss": 3.4826059468630897, + "tokens_seen": 1842413568 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003570391173520562, + "loss": 1.3253, + "theoretical_loss": 3.482595536706543, + "tokens_seen": 1842479104 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003570230692076229, + "loss": 1.35, + "theoretical_loss": 3.482585127023949, + "tokens_seen": 1842544640 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003570070210631896, + "loss": 1.3526, + "theoretical_loss": 3.482574717815269, + "tokens_seen": 1842610176 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003569909729187563, + "loss": 1.2436, + "theoretical_loss": 3.4825643090804643, + "tokens_seen": 1842675712 + }, + { + "epoch": 0.56, + "learning_rate": 0.000356974924774323, + "loss": 1.3482, + "theoretical_loss": 3.4825539008194966, + "tokens_seen": 1842741248 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003569588766298897, + "loss": 1.2419, + "theoretical_loss": 3.4825434930323276, + "tokens_seen": 1842806784 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003569428284854564, + "loss": 1.38, + "theoretical_loss": 3.482533085718919, + "tokens_seen": 1842872320 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003569267803410231, + "loss": 1.2349, + "theoretical_loss": 3.4825226788792323, + "tokens_seen": 1842937856 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003569107321965898, + "loss": 1.3301, + "theoretical_loss": 3.4825122725132287, + "tokens_seen": 1843003392 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003568946840521565, + "loss": 1.2388, + "theoretical_loss": 3.4825018666208702, + "tokens_seen": 1843068928 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003568786359077232, + "loss": 1.2916, + "theoretical_loss": 3.4824914612021187, + "tokens_seen": 1843134464 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003568625877632899, + "loss": 1.3842, + "theoretical_loss": 3.482481056256935, + "tokens_seen": 1843200000 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003568465396188566, + "loss": 1.3128, + "theoretical_loss": 3.4824706517852815, + "tokens_seen": 1843265536 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003568304914744233, + "loss": 1.2544, + "theoretical_loss": 3.4824602477871194, + "tokens_seen": 1843331072 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035681444332998996, + "loss": 1.3514, + "theoretical_loss": 3.48244984426241, + "tokens_seen": 1843396608 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035679839518555666, + "loss": 1.2024, + "theoretical_loss": 3.482439441211116, + "tokens_seen": 1843462144 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035678234704112336, + "loss": 1.2665, + "theoretical_loss": 3.4824290386331977, + "tokens_seen": 1843527680 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035676629889669007, + "loss": 1.3064, + "theoretical_loss": 3.4824186365286174, + "tokens_seen": 1843593216 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035675025075225677, + "loss": 1.2072, + "theoretical_loss": 3.4824082348973366, + "tokens_seen": 1843658752 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035673420260782347, + "loss": 1.1986, + "theoretical_loss": 3.4823978337393173, + "tokens_seen": 1843724288 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035671815446339017, + "loss": 1.3337, + "theoretical_loss": 3.4823874330545213, + "tokens_seen": 1843789824 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035670210631895687, + "loss": 1.3378, + "theoretical_loss": 3.4823770328429093, + "tokens_seen": 1843855360 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035668605817452357, + "loss": 1.4099, + "theoretical_loss": 3.4823666331044434, + "tokens_seen": 1843920896 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035667001003009027, + "loss": 1.2613, + "theoretical_loss": 3.4823562338390857, + "tokens_seen": 1843986432 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035665396188565697, + "loss": 1.3739, + "theoretical_loss": 3.4823458350467975, + "tokens_seen": 1844051968 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035663791374122367, + "loss": 1.4238, + "theoretical_loss": 3.4823354367275403, + "tokens_seen": 1844117504 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035662186559679037, + "loss": 1.3198, + "theoretical_loss": 3.482325038881276, + "tokens_seen": 1844183040 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035660581745235707, + "loss": 1.3152, + "theoretical_loss": 3.482314641507966, + "tokens_seen": 1844248576 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035658976930792377, + "loss": 1.3892, + "theoretical_loss": 3.4823042446075725, + "tokens_seen": 1844314112 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035657372116349047, + "loss": 1.3743, + "theoretical_loss": 3.4822938481800567, + "tokens_seen": 1844379648 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003565576730190572, + "loss": 1.2631, + "theoretical_loss": 3.4822834522253805, + "tokens_seen": 1844445184 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003565416248746239, + "loss": 1.3668, + "theoretical_loss": 3.482273056743506, + "tokens_seen": 1844510720 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003565255767301906, + "loss": 1.2543, + "theoretical_loss": 3.482262661734394, + "tokens_seen": 1844576256 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003565095285857573, + "loss": 1.2054, + "theoretical_loss": 3.4822522671980067, + "tokens_seen": 1844641792 + }, + { + "epoch": 0.56, + "learning_rate": 0.000356493480441324, + "loss": 1.215, + "theoretical_loss": 3.4822418731343063, + "tokens_seen": 1844707328 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003564774322968907, + "loss": 1.3121, + "theoretical_loss": 3.4822314795432536, + "tokens_seen": 1844772864 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003564613841524574, + "loss": 1.3104, + "theoretical_loss": 3.4822210864248113, + "tokens_seen": 1844838400 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003564453360080241, + "loss": 1.3972, + "theoretical_loss": 3.4822106937789394, + "tokens_seen": 1844903936 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003564292878635908, + "loss": 1.2713, + "theoretical_loss": 3.4822003016056016, + "tokens_seen": 1844969472 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003564132397191575, + "loss": 1.4351, + "theoretical_loss": 3.4821899099047586, + "tokens_seen": 1845035008 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003563971915747242, + "loss": 1.4554, + "theoretical_loss": 3.4821795186763724, + "tokens_seen": 1845100544 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003563811434302909, + "loss": 1.3021, + "theoretical_loss": 3.482169127920405, + "tokens_seen": 1845166080 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003563650952858576, + "loss": 1.2743, + "theoretical_loss": 3.4821587376368175, + "tokens_seen": 1845231616 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003563490471414243, + "loss": 1.169, + "theoretical_loss": 3.482148347825572, + "tokens_seen": 1845297152 + }, + { + "epoch": 0.56, + "learning_rate": 0.000356332998996991, + "loss": 1.3297, + "theoretical_loss": 3.4821379584866303, + "tokens_seen": 1845362688 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003563169508525577, + "loss": 1.1401, + "theoretical_loss": 3.482127569619954, + "tokens_seen": 1845428224 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003563009027081244, + "loss": 1.3374, + "theoretical_loss": 3.482117181225505, + "tokens_seen": 1845493760 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003562848545636911, + "loss": 1.3101, + "theoretical_loss": 3.482106793303245, + "tokens_seen": 1845559296 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003562688064192578, + "loss": 1.3202, + "theoretical_loss": 3.4820964058531354, + "tokens_seen": 1845624832 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003562527582748245, + "loss": 1.1543, + "theoretical_loss": 3.482086018875139, + "tokens_seen": 1845690368 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003562367101303912, + "loss": 1.2086, + "theoretical_loss": 3.4820756323692166, + "tokens_seen": 1845755904 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003562206619859579, + "loss": 1.3836, + "theoretical_loss": 3.4820652463353303, + "tokens_seen": 1845821440 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003562046138415246, + "loss": 1.3856, + "theoretical_loss": 3.4820548607734425, + "tokens_seen": 1845886976 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003561885656970913, + "loss": 1.2734, + "theoretical_loss": 3.4820444756835136, + "tokens_seen": 1845952512 + }, + { + "epoch": 0.56, + "learning_rate": 0.000356172517552658, + "loss": 1.2801, + "theoretical_loss": 3.4820340910655068, + "tokens_seen": 1846018048 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003561564694082247, + "loss": 1.2424, + "theoretical_loss": 3.482023706919383, + "tokens_seen": 1846083584 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003561404212637914, + "loss": 1.2012, + "theoretical_loss": 3.482013323245104, + "tokens_seen": 1846149120 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003561243731193581, + "loss": 1.2908, + "theoretical_loss": 3.4820029400426327, + "tokens_seen": 1846214656 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003561083249749248, + "loss": 1.3232, + "theoretical_loss": 3.481992557311929, + "tokens_seen": 1846280192 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003560922768304915, + "loss": 1.3256, + "theoretical_loss": 3.481982175052957, + "tokens_seen": 1846345728 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003560762286860582, + "loss": 1.2892, + "theoretical_loss": 3.481971793265677, + "tokens_seen": 1846411264 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003560601805416249, + "loss": 1.334, + "theoretical_loss": 3.481961411950051, + "tokens_seen": 1846476800 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003560441323971916, + "loss": 1.3386, + "theoretical_loss": 3.4819510311060418, + "tokens_seen": 1846542336 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003560280842527583, + "loss": 1.3579, + "theoretical_loss": 3.4819406507336095, + "tokens_seen": 1846607872 + }, + { + "epoch": 0.56, + "learning_rate": 0.000356012036108325, + "loss": 1.2112, + "theoretical_loss": 3.4819302708327173, + "tokens_seen": 1846673408 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003559959879638917, + "loss": 1.3334, + "theoretical_loss": 3.4819198914033267, + "tokens_seen": 1846738944 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003559799398194584, + "loss": 1.3145, + "theoretical_loss": 3.4819095124454, + "tokens_seen": 1846804480 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003559638916750251, + "loss": 1.3142, + "theoretical_loss": 3.4818991339588976, + "tokens_seen": 1846870016 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003559478435305918, + "loss": 1.2115, + "theoretical_loss": 3.4818887559437828, + "tokens_seen": 1846935552 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003559317953861585, + "loss": 1.2509, + "theoretical_loss": 3.481878378400017, + "tokens_seen": 1847001088 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003559157472417252, + "loss": 1.3842, + "theoretical_loss": 3.4818680013275625, + "tokens_seen": 1847066624 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003558996990972919, + "loss": 1.3116, + "theoretical_loss": 3.4818576247263806, + "tokens_seen": 1847132160 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003558836509528586, + "loss": 1.1526, + "theoretical_loss": 3.481847248596433, + "tokens_seen": 1847197696 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003558676028084253, + "loss": 1.3121, + "theoretical_loss": 3.4818368729376816, + "tokens_seen": 1847263232 + }, + { + "epoch": 0.56, + "learning_rate": 0.000355851554663992, + "loss": 1.2982, + "theoretical_loss": 3.4818264977500895, + "tokens_seen": 1847328768 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003558355065195587, + "loss": 1.2878, + "theoretical_loss": 3.4818161230336173, + "tokens_seen": 1847394304 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003558194583751254, + "loss": 1.2776, + "theoretical_loss": 3.481805748788227, + "tokens_seen": 1847459840 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003558034102306921, + "loss": 1.3315, + "theoretical_loss": 3.4817953750138813, + "tokens_seen": 1847525376 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003557873620862588, + "loss": 1.2859, + "theoretical_loss": 3.481785001710541, + "tokens_seen": 1847590912 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003557713139418255, + "loss": 1.3632, + "theoretical_loss": 3.481774628878169, + "tokens_seen": 1847656448 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003557552657973922, + "loss": 1.3699, + "theoretical_loss": 3.481764256516726, + "tokens_seen": 1847721984 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003557392176529589, + "loss": 1.4427, + "theoretical_loss": 3.4817538846261757, + "tokens_seen": 1847787520 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003557231695085256, + "loss": 1.3486, + "theoretical_loss": 3.481743513206479, + "tokens_seen": 1847853056 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003557071213640923, + "loss": 1.2997, + "theoretical_loss": 3.4817331422575974, + "tokens_seen": 1847918592 + }, + { + "epoch": 0.56, + "learning_rate": 0.000355691073219659, + "loss": 1.3711, + "theoretical_loss": 3.4817227717794936, + "tokens_seen": 1847984128 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003556750250752257, + "loss": 1.2703, + "theoretical_loss": 3.4817124017721293, + "tokens_seen": 1848049664 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 684326, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.214890718460083, + "objective/train/theoretical_loss": 3.481702032235466, + "objective/train/tokens_used": 1868575200, + "theoretical_loss": 3.481702032235466, + "tokens_seen": 1848115200 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003556589769307924, + "loss": 1.2149, + "theoretical_loss": 3.481702032235466, + "tokens_seen": 1848115200 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003556429287863591, + "loss": 1.2842, + "theoretical_loss": 3.4816916631694665, + "tokens_seen": 1848180736 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003556268806419258, + "loss": 1.3633, + "theoretical_loss": 3.481681294574092, + "tokens_seen": 1848246272 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003556108324974925, + "loss": 1.2495, + "theoretical_loss": 3.4816709264493046, + "tokens_seen": 1848311808 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003555947843530592, + "loss": 1.149, + "theoretical_loss": 3.481660558795067, + "tokens_seen": 1848377344 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003555787362086259, + "loss": 1.3228, + "theoretical_loss": 3.4816501916113394, + "tokens_seen": 1848442880 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003555626880641926, + "loss": 1.2859, + "theoretical_loss": 3.4816398248980858, + "tokens_seen": 1848508416 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003555466399197593, + "loss": 1.2754, + "theoretical_loss": 3.481629458655267, + "tokens_seen": 1848573952 + }, + { + "epoch": 0.56, + "learning_rate": 0.000355530591775326, + "loss": 1.3147, + "theoretical_loss": 3.4816190928828457, + "tokens_seen": 1848639488 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003555145436308927, + "loss": 1.209, + "theoretical_loss": 3.4816087275807828, + "tokens_seen": 1848705024 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003554984954864594, + "loss": 1.345, + "theoretical_loss": 3.481598362749041, + "tokens_seen": 1848770560 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003554824473420261, + "loss": 1.2357, + "theoretical_loss": 3.4815879983875826, + "tokens_seen": 1848836096 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003554663991975928, + "loss": 1.3566, + "theoretical_loss": 3.481577634496369, + "tokens_seen": 1848901632 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003554503510531595, + "loss": 1.216, + "theoretical_loss": 3.4815672710753622, + "tokens_seen": 1848967168 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003554343029087262, + "loss": 1.2654, + "theoretical_loss": 3.4815569081245243, + "tokens_seen": 1849032704 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035541825476429286, + "loss": 1.2713, + "theoretical_loss": 3.481546545643818, + "tokens_seen": 1849098240 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035540220661985956, + "loss": 1.2476, + "theoretical_loss": 3.4815361836332044, + "tokens_seen": 1849163776 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035538615847542627, + "loss": 1.3125, + "theoretical_loss": 3.481525822092646, + "tokens_seen": 1849229312 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035537011033099297, + "loss": 1.3774, + "theoretical_loss": 3.481515461022104, + "tokens_seen": 1849294848 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035535406218655967, + "loss": 1.3035, + "theoretical_loss": 3.4815051004215416, + "tokens_seen": 1849360384 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035533801404212637, + "loss": 1.2656, + "theoretical_loss": 3.4814947402909207, + "tokens_seen": 1849425920 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035532196589769307, + "loss": 1.285, + "theoretical_loss": 3.481484380630202, + "tokens_seen": 1849491456 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035530591775325977, + "loss": 1.331, + "theoretical_loss": 3.4814740214393494, + "tokens_seen": 1849556992 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035528986960882647, + "loss": 1.2469, + "theoretical_loss": 3.4814636627183235, + "tokens_seen": 1849622528 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035527382146439317, + "loss": 1.2156, + "theoretical_loss": 3.4814533044670872, + "tokens_seen": 1849688064 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035525777331995987, + "loss": 1.2121, + "theoretical_loss": 3.481442946685602, + "tokens_seen": 1849753600 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035524172517552657, + "loss": 1.2157, + "theoretical_loss": 3.4814325893738296, + "tokens_seen": 1849819136 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035522567703109327, + "loss": 1.2678, + "theoretical_loss": 3.481422232531733, + "tokens_seen": 1849884672 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035520962888665997, + "loss": 1.2395, + "theoretical_loss": 3.4814118761592745, + "tokens_seen": 1849950208 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003551935807422267, + "loss": 1.3067, + "theoretical_loss": 3.481401520256415, + "tokens_seen": 1850015744 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003551775325977934, + "loss": 1.3508, + "theoretical_loss": 3.4813911648231173, + "tokens_seen": 1850081280 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003551614844533601, + "loss": 1.2492, + "theoretical_loss": 3.481380809859343, + "tokens_seen": 1850146816 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003551454363089268, + "loss": 1.2903, + "theoretical_loss": 3.4813704553650546, + "tokens_seen": 1850212352 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003551293881644935, + "loss": 1.3944, + "theoretical_loss": 3.481360101340214, + "tokens_seen": 1850277888 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003551133400200602, + "loss": 1.3095, + "theoretical_loss": 3.4813497477847837, + "tokens_seen": 1850343424 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003550972918756269, + "loss": 1.3614, + "theoretical_loss": 3.4813393946987254, + "tokens_seen": 1850408960 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003550812437311936, + "loss": 1.3662, + "theoretical_loss": 3.481329042082001, + "tokens_seen": 1850474496 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003550651955867603, + "loss": 1.257, + "theoretical_loss": 3.481318689934573, + "tokens_seen": 1850540032 + }, + { + "epoch": 0.56, + "learning_rate": 0.000355049147442327, + "loss": 1.2776, + "theoretical_loss": 3.481308338256403, + "tokens_seen": 1850605568 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003550330992978937, + "loss": 1.4337, + "theoretical_loss": 3.481297987047454, + "tokens_seen": 1850671104 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003550170511534604, + "loss": 1.3637, + "theoretical_loss": 3.4812876363076874, + "tokens_seen": 1850736640 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003550010030090271, + "loss": 1.2678, + "theoretical_loss": 3.4812772860370655, + "tokens_seen": 1850802176 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003549849548645938, + "loss": 1.3064, + "theoretical_loss": 3.4812669362355506, + "tokens_seen": 1850867712 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003549689067201605, + "loss": 1.3333, + "theoretical_loss": 3.4812565869031045, + "tokens_seen": 1850933248 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035495285857572724, + "loss": 1.2714, + "theoretical_loss": 3.48124623803969, + "tokens_seen": 1850998784 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035493681043129394, + "loss": 1.24, + "theoretical_loss": 3.481235889645268, + "tokens_seen": 1851064320 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035492076228686064, + "loss": 1.2445, + "theoretical_loss": 3.4812255417198017, + "tokens_seen": 1851129856 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035490471414242734, + "loss": 1.3039, + "theoretical_loss": 3.4812151942632523, + "tokens_seen": 1851195392 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035488866599799404, + "loss": 1.2217, + "theoretical_loss": 3.4812048472755834, + "tokens_seen": 1851260928 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035487261785356074, + "loss": 1.2692, + "theoretical_loss": 3.4811945007567564, + "tokens_seen": 1851326464 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035485656970912744, + "loss": 1.2499, + "theoretical_loss": 3.4811841547067326, + "tokens_seen": 1851392000 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035484052156469414, + "loss": 1.248, + "theoretical_loss": 3.4811738091254756, + "tokens_seen": 1851457536 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035482447342026084, + "loss": 1.3504, + "theoretical_loss": 3.4811634640129467, + "tokens_seen": 1851523072 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035480842527582754, + "loss": 1.253, + "theoretical_loss": 3.4811531193691083, + "tokens_seen": 1851588608 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003547923771313942, + "loss": 1.3061, + "theoretical_loss": 3.4811427751939226, + "tokens_seen": 1851654144 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003547763289869609, + "loss": 1.3561, + "theoretical_loss": 3.481132431487352, + "tokens_seen": 1851719680 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003547602808425276, + "loss": 1.342, + "theoretical_loss": 3.481122088249358, + "tokens_seen": 1851785216 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003547442326980943, + "loss": 1.3394, + "theoretical_loss": 3.481111745479904, + "tokens_seen": 1851850752 + }, + { + "epoch": 0.56, + "learning_rate": 0.000354728184553661, + "loss": 1.2611, + "theoretical_loss": 3.4811014031789504, + "tokens_seen": 1851916288 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003547121364092277, + "loss": 1.2756, + "theoretical_loss": 3.4810910613464614, + "tokens_seen": 1851981824 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003546960882647944, + "loss": 1.2631, + "theoretical_loss": 3.4810807199823977, + "tokens_seen": 1852047360 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003546800401203611, + "loss": 1.216, + "theoretical_loss": 3.481070379086722, + "tokens_seen": 1852112896 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003546639919759278, + "loss": 1.2147, + "theoretical_loss": 3.481060038659397, + "tokens_seen": 1852178432 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003546479438314945, + "loss": 1.314, + "theoretical_loss": 3.4810496987003843, + "tokens_seen": 1852243968 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003546318956870612, + "loss": 1.2976, + "theoretical_loss": 3.481039359209646, + "tokens_seen": 1852309504 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003546158475426279, + "loss": 1.2663, + "theoretical_loss": 3.4810290201871448, + "tokens_seen": 1852375040 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003545997993981946, + "loss": 1.2154, + "theoretical_loss": 3.481018681632843, + "tokens_seen": 1852440576 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003545837512537613, + "loss": 1.3032, + "theoretical_loss": 3.4810083435467023, + "tokens_seen": 1852506112 + }, + { + "epoch": 0.56, + "learning_rate": 0.000354567703109328, + "loss": 1.3318, + "theoretical_loss": 3.480998005928685, + "tokens_seen": 1852571648 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003545516549648947, + "loss": 1.3694, + "theoretical_loss": 3.480987668778754, + "tokens_seen": 1852637184 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003545356068204614, + "loss": 1.2999, + "theoretical_loss": 3.480977332096871, + "tokens_seen": 1852702720 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003545195586760281, + "loss": 1.2294, + "theoretical_loss": 3.4809669958829983, + "tokens_seen": 1852768256 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003545035105315948, + "loss": 1.3826, + "theoretical_loss": 3.4809566601370983, + "tokens_seen": 1852833792 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003544874623871615, + "loss": 1.2923, + "theoretical_loss": 3.4809463248591332, + "tokens_seen": 1852899328 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003544714142427282, + "loss": 1.2508, + "theoretical_loss": 3.4809359900490655, + "tokens_seen": 1852964864 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003544553660982949, + "loss": 1.3262, + "theoretical_loss": 3.480925655706857, + "tokens_seen": 1853030400 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003544393179538616, + "loss": 1.3216, + "theoretical_loss": 3.48091532183247, + "tokens_seen": 1853095936 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003544232698094283, + "loss": 1.3556, + "theoretical_loss": 3.480904988425867, + "tokens_seen": 1853161472 + }, + { + "epoch": 0.56, + "learning_rate": 0.000354407221664995, + "loss": 1.2147, + "theoretical_loss": 3.4808946554870106, + "tokens_seen": 1853227008 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003543911735205617, + "loss": 1.173, + "theoretical_loss": 3.4808843230158626, + "tokens_seen": 1853292544 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003543751253761284, + "loss": 1.2054, + "theoretical_loss": 3.480873991012385, + "tokens_seen": 1853358080 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003543590772316951, + "loss": 1.2658, + "theoretical_loss": 3.480863659476541, + "tokens_seen": 1853423616 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003543430290872618, + "loss": 1.2984, + "theoretical_loss": 3.4808533284082928, + "tokens_seen": 1853489152 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003543269809428285, + "loss": 1.2594, + "theoretical_loss": 3.480842997807602, + "tokens_seen": 1853554688 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003543109327983952, + "loss": 1.2092, + "theoretical_loss": 3.480832667674431, + "tokens_seen": 1853620224 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003542948846539619, + "loss": 1.2243, + "theoretical_loss": 3.4808223380087426, + "tokens_seen": 1853685760 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003542788365095286, + "loss": 1.3003, + "theoretical_loss": 3.480812008810499, + "tokens_seen": 1853751296 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003542627883650953, + "loss": 1.224, + "theoretical_loss": 3.4808016800796624, + "tokens_seen": 1853816832 + }, + { + "epoch": 0.56, + "learning_rate": 0.000354246740220662, + "loss": 1.3257, + "theoretical_loss": 3.480791351816195, + "tokens_seen": 1853882368 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003542306920762287, + "loss": 1.1775, + "theoretical_loss": 3.480781024020059, + "tokens_seen": 1853947904 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003542146439317954, + "loss": 1.3897, + "theoretical_loss": 3.4807706966912173, + "tokens_seen": 1854013440 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003541985957873621, + "loss": 1.2524, + "theoretical_loss": 3.480760369829632, + "tokens_seen": 1854078976 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003541825476429288, + "loss": 1.3246, + "theoretical_loss": 3.4807500434352647, + "tokens_seen": 1854144512 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003541664994984955, + "loss": 1.2412, + "theoretical_loss": 3.4807397175080794, + "tokens_seen": 1854210048 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003541504513540622, + "loss": 1.2154, + "theoretical_loss": 3.480729392048037, + "tokens_seen": 1854275584 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003541344032096289, + "loss": 1.3361, + "theoretical_loss": 3.4807190670551007, + "tokens_seen": 1854341120 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003541183550651956, + "loss": 1.2803, + "theoretical_loss": 3.480708742529232, + "tokens_seen": 1854406656 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003541023069207623, + "loss": 1.2573, + "theoretical_loss": 3.480698418470394, + "tokens_seen": 1854472192 + }, + { + "epoch": 0.56, + "learning_rate": 0.000354086258776329, + "loss": 1.3105, + "theoretical_loss": 3.480688094878549, + "tokens_seen": 1854537728 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003540702106318957, + "loss": 1.2353, + "theoretical_loss": 3.480677771753659, + "tokens_seen": 1854603264 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 686569, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.171262502670288, + "objective/train/theoretical_loss": 3.4806674490956864, + "objective/train/tokens_used": 1875128800, + "theoretical_loss": 3.4806674490956864, + "tokens_seen": 1854668800 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003540541624874624, + "loss": 1.1713, + "theoretical_loss": 3.4806674490956864, + "tokens_seen": 1854668800 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003540381143430291, + "loss": 1.3839, + "theoretical_loss": 3.4806571269045943, + "tokens_seen": 1854734336 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035402206619859577, + "loss": 1.2926, + "theoretical_loss": 3.4806468051803443, + "tokens_seen": 1854799872 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035400601805416247, + "loss": 1.174, + "theoretical_loss": 3.480636483922899, + "tokens_seen": 1854865408 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035398996990972917, + "loss": 1.3289, + "theoretical_loss": 3.480626163132221, + "tokens_seen": 1854930944 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035397392176529587, + "loss": 1.2963, + "theoretical_loss": 3.4806158428082723, + "tokens_seen": 1854996480 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035395787362086257, + "loss": 1.2093, + "theoretical_loss": 3.480605522951016, + "tokens_seen": 1855062016 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035394182547642927, + "loss": 1.1899, + "theoretical_loss": 3.480595203560414, + "tokens_seen": 1855127552 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035392577733199597, + "loss": 1.2998, + "theoretical_loss": 3.4805848846364285, + "tokens_seen": 1855193088 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035390972918756267, + "loss": 1.2251, + "theoretical_loss": 3.480574566179022, + "tokens_seen": 1855258624 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035389368104312937, + "loss": 1.3164, + "theoretical_loss": 3.4805642481881574, + "tokens_seen": 1855324160 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035387763289869607, + "loss": 1.3181, + "theoretical_loss": 3.480553930663797, + "tokens_seen": 1855389696 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035386158475426277, + "loss": 1.3028, + "theoretical_loss": 3.480543613605903, + "tokens_seen": 1855455232 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035384553660982947, + "loss": 1.2205, + "theoretical_loss": 3.4805332970144383, + "tokens_seen": 1855520768 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035382948846539617, + "loss": 1.2519, + "theoretical_loss": 3.480522980889364, + "tokens_seen": 1855586304 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003538134403209629, + "loss": 1.2849, + "theoretical_loss": 3.4805126652306444, + "tokens_seen": 1855651840 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003537973921765296, + "loss": 1.3032, + "theoretical_loss": 3.4805023500382406, + "tokens_seen": 1855717376 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003537813440320963, + "loss": 1.3209, + "theoretical_loss": 3.480492035312116, + "tokens_seen": 1855782912 + }, + { + "epoch": 0.56, + "learning_rate": 0.000353765295887663, + "loss": 1.2771, + "theoretical_loss": 3.480481721052232, + "tokens_seen": 1855848448 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035374924774322973, + "loss": 1.3764, + "theoretical_loss": 3.480471407258552, + "tokens_seen": 1855913984 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035373319959879643, + "loss": 1.2341, + "theoretical_loss": 3.480461093931037, + "tokens_seen": 1855979520 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035371715145436313, + "loss": 1.3724, + "theoretical_loss": 3.480450781069652, + "tokens_seen": 1856045056 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035370110330992983, + "loss": 1.2677, + "theoretical_loss": 3.4804404686743577, + "tokens_seen": 1856110592 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035368505516549653, + "loss": 1.2368, + "theoretical_loss": 3.4804301567451166, + "tokens_seen": 1856176128 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035366900702106323, + "loss": 1.3266, + "theoretical_loss": 3.4804198452818915, + "tokens_seen": 1856241664 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035365295887662993, + "loss": 1.2871, + "theoretical_loss": 3.480409534284645, + "tokens_seen": 1856307200 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035363691073219663, + "loss": 1.2841, + "theoretical_loss": 3.4803992237533397, + "tokens_seen": 1856372736 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035362086258776334, + "loss": 1.3399, + "theoretical_loss": 3.4803889136879373, + "tokens_seen": 1856438272 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035360481444333004, + "loss": 1.2661, + "theoretical_loss": 3.4803786040884015, + "tokens_seen": 1856503808 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035358876629889674, + "loss": 1.298, + "theoretical_loss": 3.480368294954694, + "tokens_seen": 1856569344 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035357271815446344, + "loss": 1.2689, + "theoretical_loss": 3.4803579862867773, + "tokens_seen": 1856634880 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035355667001003014, + "loss": 1.3649, + "theoretical_loss": 3.4803476780846143, + "tokens_seen": 1856700416 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035354062186559684, + "loss": 1.2859, + "theoretical_loss": 3.4803373703481673, + "tokens_seen": 1856765952 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035352457372116354, + "loss": 1.2965, + "theoretical_loss": 3.4803270630773984, + "tokens_seen": 1856831488 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035350852557673024, + "loss": 1.2727, + "theoretical_loss": 3.480316756272271, + "tokens_seen": 1856897024 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035349247743229694, + "loss": 1.3232, + "theoretical_loss": 3.4803064499327476, + "tokens_seen": 1856962560 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035347642928786364, + "loss": 1.2901, + "theoretical_loss": 3.48029614405879, + "tokens_seen": 1857028096 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035346038114343034, + "loss": 1.3331, + "theoretical_loss": 3.480285838650361, + "tokens_seen": 1857093632 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035344433299899704, + "loss": 1.1911, + "theoretical_loss": 3.480275533707423, + "tokens_seen": 1857159168 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035342828485456374, + "loss": 1.3018, + "theoretical_loss": 3.480265229229939, + "tokens_seen": 1857224704 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035341223671013044, + "loss": 1.2752, + "theoretical_loss": 3.4802549252178716, + "tokens_seen": 1857290240 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035339618856569714, + "loss": 1.2509, + "theoretical_loss": 3.4802446216711824, + "tokens_seen": 1857355776 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003533801404212638, + "loss": 1.3688, + "theoretical_loss": 3.4802343185898352, + "tokens_seen": 1857421312 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003533640922768305, + "loss": 1.2801, + "theoretical_loss": 3.480224015973792, + "tokens_seen": 1857486848 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003533480441323972, + "loss": 1.2789, + "theoretical_loss": 3.480213713823015, + "tokens_seen": 1857552384 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003533319959879639, + "loss": 1.1852, + "theoretical_loss": 3.4802034121374676, + "tokens_seen": 1857617920 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003533159478435306, + "loss": 1.3294, + "theoretical_loss": 3.480193110917112, + "tokens_seen": 1857683456 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003532998996990973, + "loss": 1.3508, + "theoretical_loss": 3.48018281016191, + "tokens_seen": 1857748992 + }, + { + "epoch": 0.56, + "learning_rate": 0.000353283851554664, + "loss": 1.2993, + "theoretical_loss": 3.4801725098718257, + "tokens_seen": 1857814528 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003532678034102307, + "loss": 1.2875, + "theoretical_loss": 3.480162210046821, + "tokens_seen": 1857880064 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003532517552657974, + "loss": 1.2534, + "theoretical_loss": 3.4801519106868577, + "tokens_seen": 1857945600 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003532357071213641, + "loss": 1.2633, + "theoretical_loss": 3.480141611791899, + "tokens_seen": 1858011136 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003532196589769308, + "loss": 1.2704, + "theoretical_loss": 3.480131313361908, + "tokens_seen": 1858076672 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003532036108324975, + "loss": 1.3008, + "theoretical_loss": 3.480121015396847, + "tokens_seen": 1858142208 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003531875626880642, + "loss": 1.2784, + "theoretical_loss": 3.480110717896679, + "tokens_seen": 1858207744 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003531715145436309, + "loss": 1.3683, + "theoretical_loss": 3.480100420861365, + "tokens_seen": 1858273280 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003531554663991976, + "loss": 1.3289, + "theoretical_loss": 3.4800901242908693, + "tokens_seen": 1858338816 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003531394182547643, + "loss": 1.32, + "theoretical_loss": 3.480079828185154, + "tokens_seen": 1858404352 + }, + { + "epoch": 0.56, + "learning_rate": 0.000353123370110331, + "loss": 1.3658, + "theoretical_loss": 3.480069532544182, + "tokens_seen": 1858469888 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003531073219658977, + "loss": 1.3689, + "theoretical_loss": 3.4800592373679153, + "tokens_seen": 1858535424 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003530912738214644, + "loss": 1.1374, + "theoretical_loss": 3.4800489426563166, + "tokens_seen": 1858600960 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003530752256770311, + "loss": 1.4385, + "theoretical_loss": 3.4800386484093497, + "tokens_seen": 1858666496 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003530591775325978, + "loss": 1.3115, + "theoretical_loss": 3.4800283546269757, + "tokens_seen": 1858732032 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003530431293881645, + "loss": 1.162, + "theoretical_loss": 3.480018061309158, + "tokens_seen": 1858797568 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003530270812437312, + "loss": 1.3417, + "theoretical_loss": 3.4800077684558595, + "tokens_seen": 1858863104 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003530110330992979, + "loss": 1.2711, + "theoretical_loss": 3.4799974760670427, + "tokens_seen": 1858928640 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003529949849548646, + "loss": 1.265, + "theoretical_loss": 3.4799871841426695, + "tokens_seen": 1858994176 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003529789368104313, + "loss": 1.3909, + "theoretical_loss": 3.4799768926827035, + "tokens_seen": 1859059712 + }, + { + "epoch": 0.56, + "learning_rate": 0.000352962888665998, + "loss": 1.3764, + "theoretical_loss": 3.4799666016871074, + "tokens_seen": 1859125248 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003529468405215647, + "loss": 1.2939, + "theoretical_loss": 3.479956311155843, + "tokens_seen": 1859190784 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003529307923771314, + "loss": 1.2123, + "theoretical_loss": 3.4799460210888737, + "tokens_seen": 1859256320 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003529147442326981, + "loss": 1.1451, + "theoretical_loss": 3.4799357314861625, + "tokens_seen": 1859321856 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003528986960882648, + "loss": 1.2525, + "theoretical_loss": 3.479925442347671, + "tokens_seen": 1859387392 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003528826479438315, + "loss": 1.3812, + "theoretical_loss": 3.479915153673363, + "tokens_seen": 1859452928 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003528665997993982, + "loss": 1.33, + "theoretical_loss": 3.4799048654632, + "tokens_seen": 1859518464 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003528505516549649, + "loss": 1.235, + "theoretical_loss": 3.479894577717146, + "tokens_seen": 1859584000 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003528345035105316, + "loss": 1.2864, + "theoretical_loss": 3.479884290435163, + "tokens_seen": 1859649536 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003528184553660983, + "loss": 1.1966, + "theoretical_loss": 3.4798740036172138, + "tokens_seen": 1859715072 + }, + { + "epoch": 0.56, + "learning_rate": 0.000352802407221665, + "loss": 1.3244, + "theoretical_loss": 3.479863717263261, + "tokens_seen": 1859780608 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003527863590772317, + "loss": 1.212, + "theoretical_loss": 3.4798534313732676, + "tokens_seen": 1859846144 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003527703109327984, + "loss": 1.2994, + "theoretical_loss": 3.4798431459471963, + "tokens_seen": 1859911680 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003527542627883651, + "loss": 1.299, + "theoretical_loss": 3.4798328609850095, + "tokens_seen": 1859977216 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003527382146439318, + "loss": 1.3474, + "theoretical_loss": 3.4798225764866704, + "tokens_seen": 1860042752 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003527221664994985, + "loss": 1.3248, + "theoretical_loss": 3.4798122924521415, + "tokens_seen": 1860108288 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003527061183550652, + "loss": 1.2071, + "theoretical_loss": 3.4798020088813852, + "tokens_seen": 1860173824 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003526900702106319, + "loss": 1.2657, + "theoretical_loss": 3.4797917257743647, + "tokens_seen": 1860239360 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003526740220661986, + "loss": 1.2458, + "theoretical_loss": 3.479781443131043, + "tokens_seen": 1860304896 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003526579739217653, + "loss": 1.2388, + "theoretical_loss": 3.479771160951382, + "tokens_seen": 1860370432 + }, + { + "epoch": 0.56, + "learning_rate": 0.000352641925777332, + "loss": 1.2692, + "theoretical_loss": 3.479760879235345, + "tokens_seen": 1860435968 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003526258776328987, + "loss": 1.1562, + "theoretical_loss": 3.4797505979828953, + "tokens_seen": 1860501504 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035260982948846537, + "loss": 1.2273, + "theoretical_loss": 3.4797403171939942, + "tokens_seen": 1860567040 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035259378134403207, + "loss": 1.3208, + "theoretical_loss": 3.479730036868606, + "tokens_seen": 1860632576 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035257773319959877, + "loss": 1.3115, + "theoretical_loss": 3.479719757006693, + "tokens_seen": 1860698112 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035256168505516547, + "loss": 1.3599, + "theoretical_loss": 3.4797094776082176, + "tokens_seen": 1860763648 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003525456369107322, + "loss": 1.2391, + "theoretical_loss": 3.4796991986731425, + "tokens_seen": 1860829184 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003525295887662989, + "loss": 1.2411, + "theoretical_loss": 3.479688920201431, + "tokens_seen": 1860894720 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003525135406218656, + "loss": 1.2604, + "theoretical_loss": 3.4796786421930457, + "tokens_seen": 1860960256 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003524974924774323, + "loss": 1.264, + "theoretical_loss": 3.4796683646479494, + "tokens_seen": 1861025792 + }, + { + "epoch": 0.56, + "learning_rate": 0.000352481444332999, + "loss": 1.2699, + "theoretical_loss": 3.479658087566105, + "tokens_seen": 1861091328 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003524653961885657, + "loss": 1.392, + "theoretical_loss": 3.4796478109474753, + "tokens_seen": 1861156864 + }, + { + "epoch": 0.56, + "objective/train/docs_used": 689019, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3230607509613037, + "objective/train/theoretical_loss": 3.4796375347920225, + "objective/train/tokens_used": 1881682400, + "theoretical_loss": 3.4796375347920225, + "tokens_seen": 1861222400 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035244934804413243, + "loss": 1.3231, + "theoretical_loss": 3.4796375347920225, + "tokens_seen": 1861222400 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035243329989969913, + "loss": 1.321, + "theoretical_loss": 3.4796272590997104, + "tokens_seen": 1861287936 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035241725175526583, + "loss": 1.3975, + "theoretical_loss": 3.4796169838705016, + "tokens_seen": 1861353472 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035240120361083253, + "loss": 1.2708, + "theoretical_loss": 3.479606709104358, + "tokens_seen": 1861419008 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035238515546639923, + "loss": 1.1889, + "theoretical_loss": 3.4795964348012434, + "tokens_seen": 1861484544 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035236910732196593, + "loss": 1.2589, + "theoretical_loss": 3.4795861609611203, + "tokens_seen": 1861550080 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035235305917753263, + "loss": 1.3152, + "theoretical_loss": 3.4795758875839518, + "tokens_seen": 1861615616 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035233701103309933, + "loss": 1.3017, + "theoretical_loss": 3.4795656146697, + "tokens_seen": 1861681152 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035232096288866603, + "loss": 1.2813, + "theoretical_loss": 3.4795553422183287, + "tokens_seen": 1861746688 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035230491474423273, + "loss": 1.1776, + "theoretical_loss": 3.4795450702298005, + "tokens_seen": 1861812224 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035228886659979943, + "loss": 1.23, + "theoretical_loss": 3.4795347987040777, + "tokens_seen": 1861877760 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035227281845536613, + "loss": 1.2286, + "theoretical_loss": 3.4795245276411233, + "tokens_seen": 1861943296 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035225677031093283, + "loss": 1.2612, + "theoretical_loss": 3.4795142570409006, + "tokens_seen": 1862008832 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035224072216649954, + "loss": 1.2436, + "theoretical_loss": 3.4795039869033726, + "tokens_seen": 1862074368 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035222467402206624, + "loss": 1.1857, + "theoretical_loss": 3.4794937172285016, + "tokens_seen": 1862139904 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035220862587763294, + "loss": 1.3357, + "theoretical_loss": 3.479483448016251, + "tokens_seen": 1862205440 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035219257773319964, + "loss": 1.1933, + "theoretical_loss": 3.479473179266583, + "tokens_seen": 1862270976 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035217652958876634, + "loss": 1.3365, + "theoretical_loss": 3.4794629109794606, + "tokens_seen": 1862336512 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035216048144433304, + "loss": 1.2102, + "theoretical_loss": 3.479452643154848, + "tokens_seen": 1862402048 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035214443329989974, + "loss": 1.3291, + "theoretical_loss": 3.479442375792706, + "tokens_seen": 1862467584 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035212838515546644, + "loss": 1.2548, + "theoretical_loss": 3.4794321088929987, + "tokens_seen": 1862533120 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035211233701103314, + "loss": 1.2118, + "theoretical_loss": 3.4794218424556895, + "tokens_seen": 1862598656 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035209628886659984, + "loss": 1.2663, + "theoretical_loss": 3.4794115764807403, + "tokens_seen": 1862664192 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035208024072216654, + "loss": 1.2961, + "theoretical_loss": 3.479401310968114, + "tokens_seen": 1862729728 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035206419257773324, + "loss": 1.35, + "theoretical_loss": 3.479391045917774, + "tokens_seen": 1862795264 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035204814443329994, + "loss": 1.2953, + "theoretical_loss": 3.4793807813296835, + "tokens_seen": 1862860800 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035203209628886664, + "loss": 1.1915, + "theoretical_loss": 3.479370517203805, + "tokens_seen": 1862926336 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035201604814443334, + "loss": 1.323, + "theoretical_loss": 3.4793602535401016, + "tokens_seen": 1862991872 + }, + { + "epoch": 0.56, + "learning_rate": 0.00035200000000000005, + "loss": 1.1765, + "theoretical_loss": 3.4793499903385356, + "tokens_seen": 1863057408 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003519839518555667, + "loss": 1.2475, + "theoretical_loss": 3.4793397275990703, + "tokens_seen": 1863122944 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003519679037111334, + "loss": 1.2232, + "theoretical_loss": 3.479329465321669, + "tokens_seen": 1863188480 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003519518555667001, + "loss": 1.2476, + "theoretical_loss": 3.4793192035062948, + "tokens_seen": 1863254016 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003519358074222668, + "loss": 1.2676, + "theoretical_loss": 3.47930894215291, + "tokens_seen": 1863319552 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003519197592778335, + "loss": 1.3036, + "theoretical_loss": 3.4792986812614775, + "tokens_seen": 1863385088 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003519037111334002, + "loss": 1.3342, + "theoretical_loss": 3.479288420831961, + "tokens_seen": 1863450624 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003518876629889669, + "loss": 1.2744, + "theoretical_loss": 3.479278160864323, + "tokens_seen": 1863516160 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003518716148445336, + "loss": 1.2267, + "theoretical_loss": 3.4792679013585266, + "tokens_seen": 1863581696 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003518555667001003, + "loss": 1.3759, + "theoretical_loss": 3.4792576423145345, + "tokens_seen": 1863647232 + }, + { + "epoch": 0.56, + "learning_rate": 0.000351839518555667, + "loss": 1.2826, + "theoretical_loss": 3.4792473837323095, + "tokens_seen": 1863712768 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003518234704112337, + "loss": 1.2583, + "theoretical_loss": 3.4792371256118155, + "tokens_seen": 1863778304 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003518074222668004, + "loss": 1.2235, + "theoretical_loss": 3.479226867953014, + "tokens_seen": 1863843840 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003517913741223671, + "loss": 1.2901, + "theoretical_loss": 3.47921661075587, + "tokens_seen": 1863909376 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003517753259779338, + "loss": 1.3824, + "theoretical_loss": 3.4792063540203446, + "tokens_seen": 1863974912 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003517592778335005, + "loss": 1.3341, + "theoretical_loss": 3.479196097746402, + "tokens_seen": 1864040448 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003517432296890672, + "loss": 1.1491, + "theoretical_loss": 3.4791858419340045, + "tokens_seen": 1864105984 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003517271815446339, + "loss": 1.2755, + "theoretical_loss": 3.479175586583116, + "tokens_seen": 1864171520 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003517111334002006, + "loss": 1.2448, + "theoretical_loss": 3.4791653316936983, + "tokens_seen": 1864237056 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003516950852557673, + "loss": 1.1664, + "theoretical_loss": 3.479155077265715, + "tokens_seen": 1864302592 + }, + { + "epoch": 0.56, + "learning_rate": 0.000351679037111334, + "loss": 1.3257, + "theoretical_loss": 3.479144823299129, + "tokens_seen": 1864368128 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003516629889669007, + "loss": 1.2826, + "theoretical_loss": 3.479134569793904, + "tokens_seen": 1864433664 + }, + { + "epoch": 0.56, + "learning_rate": 0.0003516469408224674, + "loss": 1.3249, + "theoretical_loss": 3.479124316750002, + "tokens_seen": 1864499200 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003516308926780341, + "loss": 1.3545, + "theoretical_loss": 3.479114064167387, + "tokens_seen": 1864564736 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003516148445336008, + "loss": 1.2769, + "theoretical_loss": 3.479103812046021, + "tokens_seen": 1864630272 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003515987963891675, + "loss": 1.2034, + "theoretical_loss": 3.4790935603858677, + "tokens_seen": 1864695808 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003515827482447342, + "loss": 1.2465, + "theoretical_loss": 3.4790833091868905, + "tokens_seen": 1864761344 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003515667001003009, + "loss": 1.2934, + "theoretical_loss": 3.4790730584490515, + "tokens_seen": 1864826880 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003515506519558676, + "loss": 1.3109, + "theoretical_loss": 3.479062808172314, + "tokens_seen": 1864892416 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003515346038114343, + "loss": 1.2658, + "theoretical_loss": 3.479052558356642, + "tokens_seen": 1864957952 + }, + { + "epoch": 0.57, + "learning_rate": 0.000351518555667001, + "loss": 1.2671, + "theoretical_loss": 3.479042309001997, + "tokens_seen": 1865023488 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003515025075225677, + "loss": 1.2859, + "theoretical_loss": 3.4790320601083433, + "tokens_seen": 1865089024 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003514864593781344, + "loss": 1.2847, + "theoretical_loss": 3.4790218116756435, + "tokens_seen": 1865154560 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003514704112337011, + "loss": 1.2135, + "theoretical_loss": 3.4790115637038608, + "tokens_seen": 1865220096 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003514543630892678, + "loss": 1.2464, + "theoretical_loss": 3.4790013161929583, + "tokens_seen": 1865285632 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003514383149448345, + "loss": 1.2807, + "theoretical_loss": 3.478991069142899, + "tokens_seen": 1865351168 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003514222668004012, + "loss": 1.2362, + "theoretical_loss": 3.4789808225536456, + "tokens_seen": 1865416704 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003514062186559679, + "loss": 1.2585, + "theoretical_loss": 3.478970576425162, + "tokens_seen": 1865482240 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003513901705115346, + "loss": 1.2685, + "theoretical_loss": 3.4789603307574106, + "tokens_seen": 1865547776 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003513741223671013, + "loss": 1.2798, + "theoretical_loss": 3.478950085550355, + "tokens_seen": 1865613312 + }, + { + "epoch": 0.57, + "learning_rate": 0.000351358074222668, + "loss": 1.3524, + "theoretical_loss": 3.478939840803958, + "tokens_seen": 1865678848 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003513420260782347, + "loss": 1.171, + "theoretical_loss": 3.478929596518183, + "tokens_seen": 1865744384 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003513259779338014, + "loss": 1.1941, + "theoretical_loss": 3.4789193526929925, + "tokens_seen": 1865809920 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003513099297893681, + "loss": 1.2976, + "theoretical_loss": 3.4789091093283497, + "tokens_seen": 1865875456 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003512938816449348, + "loss": 1.2622, + "theoretical_loss": 3.4788988664242186, + "tokens_seen": 1865940992 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003512778335005015, + "loss": 1.3149, + "theoretical_loss": 3.478888623980562, + "tokens_seen": 1866006528 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003512617853560682, + "loss": 1.2865, + "theoretical_loss": 3.478878381997342, + "tokens_seen": 1866072064 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003512457372116349, + "loss": 1.2056, + "theoretical_loss": 3.478868140474523, + "tokens_seen": 1866137600 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003512296890672016, + "loss": 1.3325, + "theoretical_loss": 3.478857899412067, + "tokens_seen": 1866203136 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003512136409227683, + "loss": 1.2892, + "theoretical_loss": 3.478847658809938, + "tokens_seen": 1866268672 + }, + { + "epoch": 0.57, + "learning_rate": 0.000351197592778335, + "loss": 1.2662, + "theoretical_loss": 3.4788374186680997, + "tokens_seen": 1866334208 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003511815446339017, + "loss": 1.2435, + "theoretical_loss": 3.4788271789865135, + "tokens_seen": 1866399744 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003511654964894684, + "loss": 1.2229, + "theoretical_loss": 3.478816939765144, + "tokens_seen": 1866465280 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003511494483450351, + "loss": 1.2713, + "theoretical_loss": 3.478806701003954, + "tokens_seen": 1866530816 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003511334002006018, + "loss": 1.2267, + "theoretical_loss": 3.478796462702906, + "tokens_seen": 1866596352 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003511173520561685, + "loss": 1.3265, + "theoretical_loss": 3.4787862248619645, + "tokens_seen": 1866661888 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003511013039117352, + "loss": 1.2296, + "theoretical_loss": 3.4787759874810913, + "tokens_seen": 1866727424 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003510852557673019, + "loss": 1.2909, + "theoretical_loss": 3.4787657505602505, + "tokens_seen": 1866792960 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035106920762286863, + "loss": 1.2502, + "theoretical_loss": 3.478755514099405, + "tokens_seen": 1866858496 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035105315947843533, + "loss": 1.3001, + "theoretical_loss": 3.4787452780985175, + "tokens_seen": 1866924032 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035103711133400203, + "loss": 1.2832, + "theoretical_loss": 3.4787350425575516, + "tokens_seen": 1866989568 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035102106318956873, + "loss": 1.2774, + "theoretical_loss": 3.478724807476471, + "tokens_seen": 1867055104 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035100501504513543, + "loss": 1.2233, + "theoretical_loss": 3.478714572855238, + "tokens_seen": 1867120640 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035098896690070213, + "loss": 1.257, + "theoretical_loss": 3.4787043386938166, + "tokens_seen": 1867186176 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035097291875626883, + "loss": 1.3249, + "theoretical_loss": 3.4786941049921687, + "tokens_seen": 1867251712 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035095687061183553, + "loss": 1.3363, + "theoretical_loss": 3.4786838717502593, + "tokens_seen": 1867317248 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035094082246740223, + "loss": 1.223, + "theoretical_loss": 3.4786736389680506, + "tokens_seen": 1867382784 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035092477432296893, + "loss": 1.2329, + "theoretical_loss": 3.4786634066455058, + "tokens_seen": 1867448320 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035090872617853563, + "loss": 1.3324, + "theoretical_loss": 3.478653174782588, + "tokens_seen": 1867513856 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035089267803410233, + "loss": 1.2772, + "theoretical_loss": 3.4786429433792607, + "tokens_seen": 1867579392 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035087662988966904, + "loss": 1.2083, + "theoretical_loss": 3.4786327124354877, + "tokens_seen": 1867644928 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035086058174523574, + "loss": 1.2863, + "theoretical_loss": 3.4786224819512306, + "tokens_seen": 1867710464 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 691424, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3478171825408936, + "objective/train/theoretical_loss": 3.478612251926455, + "objective/train/tokens_used": 1888236000, + "theoretical_loss": 3.478612251926455, + "tokens_seen": 1867776000 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035084453360080244, + "loss": 1.3478, + "theoretical_loss": 3.478612251926455, + "tokens_seen": 1867776000 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035082848545636914, + "loss": 1.2721, + "theoretical_loss": 3.478602022361122, + "tokens_seen": 1867841536 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035081243731193584, + "loss": 1.3129, + "theoretical_loss": 3.4785917932551955, + "tokens_seen": 1867907072 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035079638916750254, + "loss": 1.3158, + "theoretical_loss": 3.478581564608639, + "tokens_seen": 1867972608 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035078034102306924, + "loss": 1.2087, + "theoretical_loss": 3.4785713364214166, + "tokens_seen": 1868038144 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035076429287863594, + "loss": 1.2844, + "theoretical_loss": 3.47856110869349, + "tokens_seen": 1868103680 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035074824473420264, + "loss": 1.3797, + "theoretical_loss": 3.4785508814248227, + "tokens_seen": 1868169216 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035073219658976934, + "loss": 1.255, + "theoretical_loss": 3.478540654615379, + "tokens_seen": 1868234752 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035071614844533604, + "loss": 1.3492, + "theoretical_loss": 3.4785304282651213, + "tokens_seen": 1868300288 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035070010030090274, + "loss": 1.2732, + "theoretical_loss": 3.478520202374013, + "tokens_seen": 1868365824 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035068405215646944, + "loss": 1.3561, + "theoretical_loss": 3.478509976942018, + "tokens_seen": 1868431360 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035066800401203614, + "loss": 1.3523, + "theoretical_loss": 3.478499751969098, + "tokens_seen": 1868496896 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035065195586760284, + "loss": 1.2578, + "theoretical_loss": 3.4784895274552183, + "tokens_seen": 1868562432 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035063590772316954, + "loss": 1.3431, + "theoretical_loss": 3.478479303400341, + "tokens_seen": 1868627968 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035061985957873625, + "loss": 1.3261, + "theoretical_loss": 3.4784690798044298, + "tokens_seen": 1868693504 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035060381143430295, + "loss": 1.32, + "theoretical_loss": 3.478458856667448, + "tokens_seen": 1868759040 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035058776328986965, + "loss": 1.1832, + "theoretical_loss": 3.478448633989358, + "tokens_seen": 1868824576 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003505717151454363, + "loss": 1.2849, + "theoretical_loss": 3.4784384117701244, + "tokens_seen": 1868890112 + }, + { + "epoch": 0.57, + "learning_rate": 0.000350555667001003, + "loss": 1.2624, + "theoretical_loss": 3.4784281900097103, + "tokens_seen": 1868955648 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003505396188565697, + "loss": 1.3344, + "theoretical_loss": 3.4784179687080785, + "tokens_seen": 1869021184 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003505235707121364, + "loss": 1.2609, + "theoretical_loss": 3.4784077478651922, + "tokens_seen": 1869086720 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003505075225677031, + "loss": 1.2126, + "theoretical_loss": 3.4783975274810155, + "tokens_seen": 1869152256 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003504914744232698, + "loss": 1.3278, + "theoretical_loss": 3.478387307555511, + "tokens_seen": 1869217792 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003504754262788365, + "loss": 1.177, + "theoretical_loss": 3.478377088088642, + "tokens_seen": 1869283328 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003504593781344032, + "loss": 1.2621, + "theoretical_loss": 3.4783668690803733, + "tokens_seen": 1869348864 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003504433299899699, + "loss": 1.3177, + "theoretical_loss": 3.478356650530666, + "tokens_seen": 1869414400 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003504272818455366, + "loss": 1.1913, + "theoretical_loss": 3.4783464324394853, + "tokens_seen": 1869479936 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003504112337011033, + "loss": 1.2882, + "theoretical_loss": 3.4783362148067933, + "tokens_seen": 1869545472 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035039518555667, + "loss": 1.2648, + "theoretical_loss": 3.4783259976325542, + "tokens_seen": 1869611008 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003503791374122367, + "loss": 1.3642, + "theoretical_loss": 3.4783157809167307, + "tokens_seen": 1869676544 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003503630892678034, + "loss": 1.2441, + "theoretical_loss": 3.4783055646592866, + "tokens_seen": 1869742080 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003503470411233701, + "loss": 1.2509, + "theoretical_loss": 3.4782953488601853, + "tokens_seen": 1869807616 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003503309929789368, + "loss": 1.2508, + "theoretical_loss": 3.47828513351939, + "tokens_seen": 1869873152 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003503149448345035, + "loss": 1.25, + "theoretical_loss": 3.478274918636864, + "tokens_seen": 1869938688 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003502988966900702, + "loss": 1.1993, + "theoretical_loss": 3.478264704212571, + "tokens_seen": 1870004224 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003502828485456369, + "loss": 1.2957, + "theoretical_loss": 3.4782544902464743, + "tokens_seen": 1870069760 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003502668004012036, + "loss": 1.2642, + "theoretical_loss": 3.4782442767385366, + "tokens_seen": 1870135296 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003502507522567703, + "loss": 1.2906, + "theoretical_loss": 3.4782340636887223, + "tokens_seen": 1870200832 + }, + { + "epoch": 0.57, + "learning_rate": 0.000350234704112337, + "loss": 1.2203, + "theoretical_loss": 3.4782238510969945, + "tokens_seen": 1870266368 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003502186559679037, + "loss": 1.2525, + "theoretical_loss": 3.4782136389633163, + "tokens_seen": 1870331904 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003502026078234704, + "loss": 1.3119, + "theoretical_loss": 3.478203427287651, + "tokens_seen": 1870397440 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003501865596790371, + "loss": 1.1682, + "theoretical_loss": 3.4781932160699625, + "tokens_seen": 1870462976 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003501705115346038, + "loss": 1.2177, + "theoretical_loss": 3.478183005310214, + "tokens_seen": 1870528512 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003501544633901705, + "loss": 1.2431, + "theoretical_loss": 3.478172795008369, + "tokens_seen": 1870594048 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035013841524573726, + "loss": 1.3142, + "theoretical_loss": 3.478162585164391, + "tokens_seen": 1870659584 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035012236710130397, + "loss": 1.2298, + "theoretical_loss": 3.478152375778243, + "tokens_seen": 1870725120 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035010631895687067, + "loss": 1.4479, + "theoretical_loss": 3.4781421668498886, + "tokens_seen": 1870790656 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035009027081243737, + "loss": 1.3571, + "theoretical_loss": 3.4781319583792913, + "tokens_seen": 1870856192 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035007422266800407, + "loss": 1.2829, + "theoretical_loss": 3.478121750366415, + "tokens_seen": 1870921728 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035005817452357077, + "loss": 1.2928, + "theoretical_loss": 3.478111542811222, + "tokens_seen": 1870987264 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035004212637913747, + "loss": 1.2754, + "theoretical_loss": 3.4781013357136774, + "tokens_seen": 1871052800 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035002607823470417, + "loss": 1.2801, + "theoretical_loss": 3.4780911290737433, + "tokens_seen": 1871118336 + }, + { + "epoch": 0.57, + "learning_rate": 0.00035001003009027087, + "loss": 1.3517, + "theoretical_loss": 3.478080922891383, + "tokens_seen": 1871183872 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034999398194583757, + "loss": 1.3277, + "theoretical_loss": 3.4780707171665615, + "tokens_seen": 1871249408 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034997793380140427, + "loss": 1.2915, + "theoretical_loss": 3.478060511899241, + "tokens_seen": 1871314944 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034996188565697097, + "loss": 1.2604, + "theoretical_loss": 3.478050307089385, + "tokens_seen": 1871380480 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003499458375125376, + "loss": 1.2928, + "theoretical_loss": 3.4780401027369576, + "tokens_seen": 1871446016 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003499297893681043, + "loss": 1.223, + "theoretical_loss": 3.478029898841922, + "tokens_seen": 1871511552 + }, + { + "epoch": 0.57, + "learning_rate": 0.000349913741223671, + "loss": 1.1902, + "theoretical_loss": 3.4780196954042415, + "tokens_seen": 1871577088 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003498976930792377, + "loss": 1.2327, + "theoretical_loss": 3.4780094924238796, + "tokens_seen": 1871642624 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003498816449348044, + "loss": 1.3225, + "theoretical_loss": 3.4779992899008, + "tokens_seen": 1871708160 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003498655967903711, + "loss": 1.4223, + "theoretical_loss": 3.477989087834966, + "tokens_seen": 1871773696 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003498495486459378, + "loss": 1.3396, + "theoretical_loss": 3.477978886226341, + "tokens_seen": 1871839232 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003498335005015045, + "loss": 1.2767, + "theoretical_loss": 3.477968685074889, + "tokens_seen": 1871904768 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003498174523570712, + "loss": 1.1971, + "theoretical_loss": 3.477958484380573, + "tokens_seen": 1871970304 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003498014042126379, + "loss": 1.2517, + "theoretical_loss": 3.477948284143357, + "tokens_seen": 1872035840 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003497853560682046, + "loss": 1.1795, + "theoretical_loss": 3.477938084363204, + "tokens_seen": 1872101376 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003497693079237713, + "loss": 1.29, + "theoretical_loss": 3.477927885040078, + "tokens_seen": 1872166912 + }, + { + "epoch": 0.57, + "learning_rate": 0.000349753259779338, + "loss": 1.2234, + "theoretical_loss": 3.477917686173942, + "tokens_seen": 1872232448 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003497372116349047, + "loss": 1.3511, + "theoretical_loss": 3.4779074877647593, + "tokens_seen": 1872297984 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003497211634904714, + "loss": 1.2372, + "theoretical_loss": 3.4778972898124945, + "tokens_seen": 1872363520 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034970511534603813, + "loss": 1.2238, + "theoretical_loss": 3.477887092317111, + "tokens_seen": 1872429056 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034968906720160483, + "loss": 1.1888, + "theoretical_loss": 3.4778768952785715, + "tokens_seen": 1872494592 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034967301905717153, + "loss": 1.2881, + "theoretical_loss": 3.4778666986968396, + "tokens_seen": 1872560128 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034965697091273823, + "loss": 1.2625, + "theoretical_loss": 3.4778565025718797, + "tokens_seen": 1872625664 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034964092276830493, + "loss": 1.3096, + "theoretical_loss": 3.477846306903655, + "tokens_seen": 1872691200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034962487462387163, + "loss": 1.2739, + "theoretical_loss": 3.4778361116921284, + "tokens_seen": 1872756736 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034960882647943833, + "loss": 1.3509, + "theoretical_loss": 3.477825916937264, + "tokens_seen": 1872822272 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034959277833500503, + "loss": 1.2639, + "theoretical_loss": 3.4778157226390256, + "tokens_seen": 1872887808 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034957673019057173, + "loss": 1.2537, + "theoretical_loss": 3.4778055287973766, + "tokens_seen": 1872953344 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034956068204613843, + "loss": 1.2493, + "theoretical_loss": 3.4777953354122806, + "tokens_seen": 1873018880 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034954463390170513, + "loss": 1.2911, + "theoretical_loss": 3.4777851424837007, + "tokens_seen": 1873084416 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034952858575727183, + "loss": 1.2931, + "theoretical_loss": 3.477774950011601, + "tokens_seen": 1873149952 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034951253761283853, + "loss": 1.2862, + "theoretical_loss": 3.4777647579959448, + "tokens_seen": 1873215488 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034949648946840524, + "loss": 1.3069, + "theoretical_loss": 3.477754566436696, + "tokens_seen": 1873281024 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034948044132397194, + "loss": 1.2277, + "theoretical_loss": 3.4777443753338178, + "tokens_seen": 1873346560 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034946439317953864, + "loss": 1.2917, + "theoretical_loss": 3.4777341846872742, + "tokens_seen": 1873412096 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034944834503510534, + "loss": 1.3368, + "theoretical_loss": 3.477723994497029, + "tokens_seen": 1873477632 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034943229689067204, + "loss": 1.2366, + "theoretical_loss": 3.477713804763045, + "tokens_seen": 1873543168 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034941624874623874, + "loss": 1.2716, + "theoretical_loss": 3.4777036154852863, + "tokens_seen": 1873608704 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034940020060180544, + "loss": 1.3352, + "theoretical_loss": 3.4776934266637167, + "tokens_seen": 1873674240 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034938415245737214, + "loss": 1.3851, + "theoretical_loss": 3.477683238298299, + "tokens_seen": 1873739776 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034936810431293884, + "loss": 1.3319, + "theoretical_loss": 3.477673050388998, + "tokens_seen": 1873805312 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034935205616850554, + "loss": 1.305, + "theoretical_loss": 3.4776628629357766, + "tokens_seen": 1873870848 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034933600802407224, + "loss": 1.258, + "theoretical_loss": 3.4776526759385984, + "tokens_seen": 1873936384 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034931995987963894, + "loss": 1.2321, + "theoretical_loss": 3.477642489397428, + "tokens_seen": 1874001920 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034930391173520564, + "loss": 1.2723, + "theoretical_loss": 3.477632303312227, + "tokens_seen": 1874067456 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034928786359077234, + "loss": 1.2822, + "theoretical_loss": 3.4776221176829614, + "tokens_seen": 1874132992 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034927181544633904, + "loss": 1.3026, + "theoretical_loss": 3.477611932509593, + "tokens_seen": 1874198528 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034925576730190575, + "loss": 1.2495, + "theoretical_loss": 3.4776017477920864, + "tokens_seen": 1874264064 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 694221, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.262169361114502, + "objective/train/theoretical_loss": 3.477591563530405, + "objective/train/tokens_used": 1894789600, + "theoretical_loss": 3.477591563530405, + "tokens_seen": 1874329600 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034923971915747245, + "loss": 1.2622, + "theoretical_loss": 3.477591563530405, + "tokens_seen": 1874329600 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034922367101303915, + "loss": 1.285, + "theoretical_loss": 3.477581379724513, + "tokens_seen": 1874395136 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034920762286860585, + "loss": 1.2203, + "theoretical_loss": 3.477571196374373, + "tokens_seen": 1874460672 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034919157472417255, + "loss": 1.2489, + "theoretical_loss": 3.4775610134799497, + "tokens_seen": 1874526208 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003491755265797392, + "loss": 1.3534, + "theoretical_loss": 3.477550831041206, + "tokens_seen": 1874591744 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003491594784353059, + "loss": 1.2413, + "theoretical_loss": 3.4775406490581062, + "tokens_seen": 1874657280 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003491434302908726, + "loss": 1.212, + "theoretical_loss": 3.4775304675306136, + "tokens_seen": 1874722816 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003491273821464393, + "loss": 1.3708, + "theoretical_loss": 3.4775202864586916, + "tokens_seen": 1874788352 + }, + { + "epoch": 0.57, + "learning_rate": 0.000349111334002006, + "loss": 1.2662, + "theoretical_loss": 3.477510105842305, + "tokens_seen": 1874853888 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003490952858575727, + "loss": 1.2886, + "theoretical_loss": 3.4774999256814163, + "tokens_seen": 1874919424 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003490792377131394, + "loss": 1.1862, + "theoretical_loss": 3.4774897459759897, + "tokens_seen": 1874984960 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003490631895687061, + "loss": 1.125, + "theoretical_loss": 3.477479566725989, + "tokens_seen": 1875050496 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003490471414242728, + "loss": 1.2384, + "theoretical_loss": 3.477469387931378, + "tokens_seen": 1875116032 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003490310932798395, + "loss": 1.2633, + "theoretical_loss": 3.4774592095921197, + "tokens_seen": 1875181568 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003490150451354062, + "loss": 1.4208, + "theoretical_loss": 3.4774490317081783, + "tokens_seen": 1875247104 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003489989969909729, + "loss": 1.2886, + "theoretical_loss": 3.477438854279518, + "tokens_seen": 1875312640 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003489829488465396, + "loss": 1.2225, + "theoretical_loss": 3.4774286773061016, + "tokens_seen": 1875378176 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003489669007021063, + "loss": 1.2742, + "theoretical_loss": 3.477418500787894, + "tokens_seen": 1875443712 + }, + { + "epoch": 0.57, + "learning_rate": 0.000348950852557673, + "loss": 1.2944, + "theoretical_loss": 3.4774083247248573, + "tokens_seen": 1875509248 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034893480441323976, + "loss": 1.3628, + "theoretical_loss": 3.477398149116957, + "tokens_seen": 1875574784 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034891875626880646, + "loss": 1.2624, + "theoretical_loss": 3.4773879739641553, + "tokens_seen": 1875640320 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034890270812437316, + "loss": 1.2936, + "theoretical_loss": 3.477377799266417, + "tokens_seen": 1875705856 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034888665997993986, + "loss": 1.2966, + "theoretical_loss": 3.4773676250237058, + "tokens_seen": 1875771392 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034887061183550656, + "loss": 1.1759, + "theoretical_loss": 3.477357451235984, + "tokens_seen": 1875836928 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034885456369107326, + "loss": 1.2471, + "theoretical_loss": 3.4773472779032177, + "tokens_seen": 1875902464 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034883851554663996, + "loss": 1.3168, + "theoretical_loss": 3.477337105025369, + "tokens_seen": 1875968000 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034882246740220666, + "loss": 1.3849, + "theoretical_loss": 3.477326932602402, + "tokens_seen": 1876033536 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034880641925777336, + "loss": 1.3074, + "theoretical_loss": 3.477316760634281, + "tokens_seen": 1876099072 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034879037111334006, + "loss": 1.375, + "theoretical_loss": 3.4773065891209693, + "tokens_seen": 1876164608 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034877432296890676, + "loss": 1.3131, + "theoretical_loss": 3.477296418062431, + "tokens_seen": 1876230144 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034875827482447346, + "loss": 1.3193, + "theoretical_loss": 3.477286247458629, + "tokens_seen": 1876295680 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034874222668004017, + "loss": 1.1886, + "theoretical_loss": 3.477276077309528, + "tokens_seen": 1876361216 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034872617853560687, + "loss": 1.2898, + "theoretical_loss": 3.4772659076150916, + "tokens_seen": 1876426752 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034871013039117357, + "loss": 1.3053, + "theoretical_loss": 3.477255738375283, + "tokens_seen": 1876492288 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034869408224674027, + "loss": 1.3056, + "theoretical_loss": 3.4772455695900675, + "tokens_seen": 1876557824 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034867803410230697, + "loss": 1.3349, + "theoretical_loss": 3.4772354012594073, + "tokens_seen": 1876623360 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034866198595787367, + "loss": 1.269, + "theoretical_loss": 3.4772252333832667, + "tokens_seen": 1876688896 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034864593781344037, + "loss": 1.298, + "theoretical_loss": 3.4772150659616097, + "tokens_seen": 1876754432 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034862988966900707, + "loss": 1.2762, + "theoretical_loss": 3.4772048989944, + "tokens_seen": 1876819968 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034861384152457377, + "loss": 1.2754, + "theoretical_loss": 3.4771947324816024, + "tokens_seen": 1876885504 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034859779338014047, + "loss": 1.1675, + "theoretical_loss": 3.477184566423179, + "tokens_seen": 1876951040 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034858174523570717, + "loss": 1.3217, + "theoretical_loss": 3.477174400819094, + "tokens_seen": 1877016576 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034856569709127387, + "loss": 1.3139, + "theoretical_loss": 3.4771642356693127, + "tokens_seen": 1877082112 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003485496489468405, + "loss": 1.3123, + "theoretical_loss": 3.477154070973797, + "tokens_seen": 1877147648 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003485336008024072, + "loss": 1.1647, + "theoretical_loss": 3.477143906732512, + "tokens_seen": 1877213184 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003485175526579739, + "loss": 1.2794, + "theoretical_loss": 3.477133742945421, + "tokens_seen": 1877278720 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003485015045135406, + "loss": 1.3089, + "theoretical_loss": 3.4771235796124884, + "tokens_seen": 1877344256 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003484854563691073, + "loss": 1.3343, + "theoretical_loss": 3.4771134167336775, + "tokens_seen": 1877409792 + }, + { + "epoch": 0.57, + "learning_rate": 0.000348469408224674, + "loss": 1.2996, + "theoretical_loss": 3.477103254308952, + "tokens_seen": 1877475328 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003484533600802407, + "loss": 1.3225, + "theoretical_loss": 3.4770930923382766, + "tokens_seen": 1877540864 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003484373119358074, + "loss": 1.255, + "theoretical_loss": 3.4770829308216147, + "tokens_seen": 1877606400 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003484212637913741, + "loss": 1.1961, + "theoretical_loss": 3.47707276975893, + "tokens_seen": 1877671936 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003484052156469408, + "loss": 1.2055, + "theoretical_loss": 3.4770626091501864, + "tokens_seen": 1877737472 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003483891675025075, + "loss": 1.2492, + "theoretical_loss": 3.477052448995348, + "tokens_seen": 1877803008 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003483731193580742, + "loss": 1.3222, + "theoretical_loss": 3.4770422892943786, + "tokens_seen": 1877868544 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003483570712136409, + "loss": 1.2519, + "theoretical_loss": 3.4770321300472418, + "tokens_seen": 1877934080 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034834102306920763, + "loss": 1.3722, + "theoretical_loss": 3.477021971253902, + "tokens_seen": 1877999616 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034832497492477433, + "loss": 1.3971, + "theoretical_loss": 3.4770118129143226, + "tokens_seen": 1878065152 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034830892678034103, + "loss": 1.242, + "theoretical_loss": 3.4770016550284684, + "tokens_seen": 1878130688 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034829287863590773, + "loss": 1.4241, + "theoretical_loss": 3.476991497596302, + "tokens_seen": 1878196224 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034827683049147443, + "loss": 1.2345, + "theoretical_loss": 3.476981340617788, + "tokens_seen": 1878261760 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034826078234704113, + "loss": 1.2185, + "theoretical_loss": 3.47697118409289, + "tokens_seen": 1878327296 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034824473420260783, + "loss": 1.3111, + "theoretical_loss": 3.476961028021573, + "tokens_seen": 1878392832 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034822868605817453, + "loss": 1.2658, + "theoretical_loss": 3.4769508724038, + "tokens_seen": 1878458368 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034821263791374123, + "loss": 1.2459, + "theoretical_loss": 3.4769407172395344, + "tokens_seen": 1878523904 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034819658976930793, + "loss": 1.3051, + "theoretical_loss": 3.476930562528741, + "tokens_seen": 1878589440 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034818054162487463, + "loss": 1.1838, + "theoretical_loss": 3.4769204082713836, + "tokens_seen": 1878654976 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034816449348044133, + "loss": 1.3986, + "theoretical_loss": 3.476910254467426, + "tokens_seen": 1878720512 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034814844533600803, + "loss": 1.3163, + "theoretical_loss": 3.476900101116832, + "tokens_seen": 1878786048 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034813239719157474, + "loss": 1.3352, + "theoretical_loss": 3.476889948219566, + "tokens_seen": 1878851584 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034811634904714144, + "loss": 1.2538, + "theoretical_loss": 3.4768797957755915, + "tokens_seen": 1878917120 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034810030090270814, + "loss": 1.3675, + "theoretical_loss": 3.4768696437848723, + "tokens_seen": 1878982656 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034808425275827484, + "loss": 1.3672, + "theoretical_loss": 3.4768594922473732, + "tokens_seen": 1879048192 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034806820461384154, + "loss": 1.2361, + "theoretical_loss": 3.4768493411630574, + "tokens_seen": 1879113728 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034805215646940824, + "loss": 1.3166, + "theoretical_loss": 3.476839190531889, + "tokens_seen": 1879179264 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034803610832497494, + "loss": 1.3131, + "theoretical_loss": 3.476829040353832, + "tokens_seen": 1879244800 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034802006018054164, + "loss": 1.1063, + "theoretical_loss": 3.4768188906288504, + "tokens_seen": 1879310336 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034800401203610834, + "loss": 1.2355, + "theoretical_loss": 3.4768087413569084, + "tokens_seen": 1879375872 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034798796389167504, + "loss": 1.2119, + "theoretical_loss": 3.4767985925379703, + "tokens_seen": 1879441408 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034797191574724174, + "loss": 1.2883, + "theoretical_loss": 3.4767884441719987, + "tokens_seen": 1879506944 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034795586760280844, + "loss": 1.3319, + "theoretical_loss": 3.4767782962589586, + "tokens_seen": 1879572480 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034793981945837514, + "loss": 1.352, + "theoretical_loss": 3.476768148798814, + "tokens_seen": 1879638016 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034792377131394184, + "loss": 1.2516, + "theoretical_loss": 3.476758001791529, + "tokens_seen": 1879703552 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034790772316950854, + "loss": 1.3741, + "theoretical_loss": 3.4767478552370674, + "tokens_seen": 1879769088 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034789167502507524, + "loss": 1.2092, + "theoretical_loss": 3.4767377091353926, + "tokens_seen": 1879834624 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034787562688064195, + "loss": 1.1984, + "theoretical_loss": 3.4767275634864694, + "tokens_seen": 1879900160 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034785957873620865, + "loss": 1.2939, + "theoretical_loss": 3.476717418290262, + "tokens_seen": 1879965696 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034784353059177535, + "loss": 1.3288, + "theoretical_loss": 3.476707273546733, + "tokens_seen": 1880031232 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034782748244734205, + "loss": 1.2763, + "theoretical_loss": 3.4766971292558484, + "tokens_seen": 1880096768 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034781143430290875, + "loss": 1.291, + "theoretical_loss": 3.476686985417571, + "tokens_seen": 1880162304 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034779538615847545, + "loss": 1.272, + "theoretical_loss": 3.476676842031865, + "tokens_seen": 1880227840 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003477793380140421, + "loss": 1.1985, + "theoretical_loss": 3.4766666990986947, + "tokens_seen": 1880293376 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003477632898696088, + "loss": 1.27, + "theoretical_loss": 3.476656556618024, + "tokens_seen": 1880358912 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034774724172517555, + "loss": 1.3549, + "theoretical_loss": 3.4766464145898164, + "tokens_seen": 1880424448 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034773119358074225, + "loss": 1.2647, + "theoretical_loss": 3.4766362730140368, + "tokens_seen": 1880489984 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034771514543630895, + "loss": 1.3122, + "theoretical_loss": 3.476626131890649, + "tokens_seen": 1880555520 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034769909729187565, + "loss": 1.3988, + "theoretical_loss": 3.476615991219617, + "tokens_seen": 1880621056 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034768304914744235, + "loss": 1.2812, + "theoretical_loss": 3.4766058510009046, + "tokens_seen": 1880686592 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034766700100300905, + "loss": 1.3182, + "theoretical_loss": 3.476595711234476, + "tokens_seen": 1880752128 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034765095285857575, + "loss": 1.2908, + "theoretical_loss": 3.476585571920295, + "tokens_seen": 1880817664 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 696552, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2915724515914917, + "objective/train/theoretical_loss": 3.476575433058327, + "objective/train/tokens_used": 1901343200, + "theoretical_loss": 3.476575433058327, + "tokens_seen": 1880883200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034763490471414246, + "loss": 1.2916, + "theoretical_loss": 3.476575433058327, + "tokens_seen": 1880883200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034761885656970916, + "loss": 1.3508, + "theoretical_loss": 3.476565294648535, + "tokens_seen": 1880948736 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034760280842527586, + "loss": 1.3833, + "theoretical_loss": 3.4765551566908823, + "tokens_seen": 1881014272 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034758676028084256, + "loss": 1.2934, + "theoretical_loss": 3.476545019185334, + "tokens_seen": 1881079808 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034757071213640926, + "loss": 1.2317, + "theoretical_loss": 3.4765348821318547, + "tokens_seen": 1881145344 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034755466399197596, + "loss": 1.1975, + "theoretical_loss": 3.4765247455304076, + "tokens_seen": 1881210880 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034753861584754266, + "loss": 1.206, + "theoretical_loss": 3.476514609380957, + "tokens_seen": 1881276416 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034753861584754266, + "loss": 1.3223, + "theoretical_loss": 3.476504473683467, + "tokens_seen": 1881341952 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034752256770310936, + "loss": 1.3338, + "theoretical_loss": 3.4764943384379015, + "tokens_seen": 1881407488 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034750651955867606, + "loss": 1.2629, + "theoretical_loss": 3.476484203644225, + "tokens_seen": 1881473024 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034749047141424276, + "loss": 1.2328, + "theoretical_loss": 3.4764740693024017, + "tokens_seen": 1881538560 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034747442326980946, + "loss": 1.2366, + "theoretical_loss": 3.4764639354123954, + "tokens_seen": 1881604096 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034747442326980946, + "loss": 1.2674, + "theoretical_loss": 3.47645380197417, + "tokens_seen": 1881669632 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034745837512537616, + "loss": 1.2639, + "theoretical_loss": 3.4764436689876903, + "tokens_seen": 1881735168 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034744232698094286, + "loss": 1.2238, + "theoretical_loss": 3.47643353645292, + "tokens_seen": 1881800704 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034742627883650956, + "loss": 1.2863, + "theoretical_loss": 3.476423404369823, + "tokens_seen": 1881866240 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034741023069207626, + "loss": 1.1858, + "theoretical_loss": 3.4764132727383643, + "tokens_seen": 1881931776 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034739418254764296, + "loss": 1.3431, + "theoretical_loss": 3.4764031415585066, + "tokens_seen": 1881997312 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034737813440320967, + "loss": 1.2404, + "theoretical_loss": 3.4763930108302157, + "tokens_seen": 1882062848 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034736208625877637, + "loss": 1.3191, + "theoretical_loss": 3.4763828805534547, + "tokens_seen": 1882128384 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034734603811434307, + "loss": 1.2643, + "theoretical_loss": 3.4763727507281876, + "tokens_seen": 1882193920 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034732998996990977, + "loss": 1.2985, + "theoretical_loss": 3.4763626213543795, + "tokens_seen": 1882259456 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034731394182547647, + "loss": 1.3125, + "theoretical_loss": 3.476352492431994, + "tokens_seen": 1882324992 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034729789368104317, + "loss": 1.1659, + "theoretical_loss": 3.4763423639609954, + "tokens_seen": 1882390528 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034728184553660987, + "loss": 1.3106, + "theoretical_loss": 3.4763322359413475, + "tokens_seen": 1882456064 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034726579739217657, + "loss": 1.1518, + "theoretical_loss": 3.4763221083730147, + "tokens_seen": 1882521600 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034724974924774327, + "loss": 1.3612, + "theoretical_loss": 3.4763119812559613, + "tokens_seen": 1882587136 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034723370110330997, + "loss": 1.3161, + "theoretical_loss": 3.4763018545901514, + "tokens_seen": 1882652672 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034721765295887667, + "loss": 1.2104, + "theoretical_loss": 3.476291728375549, + "tokens_seen": 1882718208 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034720160481444337, + "loss": 1.3426, + "theoretical_loss": 3.476281602612119, + "tokens_seen": 1882783744 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034718555667001007, + "loss": 1.2166, + "theoretical_loss": 3.476271477299825, + "tokens_seen": 1882849280 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003471695085255768, + "loss": 1.3175, + "theoretical_loss": 3.476261352438631, + "tokens_seen": 1882914816 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003471534603811435, + "loss": 1.3355, + "theoretical_loss": 3.4762512280285014, + "tokens_seen": 1882980352 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003471374122367101, + "loss": 1.2878, + "theoretical_loss": 3.476241104069401, + "tokens_seen": 1883045888 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003471213640922768, + "loss": 1.2183, + "theoretical_loss": 3.4762309805612928, + "tokens_seen": 1883111424 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003471053159478435, + "loss": 1.4047, + "theoretical_loss": 3.4762208575041424, + "tokens_seen": 1883176960 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003470892678034102, + "loss": 1.2854, + "theoretical_loss": 3.476210734897913, + "tokens_seen": 1883242496 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003470732196589769, + "loss": 1.3094, + "theoretical_loss": 3.476200612742569, + "tokens_seen": 1883308032 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003470571715145436, + "loss": 1.3244, + "theoretical_loss": 3.476190491038075, + "tokens_seen": 1883373568 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003470411233701103, + "loss": 1.3371, + "theoretical_loss": 3.476180369784395, + "tokens_seen": 1883439104 + }, + { + "epoch": 0.57, + "learning_rate": 0.000347025075225677, + "loss": 1.2762, + "theoretical_loss": 3.476170248981493, + "tokens_seen": 1883504640 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003470090270812437, + "loss": 1.2307, + "theoretical_loss": 3.476160128629334, + "tokens_seen": 1883570176 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003469929789368104, + "loss": 1.2623, + "theoretical_loss": 3.476150008727881, + "tokens_seen": 1883635712 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003469769307923771, + "loss": 1.1912, + "theoretical_loss": 3.4761398892771, + "tokens_seen": 1883701248 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034696088264794383, + "loss": 1.2034, + "theoretical_loss": 3.4761297702769536, + "tokens_seen": 1883766784 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034694483450351053, + "loss": 1.2141, + "theoretical_loss": 3.476119651727407, + "tokens_seen": 1883832320 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034692878635907723, + "loss": 1.2965, + "theoretical_loss": 3.476109533628424, + "tokens_seen": 1883897856 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034691273821464393, + "loss": 1.2373, + "theoretical_loss": 3.4760994159799687, + "tokens_seen": 1883963392 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034689669007021063, + "loss": 1.2487, + "theoretical_loss": 3.4760892987820062, + "tokens_seen": 1884028928 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034688064192577733, + "loss": 1.2825, + "theoretical_loss": 3.4760791820345, + "tokens_seen": 1884094464 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034686459378134403, + "loss": 1.2773, + "theoretical_loss": 3.4760690657374145, + "tokens_seen": 1884160000 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034684854563691073, + "loss": 1.2778, + "theoretical_loss": 3.476058949890714, + "tokens_seen": 1884225536 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034683249749247743, + "loss": 1.2777, + "theoretical_loss": 3.476048834494363, + "tokens_seen": 1884291072 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034681644934804413, + "loss": 1.3768, + "theoretical_loss": 3.4760387195483258, + "tokens_seen": 1884356608 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034680040120361083, + "loss": 1.2278, + "theoretical_loss": 3.4760286050525666, + "tokens_seen": 1884422144 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034678435305917753, + "loss": 1.2672, + "theoretical_loss": 3.47601849100705, + "tokens_seen": 1884487680 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034676830491474424, + "loss": 1.3021, + "theoretical_loss": 3.4760083774117394, + "tokens_seen": 1884553216 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034675225677031094, + "loss": 1.2528, + "theoretical_loss": 3.4759982642666003, + "tokens_seen": 1884618752 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034673620862587764, + "loss": 1.2287, + "theoretical_loss": 3.475988151571596, + "tokens_seen": 1884684288 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034672016048144434, + "loss": 1.1986, + "theoretical_loss": 3.475978039326691, + "tokens_seen": 1884749824 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034670411233701104, + "loss": 1.3771, + "theoretical_loss": 3.4759679275318502, + "tokens_seen": 1884815360 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034668806419257774, + "loss": 1.323, + "theoretical_loss": 3.4759578161870373, + "tokens_seen": 1884880896 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034667201604814444, + "loss": 1.3515, + "theoretical_loss": 3.475947705292217, + "tokens_seen": 1884946432 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034665596790371114, + "loss": 1.2987, + "theoretical_loss": 3.4759375948473537, + "tokens_seen": 1885011968 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034663991975927784, + "loss": 1.2322, + "theoretical_loss": 3.475927484852411, + "tokens_seen": 1885077504 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034662387161484454, + "loss": 1.3597, + "theoretical_loss": 3.475917375307354, + "tokens_seen": 1885143040 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034660782347041124, + "loss": 1.282, + "theoretical_loss": 3.475907266212147, + "tokens_seen": 1885208576 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034659177532597794, + "loss": 1.2693, + "theoretical_loss": 3.475897157566754, + "tokens_seen": 1885274112 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034657572718154464, + "loss": 1.1837, + "theoretical_loss": 3.4758870493711393, + "tokens_seen": 1885339648 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034655967903711134, + "loss": 1.334, + "theoretical_loss": 3.475876941625268, + "tokens_seen": 1885405184 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003465436308926781, + "loss": 1.2246, + "theoretical_loss": 3.4758668343291035, + "tokens_seen": 1885470720 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003465275827482448, + "loss": 1.2284, + "theoretical_loss": 3.4758567274826104, + "tokens_seen": 1885536256 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034651153460381145, + "loss": 1.3024, + "theoretical_loss": 3.475846621085754, + "tokens_seen": 1885601792 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034649548645937815, + "loss": 1.322, + "theoretical_loss": 3.4758365151384973, + "tokens_seen": 1885667328 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034647943831494485, + "loss": 1.307, + "theoretical_loss": 3.4758264096408054, + "tokens_seen": 1885732864 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034646339017051155, + "loss": 1.2899, + "theoretical_loss": 3.4758163045926427, + "tokens_seen": 1885798400 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034644734202607825, + "loss": 1.1896, + "theoretical_loss": 3.475806199993973, + "tokens_seen": 1885863936 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034643129388164495, + "loss": 1.3021, + "theoretical_loss": 3.475796095844762, + "tokens_seen": 1885929472 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034641524573721165, + "loss": 1.338, + "theoretical_loss": 3.4757859921449725, + "tokens_seen": 1885995008 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034639919759277835, + "loss": 1.2349, + "theoretical_loss": 3.47577588889457, + "tokens_seen": 1886060544 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034638314944834505, + "loss": 1.3432, + "theoretical_loss": 3.4757657860935183, + "tokens_seen": 1886126080 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034636710130391175, + "loss": 1.2913, + "theoretical_loss": 3.475755683741782, + "tokens_seen": 1886191616 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034635105315947845, + "loss": 1.2978, + "theoretical_loss": 3.4757455818393255, + "tokens_seen": 1886257152 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034633500501504515, + "loss": 1.3452, + "theoretical_loss": 3.4757354803861134, + "tokens_seen": 1886322688 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034631895687061185, + "loss": 1.2996, + "theoretical_loss": 3.4757253793821095, + "tokens_seen": 1886388224 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034630290872617855, + "loss": 1.2943, + "theoretical_loss": 3.475715278827279, + "tokens_seen": 1886453760 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034628686058174525, + "loss": 1.266, + "theoretical_loss": 3.4757051787215865, + "tokens_seen": 1886519296 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034627081243731195, + "loss": 1.2796, + "theoretical_loss": 3.4756950790649954, + "tokens_seen": 1886584832 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034625476429287866, + "loss": 1.252, + "theoretical_loss": 3.4756849798574705, + "tokens_seen": 1886650368 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034623871614844536, + "loss": 1.2658, + "theoretical_loss": 3.4756748810989766, + "tokens_seen": 1886715904 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034622266800401206, + "loss": 1.3078, + "theoretical_loss": 3.4756647827894778, + "tokens_seen": 1886781440 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034620661985957876, + "loss": 1.2751, + "theoretical_loss": 3.475654684928939, + "tokens_seen": 1886846976 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034619057171514546, + "loss": 1.2425, + "theoretical_loss": 3.475644587517324, + "tokens_seen": 1886912512 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034617452357071216, + "loss": 1.2322, + "theoretical_loss": 3.4756344905545977, + "tokens_seen": 1886978048 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034615847542627886, + "loss": 1.3459, + "theoretical_loss": 3.4756243940407243, + "tokens_seen": 1887043584 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034614242728184556, + "loss": 1.2328, + "theoretical_loss": 3.4756142979756683, + "tokens_seen": 1887109120 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034612637913741226, + "loss": 1.2816, + "theoretical_loss": 3.4756042023593947, + "tokens_seen": 1887174656 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034611033099297896, + "loss": 1.3065, + "theoretical_loss": 3.475594107191867, + "tokens_seen": 1887240192 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034609428284854566, + "loss": 1.3504, + "theoretical_loss": 3.4755840124730506, + "tokens_seen": 1887305728 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034607823470411236, + "loss": 1.1961, + "theoretical_loss": 3.475573918202909, + "tokens_seen": 1887371264 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 698896, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2869629859924316, + "objective/train/theoretical_loss": 3.4755638243814078, + "objective/train/tokens_used": 1907896800, + "theoretical_loss": 3.4755638243814078, + "tokens_seen": 1887436800 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034606218655967906, + "loss": 1.287, + "theoretical_loss": 3.4755638243814078, + "tokens_seen": 1887436800 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034604613841524576, + "loss": 1.3317, + "theoretical_loss": 3.475553731008511, + "tokens_seen": 1887502336 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034603009027081246, + "loss": 1.308, + "theoretical_loss": 3.475543638084183, + "tokens_seen": 1887567872 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034601404212637916, + "loss": 1.217, + "theoretical_loss": 3.4755335456083873, + "tokens_seen": 1887633408 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034599799398194587, + "loss": 1.2957, + "theoretical_loss": 3.4755234535810904, + "tokens_seen": 1887698944 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034598194583751257, + "loss": 1.3215, + "theoretical_loss": 3.4755133620022556, + "tokens_seen": 1887764480 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034596589769307927, + "loss": 1.2903, + "theoretical_loss": 3.4755032708718474, + "tokens_seen": 1887830016 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034594984954864597, + "loss": 1.2774, + "theoretical_loss": 3.4754931801898308, + "tokens_seen": 1887895552 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034593380140421267, + "loss": 1.2885, + "theoretical_loss": 3.4754830899561697, + "tokens_seen": 1887961088 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034591775325977937, + "loss": 1.3122, + "theoretical_loss": 3.4754730001708287, + "tokens_seen": 1888026624 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034590170511534607, + "loss": 1.2788, + "theoretical_loss": 3.475462910833773, + "tokens_seen": 1888092160 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034588565697091277, + "loss": 1.3318, + "theoretical_loss": 3.475452821944967, + "tokens_seen": 1888157696 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034586960882647947, + "loss": 1.2331, + "theoretical_loss": 3.4754427335043743, + "tokens_seen": 1888223232 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034585356068204617, + "loss": 1.2236, + "theoretical_loss": 3.47543264551196, + "tokens_seen": 1888288768 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034583751253761287, + "loss": 1.3619, + "theoretical_loss": 3.475422557967689, + "tokens_seen": 1888354304 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034582146439317957, + "loss": 1.2195, + "theoretical_loss": 3.4754124708715257, + "tokens_seen": 1888419840 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003458054162487463, + "loss": 1.2525, + "theoretical_loss": 3.4754023842234343, + "tokens_seen": 1888485376 + }, + { + "epoch": 0.57, + "learning_rate": 0.000345789368104313, + "loss": 1.1965, + "theoretical_loss": 3.4753922980233796, + "tokens_seen": 1888550912 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003457733199598797, + "loss": 1.3045, + "theoretical_loss": 3.4753822122713256, + "tokens_seen": 1888616448 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003457572718154464, + "loss": 1.3048, + "theoretical_loss": 3.4753721269672373, + "tokens_seen": 1888681984 + }, + { + "epoch": 0.57, + "learning_rate": 0.000345741223671013, + "loss": 1.3255, + "theoretical_loss": 3.4753620421110796, + "tokens_seen": 1888747520 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003457251755265797, + "loss": 1.2758, + "theoretical_loss": 3.4753519577028165, + "tokens_seen": 1888813056 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003457091273821464, + "loss": 1.1977, + "theoretical_loss": 3.475341873742413, + "tokens_seen": 1888878592 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003456930792377131, + "loss": 1.3047, + "theoretical_loss": 3.4753317902298337, + "tokens_seen": 1888944128 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003456770310932798, + "loss": 1.2818, + "theoretical_loss": 3.4753217071650426, + "tokens_seen": 1889009664 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003456609829488465, + "loss": 1.2221, + "theoretical_loss": 3.475311624548005, + "tokens_seen": 1889075200 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003456449348044132, + "loss": 1.3502, + "theoretical_loss": 3.4753015423786846, + "tokens_seen": 1889140736 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003456288866599799, + "loss": 1.2072, + "theoretical_loss": 3.475291460657047, + "tokens_seen": 1889206272 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003456128385155466, + "loss": 1.2799, + "theoretical_loss": 3.475281379383056, + "tokens_seen": 1889271808 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034559679037111333, + "loss": 1.2602, + "theoretical_loss": 3.4752712985566765, + "tokens_seen": 1889337344 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034558074222668003, + "loss": 1.4004, + "theoretical_loss": 3.475261218177873, + "tokens_seen": 1889402880 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034556469408224673, + "loss": 1.2814, + "theoretical_loss": 3.4752511382466107, + "tokens_seen": 1889468416 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034554864593781343, + "loss": 1.2801, + "theoretical_loss": 3.4752410587628533, + "tokens_seen": 1889533952 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034553259779338013, + "loss": 1.3337, + "theoretical_loss": 3.475230979726566, + "tokens_seen": 1889599488 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034551654964894683, + "loss": 1.3129, + "theoretical_loss": 3.475220901137713, + "tokens_seen": 1889665024 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034550050150451353, + "loss": 1.2793, + "theoretical_loss": 3.4752108229962593, + "tokens_seen": 1889730560 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034548445336008023, + "loss": 1.2507, + "theoretical_loss": 3.4752007453021694, + "tokens_seen": 1889796096 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034546840521564693, + "loss": 1.234, + "theoretical_loss": 3.475190668055408, + "tokens_seen": 1889861632 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034545235707121363, + "loss": 1.3438, + "theoretical_loss": 3.4751805912559393, + "tokens_seen": 1889927168 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034543630892678033, + "loss": 1.4302, + "theoretical_loss": 3.475170514903729, + "tokens_seen": 1889992704 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034542026078234703, + "loss": 1.3385, + "theoretical_loss": 3.4751604389987403, + "tokens_seen": 1890058240 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034540421263791373, + "loss": 1.3399, + "theoretical_loss": 3.475150363540939, + "tokens_seen": 1890123776 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034538816449348044, + "loss": 1.2283, + "theoretical_loss": 3.475140288530289, + "tokens_seen": 1890189312 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034537211634904714, + "loss": 1.233, + "theoretical_loss": 3.4751302139667555, + "tokens_seen": 1890254848 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034535606820461384, + "loss": 1.171, + "theoretical_loss": 3.4751201398503024, + "tokens_seen": 1890320384 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003453400200601806, + "loss": 1.2209, + "theoretical_loss": 3.4751100661808954, + "tokens_seen": 1890385920 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003453239719157473, + "loss": 1.2719, + "theoretical_loss": 3.4750999929584987, + "tokens_seen": 1890451456 + }, + { + "epoch": 0.57, + "learning_rate": 0.000345307923771314, + "loss": 1.3114, + "theoretical_loss": 3.475089920183077, + "tokens_seen": 1890516992 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003452918756268807, + "loss": 1.2376, + "theoretical_loss": 3.4750798478545946, + "tokens_seen": 1890582528 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003452758274824474, + "loss": 1.2755, + "theoretical_loss": 3.4750697759730165, + "tokens_seen": 1890648064 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003452597793380141, + "loss": 1.383, + "theoretical_loss": 3.4750597045383076, + "tokens_seen": 1890713600 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003452437311935808, + "loss": 1.2588, + "theoretical_loss": 3.475049633550432, + "tokens_seen": 1890779136 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003452276830491475, + "loss": 1.2226, + "theoretical_loss": 3.475039563009355, + "tokens_seen": 1890844672 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003452116349047142, + "loss": 1.2343, + "theoretical_loss": 3.4750294929150405, + "tokens_seen": 1890910208 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003451955867602809, + "loss": 1.2421, + "theoretical_loss": 3.4750194232674545, + "tokens_seen": 1890975744 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003451795386158476, + "loss": 1.3511, + "theoretical_loss": 3.4750093540665605, + "tokens_seen": 1891041280 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003451634904714143, + "loss": 1.3719, + "theoretical_loss": 3.4749992853123235, + "tokens_seen": 1891106816 + }, + { + "epoch": 0.57, + "learning_rate": 0.000345147442326981, + "loss": 1.2783, + "theoretical_loss": 3.4749892170047083, + "tokens_seen": 1891172352 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003451313941825477, + "loss": 1.297, + "theoretical_loss": 3.47497914914368, + "tokens_seen": 1891237888 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034511534603811435, + "loss": 1.3561, + "theoretical_loss": 3.474969081729203, + "tokens_seen": 1891303424 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034509929789368105, + "loss": 1.2071, + "theoretical_loss": 3.4749590147612417, + "tokens_seen": 1891368960 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034508324974924775, + "loss": 1.1992, + "theoretical_loss": 3.4749489482397613, + "tokens_seen": 1891434496 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034506720160481445, + "loss": 1.2269, + "theoretical_loss": 3.474938882164726, + "tokens_seen": 1891500032 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034505115346038115, + "loss": 1.2732, + "theoretical_loss": 3.474928816536101, + "tokens_seen": 1891565568 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034503510531594785, + "loss": 1.293, + "theoretical_loss": 3.4749187513538513, + "tokens_seen": 1891631104 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034501905717151455, + "loss": 1.2189, + "theoretical_loss": 3.4749086866179413, + "tokens_seen": 1891696640 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034500300902708125, + "loss": 1.2419, + "theoretical_loss": 3.474898622328335, + "tokens_seen": 1891762176 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034498696088264795, + "loss": 1.2823, + "theoretical_loss": 3.4748885584849987, + "tokens_seen": 1891827712 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034497091273821465, + "loss": 1.248, + "theoretical_loss": 3.4748784950878955, + "tokens_seen": 1891893248 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034495486459378135, + "loss": 1.2415, + "theoretical_loss": 3.474868432136991, + "tokens_seen": 1891958784 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034493881644934805, + "loss": 1.3136, + "theoretical_loss": 3.4748583696322504, + "tokens_seen": 1892024320 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034492276830491475, + "loss": 1.3177, + "theoretical_loss": 3.474848307573638, + "tokens_seen": 1892089856 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034490672016048145, + "loss": 1.3542, + "theoretical_loss": 3.4748382459611182, + "tokens_seen": 1892155392 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034489067201604816, + "loss": 1.1439, + "theoretical_loss": 3.474828184794656, + "tokens_seen": 1892220928 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034487462387161486, + "loss": 1.2241, + "theoretical_loss": 3.4748181240742166, + "tokens_seen": 1892286464 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034485857572718156, + "loss": 1.3119, + "theoretical_loss": 3.474808063799764, + "tokens_seen": 1892352000 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034484252758274826, + "loss": 1.314, + "theoretical_loss": 3.474798003971264, + "tokens_seen": 1892417536 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034482647943831496, + "loss": 1.1907, + "theoretical_loss": 3.474787944588681, + "tokens_seen": 1892483072 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034481043129388166, + "loss": 1.2587, + "theoretical_loss": 3.4747778856519793, + "tokens_seen": 1892548608 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034479438314944836, + "loss": 1.2389, + "theoretical_loss": 3.4747678271611244, + "tokens_seen": 1892614144 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034477833500501506, + "loss": 1.2424, + "theoretical_loss": 3.4747577691160805, + "tokens_seen": 1892679680 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034476228686058176, + "loss": 1.2232, + "theoretical_loss": 3.4747477115168124, + "tokens_seen": 1892745216 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034474623871614846, + "loss": 1.3329, + "theoretical_loss": 3.4747376543632855, + "tokens_seen": 1892810752 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034473019057171516, + "loss": 1.1753, + "theoretical_loss": 3.474727597655464, + "tokens_seen": 1892876288 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034471414242728186, + "loss": 1.2065, + "theoretical_loss": 3.4747175413933133, + "tokens_seen": 1892941824 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034469809428284856, + "loss": 1.332, + "theoretical_loss": 3.4747074855767974, + "tokens_seen": 1893007360 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034468204613841526, + "loss": 1.2444, + "theoretical_loss": 3.4746974302058824, + "tokens_seen": 1893072896 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034466599799398196, + "loss": 1.4647, + "theoretical_loss": 3.4746873752805314, + "tokens_seen": 1893138432 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034464994984954866, + "loss": 1.2831, + "theoretical_loss": 3.474677320800711, + "tokens_seen": 1893203968 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034463390170511537, + "loss": 1.3081, + "theoretical_loss": 3.474667266766385, + "tokens_seen": 1893269504 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034461785356068207, + "loss": 1.2823, + "theoretical_loss": 3.4746572131775184, + "tokens_seen": 1893335040 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034460180541624877, + "loss": 1.3365, + "theoretical_loss": 3.4746471600340763, + "tokens_seen": 1893400576 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034458575727181547, + "loss": 1.2733, + "theoretical_loss": 3.474637107336023, + "tokens_seen": 1893466112 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034456970912738217, + "loss": 1.3176, + "theoretical_loss": 3.474627055083324, + "tokens_seen": 1893531648 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034455366098294887, + "loss": 1.3071, + "theoretical_loss": 3.4746170032759442, + "tokens_seen": 1893597184 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034453761283851557, + "loss": 1.2824, + "theoretical_loss": 3.474606951913848, + "tokens_seen": 1893662720 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034452156469408227, + "loss": 1.2789, + "theoretical_loss": 3.4745969009969997, + "tokens_seen": 1893728256 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034450551654964897, + "loss": 1.219, + "theoretical_loss": 3.4745868505253656, + "tokens_seen": 1893793792 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034448946840521567, + "loss": 1.3548, + "theoretical_loss": 3.4745768004989097, + "tokens_seen": 1893859328 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034447342026078237, + "loss": 1.2593, + "theoretical_loss": 3.474566750917597, + "tokens_seen": 1893924864 + }, + { + "epoch": 0.57, + "objective/train/docs_used": 701135, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1987037658691406, + "objective/train/theoretical_loss": 3.474556701781392, + "objective/train/tokens_used": 1914450400, + "theoretical_loss": 3.474556701781392, + "tokens_seen": 1893990400 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034445737211634907, + "loss": 1.1987, + "theoretical_loss": 3.474556701781392, + "tokens_seen": 1893990400 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034444132397191577, + "loss": 1.289, + "theoretical_loss": 3.4745466530902607, + "tokens_seen": 1894055936 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003444252758274825, + "loss": 1.1831, + "theoretical_loss": 3.474536604844167, + "tokens_seen": 1894121472 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003444092276830492, + "loss": 1.1453, + "theoretical_loss": 3.4745265570430757, + "tokens_seen": 1894187008 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003443931795386159, + "loss": 1.2086, + "theoretical_loss": 3.4745165096869526, + "tokens_seen": 1894252544 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003443771313941826, + "loss": 1.339, + "theoretical_loss": 3.4745064627757616, + "tokens_seen": 1894318080 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003443610832497493, + "loss": 1.3197, + "theoretical_loss": 3.4744964163094685, + "tokens_seen": 1894383616 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003443450351053159, + "loss": 1.2184, + "theoretical_loss": 3.4744863702880378, + "tokens_seen": 1894449152 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003443289869608826, + "loss": 1.2786, + "theoretical_loss": 3.474476324711434, + "tokens_seen": 1894514688 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003443129388164493, + "loss": 1.3003, + "theoretical_loss": 3.474466279579623, + "tokens_seen": 1894580224 + }, + { + "epoch": 0.57, + "learning_rate": 0.000344296890672016, + "loss": 1.3143, + "theoretical_loss": 3.4744562348925685, + "tokens_seen": 1894645760 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003442808425275827, + "loss": 1.256, + "theoretical_loss": 3.4744461906502364, + "tokens_seen": 1894711296 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003442647943831494, + "loss": 1.2118, + "theoretical_loss": 3.4744361468525913, + "tokens_seen": 1894776832 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003442487462387161, + "loss": 1.2883, + "theoretical_loss": 3.474426103499598, + "tokens_seen": 1894842368 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003442326980942828, + "loss": 1.2796, + "theoretical_loss": 3.4744160605912215, + "tokens_seen": 1894907904 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034421664994984953, + "loss": 1.3148, + "theoretical_loss": 3.4744060181274268, + "tokens_seen": 1894973440 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034420060180541623, + "loss": 1.3769, + "theoretical_loss": 3.474395976108179, + "tokens_seen": 1895038976 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034418455366098293, + "loss": 1.2992, + "theoretical_loss": 3.474385934533443, + "tokens_seen": 1895104512 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034416850551654963, + "loss": 1.2483, + "theoretical_loss": 3.4743758934031836, + "tokens_seen": 1895170048 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034415245737211633, + "loss": 1.2419, + "theoretical_loss": 3.4743658527173658, + "tokens_seen": 1895235584 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003441364092276831, + "loss": 1.251, + "theoretical_loss": 3.474355812475954, + "tokens_seen": 1895301120 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003441203610832498, + "loss": 1.2445, + "theoretical_loss": 3.4743457726789146, + "tokens_seen": 1895366656 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003441043129388165, + "loss": 1.3339, + "theoretical_loss": 3.4743357333262113, + "tokens_seen": 1895432192 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003440882647943832, + "loss": 1.2799, + "theoretical_loss": 3.4743256944178094, + "tokens_seen": 1895497728 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003440722166499499, + "loss": 1.3587, + "theoretical_loss": 3.4743156559536743, + "tokens_seen": 1895563264 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003440561685055166, + "loss": 1.2047, + "theoretical_loss": 3.47430561793377, + "tokens_seen": 1895628800 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003440401203610833, + "loss": 1.4144, + "theoretical_loss": 3.4742955803580626, + "tokens_seen": 1895694336 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034402407221665, + "loss": 1.1613, + "theoretical_loss": 3.4742855432265163, + "tokens_seen": 1895759872 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003440080240722167, + "loss": 1.2908, + "theoretical_loss": 3.4742755065390964, + "tokens_seen": 1895825408 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003439919759277834, + "loss": 1.2512, + "theoretical_loss": 3.4742654702957685, + "tokens_seen": 1895890944 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003439759277833501, + "loss": 1.2706, + "theoretical_loss": 3.4742554344964964, + "tokens_seen": 1895956480 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003439598796389168, + "loss": 1.2627, + "theoretical_loss": 3.474245399141246, + "tokens_seen": 1896022016 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003439438314944835, + "loss": 1.3189, + "theoretical_loss": 3.4742353642299815, + "tokens_seen": 1896087552 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003439277833500502, + "loss": 1.3235, + "theoretical_loss": 3.474225329762669, + "tokens_seen": 1896153088 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003439117352056169, + "loss": 1.2347, + "theoretical_loss": 3.4742152957392722, + "tokens_seen": 1896218624 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003438956870611836, + "loss": 1.1606, + "theoretical_loss": 3.4742052621597574, + "tokens_seen": 1896284160 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003438796389167503, + "loss": 1.2716, + "theoretical_loss": 3.4741952290240894, + "tokens_seen": 1896349696 + }, + { + "epoch": 0.57, + "learning_rate": 0.000343863590772317, + "loss": 1.3206, + "theoretical_loss": 3.4741851963322326, + "tokens_seen": 1896415232 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003438475426278837, + "loss": 1.2075, + "theoretical_loss": 3.4741751640841523, + "tokens_seen": 1896480768 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003438314944834504, + "loss": 1.2498, + "theoretical_loss": 3.474165132279813, + "tokens_seen": 1896546304 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003438154463390171, + "loss": 1.278, + "theoretical_loss": 3.474155100919181, + "tokens_seen": 1896611840 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003437993981945838, + "loss": 1.2971, + "theoretical_loss": 3.47414507000222, + "tokens_seen": 1896677376 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003437833500501505, + "loss": 1.3248, + "theoretical_loss": 3.4741350395288966, + "tokens_seen": 1896742912 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003437673019057172, + "loss": 1.2494, + "theoretical_loss": 3.4741250094991747, + "tokens_seen": 1896808448 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003437512537612839, + "loss": 1.3454, + "theoretical_loss": 3.4741149799130193, + "tokens_seen": 1896873984 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003437352056168506, + "loss": 1.3009, + "theoretical_loss": 3.474104950770396, + "tokens_seen": 1896939520 + }, + { + "epoch": 0.57, + "learning_rate": 0.0003437191574724173, + "loss": 1.386, + "theoretical_loss": 3.474094922071269, + "tokens_seen": 1897005056 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034370310932798395, + "loss": 1.2283, + "theoretical_loss": 3.474084893815605, + "tokens_seen": 1897070592 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034368706118355065, + "loss": 1.3582, + "theoretical_loss": 3.4740748660033676, + "tokens_seen": 1897136128 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034367101303911735, + "loss": 1.4214, + "theoretical_loss": 3.474064838634522, + "tokens_seen": 1897201664 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034365496489468405, + "loss": 1.3287, + "theoretical_loss": 3.474054811709034, + "tokens_seen": 1897267200 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034363891675025075, + "loss": 1.2342, + "theoretical_loss": 3.4740447852268685, + "tokens_seen": 1897332736 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034362286860581745, + "loss": 1.2642, + "theoretical_loss": 3.4740347591879903, + "tokens_seen": 1897398272 + }, + { + "epoch": 0.57, + "learning_rate": 0.00034360682046138415, + "loss": 1.1545, + "theoretical_loss": 3.4740247335923646, + "tokens_seen": 1897463808 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034359077231695085, + "loss": 1.322, + "theoretical_loss": 3.474014708439956, + "tokens_seen": 1897529344 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034357472417251755, + "loss": 1.2458, + "theoretical_loss": 3.4740046837307306, + "tokens_seen": 1897594880 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034355867602808425, + "loss": 1.2973, + "theoretical_loss": 3.473994659464653, + "tokens_seen": 1897660416 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034354262788365095, + "loss": 1.348, + "theoretical_loss": 3.4739846356416875, + "tokens_seen": 1897725952 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034352657973921765, + "loss": 1.2573, + "theoretical_loss": 3.473974612261801, + "tokens_seen": 1897791488 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034351053159478436, + "loss": 1.3107, + "theoretical_loss": 3.4739645893249573, + "tokens_seen": 1897857024 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034349448345035106, + "loss": 1.4103, + "theoretical_loss": 3.4739545668311216, + "tokens_seen": 1897922560 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034347843530591776, + "loss": 1.3099, + "theoretical_loss": 3.4739445447802595, + "tokens_seen": 1897988096 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034346238716148446, + "loss": 1.2183, + "theoretical_loss": 3.473934523172336, + "tokens_seen": 1898053632 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034344633901705116, + "loss": 1.1528, + "theoretical_loss": 3.473924502007316, + "tokens_seen": 1898119168 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034343029087261786, + "loss": 1.4187, + "theoretical_loss": 3.4739144812851643, + "tokens_seen": 1898184704 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034341424272818456, + "loss": 1.2552, + "theoretical_loss": 3.473904461005847, + "tokens_seen": 1898250240 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034339819458375126, + "loss": 1.2272, + "theoretical_loss": 3.473894441169329, + "tokens_seen": 1898315776 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034338214643931796, + "loss": 1.2016, + "theoretical_loss": 3.4738844217755744, + "tokens_seen": 1898381312 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034336609829488466, + "loss": 1.305, + "theoretical_loss": 3.47387440282455, + "tokens_seen": 1898446848 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034335005015045136, + "loss": 1.4345, + "theoretical_loss": 3.4738643843162196, + "tokens_seen": 1898512384 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034333400200601806, + "loss": 1.2822, + "theoretical_loss": 3.4738543662505488, + "tokens_seen": 1898577920 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034331795386158476, + "loss": 1.3271, + "theoretical_loss": 3.473844348627503, + "tokens_seen": 1898643456 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034330190571715146, + "loss": 1.1909, + "theoretical_loss": 3.473834331447047, + "tokens_seen": 1898708992 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034328585757271816, + "loss": 1.2193, + "theoretical_loss": 3.4738243147091463, + "tokens_seen": 1898774528 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034326980942828487, + "loss": 1.2984, + "theoretical_loss": 3.4738142984137657, + "tokens_seen": 1898840064 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034325376128385157, + "loss": 1.2771, + "theoretical_loss": 3.473804282560871, + "tokens_seen": 1898905600 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034323771313941827, + "loss": 1.2609, + "theoretical_loss": 3.473794267150427, + "tokens_seen": 1898971136 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034322166499498497, + "loss": 1.2037, + "theoretical_loss": 3.4737842521823983, + "tokens_seen": 1899036672 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034320561685055167, + "loss": 1.2657, + "theoretical_loss": 3.473774237656751, + "tokens_seen": 1899102208 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034318956870611837, + "loss": 1.2826, + "theoretical_loss": 3.47376422357345, + "tokens_seen": 1899167744 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034317352056168507, + "loss": 1.2438, + "theoretical_loss": 3.4737542099324603, + "tokens_seen": 1899233280 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034315747241725177, + "loss": 1.2326, + "theoretical_loss": 3.4737441967337475, + "tokens_seen": 1899298816 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034314142427281847, + "loss": 1.2546, + "theoretical_loss": 3.4737341839772764, + "tokens_seen": 1899364352 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034312537612838517, + "loss": 1.2438, + "theoretical_loss": 3.4737241716630125, + "tokens_seen": 1899429888 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034310932798395187, + "loss": 1.3029, + "theoretical_loss": 3.4737141597909207, + "tokens_seen": 1899495424 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034309327983951857, + "loss": 1.3098, + "theoretical_loss": 3.4737041483609663, + "tokens_seen": 1899560960 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034307723169508527, + "loss": 1.3083, + "theoretical_loss": 3.4736941373731147, + "tokens_seen": 1899626496 + }, + { + "epoch": 0.58, + "learning_rate": 0.000343061183550652, + "loss": 1.2661, + "theoretical_loss": 3.4736841268273313, + "tokens_seen": 1899692032 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003430451354062187, + "loss": 1.2822, + "theoretical_loss": 3.473674116723581, + "tokens_seen": 1899757568 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003430290872617854, + "loss": 1.2381, + "theoretical_loss": 3.473664107061829, + "tokens_seen": 1899823104 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003430130391173521, + "loss": 1.3403, + "theoretical_loss": 3.4736540978420405, + "tokens_seen": 1899888640 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003429969909729188, + "loss": 1.3606, + "theoretical_loss": 3.473644089064181, + "tokens_seen": 1899954176 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003429809428284855, + "loss": 1.1986, + "theoretical_loss": 3.473634080728216, + "tokens_seen": 1900019712 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003429648946840522, + "loss": 1.2347, + "theoretical_loss": 3.47362407283411, + "tokens_seen": 1900085248 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003429488465396189, + "loss": 1.2984, + "theoretical_loss": 3.4736140653818284, + "tokens_seen": 1900150784 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003429327983951856, + "loss": 1.268, + "theoretical_loss": 3.473604058371337, + "tokens_seen": 1900216320 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003429167502507523, + "loss": 1.2234, + "theoretical_loss": 3.4735940518026007, + "tokens_seen": 1900281856 + }, + { + "epoch": 0.58, + "learning_rate": 0.000342900702106319, + "loss": 1.2964, + "theoretical_loss": 3.473584045675585, + "tokens_seen": 1900347392 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003428846539618857, + "loss": 1.2881, + "theoretical_loss": 3.473574039990255, + "tokens_seen": 1900412928 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003428686058174524, + "loss": 1.3262, + "theoretical_loss": 3.473564034746576, + "tokens_seen": 1900478464 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 703430, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.4034793376922607, + "objective/train/theoretical_loss": 3.4735540299445127, + "objective/train/tokens_used": 1921004000, + "theoretical_loss": 3.4735540299445127, + "tokens_seen": 1900544000 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003428525576730191, + "loss": 1.4035, + "theoretical_loss": 3.4735540299445127, + "tokens_seen": 1900544000 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003428365095285858, + "loss": 1.3106, + "theoretical_loss": 3.4735440255840313, + "tokens_seen": 1900609536 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003428204613841525, + "loss": 1.209, + "theoretical_loss": 3.4735340216650967, + "tokens_seen": 1900675072 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003428044132397192, + "loss": 1.2593, + "theoretical_loss": 3.4735240181876743, + "tokens_seen": 1900740608 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003427883650952859, + "loss": 1.1977, + "theoretical_loss": 3.473514015151729, + "tokens_seen": 1900806144 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003427723169508526, + "loss": 1.3212, + "theoretical_loss": 3.4735040125572265, + "tokens_seen": 1900871680 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003427562688064193, + "loss": 1.3039, + "theoretical_loss": 3.4734940104041323, + "tokens_seen": 1900937216 + }, + { + "epoch": 0.58, + "learning_rate": 0.000342740220661986, + "loss": 1.2263, + "theoretical_loss": 3.473484008692411, + "tokens_seen": 1901002752 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003427241725175527, + "loss": 1.2081, + "theoretical_loss": 3.4734740074220283, + "tokens_seen": 1901068288 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003427081243731194, + "loss": 1.3059, + "theoretical_loss": 3.4734640065929496, + "tokens_seen": 1901133824 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003426920762286861, + "loss": 1.2218, + "theoretical_loss": 3.47345400620514, + "tokens_seen": 1901199360 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003426760280842528, + "loss": 1.1853, + "theoretical_loss": 3.473444006258565, + "tokens_seen": 1901264896 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003426599799398195, + "loss": 1.2206, + "theoretical_loss": 3.47343400675319, + "tokens_seen": 1901330432 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003426439317953862, + "loss": 1.3727, + "theoretical_loss": 3.47342400768898, + "tokens_seen": 1901395968 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003426278836509529, + "loss": 1.2282, + "theoretical_loss": 3.473414009065901, + "tokens_seen": 1901461504 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003426118355065196, + "loss": 1.2244, + "theoretical_loss": 3.4734040108839173, + "tokens_seen": 1901527040 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003425957873620863, + "loss": 1.2011, + "theoretical_loss": 3.473394013142995, + "tokens_seen": 1901592576 + }, + { + "epoch": 0.58, + "learning_rate": 0.000342579739217653, + "loss": 1.2638, + "theoretical_loss": 3.4733840158430995, + "tokens_seen": 1901658112 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003425636910732197, + "loss": 1.3033, + "theoretical_loss": 3.4733740189841953, + "tokens_seen": 1901723648 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003425476429287864, + "loss": 1.2971, + "theoretical_loss": 3.4733640225662485, + "tokens_seen": 1901789184 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003425315947843531, + "loss": 1.2416, + "theoretical_loss": 3.4733540265892247, + "tokens_seen": 1901854720 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003425155466399198, + "loss": 1.3206, + "theoretical_loss": 3.473344031053089, + "tokens_seen": 1901920256 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003424994984954865, + "loss": 1.1989, + "theoretical_loss": 3.473334035957806, + "tokens_seen": 1901985792 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003424834503510532, + "loss": 1.3141, + "theoretical_loss": 3.473324041303342, + "tokens_seen": 1902051328 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003424674022066199, + "loss": 1.1342, + "theoretical_loss": 3.4733140470896617, + "tokens_seen": 1902116864 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003424513540621866, + "loss": 1.2022, + "theoretical_loss": 3.473304053316731, + "tokens_seen": 1902182400 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003424353059177533, + "loss": 1.244, + "theoretical_loss": 3.4732940599845152, + "tokens_seen": 1902247936 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034241925777332, + "loss": 1.259, + "theoretical_loss": 3.4732840670929797, + "tokens_seen": 1902313472 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003424032096288867, + "loss": 1.2754, + "theoretical_loss": 3.47327407464209, + "tokens_seen": 1902379008 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003423871614844534, + "loss": 1.2694, + "theoretical_loss": 3.4732640826318106, + "tokens_seen": 1902444544 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003423711133400201, + "loss": 1.2463, + "theoretical_loss": 3.473254091062108, + "tokens_seen": 1902510080 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003423550651955868, + "loss": 1.1886, + "theoretical_loss": 3.473244099932947, + "tokens_seen": 1902575616 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003423390170511535, + "loss": 1.274, + "theoretical_loss": 3.4732341092442933, + "tokens_seen": 1902641152 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003423229689067202, + "loss": 1.1526, + "theoretical_loss": 3.4732241189961117, + "tokens_seen": 1902706688 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034230692076228685, + "loss": 1.2996, + "theoretical_loss": 3.473214129188369, + "tokens_seen": 1902772224 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034229087261785355, + "loss": 1.3506, + "theoretical_loss": 3.473204139821029, + "tokens_seen": 1902837760 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034227482447342025, + "loss": 1.3303, + "theoretical_loss": 3.473194150894058, + "tokens_seen": 1902903296 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034225877632898695, + "loss": 1.3001, + "theoretical_loss": 3.473184162407421, + "tokens_seen": 1902968832 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034224272818455365, + "loss": 1.3225, + "theoretical_loss": 3.473174174361084, + "tokens_seen": 1903034368 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034222668004012035, + "loss": 1.3171, + "theoretical_loss": 3.473164186755012, + "tokens_seen": 1903099904 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034221063189568705, + "loss": 1.28, + "theoretical_loss": 3.47315419958917, + "tokens_seen": 1903165440 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034219458375125375, + "loss": 1.2224, + "theoretical_loss": 3.4731442128635246, + "tokens_seen": 1903230976 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034217853560682045, + "loss": 1.2739, + "theoretical_loss": 3.4731342265780407, + "tokens_seen": 1903296512 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034216248746238715, + "loss": 1.2223, + "theoretical_loss": 3.473124240732683, + "tokens_seen": 1903362048 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034214643931795386, + "loss": 1.3073, + "theoretical_loss": 3.473114255327418, + "tokens_seen": 1903427584 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034213039117352056, + "loss": 1.2953, + "theoretical_loss": 3.473104270362211, + "tokens_seen": 1903493120 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034211434302908726, + "loss": 1.2072, + "theoretical_loss": 3.4730942858370266, + "tokens_seen": 1903558656 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034209829488465396, + "loss": 1.2817, + "theoretical_loss": 3.473084301751831, + "tokens_seen": 1903624192 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034208224674022066, + "loss": 1.2605, + "theoretical_loss": 3.47307431810659, + "tokens_seen": 1903689728 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034206619859578736, + "loss": 1.3483, + "theoretical_loss": 3.4730643349012684, + "tokens_seen": 1903755264 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034205015045135406, + "loss": 1.2502, + "theoretical_loss": 3.4730543521358315, + "tokens_seen": 1903820800 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034203410230692076, + "loss": 1.3179, + "theoretical_loss": 3.4730443698102453, + "tokens_seen": 1903886336 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034201805416248746, + "loss": 1.2617, + "theoretical_loss": 3.473034387924475, + "tokens_seen": 1903951872 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034200200601805416, + "loss": 1.3365, + "theoretical_loss": 3.473024406478487, + "tokens_seen": 1904017408 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034198595787362086, + "loss": 1.3828, + "theoretical_loss": 3.473014425472245, + "tokens_seen": 1904082944 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034196990972918756, + "loss": 1.2961, + "theoretical_loss": 3.4730044449057162, + "tokens_seen": 1904148480 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034195386158475426, + "loss": 1.3099, + "theoretical_loss": 3.472994464778865, + "tokens_seen": 1904214016 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034193781344032096, + "loss": 1.2974, + "theoretical_loss": 3.472984485091657, + "tokens_seen": 1904279552 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034192176529588766, + "loss": 1.2139, + "theoretical_loss": 3.4729745058440584, + "tokens_seen": 1904345088 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034190571715145436, + "loss": 1.2558, + "theoretical_loss": 3.4729645270360345, + "tokens_seen": 1904410624 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034188966900702107, + "loss": 1.3469, + "theoretical_loss": 3.47295454866755, + "tokens_seen": 1904476160 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034187362086258777, + "loss": 1.3019, + "theoretical_loss": 3.472944570738571, + "tokens_seen": 1904541696 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034185757271815447, + "loss": 1.3009, + "theoretical_loss": 3.4729345932490636, + "tokens_seen": 1904607232 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034184152457372117, + "loss": 1.2546, + "theoretical_loss": 3.472924616198993, + "tokens_seen": 1904672768 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034182547642928787, + "loss": 1.4206, + "theoretical_loss": 3.472914639588324, + "tokens_seen": 1904738304 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034180942828485457, + "loss": 1.2096, + "theoretical_loss": 3.4729046634170224, + "tokens_seen": 1904803840 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034179338014042127, + "loss": 1.2869, + "theoretical_loss": 3.4728946876850544, + "tokens_seen": 1904869376 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034177733199598797, + "loss": 1.3619, + "theoretical_loss": 3.4728847123923847, + "tokens_seen": 1904934912 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034176128385155467, + "loss": 1.224, + "theoretical_loss": 3.4728747375389792, + "tokens_seen": 1905000448 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034174523570712137, + "loss": 1.3059, + "theoretical_loss": 3.4728647631248037, + "tokens_seen": 1905065984 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003417291875626881, + "loss": 1.3537, + "theoretical_loss": 3.4728547891498236, + "tokens_seen": 1905131520 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003417131394182548, + "loss": 1.2911, + "theoretical_loss": 3.4728448156140046, + "tokens_seen": 1905197056 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034169709127382153, + "loss": 1.3448, + "theoretical_loss": 3.4728348425173117, + "tokens_seen": 1905262592 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034168104312938823, + "loss": 1.328, + "theoretical_loss": 3.4728248698597106, + "tokens_seen": 1905328128 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003416649949849549, + "loss": 1.305, + "theoretical_loss": 3.4728148976411677, + "tokens_seen": 1905393664 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003416489468405216, + "loss": 1.3507, + "theoretical_loss": 3.472804925861648, + "tokens_seen": 1905459200 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003416328986960883, + "loss": 1.3572, + "theoretical_loss": 3.472794954521116, + "tokens_seen": 1905524736 + }, + { + "epoch": 0.58, + "learning_rate": 0.000341616850551655, + "loss": 1.3003, + "theoretical_loss": 3.472784983619539, + "tokens_seen": 1905590272 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003416008024072217, + "loss": 1.2811, + "theoretical_loss": 3.4727750131568818, + "tokens_seen": 1905655808 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003415847542627884, + "loss": 1.2747, + "theoretical_loss": 3.47276504313311, + "tokens_seen": 1905721344 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003415687061183551, + "loss": 1.2012, + "theoretical_loss": 3.472755073548189, + "tokens_seen": 1905786880 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003415526579739218, + "loss": 1.3361, + "theoretical_loss": 3.472745104402085, + "tokens_seen": 1905852416 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003415366098294885, + "loss": 1.3286, + "theoretical_loss": 3.472735135694763, + "tokens_seen": 1905917952 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003415205616850552, + "loss": 1.3845, + "theoretical_loss": 3.472725167426189, + "tokens_seen": 1905983488 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003415045135406219, + "loss": 1.2741, + "theoretical_loss": 3.472715199596329, + "tokens_seen": 1906049024 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003414884653961886, + "loss": 1.319, + "theoretical_loss": 3.472705232205147, + "tokens_seen": 1906114560 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003414724172517553, + "loss": 1.2442, + "theoretical_loss": 3.4726952652526104, + "tokens_seen": 1906180096 + }, + { + "epoch": 0.58, + "learning_rate": 0.000341456369107322, + "loss": 1.3555, + "theoretical_loss": 3.472685298738684, + "tokens_seen": 1906245632 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003414403209628887, + "loss": 1.2623, + "theoretical_loss": 3.472675332663333, + "tokens_seen": 1906311168 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003414242728184554, + "loss": 1.2716, + "theoretical_loss": 3.472665367026524, + "tokens_seen": 1906376704 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003414082246740221, + "loss": 1.2866, + "theoretical_loss": 3.472655401828222, + "tokens_seen": 1906442240 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003413921765295888, + "loss": 1.2643, + "theoretical_loss": 3.4726454370683926, + "tokens_seen": 1906507776 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003413761283851555, + "loss": 1.272, + "theoretical_loss": 3.472635472747002, + "tokens_seen": 1906573312 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003413600802407222, + "loss": 1.2928, + "theoretical_loss": 3.4726255088640148, + "tokens_seen": 1906638848 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003413440320962889, + "loss": 1.2468, + "theoretical_loss": 3.472615545419398, + "tokens_seen": 1906704384 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003413279839518556, + "loss": 1.265, + "theoretical_loss": 3.472605582413116, + "tokens_seen": 1906769920 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003413119358074223, + "loss": 1.2174, + "theoretical_loss": 3.4725956198451353, + "tokens_seen": 1906835456 + }, + { + "epoch": 0.58, + "learning_rate": 0.000341295887662989, + "loss": 1.343, + "theoretical_loss": 3.4725856577154213, + "tokens_seen": 1906900992 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003412798395185557, + "loss": 1.3298, + "theoretical_loss": 3.47257569602394, + "tokens_seen": 1906966528 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003412637913741224, + "loss": 1.2258, + "theoretical_loss": 3.4725657347706558, + "tokens_seen": 1907032064 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 705962, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2282001972198486, + "objective/train/theoretical_loss": 3.4725557739555355, + "objective/train/tokens_used": 1927557600, + "theoretical_loss": 3.4725557739555355, + "tokens_seen": 1907097600 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003412477432296891, + "loss": 1.2282, + "theoretical_loss": 3.4725557739555355, + "tokens_seen": 1907097600 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003412316950852558, + "loss": 1.2103, + "theoretical_loss": 3.4725458135785447, + "tokens_seen": 1907163136 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003412156469408225, + "loss": 1.3331, + "theoretical_loss": 3.4725358536396485, + "tokens_seen": 1907228672 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003411995987963892, + "loss": 1.2998, + "theoretical_loss": 3.4725258941388137, + "tokens_seen": 1907294208 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003411835506519559, + "loss": 1.3317, + "theoretical_loss": 3.4725159350760046, + "tokens_seen": 1907359744 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003411675025075226, + "loss": 1.4133, + "theoretical_loss": 3.472505976451188, + "tokens_seen": 1907425280 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003411514543630893, + "loss": 1.3426, + "theoretical_loss": 3.472496018264329, + "tokens_seen": 1907490816 + }, + { + "epoch": 0.58, + "learning_rate": 0.000341135406218656, + "loss": 1.3199, + "theoretical_loss": 3.4724860605153935, + "tokens_seen": 1907556352 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003411193580742227, + "loss": 1.3113, + "theoretical_loss": 3.4724761032043467, + "tokens_seen": 1907621888 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003411033099297894, + "loss": 1.2526, + "theoretical_loss": 3.472466146331155, + "tokens_seen": 1907687424 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003410872617853561, + "loss": 1.3631, + "theoretical_loss": 3.4724561898957838, + "tokens_seen": 1907752960 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003410712136409228, + "loss": 1.3178, + "theoretical_loss": 3.4724462338981983, + "tokens_seen": 1907818496 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003410551654964895, + "loss": 1.2683, + "theoretical_loss": 3.4724362783383658, + "tokens_seen": 1907884032 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003410391173520562, + "loss": 1.2388, + "theoretical_loss": 3.47242632321625, + "tokens_seen": 1907949568 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003410230692076229, + "loss": 1.2501, + "theoretical_loss": 3.4724163685318183, + "tokens_seen": 1908015104 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003410070210631896, + "loss": 1.1853, + "theoretical_loss": 3.472406414285035, + "tokens_seen": 1908080640 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003409909729187563, + "loss": 1.3683, + "theoretical_loss": 3.472396460475867, + "tokens_seen": 1908146176 + }, + { + "epoch": 0.58, + "learning_rate": 0.000340974924774323, + "loss": 1.2901, + "theoretical_loss": 3.4723865071042797, + "tokens_seen": 1908211712 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003409588766298897, + "loss": 1.2388, + "theoretical_loss": 3.4723765541702383, + "tokens_seen": 1908277248 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003409428284854564, + "loss": 1.2371, + "theoretical_loss": 3.472366601673709, + "tokens_seen": 1908342784 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003409267803410231, + "loss": 1.3006, + "theoretical_loss": 3.4723566496146576, + "tokens_seen": 1908408320 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003409107321965898, + "loss": 1.3058, + "theoretical_loss": 3.47234669799305, + "tokens_seen": 1908473856 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034089468405215645, + "loss": 1.2696, + "theoretical_loss": 3.4723367468088515, + "tokens_seen": 1908539392 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034087863590772315, + "loss": 1.2263, + "theoretical_loss": 3.4723267960620277, + "tokens_seen": 1908604928 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034086258776328985, + "loss": 1.4139, + "theoretical_loss": 3.4723168457525446, + "tokens_seen": 1908670464 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034084653961885655, + "loss": 1.2377, + "theoretical_loss": 3.4723068958803687, + "tokens_seen": 1908736000 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034083049147442325, + "loss": 1.2602, + "theoretical_loss": 3.4722969464454647, + "tokens_seen": 1908801536 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034081444332998995, + "loss": 1.2448, + "theoretical_loss": 3.472286997447799, + "tokens_seen": 1908867072 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034079839518555665, + "loss": 1.2949, + "theoretical_loss": 3.472277048887337, + "tokens_seen": 1908932608 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034078234704112335, + "loss": 1.2543, + "theoretical_loss": 3.4722671007640447, + "tokens_seen": 1908998144 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034076629889669006, + "loss": 1.3397, + "theoretical_loss": 3.472257153077888, + "tokens_seen": 1909063680 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034075025075225676, + "loss": 1.2987, + "theoretical_loss": 3.4722472058288325, + "tokens_seen": 1909129216 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034073420260782346, + "loss": 1.2801, + "theoretical_loss": 3.472237259016844, + "tokens_seen": 1909194752 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034071815446339016, + "loss": 1.1915, + "theoretical_loss": 3.472227312641888, + "tokens_seen": 1909260288 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034070210631895686, + "loss": 1.2421, + "theoretical_loss": 3.472217366703931, + "tokens_seen": 1909325824 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034068605817452356, + "loss": 1.1999, + "theoretical_loss": 3.4722074212029383, + "tokens_seen": 1909391360 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034067001003009026, + "loss": 1.2964, + "theoretical_loss": 3.4721974761388754, + "tokens_seen": 1909456896 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034065396188565696, + "loss": 1.3265, + "theoretical_loss": 3.4721875315117092, + "tokens_seen": 1909522432 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034063791374122366, + "loss": 1.2934, + "theoretical_loss": 3.4721775873214042, + "tokens_seen": 1909587968 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034062186559679036, + "loss": 1.255, + "theoretical_loss": 3.472167643567927, + "tokens_seen": 1909653504 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034060581745235706, + "loss": 1.3593, + "theoretical_loss": 3.472157700251244, + "tokens_seen": 1909719040 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034058976930792376, + "loss": 1.3206, + "theoretical_loss": 3.472147757371319, + "tokens_seen": 1909784576 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034057372116349046, + "loss": 1.3137, + "theoretical_loss": 3.47213781492812, + "tokens_seen": 1909850112 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034055767301905716, + "loss": 1.2547, + "theoretical_loss": 3.472127872921612, + "tokens_seen": 1909915648 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034054162487462386, + "loss": 1.2756, + "theoretical_loss": 3.4721179313517605, + "tokens_seen": 1909981184 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003405255767301906, + "loss": 1.2428, + "theoretical_loss": 3.4721079902185314, + "tokens_seen": 1910046720 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003405095285857573, + "loss": 1.2575, + "theoretical_loss": 3.4720980495218914, + "tokens_seen": 1910112256 + }, + { + "epoch": 0.58, + "learning_rate": 0.000340493480441324, + "loss": 1.4127, + "theoretical_loss": 3.4720881092618052, + "tokens_seen": 1910177792 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003404774322968907, + "loss": 1.4257, + "theoretical_loss": 3.472078169438239, + "tokens_seen": 1910243328 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003404613841524574, + "loss": 1.3165, + "theoretical_loss": 3.4720682300511596, + "tokens_seen": 1910308864 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003404453360080241, + "loss": 1.2139, + "theoretical_loss": 3.4720582911005318, + "tokens_seen": 1910374400 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003404292878635908, + "loss": 1.2445, + "theoretical_loss": 3.4720483525863215, + "tokens_seen": 1910439936 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003404132397191575, + "loss": 1.2508, + "theoretical_loss": 3.472038414508495, + "tokens_seen": 1910505472 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003403971915747242, + "loss": 1.2103, + "theoretical_loss": 3.4720284768670178, + "tokens_seen": 1910571008 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003403811434302909, + "loss": 1.2008, + "theoretical_loss": 3.4720185396618564, + "tokens_seen": 1910636544 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003403650952858576, + "loss": 1.2811, + "theoretical_loss": 3.472008602892976, + "tokens_seen": 1910702080 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003403490471414243, + "loss": 1.3181, + "theoretical_loss": 3.4719986665603426, + "tokens_seen": 1910767616 + }, + { + "epoch": 0.58, + "learning_rate": 0.000340332998996991, + "loss": 1.1999, + "theoretical_loss": 3.471988730663922, + "tokens_seen": 1910833152 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034031695085255773, + "loss": 1.2117, + "theoretical_loss": 3.4719787952036807, + "tokens_seen": 1910898688 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034030090270812443, + "loss": 1.3221, + "theoretical_loss": 3.471968860179584, + "tokens_seen": 1910964224 + }, + { + "epoch": 0.58, + "learning_rate": 0.00034028485456369113, + "loss": 1.3509, + "theoretical_loss": 3.4719589255915984, + "tokens_seen": 1911029760 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003402688064192578, + "loss": 1.2032, + "theoretical_loss": 3.471948991439689, + "tokens_seen": 1911095296 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003402527582748245, + "loss": 1.244, + "theoretical_loss": 3.471939057723822, + "tokens_seen": 1911160832 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003402367101303912, + "loss": 1.3249, + "theoretical_loss": 3.471929124443964, + "tokens_seen": 1911226368 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003402206619859579, + "loss": 1.3291, + "theoretical_loss": 3.4719191916000796, + "tokens_seen": 1911291904 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003402046138415246, + "loss": 1.3027, + "theoretical_loss": 3.471909259192136, + "tokens_seen": 1911357440 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003401885656970913, + "loss": 1.2459, + "theoretical_loss": 3.471899327220098, + "tokens_seen": 1911422976 + }, + { + "epoch": 0.58, + "learning_rate": 0.000340172517552658, + "loss": 1.3792, + "theoretical_loss": 3.4718893956839327, + "tokens_seen": 1911488512 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003401564694082247, + "loss": 1.2804, + "theoretical_loss": 3.471879464583605, + "tokens_seen": 1911554048 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003401404212637914, + "loss": 1.23, + "theoretical_loss": 3.4718695339190813, + "tokens_seen": 1911619584 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003401243731193581, + "loss": 1.3137, + "theoretical_loss": 3.4718596036903273, + "tokens_seen": 1911685120 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003401083249749248, + "loss": 1.4149, + "theoretical_loss": 3.4718496738973097, + "tokens_seen": 1911750656 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003400922768304915, + "loss": 1.3088, + "theoretical_loss": 3.4718397445399933, + "tokens_seen": 1911816192 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003400762286860582, + "loss": 1.2566, + "theoretical_loss": 3.471829815618345, + "tokens_seen": 1911881728 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003400601805416249, + "loss": 1.209, + "theoretical_loss": 3.4718198871323303, + "tokens_seen": 1911947264 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003400441323971916, + "loss": 1.3886, + "theoretical_loss": 3.471809959081915, + "tokens_seen": 1912012800 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003400280842527583, + "loss": 1.3169, + "theoretical_loss": 3.4718000314670654, + "tokens_seen": 1912078336 + }, + { + "epoch": 0.58, + "learning_rate": 0.000340012036108325, + "loss": 1.1831, + "theoretical_loss": 3.4717901042877477, + "tokens_seen": 1912143872 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003399959879638917, + "loss": 1.3357, + "theoretical_loss": 3.471780177543927, + "tokens_seen": 1912209408 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003399799398194584, + "loss": 1.3237, + "theoretical_loss": 3.47177025123557, + "tokens_seen": 1912274944 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003399638916750251, + "loss": 1.279, + "theoretical_loss": 3.4717603253626423, + "tokens_seen": 1912340480 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003399478435305918, + "loss": 1.2991, + "theoretical_loss": 3.47175039992511, + "tokens_seen": 1912406016 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003399317953861585, + "loss": 1.282, + "theoretical_loss": 3.4717404749229397, + "tokens_seen": 1912471552 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003399157472417252, + "loss": 1.2916, + "theoretical_loss": 3.471730550356096, + "tokens_seen": 1912537088 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003398996990972919, + "loss": 1.1485, + "theoretical_loss": 3.4717206262245464, + "tokens_seen": 1912602624 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003398836509528586, + "loss": 1.2795, + "theoretical_loss": 3.471710702528256, + "tokens_seen": 1912668160 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003398676028084253, + "loss": 1.3473, + "theoretical_loss": 3.4717007792671906, + "tokens_seen": 1912733696 + }, + { + "epoch": 0.58, + "learning_rate": 0.000339851554663992, + "loss": 1.2524, + "theoretical_loss": 3.4716908564413167, + "tokens_seen": 1912799232 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003398355065195587, + "loss": 1.3247, + "theoretical_loss": 3.4716809340506, + "tokens_seen": 1912864768 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003398194583751254, + "loss": 1.3566, + "theoretical_loss": 3.4716710120950074, + "tokens_seen": 1912930304 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003398034102306921, + "loss": 1.3266, + "theoretical_loss": 3.4716610905745036, + "tokens_seen": 1912995840 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003397873620862588, + "loss": 1.218, + "theoretical_loss": 3.4716511694890553, + "tokens_seen": 1913061376 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003397713139418255, + "loss": 1.3324, + "theoretical_loss": 3.471641248838629, + "tokens_seen": 1913126912 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003397552657973922, + "loss": 1.2478, + "theoretical_loss": 3.4716313286231895, + "tokens_seen": 1913192448 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003397392176529589, + "loss": 1.2882, + "theoretical_loss": 3.4716214088427035, + "tokens_seen": 1913257984 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003397231695085256, + "loss": 1.4035, + "theoretical_loss": 3.4716114894971373, + "tokens_seen": 1913323520 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003397071213640923, + "loss": 1.3109, + "theoretical_loss": 3.471601570586456, + "tokens_seen": 1913389056 + }, + { + "epoch": 0.58, + "learning_rate": 0.000339691073219659, + "loss": 1.2661, + "theoretical_loss": 3.471591652110627, + "tokens_seen": 1913454592 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003396750250752257, + "loss": 1.2848, + "theoretical_loss": 3.471581734069616, + "tokens_seen": 1913520128 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003396589769307924, + "loss": 1.3599, + "theoretical_loss": 3.471571816463388, + "tokens_seen": 1913585664 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 708581, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.235412359237671, + "objective/train/theoretical_loss": 3.4715618992919097, + "objective/train/tokens_used": 1934111200, + "theoretical_loss": 3.4715618992919097, + "tokens_seen": 1913651200 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003396429287863591, + "loss": 1.2354, + "theoretical_loss": 3.4715618992919097, + "tokens_seen": 1913651200 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003396268806419258, + "loss": 1.2537, + "theoretical_loss": 3.4715519825551477, + "tokens_seen": 1913716736 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003396108324974925, + "loss": 1.299, + "theoretical_loss": 3.4715420662530674, + "tokens_seen": 1913782272 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003395947843530592, + "loss": 1.375, + "theoretical_loss": 3.471532150385635, + "tokens_seen": 1913847808 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003395787362086259, + "loss": 1.3268, + "theoretical_loss": 3.4715222349528165, + "tokens_seen": 1913913344 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003395626880641926, + "loss": 1.2764, + "theoretical_loss": 3.471512319954578, + "tokens_seen": 1913978880 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003395466399197593, + "loss": 1.2635, + "theoretical_loss": 3.4715024053908854, + "tokens_seen": 1914044416 + }, + { + "epoch": 0.58, + "learning_rate": 0.000339530591775326, + "loss": 1.1992, + "theoretical_loss": 3.4714924912617056, + "tokens_seen": 1914109952 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003395145436308927, + "loss": 1.3044, + "theoretical_loss": 3.471482577567004, + "tokens_seen": 1914175488 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033949849548645935, + "loss": 1.2729, + "theoretical_loss": 3.4714726643067464, + "tokens_seen": 1914241024 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033948244734202605, + "loss": 1.2803, + "theoretical_loss": 3.4714627514808996, + "tokens_seen": 1914306560 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033946639919759275, + "loss": 1.2379, + "theoretical_loss": 3.4714528390894293, + "tokens_seen": 1914372096 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033945035105315945, + "loss": 1.2577, + "theoretical_loss": 3.4714429271323013, + "tokens_seen": 1914437632 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033943430290872615, + "loss": 1.2879, + "theoretical_loss": 3.471433015609482, + "tokens_seen": 1914503168 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033941825476429285, + "loss": 1.3647, + "theoretical_loss": 3.4714231045209383, + "tokens_seen": 1914568704 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033940220661985956, + "loss": 1.2678, + "theoretical_loss": 3.471413193866635, + "tokens_seen": 1914634240 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033938615847542626, + "loss": 1.3669, + "theoretical_loss": 3.4714032836465387, + "tokens_seen": 1914699776 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033937011033099296, + "loss": 1.2816, + "theoretical_loss": 3.4713933738606157, + "tokens_seen": 1914765312 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033935406218655966, + "loss": 1.2162, + "theoretical_loss": 3.4713834645088317, + "tokens_seen": 1914830848 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003393380140421264, + "loss": 1.2975, + "theoretical_loss": 3.471373555591154, + "tokens_seen": 1914896384 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003393219658976931, + "loss": 1.2153, + "theoretical_loss": 3.4713636471075473, + "tokens_seen": 1914961920 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003393059177532598, + "loss": 1.3062, + "theoretical_loss": 3.471353739057978, + "tokens_seen": 1915027456 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003392898696088265, + "loss": 1.3325, + "theoretical_loss": 3.4713438314424128, + "tokens_seen": 1915092992 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003392738214643932, + "loss": 1.318, + "theoretical_loss": 3.4713339242608177, + "tokens_seen": 1915158528 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003392577733199599, + "loss": 1.394, + "theoretical_loss": 3.471324017513159, + "tokens_seen": 1915224064 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003392417251755266, + "loss": 1.2539, + "theoretical_loss": 3.471314111199402, + "tokens_seen": 1915289600 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003392256770310933, + "loss": 1.3198, + "theoretical_loss": 3.4713042053195133, + "tokens_seen": 1915355136 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033920962888666, + "loss": 1.2767, + "theoretical_loss": 3.4712942998734597, + "tokens_seen": 1915420672 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003391935807422267, + "loss": 1.315, + "theoretical_loss": 3.4712843948612067, + "tokens_seen": 1915486208 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003391775325977934, + "loss": 1.3652, + "theoretical_loss": 3.47127449028272, + "tokens_seen": 1915551744 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003391614844533601, + "loss": 1.2147, + "theoretical_loss": 3.4712645861379667, + "tokens_seen": 1915617280 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003391454363089268, + "loss": 1.208, + "theoretical_loss": 3.4712546824269124, + "tokens_seen": 1915682816 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003391293881644935, + "loss": 1.259, + "theoretical_loss": 3.471244779149524, + "tokens_seen": 1915748352 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003391133400200602, + "loss": 1.3549, + "theoretical_loss": 3.4712348763057665, + "tokens_seen": 1915813888 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003390972918756269, + "loss": 1.3401, + "theoretical_loss": 3.471224973895607, + "tokens_seen": 1915879424 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003390812437311936, + "loss": 1.3113, + "theoretical_loss": 3.471215071919012, + "tokens_seen": 1915944960 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003390651955867603, + "loss": 1.3306, + "theoretical_loss": 3.4712051703759466, + "tokens_seen": 1916010496 + }, + { + "epoch": 0.58, + "learning_rate": 0.000339049147442327, + "loss": 1.3127, + "theoretical_loss": 3.4711952692663774, + "tokens_seen": 1916076032 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003390330992978937, + "loss": 1.2758, + "theoretical_loss": 3.471185368590271, + "tokens_seen": 1916141568 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003390170511534604, + "loss": 1.1652, + "theoretical_loss": 3.4711754683475933, + "tokens_seen": 1916207104 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003390010030090271, + "loss": 1.2971, + "theoretical_loss": 3.47116556853831, + "tokens_seen": 1916272640 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003389849548645938, + "loss": 1.2308, + "theoretical_loss": 3.471155669162388, + "tokens_seen": 1916338176 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003389689067201605, + "loss": 1.307, + "theoretical_loss": 3.4711457702197936, + "tokens_seen": 1916403712 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033895285857572723, + "loss": 1.3021, + "theoretical_loss": 3.4711358717104925, + "tokens_seen": 1916469248 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033893681043129393, + "loss": 1.3105, + "theoretical_loss": 3.471125973634451, + "tokens_seen": 1916534784 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033892076228686063, + "loss": 1.4029, + "theoretical_loss": 3.4711160759916355, + "tokens_seen": 1916600320 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033890471414242733, + "loss": 1.2602, + "theoretical_loss": 3.471106178782012, + "tokens_seen": 1916665856 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033888866599799403, + "loss": 1.3087, + "theoretical_loss": 3.4710962820055475, + "tokens_seen": 1916731392 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003388726178535607, + "loss": 1.2461, + "theoretical_loss": 3.4710863856622076, + "tokens_seen": 1916796928 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003388565697091274, + "loss": 1.3679, + "theoretical_loss": 3.471076489751958, + "tokens_seen": 1916862464 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003388405215646941, + "loss": 1.2805, + "theoretical_loss": 3.471066594274766, + "tokens_seen": 1916928000 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003388244734202608, + "loss": 1.2808, + "theoretical_loss": 3.4710566992305973, + "tokens_seen": 1916993536 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003388084252758275, + "loss": 1.2641, + "theoretical_loss": 3.4710468046194185, + "tokens_seen": 1917059072 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003387923771313942, + "loss": 1.3028, + "theoretical_loss": 3.471036910441195, + "tokens_seen": 1917124608 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003387763289869609, + "loss": 1.1388, + "theoretical_loss": 3.471027016695894, + "tokens_seen": 1917190144 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003387602808425276, + "loss": 1.2913, + "theoretical_loss": 3.471017123383481, + "tokens_seen": 1917255680 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003387442326980943, + "loss": 1.4442, + "theoretical_loss": 3.4710072305039232, + "tokens_seen": 1917321216 + }, + { + "epoch": 0.58, + "learning_rate": 0.000338728184553661, + "loss": 1.3403, + "theoretical_loss": 3.470997338057186, + "tokens_seen": 1917386752 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003387121364092277, + "loss": 1.3319, + "theoretical_loss": 3.470987446043236, + "tokens_seen": 1917452288 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003386960882647944, + "loss": 1.202, + "theoretical_loss": 3.4709775544620394, + "tokens_seen": 1917517824 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003386800401203611, + "loss": 1.3866, + "theoretical_loss": 3.4709676633135627, + "tokens_seen": 1917583360 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003386639919759278, + "loss": 1.2909, + "theoretical_loss": 3.470957772597772, + "tokens_seen": 1917648896 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003386479438314945, + "loss": 1.3307, + "theoretical_loss": 3.4709478823146336, + "tokens_seen": 1917714432 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003386318956870612, + "loss": 1.2637, + "theoretical_loss": 3.4709379924641137, + "tokens_seen": 1917779968 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003386158475426279, + "loss": 1.1747, + "theoretical_loss": 3.470928103046179, + "tokens_seen": 1917845504 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003385997993981946, + "loss": 1.2695, + "theoretical_loss": 3.470918214060795, + "tokens_seen": 1917911040 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003385837512537613, + "loss": 1.3235, + "theoretical_loss": 3.4709083255079287, + "tokens_seen": 1917976576 + }, + { + "epoch": 0.58, + "learning_rate": 0.000338567703109328, + "loss": 1.2936, + "theoretical_loss": 3.4708984373875467, + "tokens_seen": 1918042112 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003385516549648947, + "loss": 1.3394, + "theoretical_loss": 3.4708885496996142, + "tokens_seen": 1918107648 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003385356068204614, + "loss": 1.3354, + "theoretical_loss": 3.4708786624440986, + "tokens_seen": 1918173184 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003385195586760281, + "loss": 1.3611, + "theoretical_loss": 3.470868775620965, + "tokens_seen": 1918238720 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003385035105315948, + "loss": 1.2619, + "theoretical_loss": 3.470858889230181, + "tokens_seen": 1918304256 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003384874623871615, + "loss": 1.2421, + "theoretical_loss": 3.470849003271712, + "tokens_seen": 1918369792 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003384714142427282, + "loss": 1.2428, + "theoretical_loss": 3.470839117745525, + "tokens_seen": 1918435328 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003384553660982949, + "loss": 1.1957, + "theoretical_loss": 3.470829232651586, + "tokens_seen": 1918500864 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003384393179538616, + "loss": 1.2707, + "theoretical_loss": 3.4708193479898615, + "tokens_seen": 1918566400 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003384232698094283, + "loss": 1.255, + "theoretical_loss": 3.4708094637603177, + "tokens_seen": 1918631936 + }, + { + "epoch": 0.58, + "learning_rate": 0.000338407221664995, + "loss": 1.2527, + "theoretical_loss": 3.4707995799629208, + "tokens_seen": 1918697472 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003383911735205617, + "loss": 1.3719, + "theoretical_loss": 3.4707896965976373, + "tokens_seen": 1918763008 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003383751253761284, + "loss": 1.2425, + "theoretical_loss": 3.470779813664434, + "tokens_seen": 1918828544 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003383590772316951, + "loss": 1.2346, + "theoretical_loss": 3.4707699311632765, + "tokens_seen": 1918894080 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003383430290872618, + "loss": 1.3684, + "theoretical_loss": 3.4707600490941313, + "tokens_seen": 1918959616 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003383269809428285, + "loss": 1.2823, + "theoretical_loss": 3.470750167456965, + "tokens_seen": 1919025152 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003383109327983952, + "loss": 1.2747, + "theoretical_loss": 3.4707402862517442, + "tokens_seen": 1919090688 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003382948846539619, + "loss": 1.2792, + "theoretical_loss": 3.4707304054784345, + "tokens_seen": 1919156224 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003382788365095286, + "loss": 1.2465, + "theoretical_loss": 3.470720525137003, + "tokens_seen": 1919221760 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003382627883650953, + "loss": 1.315, + "theoretical_loss": 3.4707106452274155, + "tokens_seen": 1919287296 + }, + { + "epoch": 0.58, + "learning_rate": 0.000338246740220662, + "loss": 1.3334, + "theoretical_loss": 3.4707007657496387, + "tokens_seen": 1919352832 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003382306920762287, + "loss": 1.3557, + "theoretical_loss": 3.4706908867036397, + "tokens_seen": 1919418368 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003382146439317954, + "loss": 1.269, + "theoretical_loss": 3.4706810080893833, + "tokens_seen": 1919483904 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003381985957873621, + "loss": 1.3113, + "theoretical_loss": 3.4706711299068376, + "tokens_seen": 1919549440 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003381825476429288, + "loss": 1.2686, + "theoretical_loss": 3.4706612521559674, + "tokens_seen": 1919614976 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003381664994984955, + "loss": 1.3633, + "theoretical_loss": 3.47065137483674, + "tokens_seen": 1919680512 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003381504513540622, + "loss": 1.2951, + "theoretical_loss": 3.4706414979491216, + "tokens_seen": 1919746048 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003381344032096289, + "loss": 1.1877, + "theoretical_loss": 3.4706316214930792, + "tokens_seen": 1919811584 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003381183550651956, + "loss": 1.252, + "theoretical_loss": 3.470621745468578, + "tokens_seen": 1919877120 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003381023069207623, + "loss": 1.2848, + "theoretical_loss": 3.4706118698755857, + "tokens_seen": 1919942656 + }, + { + "epoch": 0.58, + "learning_rate": 0.000338086258776329, + "loss": 1.2617, + "theoretical_loss": 3.470601994714068, + "tokens_seen": 1920008192 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003380702106318957, + "loss": 1.2366, + "theoretical_loss": 3.4705921199839906, + "tokens_seen": 1920073728 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003380541624874624, + "loss": 1.3195, + "theoretical_loss": 3.4705822456853213, + "tokens_seen": 1920139264 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 710923, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2472647428512573, + "objective/train/theoretical_loss": 3.4705723718180264, + "objective/train/tokens_used": 1940664800, + "theoretical_loss": 3.4705723718180264, + "tokens_seen": 1920204800 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003380381143430291, + "loss": 1.2473, + "theoretical_loss": 3.4705723718180264, + "tokens_seen": 1920204800 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003380220661985958, + "loss": 1.276, + "theoretical_loss": 3.4705624983820718, + "tokens_seen": 1920270336 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003380060180541625, + "loss": 1.3885, + "theoretical_loss": 3.4705526253774233, + "tokens_seen": 1920335872 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003379899699097292, + "loss": 1.2276, + "theoretical_loss": 3.470542752804049, + "tokens_seen": 1920401408 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003379739217652959, + "loss": 1.2854, + "theoretical_loss": 3.4705328806619136, + "tokens_seen": 1920466944 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003379578736208626, + "loss": 1.3351, + "theoretical_loss": 3.470523008950985, + "tokens_seen": 1920532480 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003379418254764293, + "loss": 1.2711, + "theoretical_loss": 3.470513137671229, + "tokens_seen": 1920598016 + }, + { + "epoch": 0.58, + "learning_rate": 0.000337925777331996, + "loss": 1.2455, + "theoretical_loss": 3.470503266822612, + "tokens_seen": 1920663552 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003379097291875627, + "loss": 1.2351, + "theoretical_loss": 3.4704933964051, + "tokens_seen": 1920729088 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003378936810431294, + "loss": 1.2716, + "theoretical_loss": 3.470483526418661, + "tokens_seen": 1920794624 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003378776328986961, + "loss": 1.3514, + "theoretical_loss": 3.47047365686326, + "tokens_seen": 1920860160 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003378615847542628, + "loss": 1.2569, + "theoretical_loss": 3.470463787738864, + "tokens_seen": 1920925696 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003378455366098295, + "loss": 1.2445, + "theoretical_loss": 3.470453919045439, + "tokens_seen": 1920991232 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003378294884653962, + "loss": 1.1929, + "theoretical_loss": 3.4704440507829526, + "tokens_seen": 1921056768 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003378134403209629, + "loss": 1.3342, + "theoretical_loss": 3.4704341829513696, + "tokens_seen": 1921122304 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003377973921765296, + "loss": 1.2866, + "theoretical_loss": 3.4704243155506584, + "tokens_seen": 1921187840 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003377813440320963, + "loss": 1.3105, + "theoretical_loss": 3.470414448580784, + "tokens_seen": 1921253376 + }, + { + "epoch": 0.58, + "learning_rate": 0.000337765295887663, + "loss": 1.3134, + "theoretical_loss": 3.4704045820417138, + "tokens_seen": 1921318912 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003377492477432297, + "loss": 1.2431, + "theoretical_loss": 3.470394715933414, + "tokens_seen": 1921384448 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003377331995987964, + "loss": 1.2199, + "theoretical_loss": 3.4703848502558508, + "tokens_seen": 1921449984 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003377171514543631, + "loss": 1.2391, + "theoretical_loss": 3.4703749850089913, + "tokens_seen": 1921515520 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003377011033099298, + "loss": 1.3252, + "theoretical_loss": 3.4703651201928016, + "tokens_seen": 1921581056 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003376850551654965, + "loss": 1.2618, + "theoretical_loss": 3.470355255807248, + "tokens_seen": 1921646592 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003376690070210632, + "loss": 1.2952, + "theoretical_loss": 3.4703453918522973, + "tokens_seen": 1921712128 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003376529588766299, + "loss": 1.3339, + "theoretical_loss": 3.470335528327916, + "tokens_seen": 1921777664 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003376369107321966, + "loss": 1.2641, + "theoretical_loss": 3.470325665234071, + "tokens_seen": 1921843200 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003376208625877633, + "loss": 1.3276, + "theoretical_loss": 3.4703158025707284, + "tokens_seen": 1921908736 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033760481444333, + "loss": 1.2477, + "theoretical_loss": 3.4703059403378544, + "tokens_seen": 1921974272 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003375887662988967, + "loss": 1.2961, + "theoretical_loss": 3.4702960785354158, + "tokens_seen": 1922039808 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033757271815446343, + "loss": 1.2366, + "theoretical_loss": 3.4702862171633795, + "tokens_seen": 1922105344 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033755667001003013, + "loss": 1.2163, + "theoretical_loss": 3.4702763562217123, + "tokens_seen": 1922170880 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033754062186559683, + "loss": 1.3498, + "theoretical_loss": 3.47026649571038, + "tokens_seen": 1922236416 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033752457372116353, + "loss": 1.2792, + "theoretical_loss": 3.470256635629349, + "tokens_seen": 1922301952 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033750852557673023, + "loss": 1.3304, + "theoretical_loss": 3.470246775978586, + "tokens_seen": 1922367488 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033749247743229693, + "loss": 1.39, + "theoretical_loss": 3.4702369167580587, + "tokens_seen": 1922433024 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033747642928786363, + "loss": 1.2441, + "theoretical_loss": 3.4702270579677323, + "tokens_seen": 1922498560 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003374603811434303, + "loss": 1.2278, + "theoretical_loss": 3.470217199607574, + "tokens_seen": 1922564096 + }, + { + "epoch": 0.58, + "learning_rate": 0.000337444332998997, + "loss": 1.3054, + "theoretical_loss": 3.4702073416775496, + "tokens_seen": 1922629632 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003374282848545637, + "loss": 1.2651, + "theoretical_loss": 3.470197484177627, + "tokens_seen": 1922695168 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003374122367101304, + "loss": 1.2835, + "theoretical_loss": 3.470187627107772, + "tokens_seen": 1922760704 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003373961885656971, + "loss": 1.2221, + "theoretical_loss": 3.470177770467951, + "tokens_seen": 1922826240 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003373801404212638, + "loss": 1.2719, + "theoretical_loss": 3.4701679142581305, + "tokens_seen": 1922891776 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003373640922768305, + "loss": 1.2777, + "theoretical_loss": 3.470158058478278, + "tokens_seen": 1922957312 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003373480441323972, + "loss": 1.2401, + "theoretical_loss": 3.4701482031283595, + "tokens_seen": 1923022848 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003373319959879639, + "loss": 1.3022, + "theoretical_loss": 3.4701383482083408, + "tokens_seen": 1923088384 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003373159478435306, + "loss": 1.2212, + "theoretical_loss": 3.4701284937181898, + "tokens_seen": 1923153920 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003372998996990973, + "loss": 1.3291, + "theoretical_loss": 3.4701186396578727, + "tokens_seen": 1923219456 + }, + { + "epoch": 0.58, + "learning_rate": 0.000337283851554664, + "loss": 1.3678, + "theoretical_loss": 3.470108786027356, + "tokens_seen": 1923284992 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003372678034102307, + "loss": 1.4374, + "theoretical_loss": 3.4700989328266063, + "tokens_seen": 1923350528 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003372517552657974, + "loss": 1.2575, + "theoretical_loss": 3.4700890800555895, + "tokens_seen": 1923416064 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003372357071213641, + "loss": 1.3099, + "theoretical_loss": 3.470079227714274, + "tokens_seen": 1923481600 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003372196589769308, + "loss": 1.3256, + "theoretical_loss": 3.4700693758026246, + "tokens_seen": 1923547136 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003372036108324975, + "loss": 1.2699, + "theoretical_loss": 3.470059524320609, + "tokens_seen": 1923612672 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003371875626880642, + "loss": 1.211, + "theoretical_loss": 3.4700496732681936, + "tokens_seen": 1923678208 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003371715145436309, + "loss": 1.2931, + "theoretical_loss": 3.4700398226453446, + "tokens_seen": 1923743744 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003371554663991976, + "loss": 1.2463, + "theoretical_loss": 3.470029972452029, + "tokens_seen": 1923809280 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003371394182547643, + "loss": 1.3366, + "theoretical_loss": 3.4700201226882132, + "tokens_seen": 1923874816 + }, + { + "epoch": 0.58, + "learning_rate": 0.000337123370110331, + "loss": 1.34, + "theoretical_loss": 3.4700102733538642, + "tokens_seen": 1923940352 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003371073219658977, + "loss": 1.2268, + "theoretical_loss": 3.470000424448949, + "tokens_seen": 1924005888 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003370912738214644, + "loss": 1.3217, + "theoretical_loss": 3.4699905759734335, + "tokens_seen": 1924071424 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003370752256770311, + "loss": 1.3324, + "theoretical_loss": 3.4699807279272843, + "tokens_seen": 1924136960 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003370591775325978, + "loss": 1.2659, + "theoretical_loss": 3.4699708803104685, + "tokens_seen": 1924202496 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003370431293881645, + "loss": 1.3166, + "theoretical_loss": 3.469961033122953, + "tokens_seen": 1924268032 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003370270812437312, + "loss": 1.2576, + "theoretical_loss": 3.4699511863647032, + "tokens_seen": 1924333568 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003370110330992979, + "loss": 1.222, + "theoretical_loss": 3.469941340035687, + "tokens_seen": 1924399104 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003369949849548646, + "loss": 1.4019, + "theoretical_loss": 3.4699314941358708, + "tokens_seen": 1924464640 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003369789368104313, + "loss": 1.2619, + "theoretical_loss": 3.4699216486652213, + "tokens_seen": 1924530176 + }, + { + "epoch": 0.58, + "learning_rate": 0.000336962888665998, + "loss": 1.3251, + "theoretical_loss": 3.469911803623705, + "tokens_seen": 1924595712 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003369468405215647, + "loss": 1.2936, + "theoretical_loss": 3.469901959011289, + "tokens_seen": 1924661248 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033693079237713145, + "loss": 1.223, + "theoretical_loss": 3.4698921148279394, + "tokens_seen": 1924726784 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033691474423269815, + "loss": 1.2547, + "theoretical_loss": 3.4698822710736232, + "tokens_seen": 1924792320 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033689869608826485, + "loss": 1.3375, + "theoretical_loss": 3.4698724277483066, + "tokens_seen": 1924857856 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033688264794383155, + "loss": 1.2721, + "theoretical_loss": 3.469862584851957, + "tokens_seen": 1924923392 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033686659979939826, + "loss": 1.3048, + "theoretical_loss": 3.469852742384541, + "tokens_seen": 1924988928 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033685055165496496, + "loss": 1.1969, + "theoretical_loss": 3.469842900346025, + "tokens_seen": 1925054464 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003368345035105316, + "loss": 1.2091, + "theoretical_loss": 3.469833058736376, + "tokens_seen": 1925120000 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003368184553660983, + "loss": 1.3516, + "theoretical_loss": 3.4698232175555606, + "tokens_seen": 1925185536 + }, + { + "epoch": 0.58, + "learning_rate": 0.000336802407221665, + "loss": 1.3666, + "theoretical_loss": 3.469813376803545, + "tokens_seen": 1925251072 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003367863590772317, + "loss": 1.3402, + "theoretical_loss": 3.469803536480297, + "tokens_seen": 1925316608 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003367703109327984, + "loss": 1.3404, + "theoretical_loss": 3.469793696585782, + "tokens_seen": 1925382144 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003367542627883651, + "loss": 1.1909, + "theoretical_loss": 3.4697838571199684, + "tokens_seen": 1925447680 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003367382146439318, + "loss": 1.4082, + "theoretical_loss": 3.469774018082821, + "tokens_seen": 1925513216 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003367221664994985, + "loss": 1.325, + "theoretical_loss": 3.4697641794743084, + "tokens_seen": 1925578752 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003367061183550652, + "loss": 1.3643, + "theoretical_loss": 3.469754341294396, + "tokens_seen": 1925644288 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003366900702106319, + "loss": 1.2263, + "theoretical_loss": 3.469744503543051, + "tokens_seen": 1925709824 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003366740220661986, + "loss": 1.2855, + "theoretical_loss": 3.46973466622024, + "tokens_seen": 1925775360 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003366579739217653, + "loss": 1.3401, + "theoretical_loss": 3.4697248293259304, + "tokens_seen": 1925840896 + }, + { + "epoch": 0.58, + "learning_rate": 0.000336641925777332, + "loss": 1.4217, + "theoretical_loss": 3.469714992860088, + "tokens_seen": 1925906432 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003366258776328987, + "loss": 1.3153, + "theoretical_loss": 3.4697051568226804, + "tokens_seen": 1925971968 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003366098294884654, + "loss": 1.2466, + "theoretical_loss": 3.4696953212136736, + "tokens_seen": 1926037504 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003365937813440321, + "loss": 1.2229, + "theoretical_loss": 3.469685486033035, + "tokens_seen": 1926103040 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003365777331995988, + "loss": 1.3008, + "theoretical_loss": 3.469675651280731, + "tokens_seen": 1926168576 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003365616850551655, + "loss": 1.3325, + "theoretical_loss": 3.4696658169567285, + "tokens_seen": 1926234112 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003365456369107322, + "loss": 1.3193, + "theoretical_loss": 3.469655983060994, + "tokens_seen": 1926299648 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003365295887662989, + "loss": 1.3006, + "theoretical_loss": 3.469646149593495, + "tokens_seen": 1926365184 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003365135406218656, + "loss": 1.2385, + "theoretical_loss": 3.4696363165541975, + "tokens_seen": 1926430720 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003364974924774323, + "loss": 1.2494, + "theoretical_loss": 3.4696264839430686, + "tokens_seen": 1926496256 + }, + { + "epoch": 0.58, + "learning_rate": 0.000336481444332999, + "loss": 1.4095, + "theoretical_loss": 3.4696166517600755, + "tokens_seen": 1926561792 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003364653961885657, + "loss": 1.222, + "theoretical_loss": 3.4696068200051844, + "tokens_seen": 1926627328 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003364493480441324, + "loss": 1.2369, + "theoretical_loss": 3.469596988678362, + "tokens_seen": 1926692864 + }, + { + "epoch": 0.58, + "objective/train/docs_used": 713041, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1737269163131714, + "objective/train/theoretical_loss": 3.469587157779576, + "objective/train/tokens_used": 1947218400, + "theoretical_loss": 3.469587157779576, + "tokens_seen": 1926758400 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003364332998996991, + "loss": 1.1737, + "theoretical_loss": 3.469587157779576, + "tokens_seen": 1926758400 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003364172517552658, + "loss": 1.2593, + "theoretical_loss": 3.4695773273087918, + "tokens_seen": 1926823936 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003364012036108325, + "loss": 1.2268, + "theoretical_loss": 3.4695674972659774, + "tokens_seen": 1926889472 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003363851554663992, + "loss": 1.362, + "theoretical_loss": 3.469557667651099, + "tokens_seen": 1926955008 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003363691073219659, + "loss": 1.2658, + "theoretical_loss": 3.4695478384641243, + "tokens_seen": 1927020544 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003363530591775326, + "loss": 1.2785, + "theoretical_loss": 3.469538009705019, + "tokens_seen": 1927086080 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003363370110330993, + "loss": 1.3077, + "theoretical_loss": 3.46952818137375, + "tokens_seen": 1927151616 + }, + { + "epoch": 0.58, + "learning_rate": 0.000336320962888666, + "loss": 1.2524, + "theoretical_loss": 3.4695183534702854, + "tokens_seen": 1927217152 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003363049147442327, + "loss": 1.3203, + "theoretical_loss": 3.4695085259945904, + "tokens_seen": 1927282688 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003362888665997994, + "loss": 1.326, + "theoretical_loss": 3.4694986989466328, + "tokens_seen": 1927348224 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003362728184553661, + "loss": 1.3829, + "theoretical_loss": 3.4694888723263793, + "tokens_seen": 1927413760 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003362567703109328, + "loss": 1.183, + "theoretical_loss": 3.469479046133797, + "tokens_seen": 1927479296 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003362407221664995, + "loss": 1.2605, + "theoretical_loss": 3.469469220368852, + "tokens_seen": 1927544832 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003362246740220662, + "loss": 1.3358, + "theoretical_loss": 3.4694593950315116, + "tokens_seen": 1927610368 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033620862587763293, + "loss": 1.2266, + "theoretical_loss": 3.4694495701217427, + "tokens_seen": 1927675904 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033619257773319963, + "loss": 1.3225, + "theoretical_loss": 3.4694397456395123, + "tokens_seen": 1927741440 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033617652958876633, + "loss": 1.2673, + "theoretical_loss": 3.469429921584787, + "tokens_seen": 1927806976 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033616048144433303, + "loss": 1.3187, + "theoretical_loss": 3.469420097957533, + "tokens_seen": 1927872512 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033614443329989973, + "loss": 1.292, + "theoretical_loss": 3.4694102747577187, + "tokens_seen": 1927938048 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033612838515546643, + "loss": 1.3798, + "theoretical_loss": 3.46940045198531, + "tokens_seen": 1928003584 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033611233701103313, + "loss": 1.3373, + "theoretical_loss": 3.4693906296402734, + "tokens_seen": 1928069120 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033609628886659983, + "loss": 1.2827, + "theoretical_loss": 3.469380807722577, + "tokens_seen": 1928134656 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033608024072216653, + "loss": 1.3129, + "theoretical_loss": 3.469370986232186, + "tokens_seen": 1928200192 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003360641925777332, + "loss": 1.2115, + "theoretical_loss": 3.4693611651690692, + "tokens_seen": 1928265728 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003360481444332999, + "loss": 1.2919, + "theoretical_loss": 3.4693513445331923, + "tokens_seen": 1928331264 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003360320962888666, + "loss": 1.3141, + "theoretical_loss": 3.4693415243245225, + "tokens_seen": 1928396800 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003360160481444333, + "loss": 1.2769, + "theoretical_loss": 3.4693317045430265, + "tokens_seen": 1928462336 + }, + { + "epoch": 0.58, + "learning_rate": 0.000336, + "loss": 1.2827, + "theoretical_loss": 3.4693218851886716, + "tokens_seen": 1928527872 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003359839518555667, + "loss": 1.2559, + "theoretical_loss": 3.4693120662614243, + "tokens_seen": 1928593408 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003359679037111334, + "loss": 1.3468, + "theoretical_loss": 3.4693022477612514, + "tokens_seen": 1928658944 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003359518555667001, + "loss": 1.2527, + "theoretical_loss": 3.4692924296881205, + "tokens_seen": 1928724480 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003359358074222668, + "loss": 1.2263, + "theoretical_loss": 3.469282612041998, + "tokens_seen": 1928790016 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003359197592778335, + "loss": 1.2636, + "theoretical_loss": 3.469272794822851, + "tokens_seen": 1928855552 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003359037111334002, + "loss": 1.2841, + "theoretical_loss": 3.4692629780306463, + "tokens_seen": 1928921088 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003358876629889669, + "loss": 1.2439, + "theoretical_loss": 3.4692531616653506, + "tokens_seen": 1928986624 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003358716148445336, + "loss": 1.1705, + "theoretical_loss": 3.469243345726931, + "tokens_seen": 1929052160 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003358555667001003, + "loss": 1.1957, + "theoretical_loss": 3.469233530215355, + "tokens_seen": 1929117696 + }, + { + "epoch": 0.58, + "learning_rate": 0.000335839518555667, + "loss": 1.3016, + "theoretical_loss": 3.4692237151305894, + "tokens_seen": 1929183232 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003358234704112337, + "loss": 1.2484, + "theoretical_loss": 3.4692139004726004, + "tokens_seen": 1929248768 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003358074222668004, + "loss": 1.269, + "theoretical_loss": 3.469204086241355, + "tokens_seen": 1929314304 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003357913741223671, + "loss": 1.2648, + "theoretical_loss": 3.4691942724368214, + "tokens_seen": 1929379840 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003357753259779338, + "loss": 1.1707, + "theoretical_loss": 3.469184459058965, + "tokens_seen": 1929445376 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003357592778335005, + "loss": 1.2475, + "theoretical_loss": 3.4691746461077537, + "tokens_seen": 1929510912 + }, + { + "epoch": 0.58, + "learning_rate": 0.0003357432296890672, + "loss": 1.2782, + "theoretical_loss": 3.4691648335831538, + "tokens_seen": 1929576448 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033572718154463395, + "loss": 1.3305, + "theoretical_loss": 3.4691550214851334, + "tokens_seen": 1929641984 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033571113340020065, + "loss": 1.2497, + "theoretical_loss": 3.469145209813658, + "tokens_seen": 1929707520 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033569508525576735, + "loss": 1.2292, + "theoretical_loss": 3.469135398568696, + "tokens_seen": 1929773056 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033567903711133405, + "loss": 1.3015, + "theoretical_loss": 3.469125587750213, + "tokens_seen": 1929838592 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033566298896690075, + "loss": 1.2779, + "theoretical_loss": 3.469115777358177, + "tokens_seen": 1929904128 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033564694082246745, + "loss": 1.3555, + "theoretical_loss": 3.4691059673925544, + "tokens_seen": 1929969664 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033563089267803415, + "loss": 1.2714, + "theoretical_loss": 3.4690961578533126, + "tokens_seen": 1930035200 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033561484453360085, + "loss": 1.2629, + "theoretical_loss": 3.4690863487404187, + "tokens_seen": 1930100736 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033559879638916755, + "loss": 1.3768, + "theoretical_loss": 3.4690765400538393, + "tokens_seen": 1930166272 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033558274824473425, + "loss": 1.3958, + "theoretical_loss": 3.469066731793541, + "tokens_seen": 1930231808 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033556670010030095, + "loss": 1.3653, + "theoretical_loss": 3.4690569239594917, + "tokens_seen": 1930297344 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033555065195586765, + "loss": 1.3843, + "theoretical_loss": 3.469047116551658, + "tokens_seen": 1930362880 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033553460381143435, + "loss": 1.1869, + "theoretical_loss": 3.469037309570007, + "tokens_seen": 1930428416 + }, + { + "epoch": 0.58, + "learning_rate": 0.00033551855566700105, + "loss": 1.3078, + "theoretical_loss": 3.4690275030145052, + "tokens_seen": 1930493952 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033550250752256776, + "loss": 1.2643, + "theoretical_loss": 3.4690176968851203, + "tokens_seen": 1930559488 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033548645937813446, + "loss": 1.3455, + "theoretical_loss": 3.4690078911818194, + "tokens_seen": 1930625024 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033547041123370116, + "loss": 1.2896, + "theoretical_loss": 3.468998085904569, + "tokens_seen": 1930690560 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033545436308926786, + "loss": 1.3329, + "theoretical_loss": 3.4689882810533366, + "tokens_seen": 1930756096 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033543831494483456, + "loss": 1.3285, + "theoretical_loss": 3.4689784766280884, + "tokens_seen": 1930821632 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003354222668004012, + "loss": 1.2325, + "theoretical_loss": 3.4689686726287925, + "tokens_seen": 1930887168 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003354062186559679, + "loss": 1.1882, + "theoretical_loss": 3.468958869055415, + "tokens_seen": 1930952704 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003353901705115346, + "loss": 1.2721, + "theoretical_loss": 3.468949065907924, + "tokens_seen": 1931018240 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003353741223671013, + "loss": 1.2892, + "theoretical_loss": 3.468939263186285, + "tokens_seen": 1931083776 + }, + { + "epoch": 0.59, + "learning_rate": 0.000335358074222668, + "loss": 1.2575, + "theoretical_loss": 3.4689294608904664, + "tokens_seen": 1931149312 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003353420260782347, + "loss": 1.3184, + "theoretical_loss": 3.468919659020435, + "tokens_seen": 1931214848 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003353259779338014, + "loss": 1.2214, + "theoretical_loss": 3.4689098575761577, + "tokens_seen": 1931280384 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003353099297893681, + "loss": 1.3073, + "theoretical_loss": 3.4689000565576014, + "tokens_seen": 1931345920 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003352938816449348, + "loss": 1.1942, + "theoretical_loss": 3.468890255964733, + "tokens_seen": 1931411456 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003352778335005015, + "loss": 1.2625, + "theoretical_loss": 3.46888045579752, + "tokens_seen": 1931476992 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003352617853560682, + "loss": 1.2438, + "theoretical_loss": 3.46887065605593, + "tokens_seen": 1931542528 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003352457372116349, + "loss": 1.3304, + "theoretical_loss": 3.468860856739929, + "tokens_seen": 1931608064 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003352296890672016, + "loss": 1.2688, + "theoretical_loss": 3.468851057849484, + "tokens_seen": 1931673600 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003352136409227683, + "loss": 1.2829, + "theoretical_loss": 3.468841259384563, + "tokens_seen": 1931739136 + }, + { + "epoch": 0.59, + "learning_rate": 0.000335197592778335, + "loss": 1.3484, + "theoretical_loss": 3.4688314613451325, + "tokens_seen": 1931804672 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003351815446339017, + "loss": 1.3393, + "theoretical_loss": 3.46882166373116, + "tokens_seen": 1931870208 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003351654964894684, + "loss": 1.2642, + "theoretical_loss": 3.4688118665426124, + "tokens_seen": 1931935744 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003351494483450351, + "loss": 1.3255, + "theoretical_loss": 3.4688020697794566, + "tokens_seen": 1932001280 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003351334002006018, + "loss": 1.3516, + "theoretical_loss": 3.46879227344166, + "tokens_seen": 1932066816 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003351173520561685, + "loss": 1.3525, + "theoretical_loss": 3.4687824775291896, + "tokens_seen": 1932132352 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003351013039117352, + "loss": 1.262, + "theoretical_loss": 3.4687726820420117, + "tokens_seen": 1932197888 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003350852557673019, + "loss": 1.3533, + "theoretical_loss": 3.4687628869800946, + "tokens_seen": 1932263424 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003350692076228686, + "loss": 1.3384, + "theoretical_loss": 3.4687530923434053, + "tokens_seen": 1932328960 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003350531594784353, + "loss": 1.3306, + "theoretical_loss": 3.46874329813191, + "tokens_seen": 1932394496 + }, + { + "epoch": 0.59, + "learning_rate": 0.000335037111334002, + "loss": 1.2689, + "theoretical_loss": 3.468733504345577, + "tokens_seen": 1932460032 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003350210631895687, + "loss": 1.2094, + "theoretical_loss": 3.468723710984373, + "tokens_seen": 1932525568 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003350050150451354, + "loss": 1.2317, + "theoretical_loss": 3.4687139180482647, + "tokens_seen": 1932591104 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003349889669007021, + "loss": 1.345, + "theoretical_loss": 3.4687041255372195, + "tokens_seen": 1932656640 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003349729187562688, + "loss": 1.2754, + "theoretical_loss": 3.4686943334512046, + "tokens_seen": 1932722176 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003349568706118355, + "loss": 1.3517, + "theoretical_loss": 3.4686845417901866, + "tokens_seen": 1932787712 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003349408224674022, + "loss": 1.2892, + "theoretical_loss": 3.4686747505541335, + "tokens_seen": 1932853248 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003349247743229689, + "loss": 1.2968, + "theoretical_loss": 3.4686649597430126, + "tokens_seen": 1932918784 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003349087261785356, + "loss": 1.365, + "theoretical_loss": 3.46865516935679, + "tokens_seen": 1932984320 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003348926780341023, + "loss": 1.1525, + "theoretical_loss": 3.4686453793954337, + "tokens_seen": 1933049856 + }, + { + "epoch": 0.59, + "learning_rate": 0.000334876629889669, + "loss": 1.2724, + "theoretical_loss": 3.46863558985891, + "tokens_seen": 1933115392 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003348605817452357, + "loss": 1.3035, + "theoretical_loss": 3.468625800747187, + "tokens_seen": 1933180928 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003348445336008024, + "loss": 1.3535, + "theoretical_loss": 3.4686160120602314, + "tokens_seen": 1933246464 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 715349, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.27372407913208, + "objective/train/theoretical_loss": 3.4686062237980106, + "objective/train/tokens_used": 1953772000, + "theoretical_loss": 3.4686062237980106, + "tokens_seen": 1933312000 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033482848545636913, + "loss": 1.2737, + "theoretical_loss": 3.4686062237980106, + "tokens_seen": 1933312000 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033481243731193583, + "loss": 1.3479, + "theoretical_loss": 3.468596435960491, + "tokens_seen": 1933377536 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033479638916750253, + "loss": 1.2617, + "theoretical_loss": 3.468586648547641, + "tokens_seen": 1933443072 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033478034102306923, + "loss": 1.2924, + "theoretical_loss": 3.468576861559427, + "tokens_seen": 1933508608 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033476429287863593, + "loss": 1.3099, + "theoretical_loss": 3.468567074995817, + "tokens_seen": 1933574144 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033474824473420263, + "loss": 1.3578, + "theoretical_loss": 3.4685572888567764, + "tokens_seen": 1933639680 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033473219658976933, + "loss": 1.2868, + "theoretical_loss": 3.4685475031422746, + "tokens_seen": 1933705216 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033471614844533603, + "loss": 1.2014, + "theoretical_loss": 3.468537717852277, + "tokens_seen": 1933770752 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033470010030090273, + "loss": 1.2372, + "theoretical_loss": 3.4685279329867518, + "tokens_seen": 1933836288 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033468405215646943, + "loss": 1.3074, + "theoretical_loss": 3.468518148545666, + "tokens_seen": 1933901824 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033466800401203613, + "loss": 1.2442, + "theoretical_loss": 3.468508364528987, + "tokens_seen": 1933967360 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003346519558676028, + "loss": 1.2327, + "theoretical_loss": 3.4684985809366813, + "tokens_seen": 1934032896 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003346359077231695, + "loss": 1.2905, + "theoretical_loss": 3.4684887977687167, + "tokens_seen": 1934098432 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003346198595787362, + "loss": 1.308, + "theoretical_loss": 3.4684790150250606, + "tokens_seen": 1934163968 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003346038114343029, + "loss": 1.2576, + "theoretical_loss": 3.4684692327056794, + "tokens_seen": 1934229504 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003345877632898696, + "loss": 1.1917, + "theoretical_loss": 3.468459450810541, + "tokens_seen": 1934295040 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003345717151454363, + "loss": 1.2587, + "theoretical_loss": 3.4684496693396127, + "tokens_seen": 1934360576 + }, + { + "epoch": 0.59, + "learning_rate": 0.000334555667001003, + "loss": 1.2523, + "theoretical_loss": 3.4684398882928615, + "tokens_seen": 1934426112 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003345396188565697, + "loss": 1.2686, + "theoretical_loss": 3.4684301076702546, + "tokens_seen": 1934491648 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033452357071213644, + "loss": 1.2481, + "theoretical_loss": 3.468420327471759, + "tokens_seen": 1934557184 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033450752256770314, + "loss": 1.3523, + "theoretical_loss": 3.4684105476973426, + "tokens_seen": 1934622720 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033449147442326984, + "loss": 1.2564, + "theoretical_loss": 3.4684007683469718, + "tokens_seen": 1934688256 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033447542627883654, + "loss": 1.2846, + "theoretical_loss": 3.468390989420615, + "tokens_seen": 1934753792 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033445937813440324, + "loss": 1.2416, + "theoretical_loss": 3.4683812109182384, + "tokens_seen": 1934819328 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033444332998996994, + "loss": 1.2022, + "theoretical_loss": 3.4683714328398096, + "tokens_seen": 1934884864 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033442728184553664, + "loss": 1.2144, + "theoretical_loss": 3.4683616551852956, + "tokens_seen": 1934950400 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033441123370110334, + "loss": 1.24, + "theoretical_loss": 3.4683518779546647, + "tokens_seen": 1935015936 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033439518555667004, + "loss": 1.2045, + "theoretical_loss": 3.4683421011478828, + "tokens_seen": 1935081472 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033437913741223675, + "loss": 1.2401, + "theoretical_loss": 3.468332324764918, + "tokens_seen": 1935147008 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033436308926780345, + "loss": 1.2881, + "theoretical_loss": 3.4683225488057374, + "tokens_seen": 1935212544 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033434704112337015, + "loss": 1.2688, + "theoretical_loss": 3.468312773270308, + "tokens_seen": 1935278080 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033433099297893685, + "loss": 1.1724, + "theoretical_loss": 3.4683029981585975, + "tokens_seen": 1935343616 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033431494483450355, + "loss": 1.2661, + "theoretical_loss": 3.468293223470573, + "tokens_seen": 1935409152 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033429889669007025, + "loss": 1.3571, + "theoretical_loss": 3.4682834492062016, + "tokens_seen": 1935474688 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033428284854563695, + "loss": 1.3103, + "theoretical_loss": 3.4682736753654515, + "tokens_seen": 1935540224 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033426680040120365, + "loss": 1.222, + "theoretical_loss": 3.468263901948289, + "tokens_seen": 1935605760 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033425075225677035, + "loss": 1.235, + "theoretical_loss": 3.468254128954681, + "tokens_seen": 1935671296 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033423470411233705, + "loss": 1.2823, + "theoretical_loss": 3.468244356384596, + "tokens_seen": 1935736832 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033421865596790375, + "loss": 1.3096, + "theoretical_loss": 3.468234584238001, + "tokens_seen": 1935802368 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033420260782347045, + "loss": 1.3113, + "theoretical_loss": 3.468224812514863, + "tokens_seen": 1935867904 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033418655967903715, + "loss": 1.3731, + "theoretical_loss": 3.4682150412151493, + "tokens_seen": 1935933440 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033417051153460385, + "loss": 1.3096, + "theoretical_loss": 3.468205270338828, + "tokens_seen": 1935998976 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033415446339017055, + "loss": 1.3085, + "theoretical_loss": 3.468195499885865, + "tokens_seen": 1936064512 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033413841524573726, + "loss": 1.3479, + "theoretical_loss": 3.468185729856229, + "tokens_seen": 1936130048 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033412236710130396, + "loss": 1.3574, + "theoretical_loss": 3.4681759602498863, + "tokens_seen": 1936195584 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033410631895687066, + "loss": 1.3788, + "theoretical_loss": 3.4681661910668047, + "tokens_seen": 1936261120 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033409027081243736, + "loss": 1.2734, + "theoretical_loss": 3.4681564223069516, + "tokens_seen": 1936326656 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033407422266800406, + "loss": 1.3878, + "theoretical_loss": 3.4681466539702948, + "tokens_seen": 1936392192 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033405817452357076, + "loss": 1.3137, + "theoretical_loss": 3.4681368860568, + "tokens_seen": 1936457728 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033404212637913746, + "loss": 1.3181, + "theoretical_loss": 3.4681271185664366, + "tokens_seen": 1936523264 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003340260782347041, + "loss": 1.2392, + "theoretical_loss": 3.4681173514991706, + "tokens_seen": 1936588800 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003340100300902708, + "loss": 1.173, + "theoretical_loss": 3.4681075848549696, + "tokens_seen": 1936654336 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003339939819458375, + "loss": 1.279, + "theoretical_loss": 3.4680978186338014, + "tokens_seen": 1936719872 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003339779338014042, + "loss": 1.2912, + "theoretical_loss": 3.4680880528356326, + "tokens_seen": 1936785408 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003339618856569709, + "loss": 1.296, + "theoretical_loss": 3.4680782874604317, + "tokens_seen": 1936850944 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003339458375125376, + "loss": 1.2968, + "theoretical_loss": 3.4680685225081653, + "tokens_seen": 1936916480 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003339297893681043, + "loss": 1.2442, + "theoretical_loss": 3.4680587579788007, + "tokens_seen": 1936982016 + }, + { + "epoch": 0.59, + "learning_rate": 0.000333913741223671, + "loss": 1.2589, + "theoretical_loss": 3.468048993872306, + "tokens_seen": 1937047552 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003338976930792377, + "loss": 1.2989, + "theoretical_loss": 3.468039230188647, + "tokens_seen": 1937113088 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003338816449348044, + "loss": 1.2105, + "theoretical_loss": 3.468029466927793, + "tokens_seen": 1937178624 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003338655967903711, + "loss": 1.3596, + "theoretical_loss": 3.46801970408971, + "tokens_seen": 1937244160 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003338495486459378, + "loss": 1.3693, + "theoretical_loss": 3.468009941674366, + "tokens_seen": 1937309696 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003338335005015045, + "loss": 1.2823, + "theoretical_loss": 3.468000179681728, + "tokens_seen": 1937375232 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003338174523570712, + "loss": 1.2791, + "theoretical_loss": 3.4679904181117642, + "tokens_seen": 1937440768 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003338014042126379, + "loss": 1.329, + "theoretical_loss": 3.467980656964442, + "tokens_seen": 1937506304 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003337853560682046, + "loss": 1.2252, + "theoretical_loss": 3.467970896239727, + "tokens_seen": 1937571840 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003337693079237713, + "loss": 1.2615, + "theoretical_loss": 3.467961135937589, + "tokens_seen": 1937637376 + }, + { + "epoch": 0.59, + "learning_rate": 0.000333753259779338, + "loss": 1.3532, + "theoretical_loss": 3.467951376057994, + "tokens_seen": 1937702912 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003337372116349047, + "loss": 1.4247, + "theoretical_loss": 3.4679416166009096, + "tokens_seen": 1937768448 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003337211634904714, + "loss": 1.1984, + "theoretical_loss": 3.467931857566304, + "tokens_seen": 1937833984 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003337051153460381, + "loss": 1.2449, + "theoretical_loss": 3.4679220989541433, + "tokens_seen": 1937899520 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003336890672016048, + "loss": 1.3906, + "theoretical_loss": 3.4679123407643955, + "tokens_seen": 1937965056 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003336730190571715, + "loss": 1.313, + "theoretical_loss": 3.467902582997029, + "tokens_seen": 1938030592 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003336569709127382, + "loss": 1.229, + "theoretical_loss": 3.46789282565201, + "tokens_seen": 1938096128 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003336409227683049, + "loss": 1.1767, + "theoretical_loss": 3.467883068729306, + "tokens_seen": 1938161664 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003336248746238716, + "loss": 1.4409, + "theoretical_loss": 3.467873312228885, + "tokens_seen": 1938227200 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003336088264794383, + "loss": 1.2141, + "theoretical_loss": 3.4678635561507143, + "tokens_seen": 1938292736 + }, + { + "epoch": 0.59, + "learning_rate": 0.000333592778335005, + "loss": 1.3072, + "theoretical_loss": 3.467853800494761, + "tokens_seen": 1938358272 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003335767301905717, + "loss": 1.284, + "theoretical_loss": 3.467844045260993, + "tokens_seen": 1938423808 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003335606820461384, + "loss": 1.3171, + "theoretical_loss": 3.4678342904493777, + "tokens_seen": 1938489344 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003335446339017051, + "loss": 1.3194, + "theoretical_loss": 3.467824536059882, + "tokens_seen": 1938554880 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003335285857572718, + "loss": 1.2454, + "theoretical_loss": 3.4678147820924745, + "tokens_seen": 1938620416 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003335125376128385, + "loss": 1.333, + "theoretical_loss": 3.4678050285471214, + "tokens_seen": 1938685952 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003334964894684052, + "loss": 1.2859, + "theoretical_loss": 3.467795275423791, + "tokens_seen": 1938751488 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003334804413239719, + "loss": 1.2345, + "theoretical_loss": 3.4677855227224508, + "tokens_seen": 1938817024 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033346439317953863, + "loss": 1.352, + "theoretical_loss": 3.4677757704430676, + "tokens_seen": 1938882560 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033344834503510533, + "loss": 1.303, + "theoretical_loss": 3.4677660185856096, + "tokens_seen": 1938948096 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033343229689067203, + "loss": 1.2702, + "theoretical_loss": 3.467756267150044, + "tokens_seen": 1939013632 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033341624874623873, + "loss": 1.2491, + "theoretical_loss": 3.467746516136337, + "tokens_seen": 1939079168 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033340020060180543, + "loss": 1.2421, + "theoretical_loss": 3.467736765544459, + "tokens_seen": 1939144704 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033338415245737213, + "loss": 1.2687, + "theoretical_loss": 3.467727015374375, + "tokens_seen": 1939210240 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033336810431293883, + "loss": 1.3369, + "theoretical_loss": 3.467717265626054, + "tokens_seen": 1939275776 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033335205616850553, + "loss": 1.3835, + "theoretical_loss": 3.4677075162994617, + "tokens_seen": 1939341312 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033333600802407223, + "loss": 1.2551, + "theoretical_loss": 3.4676977673945677, + "tokens_seen": 1939406848 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033331995987963893, + "loss": 1.3318, + "theoretical_loss": 3.467688018911338, + "tokens_seen": 1939472384 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033330391173520563, + "loss": 1.3213, + "theoretical_loss": 3.467678270849741, + "tokens_seen": 1939537920 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033328786359077233, + "loss": 1.2471, + "theoretical_loss": 3.467668523209744, + "tokens_seen": 1939603456 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033327181544633904, + "loss": 1.3111, + "theoretical_loss": 3.467658775991315, + "tokens_seen": 1939668992 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033325576730190574, + "loss": 1.3104, + "theoretical_loss": 3.46764902919442, + "tokens_seen": 1939734528 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033323971915747244, + "loss": 1.2879, + "theoretical_loss": 3.4676392828190274, + "tokens_seen": 1939800064 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 717575, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.240422248840332, + "objective/train/theoretical_loss": 3.4676295368651053, + "objective/train/tokens_used": 1960325600, + "theoretical_loss": 3.4676295368651053, + "tokens_seen": 1939865600 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033322367101303914, + "loss": 1.2404, + "theoretical_loss": 3.4676295368651053, + "tokens_seen": 1939865600 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033320762286860584, + "loss": 1.352, + "theoretical_loss": 3.4676197913326208, + "tokens_seen": 1939931136 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033319157472417254, + "loss": 1.3858, + "theoretical_loss": 3.467610046221541, + "tokens_seen": 1939996672 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033317552657973924, + "loss": 1.3221, + "theoretical_loss": 3.467600301531834, + "tokens_seen": 1940062208 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033315947843530594, + "loss": 1.3323, + "theoretical_loss": 3.467590557263467, + "tokens_seen": 1940127744 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033314343029087264, + "loss": 1.2922, + "theoretical_loss": 3.467580813416408, + "tokens_seen": 1940193280 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033312738214643934, + "loss": 1.2653, + "theoretical_loss": 3.4675710699906244, + "tokens_seen": 1940258816 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033311133400200604, + "loss": 1.2215, + "theoretical_loss": 3.4675613269860834, + "tokens_seen": 1940324352 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033309528585757274, + "loss": 1.4431, + "theoretical_loss": 3.4675515844027522, + "tokens_seen": 1940389888 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033307923771313944, + "loss": 1.2519, + "theoretical_loss": 3.4675418422405997, + "tokens_seen": 1940455424 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033306318956870614, + "loss": 1.204, + "theoretical_loss": 3.467532100499593, + "tokens_seen": 1940520960 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033304714142427284, + "loss": 1.3147, + "theoretical_loss": 3.4675223591796986, + "tokens_seen": 1940586496 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033303109327983954, + "loss": 1.3123, + "theoretical_loss": 3.4675126182808853, + "tokens_seen": 1940652032 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033301504513540625, + "loss": 1.1793, + "theoretical_loss": 3.4675028778031205, + "tokens_seen": 1940717568 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033299899699097295, + "loss": 1.2049, + "theoretical_loss": 3.467493137746371, + "tokens_seen": 1940783104 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033298294884653965, + "loss": 1.3128, + "theoretical_loss": 3.4674833981106055, + "tokens_seen": 1940848640 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033296690070210635, + "loss": 1.3617, + "theoretical_loss": 3.4674736588957904, + "tokens_seen": 1940914176 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033295085255767305, + "loss": 1.2269, + "theoretical_loss": 3.4674639201018946, + "tokens_seen": 1940979712 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033293480441323975, + "loss": 1.2386, + "theoretical_loss": 3.4674541817288844, + "tokens_seen": 1941045248 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033291875626880645, + "loss": 1.3699, + "theoretical_loss": 3.4674444437767282, + "tokens_seen": 1941110784 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033290270812437315, + "loss": 1.2726, + "theoretical_loss": 3.4674347062453936, + "tokens_seen": 1941176320 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033288665997993985, + "loss": 1.354, + "theoretical_loss": 3.467424969134848, + "tokens_seen": 1941241856 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033287061183550655, + "loss": 1.3563, + "theoretical_loss": 3.4674152324450587, + "tokens_seen": 1941307392 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033285456369107325, + "loss": 1.3012, + "theoretical_loss": 3.467405496175994, + "tokens_seen": 1941372928 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033283851554663995, + "loss": 1.2888, + "theoretical_loss": 3.467395760327621, + "tokens_seen": 1941438464 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033282246740220665, + "loss": 1.2502, + "theoretical_loss": 3.4673860248999073, + "tokens_seen": 1941504000 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033280641925777335, + "loss": 1.3568, + "theoretical_loss": 3.467376289892821, + "tokens_seen": 1941569536 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033279037111334005, + "loss": 1.3128, + "theoretical_loss": 3.4673665553063295, + "tokens_seen": 1941635072 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033277432296890675, + "loss": 1.334, + "theoretical_loss": 3.4673568211404002, + "tokens_seen": 1941700608 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033275827482447346, + "loss": 1.2227, + "theoretical_loss": 3.4673470873950007, + "tokens_seen": 1941766144 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033274222668004016, + "loss": 1.2337, + "theoretical_loss": 3.467337354070099, + "tokens_seen": 1941831680 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033272617853560686, + "loss": 1.2007, + "theoretical_loss": 3.467327621165663, + "tokens_seen": 1941897216 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033271013039117356, + "loss": 1.3253, + "theoretical_loss": 3.46731788868166, + "tokens_seen": 1941962752 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033269408224674026, + "loss": 1.3084, + "theoretical_loss": 3.4673081566180572, + "tokens_seen": 1942028288 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033267803410230696, + "loss": 1.3702, + "theoretical_loss": 3.467298424974823, + "tokens_seen": 1942093824 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033266198595787366, + "loss": 1.3043, + "theoretical_loss": 3.4672886937519243, + "tokens_seen": 1942159360 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033264593781344036, + "loss": 1.2668, + "theoretical_loss": 3.467278962949329, + "tokens_seen": 1942224896 + }, + { + "epoch": 0.59, + "learning_rate": 0.000332629889669007, + "loss": 1.2562, + "theoretical_loss": 3.4672692325670056, + "tokens_seen": 1942290432 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003326138415245737, + "loss": 1.2636, + "theoretical_loss": 3.4672595026049207, + "tokens_seen": 1942355968 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003325977933801404, + "loss": 1.2412, + "theoretical_loss": 3.467249773063042, + "tokens_seen": 1942421504 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003325817452357071, + "loss": 1.2943, + "theoretical_loss": 3.467240043941338, + "tokens_seen": 1942487040 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003325656970912738, + "loss": 1.2821, + "theoretical_loss": 3.4672303152397763, + "tokens_seen": 1942552576 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003325496489468405, + "loss": 1.2073, + "theoretical_loss": 3.4672205869583235, + "tokens_seen": 1942618112 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003325336008024072, + "loss": 1.2823, + "theoretical_loss": 3.467210859096948, + "tokens_seen": 1942683648 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003325175526579739, + "loss": 1.2371, + "theoretical_loss": 3.467201131655618, + "tokens_seen": 1942749184 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003325015045135406, + "loss": 1.1092, + "theoretical_loss": 3.467191404634301, + "tokens_seen": 1942814720 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003324854563691073, + "loss": 1.3176, + "theoretical_loss": 3.467181678032963, + "tokens_seen": 1942880256 + }, + { + "epoch": 0.59, + "learning_rate": 0.000332469408224674, + "loss": 1.3503, + "theoretical_loss": 3.4671719518515745, + "tokens_seen": 1942945792 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003324533600802407, + "loss": 1.2564, + "theoretical_loss": 3.4671622260901014, + "tokens_seen": 1943011328 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003324373119358074, + "loss": 1.2514, + "theoretical_loss": 3.4671525007485116, + "tokens_seen": 1943076864 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003324212637913741, + "loss": 1.2854, + "theoretical_loss": 3.4671427758267725, + "tokens_seen": 1943142400 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003324052156469408, + "loss": 1.2536, + "theoretical_loss": 3.4671330513248533, + "tokens_seen": 1943207936 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003323891675025075, + "loss": 1.3095, + "theoretical_loss": 3.46712332724272, + "tokens_seen": 1943273472 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003323731193580742, + "loss": 1.262, + "theoretical_loss": 3.4671136035803416, + "tokens_seen": 1943339008 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003323570712136409, + "loss": 1.2032, + "theoretical_loss": 3.4671038803376852, + "tokens_seen": 1943404544 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003323410230692076, + "loss": 1.268, + "theoretical_loss": 3.4670941575147185, + "tokens_seen": 1943470080 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003323249749247743, + "loss": 1.3238, + "theoretical_loss": 3.4670844351114094, + "tokens_seen": 1943535616 + }, + { + "epoch": 0.59, + "learning_rate": 0.000332308926780341, + "loss": 1.346, + "theoretical_loss": 3.4670747131277255, + "tokens_seen": 1943601152 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003322928786359077, + "loss": 1.3192, + "theoretical_loss": 3.467064991563635, + "tokens_seen": 1943666688 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003322768304914744, + "loss": 1.2587, + "theoretical_loss": 3.4670552704191047, + "tokens_seen": 1943732224 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003322607823470411, + "loss": 1.3289, + "theoretical_loss": 3.4670455496941033, + "tokens_seen": 1943797760 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003322447342026078, + "loss": 1.1993, + "theoretical_loss": 3.467035829388598, + "tokens_seen": 1943863296 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003322286860581745, + "loss": 1.2316, + "theoretical_loss": 3.467026109502557, + "tokens_seen": 1943928832 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003322126379137412, + "loss": 1.3257, + "theoretical_loss": 3.4670163900359476, + "tokens_seen": 1943994368 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003321965897693079, + "loss": 1.3963, + "theoretical_loss": 3.4670066709887375, + "tokens_seen": 1944059904 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003321805416248746, + "loss": 1.2258, + "theoretical_loss": 3.466996952360895, + "tokens_seen": 1944125440 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003321644934804413, + "loss": 1.2544, + "theoretical_loss": 3.4669872341523877, + "tokens_seen": 1944190976 + }, + { + "epoch": 0.59, + "learning_rate": 0.000332148445336008, + "loss": 1.3548, + "theoretical_loss": 3.4669775163631833, + "tokens_seen": 1944256512 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003321323971915747, + "loss": 1.2685, + "theoretical_loss": 3.466967798993249, + "tokens_seen": 1944322048 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003321163490471415, + "loss": 1.2669, + "theoretical_loss": 3.466958082042553, + "tokens_seen": 1944387584 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003321003009027082, + "loss": 1.2753, + "theoretical_loss": 3.466948365511064, + "tokens_seen": 1944453120 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003320842527582749, + "loss": 1.2362, + "theoretical_loss": 3.466938649398749, + "tokens_seen": 1944518656 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003320682046138416, + "loss": 1.3303, + "theoretical_loss": 3.4669289337055753, + "tokens_seen": 1944584192 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003320521564694083, + "loss": 1.2751, + "theoretical_loss": 3.466919218431511, + "tokens_seen": 1944649728 + }, + { + "epoch": 0.59, + "learning_rate": 0.000332036108324975, + "loss": 1.1977, + "theoretical_loss": 3.4669095035765247, + "tokens_seen": 1944715264 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003320200601805417, + "loss": 1.2971, + "theoretical_loss": 3.466899789140583, + "tokens_seen": 1944780800 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003320040120361084, + "loss": 1.2284, + "theoretical_loss": 3.466890075123654, + "tokens_seen": 1944846336 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033198796389167503, + "loss": 1.2021, + "theoretical_loss": 3.466880361525707, + "tokens_seen": 1944911872 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033197191574724173, + "loss": 1.3086, + "theoretical_loss": 3.466870648346707, + "tokens_seen": 1944977408 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033195586760280843, + "loss": 1.1439, + "theoretical_loss": 3.4668609355866247, + "tokens_seen": 1945042944 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033193981945837513, + "loss": 1.2355, + "theoretical_loss": 3.466851223245426, + "tokens_seen": 1945108480 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033192377131394183, + "loss": 1.3658, + "theoretical_loss": 3.4668415113230795, + "tokens_seen": 1945174016 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033190772316950853, + "loss": 1.251, + "theoretical_loss": 3.466831799819553, + "tokens_seen": 1945239552 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033189167502507524, + "loss": 1.3193, + "theoretical_loss": 3.466822088734814, + "tokens_seen": 1945305088 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033187562688064194, + "loss": 1.1697, + "theoretical_loss": 3.4668123780688305, + "tokens_seen": 1945370624 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033185957873620864, + "loss": 1.3446, + "theoretical_loss": 3.4668026678215704, + "tokens_seen": 1945436160 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033184353059177534, + "loss": 1.2212, + "theoretical_loss": 3.4667929579930012, + "tokens_seen": 1945501696 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033182748244734204, + "loss": 1.3068, + "theoretical_loss": 3.466783248583092, + "tokens_seen": 1945567232 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033181143430290874, + "loss": 1.4102, + "theoretical_loss": 3.466773539591809, + "tokens_seen": 1945632768 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033179538615847544, + "loss": 1.3615, + "theoretical_loss": 3.4667638310191204, + "tokens_seen": 1945698304 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033177933801404214, + "loss": 1.2553, + "theoretical_loss": 3.4667541228649954, + "tokens_seen": 1945763840 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033176328986960884, + "loss": 1.1951, + "theoretical_loss": 3.4667444151294005, + "tokens_seen": 1945829376 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033174724172517554, + "loss": 1.3081, + "theoretical_loss": 3.4667347078123036, + "tokens_seen": 1945894912 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033173119358074224, + "loss": 1.2945, + "theoretical_loss": 3.466725000913673, + "tokens_seen": 1945960448 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033171514543630894, + "loss": 1.3359, + "theoretical_loss": 3.4667152944334765, + "tokens_seen": 1946025984 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033169909729187564, + "loss": 1.292, + "theoretical_loss": 3.466705588371682, + "tokens_seen": 1946091520 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033168304914744234, + "loss": 1.2825, + "theoretical_loss": 3.4666958827282572, + "tokens_seen": 1946157056 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033166700100300904, + "loss": 1.3081, + "theoretical_loss": 3.46668617750317, + "tokens_seen": 1946222592 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033165095285857574, + "loss": 1.317, + "theoretical_loss": 3.4666764726963883, + "tokens_seen": 1946288128 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033163490471414245, + "loss": 1.2659, + "theoretical_loss": 3.4666667683078805, + "tokens_seen": 1946353664 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 720153, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1807372570037842, + "objective/train/theoretical_loss": 3.466657064337614, + "objective/train/tokens_used": 1966879200, + "theoretical_loss": 3.466657064337614, + "tokens_seen": 1946419200 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033161885656970915, + "loss": 1.1807, + "theoretical_loss": 3.466657064337614, + "tokens_seen": 1946419200 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033160280842527585, + "loss": 1.2041, + "theoretical_loss": 3.466647360785556, + "tokens_seen": 1946484736 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033158676028084255, + "loss": 1.2164, + "theoretical_loss": 3.4666376576516758, + "tokens_seen": 1946550272 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033157071213640925, + "loss": 1.1961, + "theoretical_loss": 3.4666279549359404, + "tokens_seen": 1946615808 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033155466399197595, + "loss": 1.2605, + "theoretical_loss": 3.4666182526383182, + "tokens_seen": 1946681344 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033153861584754265, + "loss": 1.2425, + "theoretical_loss": 3.4666085507587767, + "tokens_seen": 1946746880 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033152256770310935, + "loss": 1.2331, + "theoretical_loss": 3.466598849297284, + "tokens_seen": 1946812416 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033150651955867605, + "loss": 1.2671, + "theoretical_loss": 3.466589148253808, + "tokens_seen": 1946877952 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033149047141424275, + "loss": 1.3531, + "theoretical_loss": 3.466579447628316, + "tokens_seen": 1946943488 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033147442326980945, + "loss": 1.2496, + "theoretical_loss": 3.4665697474207775, + "tokens_seen": 1947009024 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033145837512537615, + "loss": 1.1599, + "theoretical_loss": 3.466560047631159, + "tokens_seen": 1947074560 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033144232698094285, + "loss": 1.3309, + "theoretical_loss": 3.466550348259428, + "tokens_seen": 1947140096 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033142627883650955, + "loss": 1.2816, + "theoretical_loss": 3.4665406493055544, + "tokens_seen": 1947205632 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033141023069207625, + "loss": 1.1278, + "theoretical_loss": 3.4665309507695046, + "tokens_seen": 1947271168 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033139418254764296, + "loss": 1.2733, + "theoretical_loss": 3.466521252651247, + "tokens_seen": 1947336704 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033137813440320966, + "loss": 1.3394, + "theoretical_loss": 3.46651155495075, + "tokens_seen": 1947402240 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033136208625877636, + "loss": 1.2361, + "theoretical_loss": 3.4665018576679802, + "tokens_seen": 1947467776 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033134603811434306, + "loss": 1.2804, + "theoretical_loss": 3.466492160802907, + "tokens_seen": 1947533312 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033132998996990976, + "loss": 1.291, + "theoretical_loss": 3.466482464355498, + "tokens_seen": 1947598848 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033131394182547646, + "loss": 1.2873, + "theoretical_loss": 3.4664727683257204, + "tokens_seen": 1947664384 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033129789368104316, + "loss": 1.19, + "theoretical_loss": 3.466463072713543, + "tokens_seen": 1947729920 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033128184553660986, + "loss": 1.3908, + "theoretical_loss": 3.466453377518933, + "tokens_seen": 1947795456 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033126579739217656, + "loss": 1.2967, + "theoretical_loss": 3.466443682741859, + "tokens_seen": 1947860992 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033124974924774326, + "loss": 1.3467, + "theoretical_loss": 3.466433988382289, + "tokens_seen": 1947926528 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033123370110330996, + "loss": 1.2975, + "theoretical_loss": 3.4664242944401904, + "tokens_seen": 1947992064 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003312176529588766, + "loss": 1.2638, + "theoretical_loss": 3.466414600915532, + "tokens_seen": 1948057600 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003312016048144433, + "loss": 1.283, + "theoretical_loss": 3.466404907808281, + "tokens_seen": 1948123136 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033118555667001, + "loss": 1.2857, + "theoretical_loss": 3.466395215118406, + "tokens_seen": 1948188672 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003311695085255767, + "loss": 1.2488, + "theoretical_loss": 3.466385522845874, + "tokens_seen": 1948254208 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003311534603811434, + "loss": 1.1653, + "theoretical_loss": 3.4663758309906543, + "tokens_seen": 1948319744 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003311374122367101, + "loss": 1.2807, + "theoretical_loss": 3.466366139552714, + "tokens_seen": 1948385280 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003311213640922768, + "loss": 1.2541, + "theoretical_loss": 3.4663564485320215, + "tokens_seen": 1948450816 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003311053159478435, + "loss": 1.243, + "theoretical_loss": 3.466346757928545, + "tokens_seen": 1948516352 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003310892678034102, + "loss": 1.1892, + "theoretical_loss": 3.466337067742252, + "tokens_seen": 1948581888 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003310732196589769, + "loss": 1.3092, + "theoretical_loss": 3.4663273779731103, + "tokens_seen": 1948647424 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003310571715145436, + "loss": 1.2902, + "theoretical_loss": 3.4663176886210887, + "tokens_seen": 1948712960 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003310411233701103, + "loss": 1.2778, + "theoretical_loss": 3.4663079996861548, + "tokens_seen": 1948778496 + }, + { + "epoch": 0.59, + "learning_rate": 0.000331025075225677, + "loss": 1.2113, + "theoretical_loss": 3.4662983111682766, + "tokens_seen": 1948844032 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003310090270812437, + "loss": 1.2739, + "theoretical_loss": 3.466288623067422, + "tokens_seen": 1948909568 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003309929789368104, + "loss": 1.3458, + "theoretical_loss": 3.4662789353835595, + "tokens_seen": 1948975104 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003309769307923771, + "loss": 1.2894, + "theoretical_loss": 3.4662692481166566, + "tokens_seen": 1949040640 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003309608826479438, + "loss": 1.2238, + "theoretical_loss": 3.466259561266682, + "tokens_seen": 1949106176 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003309448345035105, + "loss": 1.1067, + "theoretical_loss": 3.466249874833603, + "tokens_seen": 1949171712 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003309287863590773, + "loss": 1.287, + "theoretical_loss": 3.4662401888173875, + "tokens_seen": 1949237248 + }, + { + "epoch": 0.59, + "learning_rate": 0.000330912738214644, + "loss": 1.3052, + "theoretical_loss": 3.4662305032180045, + "tokens_seen": 1949302784 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003308966900702107, + "loss": 1.2639, + "theoretical_loss": 3.4662208180354215, + "tokens_seen": 1949368320 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003308806419257774, + "loss": 1.2659, + "theoretical_loss": 3.4662111332696064, + "tokens_seen": 1949433856 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003308645937813441, + "loss": 1.2914, + "theoretical_loss": 3.4662014489205277, + "tokens_seen": 1949499392 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003308485456369108, + "loss": 1.2528, + "theoretical_loss": 3.466191764988153, + "tokens_seen": 1949564928 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003308324974924775, + "loss": 1.3348, + "theoretical_loss": 3.466182081472451, + "tokens_seen": 1949630464 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003308164493480442, + "loss": 1.2687, + "theoretical_loss": 3.4661723983733888, + "tokens_seen": 1949696000 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003308004012036109, + "loss": 1.1333, + "theoretical_loss": 3.4661627156909356, + "tokens_seen": 1949761536 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003307843530591776, + "loss": 1.2406, + "theoretical_loss": 3.466153033425058, + "tokens_seen": 1949827072 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003307683049147443, + "loss": 1.1973, + "theoretical_loss": 3.4661433515757256, + "tokens_seen": 1949892608 + }, + { + "epoch": 0.59, + "learning_rate": 0.000330752256770311, + "loss": 1.3457, + "theoretical_loss": 3.4661336701429057, + "tokens_seen": 1949958144 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003307362086258777, + "loss": 1.2242, + "theoretical_loss": 3.4661239891265665, + "tokens_seen": 1950023680 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003307201604814444, + "loss": 1.2432, + "theoretical_loss": 3.4661143085266763, + "tokens_seen": 1950089216 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003307041123370111, + "loss": 1.1753, + "theoretical_loss": 3.466104628343203, + "tokens_seen": 1950154752 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003306880641925778, + "loss": 1.1943, + "theoretical_loss": 3.4660949485761146, + "tokens_seen": 1950220288 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003306720160481445, + "loss": 1.326, + "theoretical_loss": 3.466085269225379, + "tokens_seen": 1950285824 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003306559679037112, + "loss": 1.3232, + "theoretical_loss": 3.4660755902909646, + "tokens_seen": 1950351360 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003306399197592779, + "loss": 1.3181, + "theoretical_loss": 3.46606591177284, + "tokens_seen": 1950416896 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003306238716148446, + "loss": 1.3089, + "theoretical_loss": 3.4660562336709724, + "tokens_seen": 1950482432 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003306078234704113, + "loss": 1.2949, + "theoretical_loss": 3.4660465559853306, + "tokens_seen": 1950547968 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033059177532597793, + "loss": 1.3202, + "theoretical_loss": 3.466036878715882, + "tokens_seen": 1950613504 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033057572718154463, + "loss": 1.3765, + "theoretical_loss": 3.4660272018625955, + "tokens_seen": 1950679040 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033055967903711133, + "loss": 1.2718, + "theoretical_loss": 3.466017525425439, + "tokens_seen": 1950744576 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033054363089267803, + "loss": 1.2464, + "theoretical_loss": 3.4660078494043804, + "tokens_seen": 1950810112 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033052758274824474, + "loss": 1.3939, + "theoretical_loss": 3.465998173799388, + "tokens_seen": 1950875648 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033051153460381144, + "loss": 1.1943, + "theoretical_loss": 3.4659884986104297, + "tokens_seen": 1950941184 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033049548645937814, + "loss": 1.3223, + "theoretical_loss": 3.4659788238374736, + "tokens_seen": 1951006720 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033047943831494484, + "loss": 1.2797, + "theoretical_loss": 3.4659691494804887, + "tokens_seen": 1951072256 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033046339017051154, + "loss": 1.245, + "theoretical_loss": 3.4659594755394423, + "tokens_seen": 1951137792 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033044734202607824, + "loss": 1.2636, + "theoretical_loss": 3.4659498020143023, + "tokens_seen": 1951203328 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033043129388164494, + "loss": 1.2173, + "theoretical_loss": 3.4659401289050376, + "tokens_seen": 1951268864 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033041524573721164, + "loss": 1.3509, + "theoretical_loss": 3.4659304562116158, + "tokens_seen": 1951334400 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033039919759277834, + "loss": 1.3844, + "theoretical_loss": 3.4659207839340054, + "tokens_seen": 1951399936 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033038314944834504, + "loss": 1.3581, + "theoretical_loss": 3.465911112072175, + "tokens_seen": 1951465472 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033036710130391174, + "loss": 1.1591, + "theoretical_loss": 3.4659014406260913, + "tokens_seen": 1951531008 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033035105315947844, + "loss": 1.1915, + "theoretical_loss": 3.465891769595724, + "tokens_seen": 1951596544 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033033500501504514, + "loss": 1.3114, + "theoretical_loss": 3.465882098981041, + "tokens_seen": 1951662080 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033031895687061184, + "loss": 1.3399, + "theoretical_loss": 3.465872428782009, + "tokens_seen": 1951727616 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033030290872617854, + "loss": 1.3943, + "theoretical_loss": 3.465862758998598, + "tokens_seen": 1951793152 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033028686058174524, + "loss": 1.3145, + "theoretical_loss": 3.4658530896307758, + "tokens_seen": 1951858688 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033027081243731195, + "loss": 1.1883, + "theoretical_loss": 3.46584342067851, + "tokens_seen": 1951924224 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033025476429287865, + "loss": 1.3343, + "theoretical_loss": 3.465833752141769, + "tokens_seen": 1951989760 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033023871614844535, + "loss": 1.3763, + "theoretical_loss": 3.465824084020521, + "tokens_seen": 1952055296 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033022266800401205, + "loss": 1.2318, + "theoretical_loss": 3.465814416314734, + "tokens_seen": 1952120832 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033020661985957875, + "loss": 1.2811, + "theoretical_loss": 3.465804749024377, + "tokens_seen": 1952186368 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033019057171514545, + "loss": 1.2305, + "theoretical_loss": 3.465795082149418, + "tokens_seen": 1952251904 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033017452357071215, + "loss": 1.3142, + "theoretical_loss": 3.4657854156898242, + "tokens_seen": 1952317440 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033015847542627885, + "loss": 1.3328, + "theoretical_loss": 3.4657757496455646, + "tokens_seen": 1952382976 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033014242728184555, + "loss": 1.3688, + "theoretical_loss": 3.4657660840166074, + "tokens_seen": 1952448512 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033012637913741225, + "loss": 1.2835, + "theoretical_loss": 3.4657564188029206, + "tokens_seen": 1952514048 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033011033099297895, + "loss": 1.3199, + "theoretical_loss": 3.465746754004473, + "tokens_seen": 1952579584 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033009428284854565, + "loss": 1.3122, + "theoretical_loss": 3.465737089621232, + "tokens_seen": 1952645120 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033007823470411235, + "loss": 1.2889, + "theoretical_loss": 3.4657274256531663, + "tokens_seen": 1952710656 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033006218655967905, + "loss": 1.3542, + "theoretical_loss": 3.465717762100244, + "tokens_seen": 1952776192 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033004613841524575, + "loss": 1.16, + "theoretical_loss": 3.465708098962433, + "tokens_seen": 1952841728 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033003009027081245, + "loss": 1.3491, + "theoretical_loss": 3.465698436239702, + "tokens_seen": 1952907264 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 722678, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3473236560821533, + "objective/train/theoretical_loss": 3.4656887739320195, + "objective/train/tokens_used": 1973432800, + "theoretical_loss": 3.4656887739320195, + "tokens_seen": 1952972800 + }, + { + "epoch": 0.59, + "learning_rate": 0.00033001404212637916, + "loss": 1.3473, + "theoretical_loss": 3.4656887739320195, + "tokens_seen": 1952972800 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032999799398194586, + "loss": 1.4197, + "theoretical_loss": 3.465679112039353, + "tokens_seen": 1953038336 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032998194583751256, + "loss": 1.22, + "theoretical_loss": 3.4656694505616716, + "tokens_seen": 1953103872 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032996589769307926, + "loss": 1.3152, + "theoretical_loss": 3.4656597894989427, + "tokens_seen": 1953169408 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032994984954864596, + "loss": 1.2542, + "theoretical_loss": 3.465650128851135, + "tokens_seen": 1953234944 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032993380140421266, + "loss": 1.2053, + "theoretical_loss": 3.4656404686182167, + "tokens_seen": 1953300480 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032991775325977936, + "loss": 1.2074, + "theoretical_loss": 3.465630808800156, + "tokens_seen": 1953366016 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032990170511534606, + "loss": 1.3017, + "theoretical_loss": 3.4656211493969216, + "tokens_seen": 1953431552 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032988565697091276, + "loss": 1.2692, + "theoretical_loss": 3.465611490408481, + "tokens_seen": 1953497088 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032986960882647946, + "loss": 1.2655, + "theoretical_loss": 3.465601831834803, + "tokens_seen": 1953562624 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032985356068204616, + "loss": 1.3737, + "theoretical_loss": 3.465592173675856, + "tokens_seen": 1953628160 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032983751253761286, + "loss": 1.2347, + "theoretical_loss": 3.465582515931608, + "tokens_seen": 1953693696 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003298214643931795, + "loss": 1.2965, + "theoretical_loss": 3.4655728586020267, + "tokens_seen": 1953759232 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003298054162487462, + "loss": 1.2068, + "theoretical_loss": 3.4655632016870817, + "tokens_seen": 1953824768 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003297893681043129, + "loss": 1.3123, + "theoretical_loss": 3.46555354518674, + "tokens_seen": 1953890304 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003297733199598796, + "loss": 1.3616, + "theoretical_loss": 3.465543889100971, + "tokens_seen": 1953955840 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003297572718154463, + "loss": 1.2515, + "theoretical_loss": 3.465534233429742, + "tokens_seen": 1954021376 + }, + { + "epoch": 0.59, + "learning_rate": 0.000329741223671013, + "loss": 1.3844, + "theoretical_loss": 3.4655245781730226, + "tokens_seen": 1954086912 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032972517552657977, + "loss": 1.1338, + "theoretical_loss": 3.4655149233307796, + "tokens_seen": 1954152448 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032970912738214647, + "loss": 1.3286, + "theoretical_loss": 3.465505268902982, + "tokens_seen": 1954217984 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032969307923771317, + "loss": 1.187, + "theoretical_loss": 3.4654956148895986, + "tokens_seen": 1954283520 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032967703109327987, + "loss": 1.23, + "theoretical_loss": 3.4654859612905966, + "tokens_seen": 1954349056 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032966098294884657, + "loss": 1.2279, + "theoretical_loss": 3.465476308105945, + "tokens_seen": 1954414592 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032964493480441327, + "loss": 1.3068, + "theoretical_loss": 3.4654666553356126, + "tokens_seen": 1954480128 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032962888665997997, + "loss": 1.3025, + "theoretical_loss": 3.4654570029795666, + "tokens_seen": 1954545664 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032961283851554667, + "loss": 1.299, + "theoretical_loss": 3.465447351037776, + "tokens_seen": 1954611200 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032959679037111337, + "loss": 1.2776, + "theoretical_loss": 3.4654376995102094, + "tokens_seen": 1954676736 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032958074222668007, + "loss": 1.2823, + "theoretical_loss": 3.4654280483968347, + "tokens_seen": 1954742272 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003295646940822468, + "loss": 1.3084, + "theoretical_loss": 3.4654183976976203, + "tokens_seen": 1954807808 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003295486459378135, + "loss": 1.3486, + "theoretical_loss": 3.4654087474125346, + "tokens_seen": 1954873344 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003295325977933802, + "loss": 1.4068, + "theoretical_loss": 3.4653990975415456, + "tokens_seen": 1954938880 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003295165496489469, + "loss": 1.1931, + "theoretical_loss": 3.4653894480846223, + "tokens_seen": 1955004416 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003295005015045136, + "loss": 1.3583, + "theoretical_loss": 3.4653797990417328, + "tokens_seen": 1955069952 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003294844533600803, + "loss": 1.3425, + "theoretical_loss": 3.465370150412845, + "tokens_seen": 1955135488 + }, + { + "epoch": 0.59, + "learning_rate": 0.000329468405215647, + "loss": 1.4005, + "theoretical_loss": 3.465360502197928, + "tokens_seen": 1955201024 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003294523570712137, + "loss": 1.2795, + "theoretical_loss": 3.4653508543969496, + "tokens_seen": 1955266560 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003294363089267804, + "loss": 1.2668, + "theoretical_loss": 3.465341207009878, + "tokens_seen": 1955332096 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003294202607823471, + "loss": 1.2943, + "theoretical_loss": 3.4653315600366827, + "tokens_seen": 1955397632 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003294042126379138, + "loss": 1.3037, + "theoretical_loss": 3.465321913477331, + "tokens_seen": 1955463168 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003293881644934805, + "loss": 1.2571, + "theoretical_loss": 3.4653122673317913, + "tokens_seen": 1955528704 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003293721163490472, + "loss": 1.0898, + "theoretical_loss": 3.4653026216000327, + "tokens_seen": 1955594240 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003293560682046139, + "loss": 1.346, + "theoretical_loss": 3.465292976282023, + "tokens_seen": 1955659776 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003293400200601806, + "loss": 1.2244, + "theoretical_loss": 3.465283331377731, + "tokens_seen": 1955725312 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003293239719157473, + "loss": 1.3129, + "theoretical_loss": 3.465273686887124, + "tokens_seen": 1955790848 + }, + { + "epoch": 0.59, + "learning_rate": 0.000329307923771314, + "loss": 1.2878, + "theoretical_loss": 3.4652640428101718, + "tokens_seen": 1955856384 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003292918756268807, + "loss": 1.344, + "theoretical_loss": 3.4652543991468425, + "tokens_seen": 1955921920 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003292758274824474, + "loss": 1.3695, + "theoretical_loss": 3.4652447558971042, + "tokens_seen": 1955987456 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003292597793380141, + "loss": 1.2563, + "theoretical_loss": 3.465235113060925, + "tokens_seen": 1956052992 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003292437311935808, + "loss": 1.2601, + "theoretical_loss": 3.4652254706382735, + "tokens_seen": 1956118528 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003292276830491475, + "loss": 1.279, + "theoretical_loss": 3.4652158286291184, + "tokens_seen": 1956184064 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003292116349047142, + "loss": 1.244, + "theoretical_loss": 3.465206187033428, + "tokens_seen": 1956249600 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003291955867602809, + "loss": 1.3361, + "theoretical_loss": 3.465196545851171, + "tokens_seen": 1956315136 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032917953861584753, + "loss": 1.2837, + "theoretical_loss": 3.465186905082315, + "tokens_seen": 1956380672 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032916349047141423, + "loss": 1.3807, + "theoretical_loss": 3.4651772647268295, + "tokens_seen": 1956446208 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032914744232698094, + "loss": 1.2853, + "theoretical_loss": 3.465167624784682, + "tokens_seen": 1956511744 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032913139418254764, + "loss": 1.2576, + "theoretical_loss": 3.4651579852558414, + "tokens_seen": 1956577280 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032911534603811434, + "loss": 1.2823, + "theoretical_loss": 3.465148346140276, + "tokens_seen": 1956642816 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032909929789368104, + "loss": 1.2477, + "theoretical_loss": 3.4651387074379545, + "tokens_seen": 1956708352 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032908324974924774, + "loss": 1.2939, + "theoretical_loss": 3.465129069148845, + "tokens_seen": 1956773888 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032906720160481444, + "loss": 1.296, + "theoretical_loss": 3.4651194312729157, + "tokens_seen": 1956839424 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032905115346038114, + "loss": 1.2641, + "theoretical_loss": 3.465109793810136, + "tokens_seen": 1956904960 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032903510531594784, + "loss": 1.2996, + "theoretical_loss": 3.4651001567604736, + "tokens_seen": 1956970496 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032901905717151454, + "loss": 1.3251, + "theoretical_loss": 3.465090520123897, + "tokens_seen": 1957036032 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032900300902708124, + "loss": 1.175, + "theoretical_loss": 3.465080883900375, + "tokens_seen": 1957101568 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032898696088264794, + "loss": 1.33, + "theoretical_loss": 3.4650712480898758, + "tokens_seen": 1957167104 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032897091273821464, + "loss": 1.2631, + "theoretical_loss": 3.4650616126923675, + "tokens_seen": 1957232640 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032895486459378134, + "loss": 1.2404, + "theoretical_loss": 3.4650519777078195, + "tokens_seen": 1957298176 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032893881644934804, + "loss": 1.2676, + "theoretical_loss": 3.4650423431362, + "tokens_seen": 1957363712 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032892276830491474, + "loss": 1.2207, + "theoretical_loss": 3.4650327089774766, + "tokens_seen": 1957429248 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032890672016048145, + "loss": 1.2915, + "theoretical_loss": 3.4650230752316187, + "tokens_seen": 1957494784 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032889067201604815, + "loss": 1.2022, + "theoretical_loss": 3.465013441898595, + "tokens_seen": 1957560320 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032887462387161485, + "loss": 1.3361, + "theoretical_loss": 3.4650038089783726, + "tokens_seen": 1957625856 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032885857572718155, + "loss": 1.2027, + "theoretical_loss": 3.4649941764709213, + "tokens_seen": 1957691392 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032884252758274825, + "loss": 1.2226, + "theoretical_loss": 3.4649845443762093, + "tokens_seen": 1957756928 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032882647943831495, + "loss": 1.2953, + "theoretical_loss": 3.4649749126942053, + "tokens_seen": 1957822464 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032881043129388165, + "loss": 1.2933, + "theoretical_loss": 3.4649652814248775, + "tokens_seen": 1957888000 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032879438314944835, + "loss": 1.4052, + "theoretical_loss": 3.4649556505681938, + "tokens_seen": 1957953536 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032877833500501505, + "loss": 1.3121, + "theoretical_loss": 3.4649460201241236, + "tokens_seen": 1958019072 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032876228686058175, + "loss": 1.291, + "theoretical_loss": 3.4649363900926353, + "tokens_seen": 1958084608 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032874623871614845, + "loss": 1.3082, + "theoretical_loss": 3.4649267604736966, + "tokens_seen": 1958150144 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032873019057171515, + "loss": 1.3731, + "theoretical_loss": 3.4649171312672777, + "tokens_seen": 1958215680 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032871414242728185, + "loss": 1.2764, + "theoretical_loss": 3.4649075024733453, + "tokens_seen": 1958281216 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032869809428284855, + "loss": 1.1892, + "theoretical_loss": 3.464897874091869, + "tokens_seen": 1958346752 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032868204613841525, + "loss": 1.324, + "theoretical_loss": 3.4648882461228174, + "tokens_seen": 1958412288 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032866599799398195, + "loss": 1.209, + "theoretical_loss": 3.464878618566158, + "tokens_seen": 1958477824 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032864994984954866, + "loss": 1.2495, + "theoretical_loss": 3.4648689914218602, + "tokens_seen": 1958543360 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032863390170511536, + "loss": 1.2889, + "theoretical_loss": 3.4648593646898926, + "tokens_seen": 1958608896 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032861785356068206, + "loss": 1.2486, + "theoretical_loss": 3.4648497383702237, + "tokens_seen": 1958674432 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032860180541624876, + "loss": 1.2997, + "theoretical_loss": 3.464840112462821, + "tokens_seen": 1958739968 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032858575727181546, + "loss": 1.2506, + "theoretical_loss": 3.4648304869676547, + "tokens_seen": 1958805504 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032856970912738216, + "loss": 1.2165, + "theoretical_loss": 3.464820861884692, + "tokens_seen": 1958871040 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032855366098294886, + "loss": 1.2739, + "theoretical_loss": 3.464811237213903, + "tokens_seen": 1958936576 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032853761283851556, + "loss": 1.2599, + "theoretical_loss": 3.464801612955254, + "tokens_seen": 1959002112 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032852156469408226, + "loss": 1.2092, + "theoretical_loss": 3.4647919891087158, + "tokens_seen": 1959067648 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032850551654964896, + "loss": 1.2931, + "theoretical_loss": 3.4647823656742553, + "tokens_seen": 1959133184 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032848946840521566, + "loss": 1.2795, + "theoretical_loss": 3.464772742651842, + "tokens_seen": 1959198720 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032847342026078236, + "loss": 1.2953, + "theoretical_loss": 3.464763120041445, + "tokens_seen": 1959264256 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032845737211634906, + "loss": 1.297, + "theoretical_loss": 3.4647534978430308, + "tokens_seen": 1959329792 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032844132397191576, + "loss": 1.2368, + "theoretical_loss": 3.4647438760565703, + "tokens_seen": 1959395328 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032842527582748246, + "loss": 1.2883, + "theoretical_loss": 3.464734254682031, + "tokens_seen": 1959460864 + }, + { + "epoch": 0.59, + "objective/train/docs_used": 724993, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3196464776992798, + "objective/train/theoretical_loss": 3.4647246337193813, + "objective/train/tokens_used": 1979986400, + "theoretical_loss": 3.4647246337193813, + "tokens_seen": 1959526400 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032840922768304916, + "loss": 1.3196, + "theoretical_loss": 3.4647246337193813, + "tokens_seen": 1959526400 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032839317953861587, + "loss": 1.245, + "theoretical_loss": 3.4647150131685898, + "tokens_seen": 1959591936 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032837713139418257, + "loss": 1.2302, + "theoretical_loss": 3.464705393029626, + "tokens_seen": 1959657472 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032836108324974927, + "loss": 1.2244, + "theoretical_loss": 3.4646957733024575, + "tokens_seen": 1959723008 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032834503510531597, + "loss": 1.2219, + "theoretical_loss": 3.4646861539870537, + "tokens_seen": 1959788544 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032832898696088267, + "loss": 1.3102, + "theoretical_loss": 3.4646765350833824, + "tokens_seen": 1959854080 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032831293881644937, + "loss": 1.3229, + "theoretical_loss": 3.464666916591413, + "tokens_seen": 1959919616 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032829689067201607, + "loss": 1.2935, + "theoretical_loss": 3.4646572985111126, + "tokens_seen": 1959985152 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032828084252758277, + "loss": 1.2116, + "theoretical_loss": 3.464647680842452, + "tokens_seen": 1960050688 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032826479438314947, + "loss": 1.3443, + "theoretical_loss": 3.464638063585398, + "tokens_seen": 1960116224 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032824874623871617, + "loss": 1.2155, + "theoretical_loss": 3.464628446739921, + "tokens_seen": 1960181760 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032823269809428287, + "loss": 1.3359, + "theoretical_loss": 3.464618830305988, + "tokens_seen": 1960247296 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032821664994984957, + "loss": 1.3148, + "theoretical_loss": 3.464609214283568, + "tokens_seen": 1960312832 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003282006018054163, + "loss": 1.2699, + "theoretical_loss": 3.4645995986726303, + "tokens_seen": 1960378368 + }, + { + "epoch": 0.59, + "learning_rate": 0.000328184553660983, + "loss": 1.3034, + "theoretical_loss": 3.464589983473143, + "tokens_seen": 1960443904 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003281685055165497, + "loss": 1.282, + "theoretical_loss": 3.4645803686850747, + "tokens_seen": 1960509440 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003281524573721164, + "loss": 1.2319, + "theoretical_loss": 3.464570754308394, + "tokens_seen": 1960574976 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003281364092276831, + "loss": 1.3192, + "theoretical_loss": 3.4645611403430703, + "tokens_seen": 1960640512 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003281203610832498, + "loss": 1.3217, + "theoretical_loss": 3.464551526789071, + "tokens_seen": 1960706048 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003281043129388165, + "loss": 1.3152, + "theoretical_loss": 3.464541913646366, + "tokens_seen": 1960771584 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003280882647943832, + "loss": 1.262, + "theoretical_loss": 3.4645323009149234, + "tokens_seen": 1960837120 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003280722166499499, + "loss": 1.3322, + "theoretical_loss": 3.4645226885947116, + "tokens_seen": 1960902656 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003280561685055166, + "loss": 1.2929, + "theoretical_loss": 3.4645130766856993, + "tokens_seen": 1960968192 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003280401203610833, + "loss": 1.2846, + "theoretical_loss": 3.464503465187856, + "tokens_seen": 1961033728 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032802407221665, + "loss": 1.3167, + "theoretical_loss": 3.464493854101149, + "tokens_seen": 1961099264 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003280080240722167, + "loss": 1.2884, + "theoretical_loss": 3.464484243425548, + "tokens_seen": 1961164800 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003279919759277834, + "loss": 1.2166, + "theoretical_loss": 3.4644746331610214, + "tokens_seen": 1961230336 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003279759277833501, + "loss": 1.207, + "theoretical_loss": 3.464465023307538, + "tokens_seen": 1961295872 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003279598796389168, + "loss": 1.3057, + "theoretical_loss": 3.4644554138650667, + "tokens_seen": 1961361408 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003279438314944835, + "loss": 1.3693, + "theoretical_loss": 3.4644458048335753, + "tokens_seen": 1961426944 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003279277833500502, + "loss": 1.3275, + "theoretical_loss": 3.4644361962130334, + "tokens_seen": 1961492480 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003279117352056169, + "loss": 1.3503, + "theoretical_loss": 3.464426588003409, + "tokens_seen": 1961558016 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003278956870611836, + "loss": 1.3225, + "theoretical_loss": 3.4644169802046716, + "tokens_seen": 1961623552 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003278796389167503, + "loss": 1.2887, + "theoretical_loss": 3.4644073728167895, + "tokens_seen": 1961689088 + }, + { + "epoch": 0.59, + "learning_rate": 0.000327863590772317, + "loss": 1.2616, + "theoretical_loss": 3.4643977658397307, + "tokens_seen": 1961754624 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003278475426278837, + "loss": 1.2322, + "theoretical_loss": 3.464388159273465, + "tokens_seen": 1961820160 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003278314944834504, + "loss": 1.2737, + "theoretical_loss": 3.4643785531179607, + "tokens_seen": 1961885696 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003278154463390171, + "loss": 1.2571, + "theoretical_loss": 3.4643689473731865, + "tokens_seen": 1961951232 + }, + { + "epoch": 0.59, + "learning_rate": 0.0003277993981945838, + "loss": 1.3048, + "theoretical_loss": 3.464359342039111, + "tokens_seen": 1962016768 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032778335005015044, + "loss": 1.3207, + "theoretical_loss": 3.464349737115703, + "tokens_seen": 1962082304 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032776730190571714, + "loss": 1.2466, + "theoretical_loss": 3.4643401326029313, + "tokens_seen": 1962147840 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032775125376128384, + "loss": 1.2047, + "theoretical_loss": 3.464330528500765, + "tokens_seen": 1962213376 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032773520561685054, + "loss": 1.2765, + "theoretical_loss": 3.464320924809172, + "tokens_seen": 1962278912 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032771915747241724, + "loss": 1.2347, + "theoretical_loss": 3.464311321528122, + "tokens_seen": 1962344448 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032770310932798394, + "loss": 1.3743, + "theoretical_loss": 3.464301718657582, + "tokens_seen": 1962409984 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032768706118355064, + "loss": 1.3099, + "theoretical_loss": 3.464292116197523, + "tokens_seen": 1962475520 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032767101303911734, + "loss": 1.3004, + "theoretical_loss": 3.4642825141479126, + "tokens_seen": 1962541056 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032765496489468404, + "loss": 1.2006, + "theoretical_loss": 3.4642729125087195, + "tokens_seen": 1962606592 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032763891675025074, + "loss": 1.321, + "theoretical_loss": 3.464263311279912, + "tokens_seen": 1962672128 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032762286860581744, + "loss": 1.3448, + "theoretical_loss": 3.4642537104614606, + "tokens_seen": 1962737664 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032760682046138414, + "loss": 1.3088, + "theoretical_loss": 3.464244110053332, + "tokens_seen": 1962803200 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032759077231695084, + "loss": 1.3091, + "theoretical_loss": 3.4642345100554963, + "tokens_seen": 1962868736 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032757472417251754, + "loss": 1.2001, + "theoretical_loss": 3.4642249104679217, + "tokens_seen": 1962934272 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032755867602808424, + "loss": 1.2544, + "theoretical_loss": 3.464215311290577, + "tokens_seen": 1962999808 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032754262788365094, + "loss": 1.2442, + "theoretical_loss": 3.4642057125234316, + "tokens_seen": 1963065344 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032752657973921765, + "loss": 1.2893, + "theoretical_loss": 3.4641961141664535, + "tokens_seen": 1963130880 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032751053159478435, + "loss": 1.3221, + "theoretical_loss": 3.4641865162196117, + "tokens_seen": 1963196416 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032749448345035105, + "loss": 1.2243, + "theoretical_loss": 3.464176918682875, + "tokens_seen": 1963261952 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032747843530591775, + "loss": 1.2074, + "theoretical_loss": 3.464167321556212, + "tokens_seen": 1963327488 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032746238716148445, + "loss": 1.2168, + "theoretical_loss": 3.464157724839592, + "tokens_seen": 1963393024 + }, + { + "epoch": 0.59, + "learning_rate": 0.00032744633901705115, + "loss": 1.407, + "theoretical_loss": 3.4641481285329836, + "tokens_seen": 1963458560 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032743029087261785, + "loss": 1.304, + "theoretical_loss": 3.4641385326363556, + "tokens_seen": 1963524096 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032741424272818455, + "loss": 1.1448, + "theoretical_loss": 3.4641289371496766, + "tokens_seen": 1963589632 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032739819458375125, + "loss": 1.2649, + "theoretical_loss": 3.464119342072915, + "tokens_seen": 1963655168 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032738214643931795, + "loss": 1.3122, + "theoretical_loss": 3.4641097474060407, + "tokens_seen": 1963720704 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032736609829488465, + "loss": 1.273, + "theoretical_loss": 3.4641001531490216, + "tokens_seen": 1963786240 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032735005015045135, + "loss": 1.2591, + "theoretical_loss": 3.464090559301827, + "tokens_seen": 1963851776 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032733400200601805, + "loss": 1.2248, + "theoretical_loss": 3.464080965864426, + "tokens_seen": 1963917312 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003273179538615848, + "loss": 1.2616, + "theoretical_loss": 3.4640713728367865, + "tokens_seen": 1963982848 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003273019057171515, + "loss": 1.1725, + "theoretical_loss": 3.464061780218878, + "tokens_seen": 1964048384 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003272858575727182, + "loss": 1.2199, + "theoretical_loss": 3.464052188010669, + "tokens_seen": 1964113920 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003272698094282849, + "loss": 1.3025, + "theoretical_loss": 3.4640425962121286, + "tokens_seen": 1964179456 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003272537612838516, + "loss": 1.2822, + "theoretical_loss": 3.464033004823225, + "tokens_seen": 1964244992 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003272377131394183, + "loss": 1.373, + "theoretical_loss": 3.464023413843928, + "tokens_seen": 1964310528 + }, + { + "epoch": 0.6, + "learning_rate": 0.000327221664994985, + "loss": 1.2095, + "theoretical_loss": 3.4640138232742066, + "tokens_seen": 1964376064 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003272056168505517, + "loss": 1.2565, + "theoretical_loss": 3.464004233114028, + "tokens_seen": 1964441600 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003271895687061184, + "loss": 1.3184, + "theoretical_loss": 3.4639946433633626, + "tokens_seen": 1964507136 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003271735205616851, + "loss": 1.3553, + "theoretical_loss": 3.463985054022179, + "tokens_seen": 1964572672 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032715747241725176, + "loss": 1.2625, + "theoretical_loss": 3.4639754650904453, + "tokens_seen": 1964638208 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032714142427281846, + "loss": 1.2704, + "theoretical_loss": 3.463965876568131, + "tokens_seen": 1964703744 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032712537612838516, + "loss": 1.2566, + "theoretical_loss": 3.463956288455205, + "tokens_seen": 1964769280 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032710932798395186, + "loss": 1.1797, + "theoretical_loss": 3.463946700751636, + "tokens_seen": 1964834816 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032709327983951856, + "loss": 1.2336, + "theoretical_loss": 3.4639371134573924, + "tokens_seen": 1964900352 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032707723169508526, + "loss": 1.2413, + "theoretical_loss": 3.463927526572444, + "tokens_seen": 1964965888 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032706118355065196, + "loss": 1.3296, + "theoretical_loss": 3.463917940096759, + "tokens_seen": 1965031424 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032704513540621866, + "loss": 1.307, + "theoretical_loss": 3.4639083540303064, + "tokens_seen": 1965096960 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032702908726178537, + "loss": 1.3638, + "theoretical_loss": 3.4638987683730553, + "tokens_seen": 1965162496 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032701303911735207, + "loss": 1.367, + "theoretical_loss": 3.4638891831249747, + "tokens_seen": 1965228032 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032699699097291877, + "loss": 1.2786, + "theoretical_loss": 3.463879598286033, + "tokens_seen": 1965293568 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032698094282848547, + "loss": 1.263, + "theoretical_loss": 3.4638700138561993, + "tokens_seen": 1965359104 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032696489468405217, + "loss": 1.3618, + "theoretical_loss": 3.4638604298354427, + "tokens_seen": 1965424640 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032694884653961887, + "loss": 1.3026, + "theoretical_loss": 3.463850846223732, + "tokens_seen": 1965490176 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032693279839518557, + "loss": 1.268, + "theoretical_loss": 3.4638412630210356, + "tokens_seen": 1965555712 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032691675025075227, + "loss": 1.2628, + "theoretical_loss": 3.4638316802273232, + "tokens_seen": 1965621248 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032690070210631897, + "loss": 1.2788, + "theoretical_loss": 3.463822097842564, + "tokens_seen": 1965686784 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032688465396188567, + "loss": 1.3096, + "theoretical_loss": 3.4638125158667252, + "tokens_seen": 1965752320 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032686860581745237, + "loss": 1.3556, + "theoretical_loss": 3.4638029342997774, + "tokens_seen": 1965817856 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032685255767301907, + "loss": 1.2882, + "theoretical_loss": 3.4637933531416882, + "tokens_seen": 1965883392 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032683650952858577, + "loss": 1.2608, + "theoretical_loss": 3.4637837723924276, + "tokens_seen": 1965948928 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003268204613841525, + "loss": 1.2603, + "theoretical_loss": 3.4637741920519645, + "tokens_seen": 1966014464 + }, + { + "debugging/Compilability": 1.0, + "debugging/distinct-1-grams": 0.7544709383461976, + "debugging/entropy-1-grams": 5.498686634297508, + "debugging/length": 461.52941176470586, + "debugging/num_segments": 17, + "debugging/score": 0.004263764143320228, + "debugging/score_std": 0.0026579881444447114, + "epoch": 0.6, + "objective/train/docs_used": 727418, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3451262712478638, + "objective/train/theoretical_loss": 3.4637646121202676, + "objective/train/tokens_used": 1986540000, + "theoretical_loss": 3.4637646121202676, + "tokens_seen": 1966080000 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003268044132397192, + "loss": 1.3451, + "theoretical_loss": 3.4637646121202676, + "tokens_seen": 1966080000 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003267883650952859, + "loss": 1.3407, + "theoretical_loss": 3.463755032597305, + "tokens_seen": 1966145536 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003267723169508526, + "loss": 1.2693, + "theoretical_loss": 3.463745453483047, + "tokens_seen": 1966211072 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003267562688064193, + "loss": 1.1658, + "theoretical_loss": 3.463735874777462, + "tokens_seen": 1966276608 + }, + { + "epoch": 0.6, + "learning_rate": 0.000326740220661986, + "loss": 1.2795, + "theoretical_loss": 3.4637262964805187, + "tokens_seen": 1966342144 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003267241725175527, + "loss": 1.2648, + "theoretical_loss": 3.4637167185921856, + "tokens_seen": 1966407680 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003267081243731194, + "loss": 1.1563, + "theoretical_loss": 3.463707141112433, + "tokens_seen": 1966473216 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003266920762286861, + "loss": 1.3762, + "theoretical_loss": 3.4636975640412286, + "tokens_seen": 1966538752 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003266760280842528, + "loss": 1.2727, + "theoretical_loss": 3.4636879873785427, + "tokens_seen": 1966604288 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003266599799398195, + "loss": 1.3064, + "theoretical_loss": 3.4636784111243424, + "tokens_seen": 1966669824 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003266439317953862, + "loss": 1.316, + "theoretical_loss": 3.4636688352785985, + "tokens_seen": 1966735360 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003266278836509529, + "loss": 1.3034, + "theoretical_loss": 3.463659259841279, + "tokens_seen": 1966800896 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003266118355065196, + "loss": 1.2547, + "theoretical_loss": 3.463649684812353, + "tokens_seen": 1966866432 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003265957873620863, + "loss": 1.3117, + "theoretical_loss": 3.463640110191789, + "tokens_seen": 1966931968 + }, + { + "epoch": 0.6, + "learning_rate": 0.000326579739217653, + "loss": 1.4056, + "theoretical_loss": 3.463630535979557, + "tokens_seen": 1966997504 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003265636910732197, + "loss": 1.2598, + "theoretical_loss": 3.463620962175625, + "tokens_seen": 1967063040 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003265476429287864, + "loss": 1.3212, + "theoretical_loss": 3.4636113887799627, + "tokens_seen": 1967128576 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003265315947843531, + "loss": 1.4057, + "theoretical_loss": 3.463601815792539, + "tokens_seen": 1967194112 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003265155466399198, + "loss": 1.2915, + "theoretical_loss": 3.4635922432133226, + "tokens_seen": 1967259648 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003264994984954865, + "loss": 1.2377, + "theoretical_loss": 3.463582671042283, + "tokens_seen": 1967325184 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003264834503510532, + "loss": 1.164, + "theoretical_loss": 3.4635730992793885, + "tokens_seen": 1967390720 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003264674022066199, + "loss": 1.3059, + "theoretical_loss": 3.463563527924608, + "tokens_seen": 1967456256 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003264513540621866, + "loss": 1.1866, + "theoretical_loss": 3.4635539569779117, + "tokens_seen": 1967521792 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003264353059177533, + "loss": 1.3403, + "theoretical_loss": 3.463544386439268, + "tokens_seen": 1967587328 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032641925777332, + "loss": 1.3037, + "theoretical_loss": 3.463534816308645, + "tokens_seen": 1967652864 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003264032096288867, + "loss": 1.2975, + "theoretical_loss": 3.4635252465860127, + "tokens_seen": 1967718400 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032638716148445334, + "loss": 1.2898, + "theoretical_loss": 3.46351567727134, + "tokens_seen": 1967783936 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032637111334002004, + "loss": 1.2048, + "theoretical_loss": 3.4635061083645957, + "tokens_seen": 1967849472 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032635506519558674, + "loss": 1.3615, + "theoretical_loss": 3.463496539865749, + "tokens_seen": 1967915008 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032633901705115344, + "loss": 1.2851, + "theoretical_loss": 3.463486971774769, + "tokens_seen": 1967980544 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032632296890672014, + "loss": 1.3204, + "theoretical_loss": 3.4634774040916243, + "tokens_seen": 1968046080 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032630692076228684, + "loss": 1.1801, + "theoretical_loss": 3.463467836816285, + "tokens_seen": 1968111616 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032629087261785354, + "loss": 1.3139, + "theoretical_loss": 3.4634582699487186, + "tokens_seen": 1968177152 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032627482447342024, + "loss": 1.2516, + "theoretical_loss": 3.463448703488895, + "tokens_seen": 1968242688 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032625877632898694, + "loss": 1.3401, + "theoretical_loss": 3.4634391374367834, + "tokens_seen": 1968308224 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032624272818455364, + "loss": 1.2926, + "theoretical_loss": 3.4634295717923527, + "tokens_seen": 1968373760 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032622668004012034, + "loss": 1.1601, + "theoretical_loss": 3.463420006555572, + "tokens_seen": 1968439296 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032621063189568704, + "loss": 1.2287, + "theoretical_loss": 3.4634104417264098, + "tokens_seen": 1968504832 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032619458375125374, + "loss": 1.2469, + "theoretical_loss": 3.463400877304836, + "tokens_seen": 1968570368 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032617853560682044, + "loss": 1.3134, + "theoretical_loss": 3.4633913132908187, + "tokens_seen": 1968635904 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032616248746238715, + "loss": 1.2615, + "theoretical_loss": 3.463381749684328, + "tokens_seen": 1968701440 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032614643931795385, + "loss": 1.2965, + "theoretical_loss": 3.463372186485332, + "tokens_seen": 1968766976 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032613039117352055, + "loss": 1.2779, + "theoretical_loss": 3.4633626236938007, + "tokens_seen": 1968832512 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003261143430290873, + "loss": 1.2233, + "theoretical_loss": 3.4633530613097028, + "tokens_seen": 1968898048 + }, + { + "epoch": 0.6, + "learning_rate": 0.000326098294884654, + "loss": 1.2871, + "theoretical_loss": 3.463343499333007, + "tokens_seen": 1968963584 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003260822467402207, + "loss": 1.2283, + "theoretical_loss": 3.463333937763683, + "tokens_seen": 1969029120 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003260661985957874, + "loss": 1.2481, + "theoretical_loss": 3.4633243766016992, + "tokens_seen": 1969094656 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003260501504513541, + "loss": 1.3261, + "theoretical_loss": 3.463314815847026, + "tokens_seen": 1969160192 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003260341023069208, + "loss": 1.2636, + "theoretical_loss": 3.4633052554996304, + "tokens_seen": 1969225728 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003260180541624875, + "loss": 1.2397, + "theoretical_loss": 3.4632956955594834, + "tokens_seen": 1969291264 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003260020060180542, + "loss": 1.2012, + "theoretical_loss": 3.4632861360265528, + "tokens_seen": 1969356800 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003259859578736209, + "loss": 1.2893, + "theoretical_loss": 3.463276576900809, + "tokens_seen": 1969422336 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003259699097291876, + "loss": 1.3712, + "theoretical_loss": 3.4632670181822203, + "tokens_seen": 1969487872 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003259538615847543, + "loss": 1.2736, + "theoretical_loss": 3.4632574598707553, + "tokens_seen": 1969553408 + }, + { + "epoch": 0.6, + "learning_rate": 0.000325937813440321, + "loss": 1.275, + "theoretical_loss": 3.463247901966384, + "tokens_seen": 1969618944 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003259217652958877, + "loss": 1.2112, + "theoretical_loss": 3.463238344469075, + "tokens_seen": 1969684480 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003259057171514544, + "loss": 1.3151, + "theoretical_loss": 3.463228787378798, + "tokens_seen": 1969750016 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003258896690070211, + "loss": 1.1951, + "theoretical_loss": 3.463219230695522, + "tokens_seen": 1969815552 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003258736208625878, + "loss": 1.1576, + "theoretical_loss": 3.4632096744192156, + "tokens_seen": 1969881088 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003258575727181545, + "loss": 1.2622, + "theoretical_loss": 3.4632001185498478, + "tokens_seen": 1969946624 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003258415245737212, + "loss": 1.2086, + "theoretical_loss": 3.4631905630873883, + "tokens_seen": 1970012160 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003258254764292879, + "loss": 1.3117, + "theoretical_loss": 3.4631810080318064, + "tokens_seen": 1970077696 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003258094282848546, + "loss": 1.2809, + "theoretical_loss": 3.4631714533830706, + "tokens_seen": 1970143232 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003257933801404213, + "loss": 1.3649, + "theoretical_loss": 3.4631618991411512, + "tokens_seen": 1970208768 + }, + { + "epoch": 0.6, + "learning_rate": 0.000325777331995988, + "loss": 1.2548, + "theoretical_loss": 3.463152345306016, + "tokens_seen": 1970274304 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003257612838515547, + "loss": 1.2626, + "theoretical_loss": 3.463142791877634, + "tokens_seen": 1970339840 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032574523570712136, + "loss": 1.2351, + "theoretical_loss": 3.463133238855976, + "tokens_seen": 1970405376 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032572918756268806, + "loss": 1.257, + "theoretical_loss": 3.46312368624101, + "tokens_seen": 1970470912 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032571313941825476, + "loss": 1.2631, + "theoretical_loss": 3.463114134032705, + "tokens_seen": 1970536448 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032569709127382146, + "loss": 1.2936, + "theoretical_loss": 3.463104582231031, + "tokens_seen": 1970601984 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032568104312938816, + "loss": 1.2501, + "theoretical_loss": 3.4630950308359565, + "tokens_seen": 1970667520 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032566499498495486, + "loss": 1.2376, + "theoretical_loss": 3.4630854798474506, + "tokens_seen": 1970733056 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032564894684052157, + "loss": 1.1947, + "theoretical_loss": 3.463075929265483, + "tokens_seen": 1970798592 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032563289869608827, + "loss": 1.3521, + "theoretical_loss": 3.4630663790900225, + "tokens_seen": 1970864128 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032561685055165497, + "loss": 1.2952, + "theoretical_loss": 3.4630568293210384, + "tokens_seen": 1970929664 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032560080240722167, + "loss": 1.3199, + "theoretical_loss": 3.4630472799585, + "tokens_seen": 1970995200 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032558475426278837, + "loss": 1.2248, + "theoretical_loss": 3.4630377310023763, + "tokens_seen": 1971060736 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032556870611835507, + "loss": 1.1952, + "theoretical_loss": 3.4630281824526365, + "tokens_seen": 1971126272 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032555265797392177, + "loss": 1.2686, + "theoretical_loss": 3.4630186343092495, + "tokens_seen": 1971191808 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032553660982948847, + "loss": 1.2935, + "theoretical_loss": 3.4630090865721854, + "tokens_seen": 1971257344 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032552056168505517, + "loss": 1.353, + "theoretical_loss": 3.462999539241413, + "tokens_seen": 1971322880 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032550451354062187, + "loss": 1.2023, + "theoretical_loss": 3.462989992316901, + "tokens_seen": 1971388416 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032548846539618857, + "loss": 1.1963, + "theoretical_loss": 3.4629804457986184, + "tokens_seen": 1971453952 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032547241725175527, + "loss": 1.2911, + "theoretical_loss": 3.462970899686536, + "tokens_seen": 1971519488 + }, + { + "epoch": 0.6, + "learning_rate": 0.000325456369107322, + "loss": 1.3499, + "theoretical_loss": 3.462961353980621, + "tokens_seen": 1971585024 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003254403209628887, + "loss": 1.2158, + "theoretical_loss": 3.4629518086808444, + "tokens_seen": 1971650560 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003254242728184554, + "loss": 1.29, + "theoretical_loss": 3.4629422637871743, + "tokens_seen": 1971716096 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003254082246740221, + "loss": 1.2269, + "theoretical_loss": 3.4629327192995802, + "tokens_seen": 1971781632 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003253921765295888, + "loss": 1.3054, + "theoretical_loss": 3.4629231752180316, + "tokens_seen": 1971847168 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003253761283851555, + "loss": 1.2532, + "theoretical_loss": 3.4629136315424973, + "tokens_seen": 1971912704 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003253600802407222, + "loss": 1.2788, + "theoretical_loss": 3.4629040882729467, + "tokens_seen": 1971978240 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003253440320962889, + "loss": 1.2394, + "theoretical_loss": 3.462894545409349, + "tokens_seen": 1972043776 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003253279839518556, + "loss": 1.2225, + "theoretical_loss": 3.462885002951674, + "tokens_seen": 1972109312 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003253119358074223, + "loss": 1.3068, + "theoretical_loss": 3.46287546089989, + "tokens_seen": 1972174848 + }, + { + "epoch": 0.6, + "learning_rate": 0.000325295887662989, + "loss": 1.2951, + "theoretical_loss": 3.462865919253967, + "tokens_seen": 1972240384 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003252798395185557, + "loss": 1.3409, + "theoretical_loss": 3.462856378013874, + "tokens_seen": 1972305920 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003252637913741224, + "loss": 1.2454, + "theoretical_loss": 3.4628468371795798, + "tokens_seen": 1972371456 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003252477432296891, + "loss": 1.2898, + "theoretical_loss": 3.4628372967510543, + "tokens_seen": 1972436992 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003252316950852558, + "loss": 1.2655, + "theoretical_loss": 3.4628277567282666, + "tokens_seen": 1972502528 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003252156469408225, + "loss": 1.2136, + "theoretical_loss": 3.4628182171111863, + "tokens_seen": 1972568064 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 729681, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2861812114715576, + "objective/train/theoretical_loss": 3.4628086778997815, + "objective/train/tokens_used": 1993093600, + "theoretical_loss": 3.4628086778997815, + "tokens_seen": 1972633600 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003251995987963892, + "loss": 1.2862, + "theoretical_loss": 3.4628086778997815, + "tokens_seen": 1972633600 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003251835506519559, + "loss": 1.2383, + "theoretical_loss": 3.462799139094023, + "tokens_seen": 1972699136 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003251675025075226, + "loss": 1.3604, + "theoretical_loss": 3.462789600693879, + "tokens_seen": 1972764672 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003251514543630893, + "loss": 1.2266, + "theoretical_loss": 3.462780062699319, + "tokens_seen": 1972830208 + }, + { + "epoch": 0.6, + "learning_rate": 0.000325135406218656, + "loss": 1.2503, + "theoretical_loss": 3.4627705251103125, + "tokens_seen": 1972895744 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003251193580742227, + "loss": 1.2868, + "theoretical_loss": 3.4627609879268286, + "tokens_seen": 1972961280 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003251033099297894, + "loss": 1.2374, + "theoretical_loss": 3.4627514511488364, + "tokens_seen": 1973026816 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003250872617853561, + "loss": 1.2827, + "theoretical_loss": 3.4627419147763057, + "tokens_seen": 1973092352 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003250712136409228, + "loss": 1.2893, + "theoretical_loss": 3.462732378809206, + "tokens_seen": 1973157888 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003250551654964895, + "loss": 1.2725, + "theoretical_loss": 3.4627228432475055, + "tokens_seen": 1973223424 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003250391173520562, + "loss": 1.1792, + "theoretical_loss": 3.4627133080911743, + "tokens_seen": 1973288960 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003250230692076229, + "loss": 1.2814, + "theoretical_loss": 3.4627037733401815, + "tokens_seen": 1973354496 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003250070210631896, + "loss": 1.2023, + "theoretical_loss": 3.462694238994497, + "tokens_seen": 1973420032 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003249909729187563, + "loss": 1.2981, + "theoretical_loss": 3.462684705054089, + "tokens_seen": 1973485568 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032497492477432294, + "loss": 1.25, + "theoretical_loss": 3.4626751715189275, + "tokens_seen": 1973551104 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032495887662988964, + "loss": 1.3286, + "theoretical_loss": 3.4626656383889816, + "tokens_seen": 1973616640 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032494282848545634, + "loss": 1.2596, + "theoretical_loss": 3.462656105664221, + "tokens_seen": 1973682176 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032492678034102304, + "loss": 1.2184, + "theoretical_loss": 3.4626465733446143, + "tokens_seen": 1973747712 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003249107321965898, + "loss": 1.2646, + "theoretical_loss": 3.4626370414301317, + "tokens_seen": 1973813248 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003248946840521565, + "loss": 1.2724, + "theoretical_loss": 3.462627509920742, + "tokens_seen": 1973878784 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003248786359077232, + "loss": 1.2841, + "theoretical_loss": 3.4626179788164144, + "tokens_seen": 1973944320 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003248625877632899, + "loss": 1.273, + "theoretical_loss": 3.462608448117119, + "tokens_seen": 1974009856 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003248465396188566, + "loss": 1.1992, + "theoretical_loss": 3.4625989178228243, + "tokens_seen": 1974075392 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003248304914744233, + "loss": 1.236, + "theoretical_loss": 3.4625893879335, + "tokens_seen": 1974140928 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032481444332999, + "loss": 1.3164, + "theoretical_loss": 3.462579858449115, + "tokens_seen": 1974206464 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003247983951855567, + "loss": 1.2003, + "theoretical_loss": 3.4625703293696395, + "tokens_seen": 1974272000 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003247823470411234, + "loss": 1.1987, + "theoretical_loss": 3.4625608006950426, + "tokens_seen": 1974337536 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003247662988966901, + "loss": 1.2591, + "theoretical_loss": 3.4625512724252934, + "tokens_seen": 1974403072 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003247502507522568, + "loss": 1.2222, + "theoretical_loss": 3.4625417445603617, + "tokens_seen": 1974468608 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003247342026078235, + "loss": 1.2597, + "theoretical_loss": 3.4625322171002155, + "tokens_seen": 1974534144 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003247181544633902, + "loss": 1.216, + "theoretical_loss": 3.462522690044826, + "tokens_seen": 1974599680 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003247021063189569, + "loss": 1.3134, + "theoretical_loss": 3.4625131633941617, + "tokens_seen": 1974665216 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003246860581745236, + "loss": 1.3323, + "theoretical_loss": 3.462503637148192, + "tokens_seen": 1974730752 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003246700100300903, + "loss": 1.2187, + "theoretical_loss": 3.462494111306886, + "tokens_seen": 1974796288 + }, + { + "epoch": 0.6, + "learning_rate": 0.000324653961885657, + "loss": 1.2612, + "theoretical_loss": 3.4624845858702136, + "tokens_seen": 1974861824 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003246379137412237, + "loss": 1.2138, + "theoretical_loss": 3.4624750608381434, + "tokens_seen": 1974927360 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003246218655967904, + "loss": 1.2347, + "theoretical_loss": 3.462465536210646, + "tokens_seen": 1974992896 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003246058174523571, + "loss": 1.206, + "theoretical_loss": 3.4624560119876904, + "tokens_seen": 1975058432 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003245897693079238, + "loss": 1.2424, + "theoretical_loss": 3.462446488169245, + "tokens_seen": 1975123968 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003245737211634905, + "loss": 1.243, + "theoretical_loss": 3.4624369647552804, + "tokens_seen": 1975189504 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003245576730190572, + "loss": 1.23, + "theoretical_loss": 3.462427441745765, + "tokens_seen": 1975255040 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003245416248746239, + "loss": 1.2343, + "theoretical_loss": 3.4624179191406697, + "tokens_seen": 1975320576 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003245255767301906, + "loss": 1.3402, + "theoretical_loss": 3.4624083969399626, + "tokens_seen": 1975386112 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003245095285857573, + "loss": 1.2171, + "theoretical_loss": 3.462398875143613, + "tokens_seen": 1975451648 + }, + { + "epoch": 0.6, + "learning_rate": 0.000324493480441324, + "loss": 1.3397, + "theoretical_loss": 3.4623893537515915, + "tokens_seen": 1975517184 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003244774322968907, + "loss": 1.2322, + "theoretical_loss": 3.462379832763866, + "tokens_seen": 1975582720 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003244613841524574, + "loss": 1.2025, + "theoretical_loss": 3.4623703121804077, + "tokens_seen": 1975648256 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003244453360080241, + "loss": 1.3558, + "theoretical_loss": 3.462360792001184, + "tokens_seen": 1975713792 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003244292878635908, + "loss": 1.0972, + "theoretical_loss": 3.462351272226166, + "tokens_seen": 1975779328 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003244132397191575, + "loss": 1.3041, + "theoretical_loss": 3.4623417528553224, + "tokens_seen": 1975844864 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003243971915747242, + "loss": 1.2776, + "theoretical_loss": 3.462332233888623, + "tokens_seen": 1975910400 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003243811434302909, + "loss": 1.241, + "theoretical_loss": 3.462322715326037, + "tokens_seen": 1975975936 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003243650952858576, + "loss": 1.3103, + "theoretical_loss": 3.462313197167534, + "tokens_seen": 1976041472 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032434904714142426, + "loss": 1.2925, + "theoretical_loss": 3.4623036794130826, + "tokens_seen": 1976107008 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032433299899699096, + "loss": 1.1652, + "theoretical_loss": 3.462294162062653, + "tokens_seen": 1976172544 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032431695085255766, + "loss": 1.2073, + "theoretical_loss": 3.462284645116215, + "tokens_seen": 1976238080 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032430090270812436, + "loss": 1.2308, + "theoretical_loss": 3.4622751285737374, + "tokens_seen": 1976303616 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032428485456369107, + "loss": 1.1103, + "theoretical_loss": 3.46226561243519, + "tokens_seen": 1976369152 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032426880641925777, + "loss": 1.1947, + "theoretical_loss": 3.462256096700542, + "tokens_seen": 1976434688 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032425275827482447, + "loss": 1.1969, + "theoretical_loss": 3.462246581369763, + "tokens_seen": 1976500224 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032423671013039117, + "loss": 1.2734, + "theoretical_loss": 3.462237066442823, + "tokens_seen": 1976565760 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032422066198595787, + "loss": 1.2957, + "theoretical_loss": 3.46222755191969, + "tokens_seen": 1976631296 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032420461384152457, + "loss": 1.2378, + "theoretical_loss": 3.4622180378003353, + "tokens_seen": 1976696832 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032418856569709127, + "loss": 1.3273, + "theoretical_loss": 3.462208524084727, + "tokens_seen": 1976762368 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032417251755265797, + "loss": 1.3348, + "theoretical_loss": 3.4621990107728355, + "tokens_seen": 1976827904 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032415646940822467, + "loss": 1.2415, + "theoretical_loss": 3.46218949786463, + "tokens_seen": 1976893440 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032414042126379137, + "loss": 1.2782, + "theoretical_loss": 3.462179985360079, + "tokens_seen": 1976958976 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032412437311935807, + "loss": 1.2449, + "theoretical_loss": 3.4621704732591536, + "tokens_seen": 1977024512 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032410832497492477, + "loss": 1.2696, + "theoretical_loss": 3.462160961561822, + "tokens_seen": 1977090048 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032409227683049147, + "loss": 1.3205, + "theoretical_loss": 3.4621514502680553, + "tokens_seen": 1977155584 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003240762286860582, + "loss": 1.3169, + "theoretical_loss": 3.4621419393778208, + "tokens_seen": 1977221120 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003240601805416249, + "loss": 1.2459, + "theoretical_loss": 3.4621324288910897, + "tokens_seen": 1977286656 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003240441323971916, + "loss": 1.3279, + "theoretical_loss": 3.462122918807831, + "tokens_seen": 1977352192 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003240280842527583, + "loss": 1.254, + "theoretical_loss": 3.4621134091280137, + "tokens_seen": 1977417728 + }, + { + "epoch": 0.6, + "learning_rate": 0.000324012036108325, + "loss": 1.1497, + "theoretical_loss": 3.4621038998516083, + "tokens_seen": 1977483264 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003239959879638917, + "loss": 1.1942, + "theoretical_loss": 3.4620943909785837, + "tokens_seen": 1977548800 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003239799398194584, + "loss": 1.2719, + "theoretical_loss": 3.4620848825089094, + "tokens_seen": 1977614336 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003239638916750251, + "loss": 1.2796, + "theoretical_loss": 3.4620753744425556, + "tokens_seen": 1977679872 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003239478435305918, + "loss": 1.2013, + "theoretical_loss": 3.462065866779491, + "tokens_seen": 1977745408 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003239317953861585, + "loss": 1.3183, + "theoretical_loss": 3.4620563595196847, + "tokens_seen": 1977810944 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003239157472417252, + "loss": 1.2432, + "theoretical_loss": 3.462046852663108, + "tokens_seen": 1977876480 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003238996990972919, + "loss": 1.2743, + "theoretical_loss": 3.462037346209729, + "tokens_seen": 1977942016 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003238836509528586, + "loss": 1.407, + "theoretical_loss": 3.462027840159517, + "tokens_seen": 1978007552 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003238676028084253, + "loss": 1.2282, + "theoretical_loss": 3.462018334512443, + "tokens_seen": 1978073088 + }, + { + "epoch": 0.6, + "learning_rate": 0.000323851554663992, + "loss": 1.3075, + "theoretical_loss": 3.462008829268475, + "tokens_seen": 1978138624 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003238355065195587, + "loss": 1.2613, + "theoretical_loss": 3.461999324427584, + "tokens_seen": 1978204160 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003238194583751254, + "loss": 1.1914, + "theoretical_loss": 3.4619898199897388, + "tokens_seen": 1978269696 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003238034102306921, + "loss": 1.2806, + "theoretical_loss": 3.461980315954909, + "tokens_seen": 1978335232 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003237873620862588, + "loss": 1.2199, + "theoretical_loss": 3.461970812323064, + "tokens_seen": 1978400768 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003237713139418255, + "loss": 1.3138, + "theoretical_loss": 3.461961309094173, + "tokens_seen": 1978466304 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003237552657973922, + "loss": 1.2722, + "theoretical_loss": 3.4619518062682064, + "tokens_seen": 1978531840 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003237392176529589, + "loss": 1.1866, + "theoretical_loss": 3.4619423038451336, + "tokens_seen": 1978597376 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003237231695085256, + "loss": 1.302, + "theoretical_loss": 3.461932801824924, + "tokens_seen": 1978662912 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003237071213640923, + "loss": 1.3537, + "theoretical_loss": 3.4619233002075473, + "tokens_seen": 1978728448 + }, + { + "epoch": 0.6, + "learning_rate": 0.000323691073219659, + "loss": 1.2304, + "theoretical_loss": 3.4619137989929727, + "tokens_seen": 1978793984 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003236750250752257, + "loss": 1.2971, + "theoretical_loss": 3.46190429818117, + "tokens_seen": 1978859520 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003236589769307924, + "loss": 1.2848, + "theoretical_loss": 3.4618947977721097, + "tokens_seen": 1978925056 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003236429287863591, + "loss": 1.2117, + "theoretical_loss": 3.4618852977657593, + "tokens_seen": 1978990592 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003236268806419258, + "loss": 1.2686, + "theoretical_loss": 3.4618757981620907, + "tokens_seen": 1979056128 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003236108324974925, + "loss": 1.2459, + "theoretical_loss": 3.461866298961072, + "tokens_seen": 1979121664 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 732490, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3188607692718506, + "objective/train/theoretical_loss": 3.461856800162673, + "objective/train/tokens_used": 1999647200, + "theoretical_loss": 3.461856800162673, + "tokens_seen": 1979187200 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003235947843530592, + "loss": 1.3189, + "theoretical_loss": 3.461856800162673, + "tokens_seen": 1979187200 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003235787362086259, + "loss": 1.2303, + "theoretical_loss": 3.461847301766864, + "tokens_seen": 1979252736 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003235626880641926, + "loss": 1.1775, + "theoretical_loss": 3.461837803773614, + "tokens_seen": 1979318272 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003235466399197593, + "loss": 1.3189, + "theoretical_loss": 3.461828306182893, + "tokens_seen": 1979383808 + }, + { + "epoch": 0.6, + "learning_rate": 0.000323530591775326, + "loss": 1.171, + "theoretical_loss": 3.4618188089946695, + "tokens_seen": 1979449344 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003235145436308927, + "loss": 1.31, + "theoretical_loss": 3.461809312208915, + "tokens_seen": 1979514880 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003234984954864594, + "loss": 1.1628, + "theoretical_loss": 3.4617998158255974, + "tokens_seen": 1979580416 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003234824473420261, + "loss": 1.4201, + "theoretical_loss": 3.4617903198446878, + "tokens_seen": 1979645952 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003234663991975928, + "loss": 1.2192, + "theoretical_loss": 3.4617808242661545, + "tokens_seen": 1979711488 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003234503510531595, + "loss": 1.3009, + "theoretical_loss": 3.461771329089968, + "tokens_seen": 1979777024 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003234343029087262, + "loss": 1.2551, + "theoretical_loss": 3.461761834316097, + "tokens_seen": 1979842560 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003234182547642929, + "loss": 1.2038, + "theoretical_loss": 3.4617523399445123, + "tokens_seen": 1979908096 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003234022066198596, + "loss": 1.218, + "theoretical_loss": 3.4617428459751833, + "tokens_seen": 1979973632 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003233861584754263, + "loss": 1.2814, + "theoretical_loss": 3.4617333524080784, + "tokens_seen": 1980039168 + }, + { + "epoch": 0.6, + "learning_rate": 0.000323370110330993, + "loss": 1.3134, + "theoretical_loss": 3.461723859243169, + "tokens_seen": 1980104704 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003233540621865597, + "loss": 1.2812, + "theoretical_loss": 3.461714366480424, + "tokens_seen": 1980170240 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003233380140421264, + "loss": 1.244, + "theoretical_loss": 3.461704874119813, + "tokens_seen": 1980235776 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003233219658976931, + "loss": 1.3786, + "theoretical_loss": 3.461695382161305, + "tokens_seen": 1980301312 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003233059177532598, + "loss": 1.2194, + "theoretical_loss": 3.461685890604871, + "tokens_seen": 1980366848 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003232898696088265, + "loss": 1.2627, + "theoretical_loss": 3.4616763994504796, + "tokens_seen": 1980432384 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003232738214643932, + "loss": 1.2556, + "theoretical_loss": 3.461666908698101, + "tokens_seen": 1980497920 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003232577733199599, + "loss": 1.2818, + "theoretical_loss": 3.4616574183477047, + "tokens_seen": 1980563456 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003232417251755266, + "loss": 1.2674, + "theoretical_loss": 3.4616479283992607, + "tokens_seen": 1980628992 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003232256770310933, + "loss": 1.2898, + "theoretical_loss": 3.461638438852738, + "tokens_seen": 1980694528 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032320962888666, + "loss": 1.2801, + "theoretical_loss": 3.461628949708106, + "tokens_seen": 1980760064 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003231935807422267, + "loss": 1.3563, + "theoretical_loss": 3.461619460965336, + "tokens_seen": 1980825600 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003231775325977934, + "loss": 1.352, + "theoretical_loss": 3.4616099726243963, + "tokens_seen": 1980891136 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003231614844533601, + "loss": 1.329, + "theoretical_loss": 3.4616004846852575, + "tokens_seen": 1980956672 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003231454363089268, + "loss": 1.2957, + "theoretical_loss": 3.461590997147889, + "tokens_seen": 1981022208 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003231293881644935, + "loss": 1.2536, + "theoretical_loss": 3.4615815100122593, + "tokens_seen": 1981087744 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003231133400200602, + "loss": 1.2352, + "theoretical_loss": 3.4615720232783396, + "tokens_seen": 1981153280 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003230972918756269, + "loss": 1.2444, + "theoretical_loss": 3.461562536946099, + "tokens_seen": 1981218816 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003230812437311936, + "loss": 1.2318, + "theoretical_loss": 3.4615530510155077, + "tokens_seen": 1981284352 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003230651955867603, + "loss": 1.3292, + "theoretical_loss": 3.4615435654865347, + "tokens_seen": 1981349888 + }, + { + "epoch": 0.6, + "learning_rate": 0.000323049147442327, + "loss": 1.3002, + "theoretical_loss": 3.46153408035915, + "tokens_seen": 1981415424 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003230330992978937, + "loss": 1.2232, + "theoretical_loss": 3.4615245956333234, + "tokens_seen": 1981480960 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003230170511534604, + "loss": 1.2572, + "theoretical_loss": 3.4615151113090246, + "tokens_seen": 1981546496 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003230010030090271, + "loss": 1.3798, + "theoretical_loss": 3.4615056273862232, + "tokens_seen": 1981612032 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003229849548645938, + "loss": 1.2849, + "theoretical_loss": 3.4614961438648892, + "tokens_seen": 1981677568 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003229689067201605, + "loss": 1.2129, + "theoretical_loss": 3.461486660744992, + "tokens_seen": 1981743104 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032295285857572716, + "loss": 1.2547, + "theoretical_loss": 3.4614771780265015, + "tokens_seen": 1981808640 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032293681043129386, + "loss": 1.2885, + "theoretical_loss": 3.4614676957093873, + "tokens_seen": 1981874176 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032292076228686056, + "loss": 1.251, + "theoretical_loss": 3.4614582137936196, + "tokens_seen": 1981939712 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032290471414242727, + "loss": 1.2831, + "theoretical_loss": 3.4614487322791674, + "tokens_seen": 1982005248 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032288866599799397, + "loss": 1.3164, + "theoretical_loss": 3.4614392511660013, + "tokens_seen": 1982070784 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032287261785356067, + "loss": 1.2139, + "theoretical_loss": 3.46142977045409, + "tokens_seen": 1982136320 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032285656970912737, + "loss": 1.2079, + "theoretical_loss": 3.461420290143404, + "tokens_seen": 1982201856 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032284052156469407, + "loss": 1.3113, + "theoretical_loss": 3.461410810233913, + "tokens_seen": 1982267392 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032282447342026077, + "loss": 1.2639, + "theoretical_loss": 3.461401330725587, + "tokens_seen": 1982332928 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032280842527582747, + "loss": 1.3377, + "theoretical_loss": 3.4613918516183952, + "tokens_seen": 1982398464 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032279237713139417, + "loss": 1.3616, + "theoretical_loss": 3.4613823729123077, + "tokens_seen": 1982464000 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032277632898696087, + "loss": 1.2614, + "theoretical_loss": 3.461372894607294, + "tokens_seen": 1982529536 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032276028084252757, + "loss": 1.3428, + "theoretical_loss": 3.461363416703324, + "tokens_seen": 1982595072 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032274423269809427, + "loss": 1.3031, + "theoretical_loss": 3.461353939200367, + "tokens_seen": 1982660608 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032272818455366097, + "loss": 1.3066, + "theoretical_loss": 3.4613444620983937, + "tokens_seen": 1982726144 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003227121364092277, + "loss": 1.2935, + "theoretical_loss": 3.461334985397374, + "tokens_seen": 1982791680 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003226960882647944, + "loss": 1.2075, + "theoretical_loss": 3.4613255090972768, + "tokens_seen": 1982857216 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003226800401203611, + "loss": 1.224, + "theoretical_loss": 3.461316033198072, + "tokens_seen": 1982922752 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003226639919759278, + "loss": 1.2229, + "theoretical_loss": 3.4613065576997295, + "tokens_seen": 1982988288 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003226479438314945, + "loss": 1.2309, + "theoretical_loss": 3.4612970826022194, + "tokens_seen": 1983053824 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003226318956870612, + "loss": 1.355, + "theoretical_loss": 3.4612876079055113, + "tokens_seen": 1983119360 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003226158475426279, + "loss": 1.2949, + "theoretical_loss": 3.461278133609575, + "tokens_seen": 1983184896 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003225997993981946, + "loss": 1.2195, + "theoretical_loss": 3.461268659714381, + "tokens_seen": 1983250432 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003225837512537613, + "loss": 1.3531, + "theoretical_loss": 3.4612591862198974, + "tokens_seen": 1983315968 + }, + { + "epoch": 0.6, + "learning_rate": 0.000322567703109328, + "loss": 1.2515, + "theoretical_loss": 3.4612497131260955, + "tokens_seen": 1983381504 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003225516549648947, + "loss": 1.229, + "theoretical_loss": 3.4612402404329448, + "tokens_seen": 1983447040 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003225356068204614, + "loss": 1.2394, + "theoretical_loss": 3.461230768140415, + "tokens_seen": 1983512576 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032251955867602813, + "loss": 1.2734, + "theoretical_loss": 3.4612212962484756, + "tokens_seen": 1983578112 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032250351053159484, + "loss": 1.1844, + "theoretical_loss": 3.461211824757097, + "tokens_seen": 1983643648 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032248746238716154, + "loss": 1.326, + "theoretical_loss": 3.4612023536662484, + "tokens_seen": 1983709184 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032247141424272824, + "loss": 1.2999, + "theoretical_loss": 3.4611928829759004, + "tokens_seen": 1983774720 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032245536609829494, + "loss": 1.2224, + "theoretical_loss": 3.4611834126860224, + "tokens_seen": 1983840256 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032243931795386164, + "loss": 1.2188, + "theoretical_loss": 3.4611739427965835, + "tokens_seen": 1983905792 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032242326980942834, + "loss": 1.1813, + "theoretical_loss": 3.461164473307555, + "tokens_seen": 1983971328 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032240722166499504, + "loss": 1.2103, + "theoretical_loss": 3.461155004218906, + "tokens_seen": 1984036864 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032239117352056174, + "loss": 1.18, + "theoretical_loss": 3.4611455355306067, + "tokens_seen": 1984102400 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032237512537612844, + "loss": 1.3016, + "theoretical_loss": 3.461136067242626, + "tokens_seen": 1984167936 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032235907723169514, + "loss": 1.1607, + "theoretical_loss": 3.4611265993549347, + "tokens_seen": 1984233472 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032234302908726184, + "loss": 1.2267, + "theoretical_loss": 3.461117131867502, + "tokens_seen": 1984299008 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032232698094282854, + "loss": 1.3108, + "theoretical_loss": 3.4611076647802985, + "tokens_seen": 1984364544 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003223109327983952, + "loss": 1.2402, + "theoretical_loss": 3.4610981980932936, + "tokens_seen": 1984430080 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003222948846539619, + "loss": 1.2684, + "theoretical_loss": 3.461088731806457, + "tokens_seen": 1984495616 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003222788365095286, + "loss": 1.31, + "theoretical_loss": 3.461079265919759, + "tokens_seen": 1984561152 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003222627883650953, + "loss": 1.2289, + "theoretical_loss": 3.4610698004331693, + "tokens_seen": 1984626688 + }, + { + "epoch": 0.6, + "learning_rate": 0.000322246740220662, + "loss": 1.2123, + "theoretical_loss": 3.461060335346658, + "tokens_seen": 1984692224 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003222306920762287, + "loss": 1.3218, + "theoretical_loss": 3.461050870660194, + "tokens_seen": 1984757760 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003222146439317954, + "loss": 1.1713, + "theoretical_loss": 3.4610414063737487, + "tokens_seen": 1984823296 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003221985957873621, + "loss": 1.3318, + "theoretical_loss": 3.46103194248729, + "tokens_seen": 1984888832 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003221825476429288, + "loss": 1.2769, + "theoretical_loss": 3.4610224790007904, + "tokens_seen": 1984954368 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003221664994984955, + "loss": 1.3018, + "theoretical_loss": 3.461013015914218, + "tokens_seen": 1985019904 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003221504513540622, + "loss": 1.2834, + "theoretical_loss": 3.4610035532275427, + "tokens_seen": 1985085440 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003221344032096289, + "loss": 1.1979, + "theoretical_loss": 3.4609940909407353, + "tokens_seen": 1985150976 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003221183550651956, + "loss": 1.2263, + "theoretical_loss": 3.4609846290537645, + "tokens_seen": 1985216512 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003221023069207623, + "loss": 1.268, + "theoretical_loss": 3.4609751675666014, + "tokens_seen": 1985282048 + }, + { + "epoch": 0.6, + "learning_rate": 0.000322086258776329, + "loss": 1.2984, + "theoretical_loss": 3.4609657064792154, + "tokens_seen": 1985347584 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003220702106318957, + "loss": 1.2744, + "theoretical_loss": 3.460956245791576, + "tokens_seen": 1985413120 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003220541624874624, + "loss": 1.2073, + "theoretical_loss": 3.460946785503654, + "tokens_seen": 1985478656 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003220381143430291, + "loss": 1.2475, + "theoretical_loss": 3.4609373256154186, + "tokens_seen": 1985544192 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003220220661985958, + "loss": 1.1307, + "theoretical_loss": 3.4609278661268403, + "tokens_seen": 1985609728 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003220060180541625, + "loss": 1.1906, + "theoretical_loss": 3.4609184070378882, + "tokens_seen": 1985675264 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 734819, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3523163795471191, + "objective/train/theoretical_loss": 3.460908948348533, + "objective/train/tokens_used": 2006200800, + "theoretical_loss": 3.460908948348533, + "tokens_seen": 1985740800 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003219899699097292, + "loss": 1.3523, + "theoretical_loss": 3.460908948348533, + "tokens_seen": 1985740800 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003219739217652959, + "loss": 1.2167, + "theoretical_loss": 3.460899490058744, + "tokens_seen": 1985806336 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003219578736208626, + "loss": 1.4006, + "theoretical_loss": 3.460890032168492, + "tokens_seen": 1985871872 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003219418254764293, + "loss": 1.2607, + "theoretical_loss": 3.4608805746777467, + "tokens_seen": 1985937408 + }, + { + "epoch": 0.6, + "learning_rate": 0.000321925777331996, + "loss": 1.242, + "theoretical_loss": 3.460871117586477, + "tokens_seen": 1986002944 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003219097291875627, + "loss": 1.2586, + "theoretical_loss": 3.460861660894654, + "tokens_seen": 1986068480 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003218936810431294, + "loss": 1.344, + "theoretical_loss": 3.460852204602247, + "tokens_seen": 1986134016 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003218776328986961, + "loss": 1.223, + "theoretical_loss": 3.4608427487092266, + "tokens_seen": 1986199552 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003218615847542628, + "loss": 1.2199, + "theoretical_loss": 3.460833293215562, + "tokens_seen": 1986265088 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003218455366098295, + "loss": 1.1704, + "theoretical_loss": 3.4608238381212235, + "tokens_seen": 1986330624 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003218294884653962, + "loss": 1.2921, + "theoretical_loss": 3.4608143834261815, + "tokens_seen": 1986396160 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003218134403209629, + "loss": 1.215, + "theoretical_loss": 3.4608049291304055, + "tokens_seen": 1986461696 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003217973921765296, + "loss": 1.1792, + "theoretical_loss": 3.4607954752338657, + "tokens_seen": 1986527232 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003217813440320963, + "loss": 1.3989, + "theoretical_loss": 3.4607860217365314, + "tokens_seen": 1986592768 + }, + { + "epoch": 0.6, + "learning_rate": 0.000321765295887663, + "loss": 1.3139, + "theoretical_loss": 3.460776568638373, + "tokens_seen": 1986658304 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003217492477432297, + "loss": 1.3134, + "theoretical_loss": 3.460767115939361, + "tokens_seen": 1986723840 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003217331995987964, + "loss": 1.2275, + "theoretical_loss": 3.4607576636394644, + "tokens_seen": 1986789376 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003217171514543631, + "loss": 1.3099, + "theoretical_loss": 3.460748211738654, + "tokens_seen": 1986854912 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003217011033099298, + "loss": 1.2472, + "theoretical_loss": 3.4607387602368997, + "tokens_seen": 1986920448 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003216850551654965, + "loss": 1.282, + "theoretical_loss": 3.4607293091341713, + "tokens_seen": 1986985984 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003216690070210632, + "loss": 1.2253, + "theoretical_loss": 3.4607198584304384, + "tokens_seen": 1987051520 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003216529588766299, + "loss": 1.303, + "theoretical_loss": 3.4607104081256717, + "tokens_seen": 1987117056 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003216369107321966, + "loss": 1.2936, + "theoretical_loss": 3.4607009582198405, + "tokens_seen": 1987182592 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003216208625877633, + "loss": 1.3764, + "theoretical_loss": 3.4606915087129155, + "tokens_seen": 1987248128 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032160481444333, + "loss": 1.2342, + "theoretical_loss": 3.460682059604866, + "tokens_seen": 1987313664 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003215887662988967, + "loss": 1.4077, + "theoretical_loss": 3.4606726108956627, + "tokens_seen": 1987379200 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003215727181544634, + "loss": 1.2929, + "theoretical_loss": 3.4606631625852757, + "tokens_seen": 1987444736 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003215566700100301, + "loss": 1.2755, + "theoretical_loss": 3.460653714673674, + "tokens_seen": 1987510272 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032154062186559677, + "loss": 1.3061, + "theoretical_loss": 3.460644267160829, + "tokens_seen": 1987575808 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032152457372116347, + "loss": 1.2224, + "theoretical_loss": 3.4606348200467094, + "tokens_seen": 1987641344 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032150852557673017, + "loss": 1.3256, + "theoretical_loss": 3.4606253733312857, + "tokens_seen": 1987706880 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032149247743229687, + "loss": 1.3212, + "theoretical_loss": 3.460615927014528, + "tokens_seen": 1987772416 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032147642928786357, + "loss": 1.252, + "theoretical_loss": 3.4606064810964066, + "tokens_seen": 1987837952 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032146038114343027, + "loss": 1.2974, + "theoretical_loss": 3.4605970355768916, + "tokens_seen": 1987903488 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032144433299899697, + "loss": 1.3922, + "theoretical_loss": 3.460587590455952, + "tokens_seen": 1987969024 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032142828485456367, + "loss": 1.1854, + "theoretical_loss": 3.4605781457335594, + "tokens_seen": 1988034560 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032141223671013037, + "loss": 1.2731, + "theoretical_loss": 3.4605687014096826, + "tokens_seen": 1988100096 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032139618856569707, + "loss": 1.3992, + "theoretical_loss": 3.4605592574842916, + "tokens_seen": 1988165632 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032138014042126377, + "loss": 1.2792, + "theoretical_loss": 3.460549813957358, + "tokens_seen": 1988231168 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032136409227683047, + "loss": 1.3066, + "theoretical_loss": 3.46054037082885, + "tokens_seen": 1988296704 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032134804413239717, + "loss": 1.3393, + "theoretical_loss": 3.4605309280987386, + "tokens_seen": 1988362240 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003213319959879639, + "loss": 1.278, + "theoretical_loss": 3.4605214857669937, + "tokens_seen": 1988427776 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032131594784353063, + "loss": 1.2478, + "theoretical_loss": 3.4605120438335852, + "tokens_seen": 1988493312 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032129989969909733, + "loss": 1.1878, + "theoretical_loss": 3.4605026022984835, + "tokens_seen": 1988558848 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032128385155466403, + "loss": 1.2493, + "theoretical_loss": 3.4604931611616587, + "tokens_seen": 1988624384 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032126780341023073, + "loss": 1.2521, + "theoretical_loss": 3.4604837204230803, + "tokens_seen": 1988689920 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032125175526579743, + "loss": 1.2633, + "theoretical_loss": 3.460474280082719, + "tokens_seen": 1988755456 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032123570712136413, + "loss": 1.2896, + "theoretical_loss": 3.4604648401405447, + "tokens_seen": 1988820992 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032121965897693083, + "loss": 1.2889, + "theoretical_loss": 3.460455400596527, + "tokens_seen": 1988886528 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032120361083249753, + "loss": 1.3066, + "theoretical_loss": 3.460445961450637, + "tokens_seen": 1988952064 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032118756268806423, + "loss": 1.3221, + "theoretical_loss": 3.4604365227028437, + "tokens_seen": 1989017600 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032117151454363093, + "loss": 1.3245, + "theoretical_loss": 3.460427084353118, + "tokens_seen": 1989083136 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032115546639919763, + "loss": 1.2391, + "theoretical_loss": 3.4604176464014293, + "tokens_seen": 1989148672 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032113941825476434, + "loss": 1.2866, + "theoretical_loss": 3.4604082088477486, + "tokens_seen": 1989214208 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032112337011033104, + "loss": 1.1949, + "theoretical_loss": 3.460398771692045, + "tokens_seen": 1989279744 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032110732196589774, + "loss": 1.324, + "theoretical_loss": 3.4603893349342894, + "tokens_seen": 1989345280 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032109127382146444, + "loss": 1.2141, + "theoretical_loss": 3.460379898574452, + "tokens_seen": 1989410816 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032107522567703114, + "loss": 1.2368, + "theoretical_loss": 3.4603704626125023, + "tokens_seen": 1989476352 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032105917753259784, + "loss": 1.244, + "theoretical_loss": 3.4603610270484104, + "tokens_seen": 1989541888 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032104312938816454, + "loss": 1.2956, + "theoretical_loss": 3.4603515918821466, + "tokens_seen": 1989607424 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032102708124373124, + "loss": 1.2711, + "theoretical_loss": 3.460342157113681, + "tokens_seen": 1989672960 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032101103309929794, + "loss": 1.3443, + "theoretical_loss": 3.4603327227429843, + "tokens_seen": 1989738496 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032099498495486464, + "loss": 1.27, + "theoretical_loss": 3.460323288770026, + "tokens_seen": 1989804032 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032097893681043134, + "loss": 1.2329, + "theoretical_loss": 3.460313855194776, + "tokens_seen": 1989869568 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032096288866599804, + "loss": 1.3296, + "theoretical_loss": 3.460304422017205, + "tokens_seen": 1989935104 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032094684052156474, + "loss": 1.2265, + "theoretical_loss": 3.4602949892372834, + "tokens_seen": 1990000640 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032093079237713144, + "loss": 1.2507, + "theoretical_loss": 3.4602855568549806, + "tokens_seen": 1990066176 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003209147442326981, + "loss": 1.2879, + "theoretical_loss": 3.4602761248702665, + "tokens_seen": 1990131712 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003208986960882648, + "loss": 1.2363, + "theoretical_loss": 3.4602666932831125, + "tokens_seen": 1990197248 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003208826479438315, + "loss": 1.2853, + "theoretical_loss": 3.4602572620934877, + "tokens_seen": 1990262784 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003208665997993982, + "loss": 1.3151, + "theoretical_loss": 3.4602478313013627, + "tokens_seen": 1990328320 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003208505516549649, + "loss": 1.3522, + "theoretical_loss": 3.4602384009067073, + "tokens_seen": 1990393856 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003208345035105316, + "loss": 1.2648, + "theoretical_loss": 3.460228970909492, + "tokens_seen": 1990459392 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003208184553660983, + "loss": 1.2363, + "theoretical_loss": 3.4602195413096877, + "tokens_seen": 1990524928 + }, + { + "epoch": 0.6, + "learning_rate": 0.000320802407221665, + "loss": 1.2931, + "theoretical_loss": 3.460210112107263, + "tokens_seen": 1990590464 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003207863590772317, + "loss": 1.2845, + "theoretical_loss": 3.4602006833021886, + "tokens_seen": 1990656000 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003207703109327984, + "loss": 1.3696, + "theoretical_loss": 3.4601912548944354, + "tokens_seen": 1990721536 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003207542627883651, + "loss": 1.2783, + "theoretical_loss": 3.4601818268839732, + "tokens_seen": 1990787072 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003207382146439318, + "loss": 1.2377, + "theoretical_loss": 3.4601723992707716, + "tokens_seen": 1990852608 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003207221664994985, + "loss": 1.2655, + "theoretical_loss": 3.4601629720548015, + "tokens_seen": 1990918144 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003207061183550652, + "loss": 1.3213, + "theoretical_loss": 3.460153545236033, + "tokens_seen": 1990983680 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003206900702106319, + "loss": 1.2837, + "theoretical_loss": 3.4601441188144353, + "tokens_seen": 1991049216 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003206740220661986, + "loss": 1.4595, + "theoretical_loss": 3.46013469278998, + "tokens_seen": 1991114752 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003206579739217653, + "loss": 1.3384, + "theoretical_loss": 3.4601252671626366, + "tokens_seen": 1991180288 + }, + { + "epoch": 0.6, + "learning_rate": 0.000320641925777332, + "loss": 1.2612, + "theoretical_loss": 3.460115841932376, + "tokens_seen": 1991245824 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003206258776328987, + "loss": 1.22, + "theoretical_loss": 3.4601064170991673, + "tokens_seen": 1991311360 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003206098294884654, + "loss": 1.2003, + "theoretical_loss": 3.460096992662981, + "tokens_seen": 1991376896 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003205937813440321, + "loss": 1.2062, + "theoretical_loss": 3.4600875686237877, + "tokens_seen": 1991442432 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003205777331995988, + "loss": 1.2861, + "theoretical_loss": 3.4600781449815576, + "tokens_seen": 1991507968 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003205616850551655, + "loss": 1.3836, + "theoretical_loss": 3.4600687217362607, + "tokens_seen": 1991573504 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003205456369107322, + "loss": 1.3363, + "theoretical_loss": 3.4600592988878676, + "tokens_seen": 1991639040 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003205295887662989, + "loss": 1.2489, + "theoretical_loss": 3.4600498764363476, + "tokens_seen": 1991704576 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003205135406218656, + "loss": 1.263, + "theoretical_loss": 3.460040454381672, + "tokens_seen": 1991770112 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003204974924774323, + "loss": 1.2736, + "theoretical_loss": 3.46003103272381, + "tokens_seen": 1991835648 + }, + { + "epoch": 0.6, + "learning_rate": 0.000320481444332999, + "loss": 1.2367, + "theoretical_loss": 3.460021611462733, + "tokens_seen": 1991901184 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003204653961885657, + "loss": 1.2444, + "theoretical_loss": 3.4600121905984103, + "tokens_seen": 1991966720 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003204493480441324, + "loss": 1.2789, + "theoretical_loss": 3.4600027701308127, + "tokens_seen": 1992032256 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003204332998996991, + "loss": 1.3087, + "theoretical_loss": 3.45999335005991, + "tokens_seen": 1992097792 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003204172517552658, + "loss": 1.2514, + "theoretical_loss": 3.4599839303856728, + "tokens_seen": 1992163328 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003204012036108325, + "loss": 1.2768, + "theoretical_loss": 3.4599745111080713, + "tokens_seen": 1992228864 + }, + { + "epoch": 0.6, + "objective/train/docs_used": 736978, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.255151629447937, + "objective/train/theoretical_loss": 3.4599650922270753, + "objective/train/tokens_used": 2012754400, + "theoretical_loss": 3.4599650922270753, + "tokens_seen": 1992294400 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003203851554663992, + "loss": 1.2552, + "theoretical_loss": 3.4599650922270753, + "tokens_seen": 1992294400 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003203691073219659, + "loss": 1.2964, + "theoretical_loss": 3.4599556737426553, + "tokens_seen": 1992359936 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003203530591775326, + "loss": 1.3165, + "theoretical_loss": 3.459946255654782, + "tokens_seen": 1992425472 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003203370110330993, + "loss": 1.1964, + "theoretical_loss": 3.4599368379634257, + "tokens_seen": 1992491008 + }, + { + "epoch": 0.6, + "learning_rate": 0.000320320962888666, + "loss": 1.2366, + "theoretical_loss": 3.4599274206685555, + "tokens_seen": 1992556544 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003203049147442327, + "loss": 1.2652, + "theoretical_loss": 3.4599180037701434, + "tokens_seen": 1992622080 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003202888665997994, + "loss": 1.316, + "theoretical_loss": 3.459908587268158, + "tokens_seen": 1992687616 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003202728184553661, + "loss": 1.2976, + "theoretical_loss": 3.4598991711625704, + "tokens_seen": 1992753152 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003202567703109328, + "loss": 1.3362, + "theoretical_loss": 3.459889755453351, + "tokens_seen": 1992818688 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003202407221664995, + "loss": 1.3377, + "theoretical_loss": 3.45988034014047, + "tokens_seen": 1992884224 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003202246740220662, + "loss": 1.3596, + "theoretical_loss": 3.459870925223897, + "tokens_seen": 1992949760 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003202086258776329, + "loss": 1.2893, + "theoretical_loss": 3.4598615107036035, + "tokens_seen": 1993015296 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003201925777331996, + "loss": 1.3324, + "theoretical_loss": 3.4598520965795587, + "tokens_seen": 1993080832 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003201765295887663, + "loss": 1.2796, + "theoretical_loss": 3.4598426828517335, + "tokens_seen": 1993146368 + }, + { + "epoch": 0.6, + "learning_rate": 0.000320160481444333, + "loss": 1.2535, + "theoretical_loss": 3.4598332695200984, + "tokens_seen": 1993211904 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032014443329989967, + "loss": 1.3066, + "theoretical_loss": 3.459823856584623, + "tokens_seen": 1993277440 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032012838515546637, + "loss": 1.2186, + "theoretical_loss": 3.4598144440452776, + "tokens_seen": 1993342976 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003201123370110331, + "loss": 1.1672, + "theoretical_loss": 3.459805031902033, + "tokens_seen": 1993408512 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003200962888665998, + "loss": 1.1741, + "theoretical_loss": 3.459795620154859, + "tokens_seen": 1993474048 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003200802407221665, + "loss": 1.2711, + "theoretical_loss": 3.4597862088037266, + "tokens_seen": 1993539584 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003200641925777332, + "loss": 1.1859, + "theoretical_loss": 3.4597767978486065, + "tokens_seen": 1993605120 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003200481444332999, + "loss": 1.2443, + "theoretical_loss": 3.4597673872894674, + "tokens_seen": 1993670656 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003200320962888666, + "loss": 1.2957, + "theoretical_loss": 3.459757977126281, + "tokens_seen": 1993736192 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003200160481444333, + "loss": 1.3592, + "theoretical_loss": 3.4597485673590165, + "tokens_seen": 1993801728 + }, + { + "epoch": 0.6, + "learning_rate": 0.00032, + "loss": 1.2959, + "theoretical_loss": 3.459739157987645, + "tokens_seen": 1993867264 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003199839518555667, + "loss": 1.2654, + "theoretical_loss": 3.459729749012137, + "tokens_seen": 1993932800 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031996790371113343, + "loss": 1.3333, + "theoretical_loss": 3.4597203404324626, + "tokens_seen": 1993998336 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031995185556670013, + "loss": 1.2235, + "theoretical_loss": 3.459710932248592, + "tokens_seen": 1994063872 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031993580742226683, + "loss": 1.2287, + "theoretical_loss": 3.4597015244604954, + "tokens_seen": 1994129408 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031991975927783353, + "loss": 1.3032, + "theoretical_loss": 3.459692117068144, + "tokens_seen": 1994194944 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031990371113340023, + "loss": 1.2327, + "theoretical_loss": 3.459682710071507, + "tokens_seen": 1994260480 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031988766298896693, + "loss": 1.3139, + "theoretical_loss": 3.459673303470555, + "tokens_seen": 1994326016 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031987161484453363, + "loss": 1.1908, + "theoretical_loss": 3.4596638972652594, + "tokens_seen": 1994391552 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031985556670010033, + "loss": 1.26, + "theoretical_loss": 3.4596544914555887, + "tokens_seen": 1994457088 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031983951855566703, + "loss": 1.3166, + "theoretical_loss": 3.4596450860415153, + "tokens_seen": 1994522624 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031982347041123373, + "loss": 1.2388, + "theoretical_loss": 3.4596356810230082, + "tokens_seen": 1994588160 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031980742226680043, + "loss": 1.1927, + "theoretical_loss": 3.459626276400038, + "tokens_seen": 1994653696 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031979137412236713, + "loss": 1.3629, + "theoretical_loss": 3.4596168721725755, + "tokens_seen": 1994719232 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031977532597793384, + "loss": 1.3459, + "theoretical_loss": 3.459607468340591, + "tokens_seen": 1994784768 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031975927783350054, + "loss": 1.3522, + "theoretical_loss": 3.4595980649040543, + "tokens_seen": 1994850304 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031974322968906724, + "loss": 1.2396, + "theoretical_loss": 3.459588661862936, + "tokens_seen": 1994915840 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031972718154463394, + "loss": 1.2453, + "theoretical_loss": 3.459579259217207, + "tokens_seen": 1994981376 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031971113340020064, + "loss": 1.2894, + "theoretical_loss": 3.459569856966837, + "tokens_seen": 1995046912 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031969508525576734, + "loss": 1.226, + "theoretical_loss": 3.4595604551117973, + "tokens_seen": 1995112448 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031967903711133404, + "loss": 1.3006, + "theoretical_loss": 3.4595510536520573, + "tokens_seen": 1995177984 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031966298896690074, + "loss": 1.2572, + "theoretical_loss": 3.459541652587588, + "tokens_seen": 1995243520 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031964694082246744, + "loss": 1.2319, + "theoretical_loss": 3.4595322519183593, + "tokens_seen": 1995309056 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031963089267803414, + "loss": 1.2981, + "theoretical_loss": 3.4595228516443424, + "tokens_seen": 1995374592 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031961484453360084, + "loss": 1.2888, + "theoretical_loss": 3.459513451765507, + "tokens_seen": 1995440128 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031959879638916754, + "loss": 1.3664, + "theoretical_loss": 3.4595040522818232, + "tokens_seen": 1995505664 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031958274824473424, + "loss": 1.2444, + "theoretical_loss": 3.4594946531932624, + "tokens_seen": 1995571200 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031956670010030094, + "loss": 1.127, + "theoretical_loss": 3.459485254499795, + "tokens_seen": 1995636736 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031955065195586764, + "loss": 1.3176, + "theoretical_loss": 3.4594758562013905, + "tokens_seen": 1995702272 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031953460381143434, + "loss": 1.2219, + "theoretical_loss": 3.45946645829802, + "tokens_seen": 1995767808 + }, + { + "epoch": 0.6, + "learning_rate": 0.00031951855566700105, + "loss": 1.3396, + "theoretical_loss": 3.4594570607896538, + "tokens_seen": 1995833344 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003195025075225677, + "loss": 1.3509, + "theoretical_loss": 3.4594476636762614, + "tokens_seen": 1995898880 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003194864593781344, + "loss": 1.2587, + "theoretical_loss": 3.459438266957815, + "tokens_seen": 1995964416 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003194704112337011, + "loss": 1.3032, + "theoretical_loss": 3.4594288706342837, + "tokens_seen": 1996029952 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003194543630892678, + "loss": 1.2743, + "theoretical_loss": 3.459419474705639, + "tokens_seen": 1996095488 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003194383149448345, + "loss": 1.2939, + "theoretical_loss": 3.4594100791718496, + "tokens_seen": 1996161024 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003194222668004012, + "loss": 1.3698, + "theoretical_loss": 3.4594006840328877, + "tokens_seen": 1996226560 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003194062186559679, + "loss": 1.2141, + "theoretical_loss": 3.459391289288723, + "tokens_seen": 1996292096 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003193901705115346, + "loss": 1.3338, + "theoretical_loss": 3.459381894939326, + "tokens_seen": 1996357632 + }, + { + "epoch": 0.6, + "learning_rate": 0.0003193741223671013, + "loss": 1.3759, + "theoretical_loss": 3.459372500984667, + "tokens_seen": 1996423168 + }, + { + "epoch": 0.6, + "learning_rate": 0.000319358074222668, + "loss": 1.241, + "theoretical_loss": 3.459363107424717, + "tokens_seen": 1996488704 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003193420260782347, + "loss": 1.2835, + "theoretical_loss": 3.459353714259446, + "tokens_seen": 1996554240 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003193259779338014, + "loss": 1.3492, + "theoretical_loss": 3.4593443214888246, + "tokens_seen": 1996619776 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003193099297893681, + "loss": 1.2113, + "theoretical_loss": 3.4593349291128233, + "tokens_seen": 1996685312 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003192938816449348, + "loss": 1.3885, + "theoretical_loss": 3.459325537131412, + "tokens_seen": 1996750848 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003192778335005015, + "loss": 1.4077, + "theoretical_loss": 3.459316145544562, + "tokens_seen": 1996816384 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003192617853560682, + "loss": 1.2605, + "theoretical_loss": 3.4593067543522436, + "tokens_seen": 1996881920 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003192457372116349, + "loss": 1.2925, + "theoretical_loss": 3.459297363554427, + "tokens_seen": 1996947456 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003192296890672016, + "loss": 1.2156, + "theoretical_loss": 3.4592879731510826, + "tokens_seen": 1997012992 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003192136409227683, + "loss": 1.2916, + "theoretical_loss": 3.459278583142181, + "tokens_seen": 1997078528 + }, + { + "epoch": 0.61, + "learning_rate": 0.000319197592778335, + "loss": 1.1914, + "theoretical_loss": 3.4592691935276934, + "tokens_seen": 1997144064 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003191815446339017, + "loss": 1.3507, + "theoretical_loss": 3.4592598043075893, + "tokens_seen": 1997209600 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003191654964894684, + "loss": 1.2118, + "theoretical_loss": 3.4592504154818395, + "tokens_seen": 1997275136 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003191494483450351, + "loss": 1.2432, + "theoretical_loss": 3.459241027050415, + "tokens_seen": 1997340672 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003191334002006018, + "loss": 1.1282, + "theoretical_loss": 3.4592316390132853, + "tokens_seen": 1997406208 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003191173520561685, + "loss": 1.3246, + "theoretical_loss": 3.459222251370422, + "tokens_seen": 1997471744 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003191013039117352, + "loss": 1.3008, + "theoretical_loss": 3.4592128641217945, + "tokens_seen": 1997537280 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003190852557673019, + "loss": 1.3357, + "theoretical_loss": 3.459203477267374, + "tokens_seen": 1997602816 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003190692076228686, + "loss": 1.2207, + "theoretical_loss": 3.459194090807131, + "tokens_seen": 1997668352 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003190531594784353, + "loss": 1.2742, + "theoretical_loss": 3.4591847047410362, + "tokens_seen": 1997733888 + }, + { + "epoch": 0.61, + "learning_rate": 0.000319037111334002, + "loss": 1.3043, + "theoretical_loss": 3.4591753190690593, + "tokens_seen": 1997799424 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003190210631895687, + "loss": 1.2694, + "theoretical_loss": 3.459165933791172, + "tokens_seen": 1997864960 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003190050150451354, + "loss": 1.3677, + "theoretical_loss": 3.4591565489073437, + "tokens_seen": 1997930496 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003189889669007021, + "loss": 1.3346, + "theoretical_loss": 3.4591471644175456, + "tokens_seen": 1997996032 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003189729187562688, + "loss": 1.2043, + "theoretical_loss": 3.4591377803217482, + "tokens_seen": 1998061568 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003189568706118355, + "loss": 1.2776, + "theoretical_loss": 3.4591283966199216, + "tokens_seen": 1998127104 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003189408224674022, + "loss": 1.2928, + "theoretical_loss": 3.4591190133120366, + "tokens_seen": 1998192640 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003189247743229689, + "loss": 1.3398, + "theoretical_loss": 3.459109630398064, + "tokens_seen": 1998258176 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031890872617853567, + "loss": 1.2458, + "theoretical_loss": 3.4591002478779744, + "tokens_seen": 1998323712 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031889267803410237, + "loss": 1.2508, + "theoretical_loss": 3.4590908657517376, + "tokens_seen": 1998389248 + }, + { + "epoch": 0.61, + "learning_rate": 0.000318876629889669, + "loss": 1.2242, + "theoretical_loss": 3.4590814840193245, + "tokens_seen": 1998454784 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003188605817452357, + "loss": 1.2289, + "theoretical_loss": 3.459072102680706, + "tokens_seen": 1998520320 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003188445336008024, + "loss": 1.3504, + "theoretical_loss": 3.4590627217358523, + "tokens_seen": 1998585856 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003188284854563691, + "loss": 1.1945, + "theoretical_loss": 3.4590533411847346, + "tokens_seen": 1998651392 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003188124373119358, + "loss": 1.2568, + "theoretical_loss": 3.459043961027322, + "tokens_seen": 1998716928 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003187963891675025, + "loss": 1.3015, + "theoretical_loss": 3.4590345812635865, + "tokens_seen": 1998782464 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 739305, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2543718814849854, + "objective/train/theoretical_loss": 3.459025201893499, + "objective/train/tokens_used": 2019308000, + "theoretical_loss": 3.459025201893499, + "tokens_seen": 1998848000 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003187803410230692, + "loss": 1.2544, + "theoretical_loss": 3.459025201893499, + "tokens_seen": 1998848000 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003187642928786359, + "loss": 1.2744, + "theoretical_loss": 3.459015822917028, + "tokens_seen": 1998913536 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003187482447342026, + "loss": 1.2648, + "theoretical_loss": 3.459006444334146, + "tokens_seen": 1998979072 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003187321965897693, + "loss": 1.3072, + "theoretical_loss": 3.4589970661448226, + "tokens_seen": 1999044608 + }, + { + "epoch": 0.61, + "learning_rate": 0.000318716148445336, + "loss": 1.122, + "theoretical_loss": 3.458987688349029, + "tokens_seen": 1999110144 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003187001003009027, + "loss": 1.3077, + "theoretical_loss": 3.458978310946735, + "tokens_seen": 1999175680 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003186840521564694, + "loss": 1.2983, + "theoretical_loss": 3.4589689339379124, + "tokens_seen": 1999241216 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003186680040120361, + "loss": 1.3442, + "theoretical_loss": 3.4589595573225305, + "tokens_seen": 1999306752 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003186519558676028, + "loss": 1.3134, + "theoretical_loss": 3.458950181100561, + "tokens_seen": 1999372288 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003186359077231695, + "loss": 1.3112, + "theoretical_loss": 3.4589408052719737, + "tokens_seen": 1999437824 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003186198595787362, + "loss": 1.223, + "theoretical_loss": 3.4589314298367393, + "tokens_seen": 1999503360 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031860381143430293, + "loss": 1.2787, + "theoretical_loss": 3.4589220547948285, + "tokens_seen": 1999568896 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031858776328986963, + "loss": 1.2126, + "theoretical_loss": 3.4589126801462124, + "tokens_seen": 1999634432 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031857171514543633, + "loss": 1.281, + "theoretical_loss": 3.458903305890861, + "tokens_seen": 1999699968 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031855566700100303, + "loss": 1.2313, + "theoretical_loss": 3.4588939320287455, + "tokens_seen": 1999765504 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031853961885656973, + "loss": 1.278, + "theoretical_loss": 3.458884558559835, + "tokens_seen": 1999831040 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031852357071213643, + "loss": 1.3305, + "theoretical_loss": 3.4588751854841027, + "tokens_seen": 1999896576 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031850752256770313, + "loss": 1.2811, + "theoretical_loss": 3.4588658128015166, + "tokens_seen": 1999962112 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031849147442326983, + "loss": 1.2382, + "theoretical_loss": 3.458856440512049, + "tokens_seen": 2000027648 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031847542627883653, + "loss": 1.38, + "theoretical_loss": 3.45884706861567, + "tokens_seen": 2000093184 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031845937813440323, + "loss": 1.2188, + "theoretical_loss": 3.4588376971123505, + "tokens_seen": 2000158720 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031844332998996993, + "loss": 1.2508, + "theoretical_loss": 3.4588283260020605, + "tokens_seen": 2000224256 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031842728184553663, + "loss": 1.2635, + "theoretical_loss": 3.4588189552847712, + "tokens_seen": 2000289792 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031841123370110333, + "loss": 1.2964, + "theoretical_loss": 3.458809584960453, + "tokens_seen": 2000355328 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031839518555667004, + "loss": 1.2966, + "theoretical_loss": 3.458800215029077, + "tokens_seen": 2000420864 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031837913741223674, + "loss": 1.315, + "theoretical_loss": 3.458790845490613, + "tokens_seen": 2000486400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031836308926780344, + "loss": 1.233, + "theoretical_loss": 3.458781476345033, + "tokens_seen": 2000551936 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031834704112337014, + "loss": 1.1692, + "theoretical_loss": 3.4587721075923055, + "tokens_seen": 2000617472 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031833099297893684, + "loss": 1.2239, + "theoretical_loss": 3.4587627392324034, + "tokens_seen": 2000683008 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031831494483450354, + "loss": 1.3584, + "theoretical_loss": 3.458753371265296, + "tokens_seen": 2000748544 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031829889669007024, + "loss": 1.2326, + "theoretical_loss": 3.4587440036909545, + "tokens_seen": 2000814080 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031828284854563694, + "loss": 1.3549, + "theoretical_loss": 3.4587346365093494, + "tokens_seen": 2000879616 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031826680040120364, + "loss": 1.2319, + "theoretical_loss": 3.4587252697204516, + "tokens_seen": 2000945152 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031825075225677034, + "loss": 1.356, + "theoretical_loss": 3.458715903324231, + "tokens_seen": 2001010688 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031823470411233704, + "loss": 1.2303, + "theoretical_loss": 3.45870653732066, + "tokens_seen": 2001076224 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031821865596790374, + "loss": 1.216, + "theoretical_loss": 3.4586971717097077, + "tokens_seen": 2001141760 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031820260782347044, + "loss": 1.3014, + "theoretical_loss": 3.4586878064913447, + "tokens_seen": 2001207296 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031818655967903714, + "loss": 1.1893, + "theoretical_loss": 3.4586784416655423, + "tokens_seen": 2001272832 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031817051153460384, + "loss": 1.3145, + "theoretical_loss": 3.4586690772322717, + "tokens_seen": 2001338368 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031815446339017054, + "loss": 1.3652, + "theoretical_loss": 3.4586597131915027, + "tokens_seen": 2001403904 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031813841524573725, + "loss": 1.3416, + "theoretical_loss": 3.4586503495432064, + "tokens_seen": 2001469440 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031812236710130395, + "loss": 1.1794, + "theoretical_loss": 3.458640986287353, + "tokens_seen": 2001534976 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003181063189568706, + "loss": 1.3601, + "theoretical_loss": 3.4586316234239143, + "tokens_seen": 2001600512 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003180902708124373, + "loss": 1.1954, + "theoretical_loss": 3.45862226095286, + "tokens_seen": 2001666048 + }, + { + "epoch": 0.61, + "learning_rate": 0.000318074222668004, + "loss": 1.3426, + "theoretical_loss": 3.458612898874161, + "tokens_seen": 2001731584 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003180581745235707, + "loss": 1.4262, + "theoretical_loss": 3.458603537187788, + "tokens_seen": 2001797120 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003180421263791374, + "loss": 1.3272, + "theoretical_loss": 3.458594175893712, + "tokens_seen": 2001862656 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003180260782347041, + "loss": 1.3773, + "theoretical_loss": 3.458584814991904, + "tokens_seen": 2001928192 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003180100300902708, + "loss": 1.3031, + "theoretical_loss": 3.4585754544823337, + "tokens_seen": 2001993728 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003179939819458375, + "loss": 1.4019, + "theoretical_loss": 3.458566094364973, + "tokens_seen": 2002059264 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003179779338014042, + "loss": 1.3251, + "theoretical_loss": 3.4585567346397914, + "tokens_seen": 2002124800 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003179618856569709, + "loss": 1.2274, + "theoretical_loss": 3.458547375306761, + "tokens_seen": 2002190336 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003179458375125376, + "loss": 1.3004, + "theoretical_loss": 3.4585380163658512, + "tokens_seen": 2002255872 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003179297893681043, + "loss": 1.2329, + "theoretical_loss": 3.4585286578170336, + "tokens_seen": 2002321408 + }, + { + "epoch": 0.61, + "learning_rate": 0.000317913741223671, + "loss": 1.2389, + "theoretical_loss": 3.4585192996602787, + "tokens_seen": 2002386944 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003178976930792377, + "loss": 1.3032, + "theoretical_loss": 3.4585099418955574, + "tokens_seen": 2002452480 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003178816449348044, + "loss": 1.2607, + "theoretical_loss": 3.45850058452284, + "tokens_seen": 2002518016 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003178655967903711, + "loss": 1.2743, + "theoretical_loss": 3.4584912275420976, + "tokens_seen": 2002583552 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003178495486459378, + "loss": 1.2058, + "theoretical_loss": 3.458481870953301, + "tokens_seen": 2002649088 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003178335005015045, + "loss": 1.2346, + "theoretical_loss": 3.4584725147564206, + "tokens_seen": 2002714624 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003178174523570712, + "loss": 1.2213, + "theoretical_loss": 3.4584631589514276, + "tokens_seen": 2002780160 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003178014042126379, + "loss": 1.2964, + "theoretical_loss": 3.4584538035382923, + "tokens_seen": 2002845696 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003177853560682046, + "loss": 1.2133, + "theoretical_loss": 3.4584444485169863, + "tokens_seen": 2002911232 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003177693079237713, + "loss": 1.3787, + "theoretical_loss": 3.4584350938874797, + "tokens_seen": 2002976768 + }, + { + "epoch": 0.61, + "learning_rate": 0.000317753259779338, + "loss": 1.2598, + "theoretical_loss": 3.4584257396497433, + "tokens_seen": 2003042304 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003177372116349047, + "loss": 1.3108, + "theoretical_loss": 3.4584163858037478, + "tokens_seen": 2003107840 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003177211634904714, + "loss": 1.3231, + "theoretical_loss": 3.458407032349464, + "tokens_seen": 2003173376 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031770511534603816, + "loss": 1.2226, + "theoretical_loss": 3.458397679286863, + "tokens_seen": 2003238912 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031768906720160486, + "loss": 1.3067, + "theoretical_loss": 3.4583883266159154, + "tokens_seen": 2003304448 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031767301905717156, + "loss": 1.312, + "theoretical_loss": 3.458378974336592, + "tokens_seen": 2003369984 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031765697091273826, + "loss": 1.2919, + "theoretical_loss": 3.4583696224488634, + "tokens_seen": 2003435520 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031764092276830497, + "loss": 1.3702, + "theoretical_loss": 3.458360270952701, + "tokens_seen": 2003501056 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031762487462387167, + "loss": 1.306, + "theoretical_loss": 3.4583509198480744, + "tokens_seen": 2003566592 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031760882647943837, + "loss": 1.2725, + "theoretical_loss": 3.458341569134956, + "tokens_seen": 2003632128 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031759277833500507, + "loss": 1.2994, + "theoretical_loss": 3.4583322188133154, + "tokens_seen": 2003697664 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031757673019057177, + "loss": 1.3138, + "theoretical_loss": 3.458322868883124, + "tokens_seen": 2003763200 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031756068204613847, + "loss": 1.31, + "theoretical_loss": 3.458313519344352, + "tokens_seen": 2003828736 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031754463390170517, + "loss": 1.2328, + "theoretical_loss": 3.4583041701969712, + "tokens_seen": 2003894272 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031752858575727187, + "loss": 1.2708, + "theoretical_loss": 3.4582948214409512, + "tokens_seen": 2003959808 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031751253761283857, + "loss": 1.1165, + "theoretical_loss": 3.4582854730762635, + "tokens_seen": 2004025344 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031749648946840527, + "loss": 1.2844, + "theoretical_loss": 3.4582761251028793, + "tokens_seen": 2004090880 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003174804413239719, + "loss": 1.3743, + "theoretical_loss": 3.4582667775207687, + "tokens_seen": 2004156416 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003174643931795386, + "loss": 1.2508, + "theoretical_loss": 3.458257430329903, + "tokens_seen": 2004221952 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003174483450351053, + "loss": 1.2908, + "theoretical_loss": 3.4582480835302527, + "tokens_seen": 2004287488 + }, + { + "epoch": 0.61, + "learning_rate": 0.000317432296890672, + "loss": 1.2689, + "theoretical_loss": 3.458238737121789, + "tokens_seen": 2004353024 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003174162487462387, + "loss": 1.281, + "theoretical_loss": 3.4582293911044824, + "tokens_seen": 2004418560 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003174002006018054, + "loss": 1.2874, + "theoretical_loss": 3.4582200454783036, + "tokens_seen": 2004484096 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003173841524573721, + "loss": 1.189, + "theoretical_loss": 3.4582107002432236, + "tokens_seen": 2004549632 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003173681043129388, + "loss": 1.2939, + "theoretical_loss": 3.458201355399214, + "tokens_seen": 2004615168 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003173520561685055, + "loss": 1.2081, + "theoretical_loss": 3.4581920109462443, + "tokens_seen": 2004680704 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003173360080240722, + "loss": 1.2949, + "theoretical_loss": 3.4581826668842863, + "tokens_seen": 2004746240 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003173199598796389, + "loss": 1.2299, + "theoretical_loss": 3.4581733232133107, + "tokens_seen": 2004811776 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003173039117352056, + "loss": 1.2939, + "theoretical_loss": 3.4581639799332886, + "tokens_seen": 2004877312 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003172878635907723, + "loss": 1.211, + "theoretical_loss": 3.4581546370441902, + "tokens_seen": 2004942848 + }, + { + "epoch": 0.61, + "learning_rate": 0.000317271815446339, + "loss": 1.3901, + "theoretical_loss": 3.4581452945459867, + "tokens_seen": 2005008384 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003172557673019057, + "loss": 1.239, + "theoretical_loss": 3.4581359524386492, + "tokens_seen": 2005073920 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003172397191574724, + "loss": 1.2903, + "theoretical_loss": 3.4581266107221484, + "tokens_seen": 2005139456 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031722367101303913, + "loss": 1.2795, + "theoretical_loss": 3.458117269396455, + "tokens_seen": 2005204992 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031720762286860583, + "loss": 1.1842, + "theoretical_loss": 3.4581079284615397, + "tokens_seen": 2005270528 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031719157472417253, + "loss": 1.2897, + "theoretical_loss": 3.458098587917374, + "tokens_seen": 2005336064 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 741632, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.268796443939209, + "objective/train/theoretical_loss": 3.4580892477639282, + "objective/train/tokens_used": 2025861600, + "theoretical_loss": 3.4580892477639282, + "tokens_seen": 2005401600 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031717552657973923, + "loss": 1.2688, + "theoretical_loss": 3.4580892477639282, + "tokens_seen": 2005401600 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031715947843530593, + "loss": 1.3568, + "theoretical_loss": 3.4580799080011735, + "tokens_seen": 2005467136 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031714343029087263, + "loss": 1.2693, + "theoretical_loss": 3.458070568629081, + "tokens_seen": 2005532672 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031712738214643933, + "loss": 1.2991, + "theoretical_loss": 3.4580612296476207, + "tokens_seen": 2005598208 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031711133400200603, + "loss": 1.3053, + "theoretical_loss": 3.458051891056765, + "tokens_seen": 2005663744 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031709528585757273, + "loss": 1.1906, + "theoretical_loss": 3.458042552856484, + "tokens_seen": 2005729280 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031707923771313943, + "loss": 1.2271, + "theoretical_loss": 3.458033215046748, + "tokens_seen": 2005794816 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031706318956870613, + "loss": 1.3801, + "theoretical_loss": 3.4580238776275283, + "tokens_seen": 2005860352 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031704714142427283, + "loss": 1.3002, + "theoretical_loss": 3.4580145405987963, + "tokens_seen": 2005925888 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031703109327983954, + "loss": 1.1843, + "theoretical_loss": 3.4580052039605222, + "tokens_seen": 2005991424 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031701504513540624, + "loss": 1.3477, + "theoretical_loss": 3.457995867712678, + "tokens_seen": 2006056960 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031699899699097294, + "loss": 1.1914, + "theoretical_loss": 3.4579865318552336, + "tokens_seen": 2006122496 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031698294884653964, + "loss": 1.2495, + "theoretical_loss": 3.45797719638816, + "tokens_seen": 2006188032 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031696690070210634, + "loss": 1.2119, + "theoretical_loss": 3.4579678613114284, + "tokens_seen": 2006253568 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031695085255767304, + "loss": 1.3237, + "theoretical_loss": 3.45795852662501, + "tokens_seen": 2006319104 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031693480441323974, + "loss": 1.2658, + "theoretical_loss": 3.457949192328875, + "tokens_seen": 2006384640 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031691875626880644, + "loss": 1.3272, + "theoretical_loss": 3.4579398584229954, + "tokens_seen": 2006450176 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031690270812437314, + "loss": 1.3099, + "theoretical_loss": 3.457930524907341, + "tokens_seen": 2006515712 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031688665997993984, + "loss": 1.2127, + "theoretical_loss": 3.457921191781883, + "tokens_seen": 2006581248 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031687061183550654, + "loss": 1.231, + "theoretical_loss": 3.457911859046593, + "tokens_seen": 2006646784 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031685456369107324, + "loss": 1.2764, + "theoretical_loss": 3.4579025267014414, + "tokens_seen": 2006712320 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031683851554663994, + "loss": 1.2323, + "theoretical_loss": 3.4578931947464, + "tokens_seen": 2006777856 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031682246740220664, + "loss": 1.3127, + "theoretical_loss": 3.4578838631814377, + "tokens_seen": 2006843392 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031680641925777334, + "loss": 1.3427, + "theoretical_loss": 3.457874532006527, + "tokens_seen": 2006908928 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031679037111334004, + "loss": 1.3209, + "theoretical_loss": 3.4578652012216393, + "tokens_seen": 2006974464 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031677432296890675, + "loss": 1.3368, + "theoretical_loss": 3.457855870826745, + "tokens_seen": 2007040000 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031675827482447345, + "loss": 1.3326, + "theoretical_loss": 3.4578465408218144, + "tokens_seen": 2007105536 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031674222668004015, + "loss": 1.2934, + "theoretical_loss": 3.4578372112068196, + "tokens_seen": 2007171072 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031672617853560685, + "loss": 1.3208, + "theoretical_loss": 3.4578278819817303, + "tokens_seen": 2007236608 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003167101303911735, + "loss": 1.2803, + "theoretical_loss": 3.4578185531465184, + "tokens_seen": 2007302144 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003166940822467402, + "loss": 1.3325, + "theoretical_loss": 3.457809224701155, + "tokens_seen": 2007367680 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003166780341023069, + "loss": 1.2606, + "theoretical_loss": 3.4577998966456107, + "tokens_seen": 2007433216 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003166619859578736, + "loss": 1.1897, + "theoretical_loss": 3.4577905689798563, + "tokens_seen": 2007498752 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003166459378134403, + "loss": 1.2684, + "theoretical_loss": 3.4577812417038634, + "tokens_seen": 2007564288 + }, + { + "epoch": 0.61, + "learning_rate": 0.000316629889669007, + "loss": 1.2181, + "theoretical_loss": 3.4577719148176023, + "tokens_seen": 2007629824 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003166138415245737, + "loss": 1.2563, + "theoretical_loss": 3.457762588321044, + "tokens_seen": 2007695360 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003165977933801404, + "loss": 1.2793, + "theoretical_loss": 3.4577532622141605, + "tokens_seen": 2007760896 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003165817452357071, + "loss": 1.2137, + "theoretical_loss": 3.4577439364969216, + "tokens_seen": 2007826432 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003165656970912738, + "loss": 1.4274, + "theoretical_loss": 3.4577346111692995, + "tokens_seen": 2007891968 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003165496489468405, + "loss": 1.1953, + "theoretical_loss": 3.4577252862312635, + "tokens_seen": 2007957504 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003165336008024072, + "loss": 1.2833, + "theoretical_loss": 3.457715961682786, + "tokens_seen": 2008023040 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003165175526579739, + "loss": 1.2279, + "theoretical_loss": 3.457706637523838, + "tokens_seen": 2008088576 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031650150451354066, + "loss": 1.2377, + "theoretical_loss": 3.45769731375439, + "tokens_seen": 2008154112 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031648545636910736, + "loss": 1.2616, + "theoretical_loss": 3.457687990374413, + "tokens_seen": 2008219648 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031646940822467406, + "loss": 1.2991, + "theoretical_loss": 3.4576786673838784, + "tokens_seen": 2008285184 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031645336008024076, + "loss": 1.2217, + "theoretical_loss": 3.457669344782757, + "tokens_seen": 2008350720 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031643731193580746, + "loss": 1.2495, + "theoretical_loss": 3.4576600225710195, + "tokens_seen": 2008416256 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031642126379137416, + "loss": 1.1789, + "theoretical_loss": 3.457650700748638, + "tokens_seen": 2008481792 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031640521564694086, + "loss": 1.2351, + "theoretical_loss": 3.4576413793155822, + "tokens_seen": 2008547328 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031638916750250756, + "loss": 1.2401, + "theoretical_loss": 3.457632058271824, + "tokens_seen": 2008612864 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031637311935807426, + "loss": 1.3336, + "theoretical_loss": 3.457622737617334, + "tokens_seen": 2008678400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031635707121364096, + "loss": 1.2722, + "theoretical_loss": 3.4576134173520834, + "tokens_seen": 2008743936 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031634102306920766, + "loss": 1.3046, + "theoretical_loss": 3.457604097476043, + "tokens_seen": 2008809472 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031632497492477436, + "loss": 1.1686, + "theoretical_loss": 3.457594777989185, + "tokens_seen": 2008875008 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031630892678034106, + "loss": 1.4153, + "theoretical_loss": 3.457585458891479, + "tokens_seen": 2008940544 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031629287863590776, + "loss": 1.28, + "theoretical_loss": 3.457576140182897, + "tokens_seen": 2009006080 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031627683049147446, + "loss": 1.2321, + "theoretical_loss": 3.4575668218634092, + "tokens_seen": 2009071616 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031626078234704117, + "loss": 1.1976, + "theoretical_loss": 3.4575575039329873, + "tokens_seen": 2009137152 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031624473420260787, + "loss": 1.328, + "theoretical_loss": 3.457548186391602, + "tokens_seen": 2009202688 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031622868605817457, + "loss": 1.2257, + "theoretical_loss": 3.457538869239225, + "tokens_seen": 2009268224 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031621263791374127, + "loss": 1.3402, + "theoretical_loss": 3.457529552475827, + "tokens_seen": 2009333760 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031619658976930797, + "loss": 1.3134, + "theoretical_loss": 3.457520236101379, + "tokens_seen": 2009399296 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031618054162487467, + "loss": 1.2435, + "theoretical_loss": 3.457510920115852, + "tokens_seen": 2009464832 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031616449348044137, + "loss": 1.36, + "theoretical_loss": 3.4575016045192175, + "tokens_seen": 2009530368 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031614844533600807, + "loss": 1.2465, + "theoretical_loss": 3.457492289311446, + "tokens_seen": 2009595904 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031613239719157477, + "loss": 1.3923, + "theoretical_loss": 3.4574829744925086, + "tokens_seen": 2009661440 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031611634904714147, + "loss": 1.2968, + "theoretical_loss": 3.4574736600623774, + "tokens_seen": 2009726976 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031610030090270817, + "loss": 1.2672, + "theoretical_loss": 3.457464346021022, + "tokens_seen": 2009792512 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031608425275827487, + "loss": 1.3641, + "theoretical_loss": 3.4574550323684146, + "tokens_seen": 2009858048 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003160682046138415, + "loss": 1.1842, + "theoretical_loss": 3.457445719104526, + "tokens_seen": 2009923584 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003160521564694082, + "loss": 1.2718, + "theoretical_loss": 3.457436406229327, + "tokens_seen": 2009989120 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003160361083249749, + "loss": 1.2549, + "theoretical_loss": 3.457427093742789, + "tokens_seen": 2010054656 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003160200601805416, + "loss": 1.3155, + "theoretical_loss": 3.4574177816448834, + "tokens_seen": 2010120192 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003160040120361083, + "loss": 1.3007, + "theoretical_loss": 3.4574084699355807, + "tokens_seen": 2010185728 + }, + { + "epoch": 0.61, + "learning_rate": 0.000315987963891675, + "loss": 1.206, + "theoretical_loss": 3.4573991586148525, + "tokens_seen": 2010251264 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003159719157472417, + "loss": 1.1863, + "theoretical_loss": 3.4573898476826694, + "tokens_seen": 2010316800 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003159558676028084, + "loss": 1.2985, + "theoretical_loss": 3.457380537139003, + "tokens_seen": 2010382336 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003159398194583751, + "loss": 1.1963, + "theoretical_loss": 3.457371226983824, + "tokens_seen": 2010447872 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003159237713139418, + "loss": 1.3541, + "theoretical_loss": 3.457361917217104, + "tokens_seen": 2010513408 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003159077231695085, + "loss": 1.2367, + "theoretical_loss": 3.457352607838814, + "tokens_seen": 2010578944 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003158916750250752, + "loss": 1.2755, + "theoretical_loss": 3.457343298848925, + "tokens_seen": 2010644480 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003158756268806419, + "loss": 1.3378, + "theoretical_loss": 3.4573339902474083, + "tokens_seen": 2010710016 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031585957873620863, + "loss": 1.2712, + "theoretical_loss": 3.4573246820342347, + "tokens_seen": 2010775552 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031584353059177533, + "loss": 1.3906, + "theoretical_loss": 3.457315374209376, + "tokens_seen": 2010841088 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031582748244734203, + "loss": 1.2473, + "theoretical_loss": 3.4573060667728024, + "tokens_seen": 2010906624 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031581143430290873, + "loss": 1.215, + "theoretical_loss": 3.4572967597244855, + "tokens_seen": 2010972160 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031579538615847543, + "loss": 1.4459, + "theoretical_loss": 3.4572874530643967, + "tokens_seen": 2011037696 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031577933801404213, + "loss": 1.2933, + "theoretical_loss": 3.4572781467925067, + "tokens_seen": 2011103232 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031576328986960883, + "loss": 1.2848, + "theoretical_loss": 3.4572688409087875, + "tokens_seen": 2011168768 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031574724172517553, + "loss": 1.1959, + "theoretical_loss": 3.4572595354132094, + "tokens_seen": 2011234304 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031573119358074223, + "loss": 1.2002, + "theoretical_loss": 3.4572502303057435, + "tokens_seen": 2011299840 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031571514543630893, + "loss": 1.3341, + "theoretical_loss": 3.457240925586362, + "tokens_seen": 2011365376 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031569909729187563, + "loss": 1.2761, + "theoretical_loss": 3.457231621255035, + "tokens_seen": 2011430912 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031568304914744233, + "loss": 1.3832, + "theoretical_loss": 3.457222317311734, + "tokens_seen": 2011496448 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031566700100300903, + "loss": 1.374, + "theoretical_loss": 3.45721301375643, + "tokens_seen": 2011561984 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031565095285857574, + "loss": 1.3302, + "theoretical_loss": 3.457203710589095, + "tokens_seen": 2011627520 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031563490471414244, + "loss": 1.2931, + "theoretical_loss": 3.457194407809699, + "tokens_seen": 2011693056 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031561885656970914, + "loss": 1.3372, + "theoretical_loss": 3.4571851054182146, + "tokens_seen": 2011758592 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031560280842527584, + "loss": 1.3291, + "theoretical_loss": 3.4571758034146116, + "tokens_seen": 2011824128 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031558676028084254, + "loss": 1.2381, + "theoretical_loss": 3.457166501798862, + "tokens_seen": 2011889664 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 744364, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.247430682182312, + "objective/train/theoretical_loss": 3.4571572005709363, + "objective/train/tokens_used": 2032415200, + "theoretical_loss": 3.4571572005709363, + "tokens_seen": 2011955200 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031557071213640924, + "loss": 1.2474, + "theoretical_loss": 3.4571572005709363, + "tokens_seen": 2011955200 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031555466399197594, + "loss": 1.4138, + "theoretical_loss": 3.4571478997308063, + "tokens_seen": 2012020736 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031553861584754264, + "loss": 1.3697, + "theoretical_loss": 3.4571385992784434, + "tokens_seen": 2012086272 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031552256770310934, + "loss": 1.2973, + "theoretical_loss": 3.4571292992138183, + "tokens_seen": 2012151808 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031550651955867604, + "loss": 1.3064, + "theoretical_loss": 3.457119999536902, + "tokens_seen": 2012217344 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031549047141424274, + "loss": 1.2905, + "theoretical_loss": 3.4571107002476666, + "tokens_seen": 2012282880 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031547442326980944, + "loss": 1.3105, + "theoretical_loss": 3.457101401346083, + "tokens_seen": 2012348416 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031545837512537614, + "loss": 1.2567, + "theoretical_loss": 3.4570921028321218, + "tokens_seen": 2012413952 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031544232698094284, + "loss": 1.3771, + "theoretical_loss": 3.4570828047057542, + "tokens_seen": 2012479488 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031542627883650954, + "loss": 1.2832, + "theoretical_loss": 3.4570735069669523, + "tokens_seen": 2012545024 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031541023069207625, + "loss": 1.3052, + "theoretical_loss": 3.457064209615687, + "tokens_seen": 2012610560 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031539418254764295, + "loss": 1.2329, + "theoretical_loss": 3.4570549126519294, + "tokens_seen": 2012676096 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031537813440320965, + "loss": 1.2624, + "theoretical_loss": 3.457045616075651, + "tokens_seen": 2012741632 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031536208625877635, + "loss": 1.2637, + "theoretical_loss": 3.457036319886822, + "tokens_seen": 2012807168 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031534603811434305, + "loss": 1.2472, + "theoretical_loss": 3.4570270240854146, + "tokens_seen": 2012872704 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031532998996990975, + "loss": 1.3521, + "theoretical_loss": 3.4570177286714, + "tokens_seen": 2012938240 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031531394182547645, + "loss": 1.245, + "theoretical_loss": 3.4570084336447495, + "tokens_seen": 2013003776 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031529789368104315, + "loss": 1.2296, + "theoretical_loss": 3.4569991390054335, + "tokens_seen": 2013069312 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031528184553660985, + "loss": 1.2707, + "theoretical_loss": 3.4569898447534246, + "tokens_seen": 2013134848 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031526579739217655, + "loss": 1.2021, + "theoretical_loss": 3.4569805508886935, + "tokens_seen": 2013200384 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031524974924774325, + "loss": 1.3048, + "theoretical_loss": 3.456971257411211, + "tokens_seen": 2013265920 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031523370110330995, + "loss": 1.3649, + "theoretical_loss": 3.456961964320948, + "tokens_seen": 2013331456 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031521765295887665, + "loss": 1.3115, + "theoretical_loss": 3.456952671617877, + "tokens_seen": 2013396992 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031520160481444335, + "loss": 1.1685, + "theoretical_loss": 3.4569433793019693, + "tokens_seen": 2013462528 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031518555667001005, + "loss": 1.2237, + "theoretical_loss": 3.4569340873731944, + "tokens_seen": 2013528064 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031516950852557675, + "loss": 1.3532, + "theoretical_loss": 3.4569247958315255, + "tokens_seen": 2013593600 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031515346038114346, + "loss": 1.2192, + "theoretical_loss": 3.4569155046769326, + "tokens_seen": 2013659136 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031513741223671016, + "loss": 1.2496, + "theoretical_loss": 3.456906213909388, + "tokens_seen": 2013724672 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031512136409227686, + "loss": 1.2898, + "theoretical_loss": 3.4568969235288627, + "tokens_seen": 2013790208 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031510531594784356, + "loss": 1.3098, + "theoretical_loss": 3.456887633535327, + "tokens_seen": 2013855744 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031508926780341026, + "loss": 1.2898, + "theoretical_loss": 3.456878343928753, + "tokens_seen": 2013921280 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031507321965897696, + "loss": 1.2656, + "theoretical_loss": 3.456869054709112, + "tokens_seen": 2013986816 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031505717151454366, + "loss": 1.1588, + "theoretical_loss": 3.4568597658763753, + "tokens_seen": 2014052352 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031504112337011036, + "loss": 1.2436, + "theoretical_loss": 3.4568504774305144, + "tokens_seen": 2014117888 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031502507522567706, + "loss": 1.1702, + "theoretical_loss": 3.4568411893715, + "tokens_seen": 2014183424 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031500902708124376, + "loss": 1.2692, + "theoretical_loss": 3.456831901699304, + "tokens_seen": 2014248960 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031499297893681046, + "loss": 1.3248, + "theoretical_loss": 3.4568226144138974, + "tokens_seen": 2014314496 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031497693079237716, + "loss": 1.2589, + "theoretical_loss": 3.4568133275152513, + "tokens_seen": 2014380032 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031496088264794386, + "loss": 1.2597, + "theoretical_loss": 3.456804041003337, + "tokens_seen": 2014445568 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031494483450351056, + "loss": 1.2128, + "theoretical_loss": 3.4567947548781266, + "tokens_seen": 2014511104 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031492878635907726, + "loss": 1.2107, + "theoretical_loss": 3.4567854691395903, + "tokens_seen": 2014576640 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031491273821464396, + "loss": 1.2161, + "theoretical_loss": 3.4567761837877002, + "tokens_seen": 2014642176 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031489669007021067, + "loss": 1.215, + "theoretical_loss": 3.456766898822428, + "tokens_seen": 2014707712 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031488064192577737, + "loss": 1.0987, + "theoretical_loss": 3.456757614243744, + "tokens_seen": 2014773248 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031486459378134407, + "loss": 1.2806, + "theoretical_loss": 3.45674833005162, + "tokens_seen": 2014838784 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031484854563691077, + "loss": 1.195, + "theoretical_loss": 3.4567390462460272, + "tokens_seen": 2014904320 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031483249749247747, + "loss": 1.2144, + "theoretical_loss": 3.456729762826937, + "tokens_seen": 2014969856 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031481644934804417, + "loss": 1.1431, + "theoretical_loss": 3.456720479794321, + "tokens_seen": 2015035392 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031480040120361087, + "loss": 1.2292, + "theoretical_loss": 3.45671119714815, + "tokens_seen": 2015100928 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031478435305917757, + "loss": 1.3939, + "theoretical_loss": 3.456701914888396, + "tokens_seen": 2015166464 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031476830491474427, + "loss": 1.2224, + "theoretical_loss": 3.45669263301503, + "tokens_seen": 2015232000 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031475225677031097, + "loss": 1.3605, + "theoretical_loss": 3.4566833515280235, + "tokens_seen": 2015297536 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031473620862587767, + "loss": 1.3428, + "theoretical_loss": 3.456674070427347, + "tokens_seen": 2015363072 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031472016048144437, + "loss": 1.2833, + "theoretical_loss": 3.456664789712973, + "tokens_seen": 2015428608 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003147041123370111, + "loss": 1.2531, + "theoretical_loss": 3.4566555093848725, + "tokens_seen": 2015494144 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003146880641925778, + "loss": 1.3309, + "theoretical_loss": 3.456646229443017, + "tokens_seen": 2015559680 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003146720160481444, + "loss": 1.3275, + "theoretical_loss": 3.4566369498873777, + "tokens_seen": 2015625216 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003146559679037111, + "loss": 1.3551, + "theoretical_loss": 3.4566276707179258, + "tokens_seen": 2015690752 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003146399197592778, + "loss": 1.1577, + "theoretical_loss": 3.456618391934633, + "tokens_seen": 2015756288 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003146238716148445, + "loss": 1.206, + "theoretical_loss": 3.45660911353747, + "tokens_seen": 2015821824 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003146078234704112, + "loss": 1.4122, + "theoretical_loss": 3.4565998355264087, + "tokens_seen": 2015887360 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003145917753259779, + "loss": 1.1988, + "theoretical_loss": 3.456590557901421, + "tokens_seen": 2015952896 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003145757271815446, + "loss": 1.3103, + "theoretical_loss": 3.456581280662477, + "tokens_seen": 2016018432 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003145596790371113, + "loss": 1.1597, + "theoretical_loss": 3.4565720038095495, + "tokens_seen": 2016083968 + }, + { + "epoch": 0.61, + "learning_rate": 0.000314543630892678, + "loss": 1.354, + "theoretical_loss": 3.456562727342609, + "tokens_seen": 2016149504 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003145275827482447, + "loss": 1.2856, + "theoretical_loss": 3.4565534512616267, + "tokens_seen": 2016215040 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003145115346038114, + "loss": 1.2602, + "theoretical_loss": 3.4565441755665747, + "tokens_seen": 2016280576 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003144954864593781, + "loss": 1.3054, + "theoretical_loss": 3.4565349002574246, + "tokens_seen": 2016346112 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031447943831494483, + "loss": 1.3319, + "theoretical_loss": 3.456525625334147, + "tokens_seen": 2016411648 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031446339017051153, + "loss": 1.3715, + "theoretical_loss": 3.456516350796713, + "tokens_seen": 2016477184 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031444734202607823, + "loss": 1.2958, + "theoretical_loss": 3.4565070766450954, + "tokens_seen": 2016542720 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031443129388164493, + "loss": 1.2651, + "theoretical_loss": 3.4564978028792646, + "tokens_seen": 2016608256 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031441524573721163, + "loss": 1.1711, + "theoretical_loss": 3.4564885294991923, + "tokens_seen": 2016673792 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031439919759277833, + "loss": 1.1931, + "theoretical_loss": 3.4564792565048497, + "tokens_seen": 2016739328 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031438314944834503, + "loss": 1.3449, + "theoretical_loss": 3.4564699838962083, + "tokens_seen": 2016804864 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031436710130391173, + "loss": 1.3999, + "theoretical_loss": 3.45646071167324, + "tokens_seen": 2016870400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031435105315947843, + "loss": 1.2315, + "theoretical_loss": 3.4564514398359156, + "tokens_seen": 2016935936 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031433500501504513, + "loss": 1.2701, + "theoretical_loss": 3.456442168384207, + "tokens_seen": 2017001472 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031431895687061183, + "loss": 1.3196, + "theoretical_loss": 3.456432897318085, + "tokens_seen": 2017067008 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031430290872617853, + "loss": 1.2923, + "theoretical_loss": 3.456423626637522, + "tokens_seen": 2017132544 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031428686058174524, + "loss": 1.2729, + "theoretical_loss": 3.4564143563424885, + "tokens_seen": 2017198080 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031427081243731194, + "loss": 1.298, + "theoretical_loss": 3.456405086432956, + "tokens_seen": 2017263616 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031425476429287864, + "loss": 1.2303, + "theoretical_loss": 3.456395816908897, + "tokens_seen": 2017329152 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031423871614844534, + "loss": 1.3675, + "theoretical_loss": 3.4563865477702818, + "tokens_seen": 2017394688 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031422266800401204, + "loss": 1.2428, + "theoretical_loss": 3.4563772790170826, + "tokens_seen": 2017460224 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031420661985957874, + "loss": 1.2928, + "theoretical_loss": 3.4563680106492702, + "tokens_seen": 2017525760 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031419057171514544, + "loss": 1.2768, + "theoretical_loss": 3.4563587426668168, + "tokens_seen": 2017591296 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031417452357071214, + "loss": 1.2654, + "theoretical_loss": 3.456349475069693, + "tokens_seen": 2017656832 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031415847542627884, + "loss": 1.3862, + "theoretical_loss": 3.4563402078578704, + "tokens_seen": 2017722368 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031414242728184554, + "loss": 1.2427, + "theoretical_loss": 3.4563309410313217, + "tokens_seen": 2017787904 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031412637913741224, + "loss": 1.3109, + "theoretical_loss": 3.4563216745900167, + "tokens_seen": 2017853440 + }, + { + "epoch": 0.61, + "learning_rate": 0.000314110330992979, + "loss": 1.194, + "theoretical_loss": 3.456312408533928, + "tokens_seen": 2017918976 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003140942828485457, + "loss": 1.3121, + "theoretical_loss": 3.456303142863027, + "tokens_seen": 2017984512 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003140782347041124, + "loss": 1.2024, + "theoretical_loss": 3.4562938775772842, + "tokens_seen": 2018050048 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003140621865596791, + "loss": 1.2455, + "theoretical_loss": 3.456284612676672, + "tokens_seen": 2018115584 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003140461384152458, + "loss": 1.2897, + "theoretical_loss": 3.456275348161162, + "tokens_seen": 2018181120 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031403009027081245, + "loss": 1.3099, + "theoretical_loss": 3.456266084030725, + "tokens_seen": 2018246656 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031401404212637915, + "loss": 1.2305, + "theoretical_loss": 3.4562568202853328, + "tokens_seen": 2018312192 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031399799398194585, + "loss": 1.2088, + "theoretical_loss": 3.456247556924957, + "tokens_seen": 2018377728 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031398194583751255, + "loss": 1.2088, + "theoretical_loss": 3.456238293949569, + "tokens_seen": 2018443264 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 746800, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2589595317840576, + "objective/train/theoretical_loss": 3.4562290313591406, + "objective/train/tokens_used": 2038968800, + "theoretical_loss": 3.4562290313591406, + "tokens_seen": 2018508800 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031396589769307925, + "loss": 1.259, + "theoretical_loss": 3.4562290313591406, + "tokens_seen": 2018508800 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031394984954864595, + "loss": 1.3024, + "theoretical_loss": 3.4562197691536425, + "tokens_seen": 2018574336 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031393380140421265, + "loss": 1.2437, + "theoretical_loss": 3.456210507333047, + "tokens_seen": 2018639872 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031391775325977935, + "loss": 1.2905, + "theoretical_loss": 3.4562012458973257, + "tokens_seen": 2018705408 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031390170511534605, + "loss": 1.1223, + "theoretical_loss": 3.4561919848464493, + "tokens_seen": 2018770944 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031388565697091275, + "loss": 1.3004, + "theoretical_loss": 3.4561827241803895, + "tokens_seen": 2018836480 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031386960882647945, + "loss": 1.2187, + "theoretical_loss": 3.456173463899119, + "tokens_seen": 2018902016 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031385356068204615, + "loss": 1.2163, + "theoretical_loss": 3.4561642040026075, + "tokens_seen": 2018967552 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031383751253761285, + "loss": 1.2215, + "theoretical_loss": 3.4561549444908275, + "tokens_seen": 2019033088 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031382146439317955, + "loss": 1.2435, + "theoretical_loss": 3.456145685363751, + "tokens_seen": 2019098624 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031380541624874625, + "loss": 1.2488, + "theoretical_loss": 3.456136426621349, + "tokens_seen": 2019164160 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031378936810431295, + "loss": 1.2875, + "theoretical_loss": 3.456127168263593, + "tokens_seen": 2019229696 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031377331995987966, + "loss": 1.31, + "theoretical_loss": 3.4561179102904536, + "tokens_seen": 2019295232 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031375727181544636, + "loss": 1.3035, + "theoretical_loss": 3.4561086527019045, + "tokens_seen": 2019360768 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031374122367101306, + "loss": 1.1288, + "theoretical_loss": 3.4560993954979153, + "tokens_seen": 2019426304 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031372517552657976, + "loss": 1.3066, + "theoretical_loss": 3.4560901386784586, + "tokens_seen": 2019491840 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031370912738214646, + "loss": 1.3476, + "theoretical_loss": 3.456080882243506, + "tokens_seen": 2019557376 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031369307923771316, + "loss": 1.4155, + "theoretical_loss": 3.456071626193028, + "tokens_seen": 2019622912 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031367703109327986, + "loss": 1.234, + "theoretical_loss": 3.4560623705269973, + "tokens_seen": 2019688448 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031366098294884656, + "loss": 1.2268, + "theoretical_loss": 3.4560531152453846, + "tokens_seen": 2019753984 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031364493480441326, + "loss": 1.2376, + "theoretical_loss": 3.4560438603481622, + "tokens_seen": 2019819520 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031362888665997996, + "loss": 1.2781, + "theoretical_loss": 3.4560346058353018, + "tokens_seen": 2019885056 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031361283851554666, + "loss": 1.2895, + "theoretical_loss": 3.4560253517067734, + "tokens_seen": 2019950592 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031359679037111336, + "loss": 1.2073, + "theoretical_loss": 3.4560160979625505, + "tokens_seen": 2020016128 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031358074222668006, + "loss": 1.3177, + "theoretical_loss": 3.4560068446026033, + "tokens_seen": 2020081664 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031356469408224676, + "loss": 1.2303, + "theoretical_loss": 3.455997591626905, + "tokens_seen": 2020147200 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031354864593781346, + "loss": 1.3169, + "theoretical_loss": 3.455988339035425, + "tokens_seen": 2020212736 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031353259779338017, + "loss": 1.2499, + "theoretical_loss": 3.455979086828136, + "tokens_seen": 2020278272 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031351654964894687, + "loss": 1.3143, + "theoretical_loss": 3.45596983500501, + "tokens_seen": 2020343808 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031350050150451357, + "loss": 1.3255, + "theoretical_loss": 3.455960583566018, + "tokens_seen": 2020409344 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031348445336008027, + "loss": 1.2678, + "theoretical_loss": 3.4559513325111317, + "tokens_seen": 2020474880 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031346840521564697, + "loss": 1.3282, + "theoretical_loss": 3.4559420818403233, + "tokens_seen": 2020540416 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031345235707121367, + "loss": 1.255, + "theoretical_loss": 3.455932831553563, + "tokens_seen": 2020605952 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031343630892678037, + "loss": 1.2432, + "theoretical_loss": 3.455923581650824, + "tokens_seen": 2020671488 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031342026078234707, + "loss": 1.1592, + "theoretical_loss": 3.4559143321320764, + "tokens_seen": 2020737024 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031340421263791377, + "loss": 1.3578, + "theoretical_loss": 3.455905082997293, + "tokens_seen": 2020802560 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031338816449348047, + "loss": 1.1917, + "theoretical_loss": 3.4558958342464448, + "tokens_seen": 2020868096 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031337211634904717, + "loss": 1.3517, + "theoretical_loss": 3.4558865858795036, + "tokens_seen": 2020933632 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031335606820461387, + "loss": 1.2122, + "theoretical_loss": 3.4558773378964407, + "tokens_seen": 2020999168 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031334002006018057, + "loss": 1.1758, + "theoretical_loss": 3.4558680902972285, + "tokens_seen": 2021064704 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003133239719157473, + "loss": 1.1891, + "theoretical_loss": 3.455858843081838, + "tokens_seen": 2021130240 + }, + { + "epoch": 0.61, + "learning_rate": 0.000313307923771314, + "loss": 1.2445, + "theoretical_loss": 3.455849596250241, + "tokens_seen": 2021195776 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003132918756268807, + "loss": 1.3165, + "theoretical_loss": 3.455840349802409, + "tokens_seen": 2021261312 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003132758274824474, + "loss": 1.3774, + "theoretical_loss": 3.455831103738314, + "tokens_seen": 2021326848 + }, + { + "epoch": 0.61, + "learning_rate": 0.000313259779338014, + "loss": 1.2043, + "theoretical_loss": 3.4558218580579263, + "tokens_seen": 2021392384 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003132437311935807, + "loss": 1.2756, + "theoretical_loss": 3.4558126127612194, + "tokens_seen": 2021457920 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003132276830491474, + "loss": 1.2638, + "theoretical_loss": 3.4558033678481643, + "tokens_seen": 2021523456 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003132116349047141, + "loss": 1.3795, + "theoretical_loss": 3.455794123318732, + "tokens_seen": 2021588992 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003131955867602808, + "loss": 1.3553, + "theoretical_loss": 3.455784879172895, + "tokens_seen": 2021654528 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003131795386158475, + "loss": 1.3463, + "theoretical_loss": 3.4557756354106246, + "tokens_seen": 2021720064 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003131634904714142, + "loss": 1.2574, + "theoretical_loss": 3.455766392031892, + "tokens_seen": 2021785600 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003131474423269809, + "loss": 1.1558, + "theoretical_loss": 3.4557571490366694, + "tokens_seen": 2021851136 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003131313941825476, + "loss": 1.3427, + "theoretical_loss": 3.4557479064249286, + "tokens_seen": 2021916672 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031311534603811433, + "loss": 1.2338, + "theoretical_loss": 3.45573866419664, + "tokens_seen": 2021982208 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031309929789368103, + "loss": 1.209, + "theoretical_loss": 3.4557294223517774, + "tokens_seen": 2022047744 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031308324974924773, + "loss": 1.2875, + "theoretical_loss": 3.455720180890311, + "tokens_seen": 2022113280 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031306720160481443, + "loss": 1.2909, + "theoretical_loss": 3.455710939812213, + "tokens_seen": 2022178816 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031305115346038113, + "loss": 1.2142, + "theoretical_loss": 3.455701699117454, + "tokens_seen": 2022244352 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031303510531594783, + "loss": 1.3984, + "theoretical_loss": 3.4556924588060074, + "tokens_seen": 2022309888 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031301905717151453, + "loss": 1.1445, + "theoretical_loss": 3.4556832188778435, + "tokens_seen": 2022375424 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031300300902708123, + "loss": 1.2685, + "theoretical_loss": 3.4556739793329347, + "tokens_seen": 2022440960 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031298696088264793, + "loss": 1.1406, + "theoretical_loss": 3.455664740171253, + "tokens_seen": 2022506496 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031297091273821463, + "loss": 1.2813, + "theoretical_loss": 3.4556555013927683, + "tokens_seen": 2022572032 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031295486459378133, + "loss": 1.3299, + "theoretical_loss": 3.4556462629974547, + "tokens_seen": 2022637568 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031293881644934803, + "loss": 1.2202, + "theoretical_loss": 3.455637024985282, + "tokens_seen": 2022703104 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031292276830491473, + "loss": 1.3007, + "theoretical_loss": 3.455627787356223, + "tokens_seen": 2022768640 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003129067201604815, + "loss": 1.2929, + "theoretical_loss": 3.455618550110249, + "tokens_seen": 2022834176 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003128906720160482, + "loss": 1.303, + "theoretical_loss": 3.455609313247332, + "tokens_seen": 2022899712 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003128746238716149, + "loss": 1.3308, + "theoretical_loss": 3.4556000767674435, + "tokens_seen": 2022965248 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003128585757271816, + "loss": 1.32, + "theoretical_loss": 3.4555908406705544, + "tokens_seen": 2023030784 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003128425275827483, + "loss": 1.38, + "theoretical_loss": 3.455581604956638, + "tokens_seen": 2023096320 + }, + { + "epoch": 0.61, + "learning_rate": 0.000312826479438315, + "loss": 1.2681, + "theoretical_loss": 3.455572369625665, + "tokens_seen": 2023161856 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003128104312938817, + "loss": 1.3561, + "theoretical_loss": 3.455563134677607, + "tokens_seen": 2023227392 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003127943831494484, + "loss": 1.2909, + "theoretical_loss": 3.4555539001124362, + "tokens_seen": 2023292928 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003127783350050151, + "loss": 1.2044, + "theoretical_loss": 3.455544665930124, + "tokens_seen": 2023358464 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003127622868605818, + "loss": 1.2356, + "theoretical_loss": 3.4555354321306426, + "tokens_seen": 2023424000 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003127462387161485, + "loss": 1.302, + "theoretical_loss": 3.4555261987139634, + "tokens_seen": 2023489536 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003127301905717152, + "loss": 1.3063, + "theoretical_loss": 3.455516965680058, + "tokens_seen": 2023555072 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003127141424272819, + "loss": 1.2097, + "theoretical_loss": 3.4555077330288984, + "tokens_seen": 2023620608 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003126980942828486, + "loss": 1.2554, + "theoretical_loss": 3.455498500760456, + "tokens_seen": 2023686144 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003126820461384153, + "loss": 1.2754, + "theoretical_loss": 3.455489268874703, + "tokens_seen": 2023751680 + }, + { + "epoch": 0.61, + "learning_rate": 0.000312665997993982, + "loss": 1.221, + "theoretical_loss": 3.455480037371611, + "tokens_seen": 2023817216 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003126499498495487, + "loss": 1.3095, + "theoretical_loss": 3.455470806251151, + "tokens_seen": 2023882752 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031263390170511535, + "loss": 1.2492, + "theoretical_loss": 3.4554615755132962, + "tokens_seen": 2023948288 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031261785356068205, + "loss": 1.1959, + "theoretical_loss": 3.455452345158017, + "tokens_seen": 2024013824 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031260180541624875, + "loss": 1.2251, + "theoretical_loss": 3.455443115185286, + "tokens_seen": 2024079360 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031258575727181545, + "loss": 1.2024, + "theoretical_loss": 3.455433885595075, + "tokens_seen": 2024144896 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031256970912738215, + "loss": 1.3111, + "theoretical_loss": 3.455424656387355, + "tokens_seen": 2024210432 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031255366098294885, + "loss": 1.2545, + "theoretical_loss": 3.455415427562098, + "tokens_seen": 2024275968 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031253761283851555, + "loss": 1.2961, + "theoretical_loss": 3.4554061991192766, + "tokens_seen": 2024341504 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031252156469408225, + "loss": 1.228, + "theoretical_loss": 3.4553969710588612, + "tokens_seen": 2024407040 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031250551654964895, + "loss": 1.3017, + "theoretical_loss": 3.4553877433808253, + "tokens_seen": 2024472576 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031248946840521565, + "loss": 1.2221, + "theoretical_loss": 3.4553785160851387, + "tokens_seen": 2024538112 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031247342026078235, + "loss": 1.2932, + "theoretical_loss": 3.4553692891717747, + "tokens_seen": 2024603648 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031245737211634905, + "loss": 1.1711, + "theoretical_loss": 3.455360062640705, + "tokens_seen": 2024669184 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031244132397191575, + "loss": 1.3487, + "theoretical_loss": 3.4553508364919, + "tokens_seen": 2024734720 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031242527582748245, + "loss": 1.2312, + "theoretical_loss": 3.4553416107253327, + "tokens_seen": 2024800256 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031240922768304916, + "loss": 1.1635, + "theoretical_loss": 3.455332385340975, + "tokens_seen": 2024865792 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031239317953861586, + "loss": 1.303, + "theoretical_loss": 3.455323160338798, + "tokens_seen": 2024931328 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031237713139418256, + "loss": 1.1828, + "theoretical_loss": 3.455313935718774, + "tokens_seen": 2024996864 + }, + { + "epoch": 0.61, + "objective/train/docs_used": 748325, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1842286586761475, + "objective/train/theoretical_loss": 3.4553047114808746, + "objective/train/tokens_used": 2043771360, + "theoretical_loss": 3.4553047114808746, + "tokens_seen": 2025062400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031236108324974926, + "loss": 1.1842, + "theoretical_loss": 3.4553047114808746, + "tokens_seen": 2025062400 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031234503510531596, + "loss": 1.2737, + "theoretical_loss": 3.455295487625072, + "tokens_seen": 2025127936 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031232898696088266, + "loss": 1.2762, + "theoretical_loss": 3.4552862641513373, + "tokens_seen": 2025193472 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031231293881644936, + "loss": 1.2972, + "theoretical_loss": 3.4552770410596425, + "tokens_seen": 2025259008 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031229689067201606, + "loss": 1.1611, + "theoretical_loss": 3.4552678183499594, + "tokens_seen": 2025324544 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031228084252758276, + "loss": 1.2253, + "theoretical_loss": 3.4552585960222606, + "tokens_seen": 2025390080 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031226479438314946, + "loss": 1.3518, + "theoretical_loss": 3.455249374076517, + "tokens_seen": 2025455616 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031224874623871616, + "loss": 1.2748, + "theoretical_loss": 3.455240152512701, + "tokens_seen": 2025521152 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031223269809428286, + "loss": 1.2412, + "theoretical_loss": 3.455230931330784, + "tokens_seen": 2025586688 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031221664994984956, + "loss": 1.2539, + "theoretical_loss": 3.4552217105307377, + "tokens_seen": 2025652224 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031220060180541626, + "loss": 1.229, + "theoretical_loss": 3.4552124901125345, + "tokens_seen": 2025717760 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031218455366098296, + "loss": 1.3265, + "theoretical_loss": 3.4552032700761455, + "tokens_seen": 2025783296 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031216850551654966, + "loss": 1.2504, + "theoretical_loss": 3.455194050421544, + "tokens_seen": 2025848832 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031215245737211637, + "loss": 1.1783, + "theoretical_loss": 3.4551848311487, + "tokens_seen": 2025914368 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031213640922768307, + "loss": 1.266, + "theoretical_loss": 3.4551756122575865, + "tokens_seen": 2025979904 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031212036108324977, + "loss": 1.2629, + "theoretical_loss": 3.4551663937481747, + "tokens_seen": 2026045440 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031210431293881647, + "loss": 1.3615, + "theoretical_loss": 3.455157175620437, + "tokens_seen": 2026110976 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031208826479438317, + "loss": 1.2936, + "theoretical_loss": 3.455147957874345, + "tokens_seen": 2026176512 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031207221664994987, + "loss": 1.4053, + "theoretical_loss": 3.45513874050987, + "tokens_seen": 2026242048 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031205616850551657, + "loss": 1.3108, + "theoretical_loss": 3.4551295235269857, + "tokens_seen": 2026307584 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031204012036108327, + "loss": 1.244, + "theoretical_loss": 3.4551203069256617, + "tokens_seen": 2026373120 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031202407221664997, + "loss": 1.3019, + "theoretical_loss": 3.4551110907058713, + "tokens_seen": 2026438656 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031200802407221667, + "loss": 1.3738, + "theoretical_loss": 3.455101874867586, + "tokens_seen": 2026504192 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031199197592778337, + "loss": 1.4389, + "theoretical_loss": 3.455092659410777, + "tokens_seen": 2026569728 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031197592778335007, + "loss": 1.3336, + "theoretical_loss": 3.4550834443354175, + "tokens_seen": 2026635264 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003119598796389168, + "loss": 1.2137, + "theoretical_loss": 3.4550742296414785, + "tokens_seen": 2026700800 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003119438314944835, + "loss": 1.3164, + "theoretical_loss": 3.455065015328932, + "tokens_seen": 2026766336 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003119277833500502, + "loss": 1.3729, + "theoretical_loss": 3.4550558013977497, + "tokens_seen": 2026831872 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003119117352056169, + "loss": 1.2694, + "theoretical_loss": 3.455046587847904, + "tokens_seen": 2026897408 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003118956870611836, + "loss": 1.178, + "theoretical_loss": 3.455037374679366, + "tokens_seen": 2026962944 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003118796389167503, + "loss": 1.3475, + "theoretical_loss": 3.455028161892108, + "tokens_seen": 2027028480 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003118635907723169, + "loss": 1.1938, + "theoretical_loss": 3.455018949486103, + "tokens_seen": 2027094016 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003118475426278836, + "loss": 1.2362, + "theoretical_loss": 3.455009737461321, + "tokens_seen": 2027159552 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003118314944834503, + "loss": 1.2418, + "theoretical_loss": 3.455000525817735, + "tokens_seen": 2027225088 + }, + { + "epoch": 0.61, + "learning_rate": 0.000311815446339017, + "loss": 1.286, + "theoretical_loss": 3.4549913145553166, + "tokens_seen": 2027290624 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003117993981945837, + "loss": 1.2823, + "theoretical_loss": 3.4549821036740385, + "tokens_seen": 2027356160 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003117833500501504, + "loss": 1.2303, + "theoretical_loss": 3.454972893173871, + "tokens_seen": 2027421696 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003117673019057171, + "loss": 1.2744, + "theoretical_loss": 3.454963683054787, + "tokens_seen": 2027487232 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003117512537612838, + "loss": 1.3059, + "theoretical_loss": 3.4549544733167585, + "tokens_seen": 2027552768 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031173520561685053, + "loss": 1.2811, + "theoretical_loss": 3.4549452639597575, + "tokens_seen": 2027618304 + }, + { + "epoch": 0.61, + "learning_rate": 0.00031171915747241723, + "loss": 1.2155, + "theoretical_loss": 3.4549360549837553, + "tokens_seen": 2027683840 + }, + { + "epoch": 0.61, + "learning_rate": 0.000311703109327984, + "loss": 1.278, + "theoretical_loss": 3.4549268463887244, + "tokens_seen": 2027749376 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003116870611835507, + "loss": 1.1828, + "theoretical_loss": 3.4549176381746367, + "tokens_seen": 2027814912 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003116710130391174, + "loss": 1.2783, + "theoretical_loss": 3.4549084303414634, + "tokens_seen": 2027880448 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003116549648946841, + "loss": 1.2231, + "theoretical_loss": 3.4548992228891775, + "tokens_seen": 2027945984 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003116389167502508, + "loss": 1.2807, + "theoretical_loss": 3.45489001581775, + "tokens_seen": 2028011520 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003116228686058175, + "loss": 1.2301, + "theoretical_loss": 3.4548808091271535, + "tokens_seen": 2028077056 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003116068204613842, + "loss": 1.2373, + "theoretical_loss": 3.4548716028173594, + "tokens_seen": 2028142592 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003115907723169509, + "loss": 1.3019, + "theoretical_loss": 3.4548623968883403, + "tokens_seen": 2028208128 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003115747241725176, + "loss": 1.3519, + "theoretical_loss": 3.4548531913400677, + "tokens_seen": 2028273664 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003115586760280843, + "loss": 1.2788, + "theoretical_loss": 3.4548439861725138, + "tokens_seen": 2028339200 + }, + { + "epoch": 0.61, + "learning_rate": 0.000311542627883651, + "loss": 1.2725, + "theoretical_loss": 3.45483478138565, + "tokens_seen": 2028404736 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003115265797392177, + "loss": 1.2524, + "theoretical_loss": 3.4548255769794487, + "tokens_seen": 2028470272 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003115105315947844, + "loss": 1.2054, + "theoretical_loss": 3.454816372953882, + "tokens_seen": 2028535808 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003114944834503511, + "loss": 1.1962, + "theoretical_loss": 3.4548071693089217, + "tokens_seen": 2028601344 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003114784353059178, + "loss": 1.3397, + "theoretical_loss": 3.4547979660445396, + "tokens_seen": 2028666880 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003114623871614845, + "loss": 1.1765, + "theoretical_loss": 3.4547887631607077, + "tokens_seen": 2028732416 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003114463390170512, + "loss": 1.2061, + "theoretical_loss": 3.4547795606573986, + "tokens_seen": 2028797952 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003114302908726179, + "loss": 1.3024, + "theoretical_loss": 3.4547703585345833, + "tokens_seen": 2028863488 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003114142427281846, + "loss": 1.2061, + "theoretical_loss": 3.4547611567922347, + "tokens_seen": 2028929024 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003113981945837513, + "loss": 1.1976, + "theoretical_loss": 3.4547519554303237, + "tokens_seen": 2028994560 + }, + { + "epoch": 0.61, + "learning_rate": 0.000311382146439318, + "loss": 1.2331, + "theoretical_loss": 3.454742754448823, + "tokens_seen": 2029060096 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003113660982948847, + "loss": 1.2332, + "theoretical_loss": 3.4547335538477046, + "tokens_seen": 2029125632 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003113500501504514, + "loss": 1.2161, + "theoretical_loss": 3.4547243536269407, + "tokens_seen": 2029191168 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003113340020060181, + "loss": 1.2903, + "theoretical_loss": 3.454715153786503, + "tokens_seen": 2029256704 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003113179538615848, + "loss": 1.2372, + "theoretical_loss": 3.454705954326363, + "tokens_seen": 2029322240 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003113019057171515, + "loss": 1.2721, + "theoretical_loss": 3.454696755246493, + "tokens_seen": 2029387776 + }, + { + "epoch": 0.61, + "learning_rate": 0.0003112858575727182, + "loss": 1.2315, + "theoretical_loss": 3.4546875565468658, + "tokens_seen": 2029453312 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003112698094282849, + "loss": 1.2355, + "theoretical_loss": 3.4546783582274525, + "tokens_seen": 2029518848 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003112537612838516, + "loss": 1.2364, + "theoretical_loss": 3.4546691602882253, + "tokens_seen": 2029584384 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031123771313941825, + "loss": 1.3577, + "theoretical_loss": 3.4546599627291563, + "tokens_seen": 2029649920 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031122166499498495, + "loss": 1.1864, + "theoretical_loss": 3.454650765550218, + "tokens_seen": 2029715456 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031120561685055165, + "loss": 1.2212, + "theoretical_loss": 3.4546415687513807, + "tokens_seen": 2029780992 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031118956870611835, + "loss": 1.2625, + "theoretical_loss": 3.4546323723326187, + "tokens_seen": 2029846528 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031117352056168505, + "loss": 1.2844, + "theoretical_loss": 3.454623176293903, + "tokens_seen": 2029912064 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031115747241725175, + "loss": 1.2428, + "theoretical_loss": 3.4546139806352048, + "tokens_seen": 2029977600 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031114142427281845, + "loss": 1.2482, + "theoretical_loss": 3.4546047853564974, + "tokens_seen": 2030043136 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031112537612838515, + "loss": 1.295, + "theoretical_loss": 3.4545955904577523, + "tokens_seen": 2030108672 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031110932798395185, + "loss": 1.2135, + "theoretical_loss": 3.4545863959389416, + "tokens_seen": 2030174208 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031109327983951855, + "loss": 1.2986, + "theoretical_loss": 3.4545772018000376, + "tokens_seen": 2030239744 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031107723169508525, + "loss": 1.3142, + "theoretical_loss": 3.4545680080410115, + "tokens_seen": 2030305280 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031106118355065195, + "loss": 1.3238, + "theoretical_loss": 3.4545588146618362, + "tokens_seen": 2030370816 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031104513540621866, + "loss": 1.2966, + "theoretical_loss": 3.4545496216624834, + "tokens_seen": 2030436352 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031102908726178536, + "loss": 1.1892, + "theoretical_loss": 3.454540429042925, + "tokens_seen": 2030501888 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031101303911735206, + "loss": 1.1766, + "theoretical_loss": 3.454531236803134, + "tokens_seen": 2030567424 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031099699097291876, + "loss": 1.1977, + "theoretical_loss": 3.4545220449430807, + "tokens_seen": 2030632960 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031098094282848546, + "loss": 1.363, + "theoretical_loss": 3.4545128534627385, + "tokens_seen": 2030698496 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031096489468405216, + "loss": 1.2642, + "theoretical_loss": 3.4545036623620793, + "tokens_seen": 2030764032 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031094884653961886, + "loss": 1.2183, + "theoretical_loss": 3.454494471641075, + "tokens_seen": 2030829568 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031093279839518556, + "loss": 1.2211, + "theoretical_loss": 3.4544852812996973, + "tokens_seen": 2030895104 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031091675025075226, + "loss": 1.1643, + "theoretical_loss": 3.454476091337919, + "tokens_seen": 2030960640 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031090070210631896, + "loss": 1.192, + "theoretical_loss": 3.4544669017557115, + "tokens_seen": 2031026176 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031088465396188566, + "loss": 1.1091, + "theoretical_loss": 3.4544577125530473, + "tokens_seen": 2031091712 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031086860581745236, + "loss": 1.2811, + "theoretical_loss": 3.4544485237298987, + "tokens_seen": 2031157248 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031085255767301906, + "loss": 1.2348, + "theoretical_loss": 3.454439335286237, + "tokens_seen": 2031222784 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031083650952858576, + "loss": 1.2158, + "theoretical_loss": 3.454430147222035, + "tokens_seen": 2031288320 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031082046138415246, + "loss": 1.2151, + "theoretical_loss": 3.4544209595372637, + "tokens_seen": 2031353856 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031080441323971916, + "loss": 1.3703, + "theoretical_loss": 3.4544117722318965, + "tokens_seen": 2031419392 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031078836509528587, + "loss": 1.2124, + "theoretical_loss": 3.4544025853059055, + "tokens_seen": 2031484928 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031077231695085257, + "loss": 1.2586, + "theoretical_loss": 3.4543933987592617, + "tokens_seen": 2031550464 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 748325, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.248844861984253, + "objective/train/theoretical_loss": 3.454384212591938, + "objective/train/tokens_used": 2043771360, + "theoretical_loss": 3.454384212591938, + "tokens_seen": 2031616000 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031075626880641927, + "loss": 1.2488, + "theoretical_loss": 3.454384212591938, + "tokens_seen": 2031616000 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031074022066198597, + "loss": 1.2702, + "theoretical_loss": 3.4543750268039064, + "tokens_seen": 2031681536 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031072417251755267, + "loss": 1.2528, + "theoretical_loss": 3.4543658413951386, + "tokens_seen": 2031747072 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031070812437311937, + "loss": 1.2999, + "theoretical_loss": 3.454356656365607, + "tokens_seen": 2031812608 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031069207622868607, + "loss": 1.307, + "theoretical_loss": 3.454347471715284, + "tokens_seen": 2031878144 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031067602808425277, + "loss": 1.3716, + "theoretical_loss": 3.4543382874441413, + "tokens_seen": 2031943680 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031065997993981947, + "loss": 1.235, + "theoretical_loss": 3.454329103552151, + "tokens_seen": 2032009216 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031064393179538617, + "loss": 1.2523, + "theoretical_loss": 3.454319920039285, + "tokens_seen": 2032074752 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031062788365095287, + "loss": 1.2706, + "theoretical_loss": 3.4543107369055166, + "tokens_seen": 2032140288 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031061183550651957, + "loss": 1.2127, + "theoretical_loss": 3.4543015541508164, + "tokens_seen": 2032205824 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031059578736208627, + "loss": 1.1383, + "theoretical_loss": 3.454292371775158, + "tokens_seen": 2032271360 + }, + { + "epoch": 0.62, + "learning_rate": 0.000310579739217653, + "loss": 1.2529, + "theoretical_loss": 3.4542831897785122, + "tokens_seen": 2032336896 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003105636910732197, + "loss": 1.2534, + "theoretical_loss": 3.4542740081608514, + "tokens_seen": 2032402432 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003105476429287864, + "loss": 1.2834, + "theoretical_loss": 3.4542648269221488, + "tokens_seen": 2032467968 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003105315947843531, + "loss": 1.1769, + "theoretical_loss": 3.454255646062375, + "tokens_seen": 2032533504 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003105155466399198, + "loss": 1.2431, + "theoretical_loss": 3.4542464655815035, + "tokens_seen": 2032599040 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003104994984954865, + "loss": 1.3129, + "theoretical_loss": 3.454237285479506, + "tokens_seen": 2032664576 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003104834503510532, + "loss": 1.2508, + "theoretical_loss": 3.454228105756354, + "tokens_seen": 2032730112 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003104674022066199, + "loss": 1.3674, + "theoretical_loss": 3.4542189264120204, + "tokens_seen": 2032795648 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003104513540621866, + "loss": 1.2445, + "theoretical_loss": 3.4542097474464772, + "tokens_seen": 2032861184 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003104353059177533, + "loss": 1.3185, + "theoretical_loss": 3.4542005688596964, + "tokens_seen": 2032926720 + }, + { + "epoch": 0.62, + "learning_rate": 0.00031041925777332, + "loss": 1.2443, + "theoretical_loss": 3.4541913906516504, + "tokens_seen": 2032992256 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003104032096288867, + "loss": 1.3123, + "theoretical_loss": 3.454182212822311, + "tokens_seen": 2033057792 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003103871614844534, + "loss": 1.3361, + "theoretical_loss": 3.4541730353716504, + "tokens_seen": 2033123328 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003103711133400201, + "loss": 1.3737, + "theoretical_loss": 3.4541638582996415, + "tokens_seen": 2033188864 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003103550651955868, + "loss": 1.3298, + "theoretical_loss": 3.454154681606256, + "tokens_seen": 2033254400 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003103390170511535, + "loss": 1.21, + "theoretical_loss": 3.454145505291465, + "tokens_seen": 2033319936 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003103229689067202, + "loss": 1.2863, + "theoretical_loss": 3.4541363293552427, + "tokens_seen": 2033385472 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003103069207622869, + "loss": 1.2966, + "theoretical_loss": 3.4541271537975593, + "tokens_seen": 2033451008 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003102908726178536, + "loss": 1.295, + "theoretical_loss": 3.4541179786183887, + "tokens_seen": 2033516544 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003102748244734203, + "loss": 1.2149, + "theoretical_loss": 3.454108803817702, + "tokens_seen": 2033582080 + }, + { + "epoch": 0.62, + "learning_rate": 0.000310258776328987, + "loss": 1.3295, + "theoretical_loss": 3.454099629395472, + "tokens_seen": 2033647616 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003102427281845537, + "loss": 1.257, + "theoretical_loss": 3.45409045535167, + "tokens_seen": 2033713152 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003102266800401204, + "loss": 1.3169, + "theoretical_loss": 3.4540812816862694, + "tokens_seen": 2033778688 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003102106318956871, + "loss": 1.1985, + "theoretical_loss": 3.4540721083992416, + "tokens_seen": 2033844224 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003101945837512538, + "loss": 1.3425, + "theoretical_loss": 3.454062935490559, + "tokens_seen": 2033909760 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003101785356068205, + "loss": 1.3239, + "theoretical_loss": 3.454053762960194, + "tokens_seen": 2033975296 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003101624874623872, + "loss": 1.178, + "theoretical_loss": 3.4540445908081185, + "tokens_seen": 2034040832 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003101464393179539, + "loss": 1.2397, + "theoretical_loss": 3.454035419034305, + "tokens_seen": 2034106368 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003101303911735206, + "loss": 1.291, + "theoretical_loss": 3.4540262476387253, + "tokens_seen": 2034171904 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003101143430290873, + "loss": 1.2186, + "theoretical_loss": 3.4540170766213523, + "tokens_seen": 2034237440 + }, + { + "epoch": 0.62, + "learning_rate": 0.000310098294884654, + "loss": 1.2026, + "theoretical_loss": 3.4540079059821576, + "tokens_seen": 2034302976 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003100822467402207, + "loss": 1.2587, + "theoretical_loss": 3.453998735721113, + "tokens_seen": 2034368512 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003100661985957874, + "loss": 1.1749, + "theoretical_loss": 3.453989565838192, + "tokens_seen": 2034434048 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003100501504513541, + "loss": 1.3034, + "theoretical_loss": 3.4539803963333657, + "tokens_seen": 2034499584 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003100341023069208, + "loss": 1.2167, + "theoretical_loss": 3.4539712272066074, + "tokens_seen": 2034565120 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003100180541624875, + "loss": 1.2012, + "theoretical_loss": 3.453962058457888, + "tokens_seen": 2034630656 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003100020060180542, + "loss": 1.2729, + "theoretical_loss": 3.453952890087181, + "tokens_seen": 2034696192 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003099859578736209, + "loss": 1.1821, + "theoretical_loss": 3.4539437220944578, + "tokens_seen": 2034761728 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003099699097291876, + "loss": 1.298, + "theoretical_loss": 3.453934554479691, + "tokens_seen": 2034827264 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003099538615847543, + "loss": 1.3544, + "theoretical_loss": 3.453925387242853, + "tokens_seen": 2034892800 + }, + { + "epoch": 0.62, + "learning_rate": 0.000309937813440321, + "loss": 1.2859, + "theoretical_loss": 3.453916220383916, + "tokens_seen": 2034958336 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003099217652958877, + "loss": 1.223, + "theoretical_loss": 3.4539070539028516, + "tokens_seen": 2035023872 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003099057171514544, + "loss": 1.1844, + "theoretical_loss": 3.4538978877996325, + "tokens_seen": 2035089408 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003098896690070211, + "loss": 1.2227, + "theoretical_loss": 3.4538887220742316, + "tokens_seen": 2035154944 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003098736208625878, + "loss": 1.2169, + "theoretical_loss": 3.45387955672662, + "tokens_seen": 2035220480 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003098575727181545, + "loss": 1.3197, + "theoretical_loss": 3.4538703917567712, + "tokens_seen": 2035286016 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003098415245737212, + "loss": 1.1107, + "theoretical_loss": 3.4538612271646563, + "tokens_seen": 2035351552 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030982547642928785, + "loss": 1.3338, + "theoretical_loss": 3.453852062950248, + "tokens_seen": 2035417088 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030980942828485455, + "loss": 1.3888, + "theoretical_loss": 3.453842899113519, + "tokens_seen": 2035482624 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030979338014042125, + "loss": 1.2035, + "theoretical_loss": 3.453833735654441, + "tokens_seen": 2035548160 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030977733199598795, + "loss": 1.3212, + "theoretical_loss": 3.453824572572987, + "tokens_seen": 2035613696 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030976128385155465, + "loss": 1.2426, + "theoretical_loss": 3.453815409869128, + "tokens_seen": 2035679232 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030974523570712135, + "loss": 1.2831, + "theoretical_loss": 3.4538062475428375, + "tokens_seen": 2035744768 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030972918756268805, + "loss": 1.2996, + "theoretical_loss": 3.453797085594087, + "tokens_seen": 2035810304 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030971313941825475, + "loss": 1.3309, + "theoretical_loss": 3.4537879240228495, + "tokens_seen": 2035875840 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030969709127382145, + "loss": 1.2998, + "theoretical_loss": 3.453778762829097, + "tokens_seen": 2035941376 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030968104312938815, + "loss": 1.2526, + "theoretical_loss": 3.4537696020128017, + "tokens_seen": 2036006912 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030966499498495486, + "loss": 1.3471, + "theoretical_loss": 3.4537604415739356, + "tokens_seen": 2036072448 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030964894684052156, + "loss": 1.3294, + "theoretical_loss": 3.4537512815124716, + "tokens_seen": 2036137984 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030963289869608826, + "loss": 1.3333, + "theoretical_loss": 3.4537421218283817, + "tokens_seen": 2036203520 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030961685055165496, + "loss": 1.2844, + "theoretical_loss": 3.4537329625216384, + "tokens_seen": 2036269056 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030960080240722166, + "loss": 1.2049, + "theoretical_loss": 3.4537238035922133, + "tokens_seen": 2036334592 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030958475426278836, + "loss": 1.2132, + "theoretical_loss": 3.4537146450400797, + "tokens_seen": 2036400128 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030956870611835506, + "loss": 1.2961, + "theoretical_loss": 3.4537054868652097, + "tokens_seen": 2036465664 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030955265797392176, + "loss": 1.2928, + "theoretical_loss": 3.4536963290675753, + "tokens_seen": 2036531200 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030953660982948846, + "loss": 1.3294, + "theoretical_loss": 3.453687171647149, + "tokens_seen": 2036596736 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030952056168505516, + "loss": 1.2169, + "theoretical_loss": 3.4536780146039026, + "tokens_seen": 2036662272 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030950451354062186, + "loss": 1.1176, + "theoretical_loss": 3.453668857937809, + "tokens_seen": 2036727808 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030948846539618856, + "loss": 1.1568, + "theoretical_loss": 3.4536597016488404, + "tokens_seen": 2036793344 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030947241725175526, + "loss": 1.3438, + "theoretical_loss": 3.4536505457369695, + "tokens_seen": 2036858880 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030945636910732196, + "loss": 1.2419, + "theoretical_loss": 3.453641390202168, + "tokens_seen": 2036924416 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030944032096288866, + "loss": 1.2828, + "theoretical_loss": 3.4536322350444086, + "tokens_seen": 2036989952 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030942427281845536, + "loss": 1.2942, + "theoretical_loss": 3.4536230802636636, + "tokens_seen": 2037055488 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030940822467402207, + "loss": 1.2796, + "theoretical_loss": 3.4536139258599055, + "tokens_seen": 2037121024 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030939217652958877, + "loss": 1.3005, + "theoretical_loss": 3.453604771833106, + "tokens_seen": 2037186560 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030937612838515547, + "loss": 1.201, + "theoretical_loss": 3.4535956181832383, + "tokens_seen": 2037252096 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030936008024072217, + "loss": 1.3613, + "theoretical_loss": 3.4535864649102743, + "tokens_seen": 2037317632 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030934403209628887, + "loss": 1.223, + "theoretical_loss": 3.453577312014186, + "tokens_seen": 2037383168 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030932798395185557, + "loss": 1.3272, + "theoretical_loss": 3.4535681594949468, + "tokens_seen": 2037448704 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030931193580742227, + "loss": 1.3372, + "theoretical_loss": 3.4535590073525286, + "tokens_seen": 2037514240 + }, + { + "epoch": 0.62, + "learning_rate": 0.000309295887662989, + "loss": 1.4164, + "theoretical_loss": 3.4535498555869033, + "tokens_seen": 2037579776 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003092798395185557, + "loss": 1.2438, + "theoretical_loss": 3.4535407041980433, + "tokens_seen": 2037645312 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003092637913741224, + "loss": 1.269, + "theoretical_loss": 3.453531553185922, + "tokens_seen": 2037710848 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003092477432296891, + "loss": 1.216, + "theoretical_loss": 3.45352240255051, + "tokens_seen": 2037776384 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003092316950852558, + "loss": 1.2753, + "theoretical_loss": 3.4535132522917813, + "tokens_seen": 2037841920 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030921564694082253, + "loss": 1.3348, + "theoretical_loss": 3.4535041024097075, + "tokens_seen": 2037907456 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003091995987963892, + "loss": 1.2529, + "theoretical_loss": 3.4534949529042613, + "tokens_seen": 2037972992 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003091835506519559, + "loss": 1.3545, + "theoretical_loss": 3.4534858037754153, + "tokens_seen": 2038038528 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003091675025075226, + "loss": 1.2761, + "theoretical_loss": 3.453476655023141, + "tokens_seen": 2038104064 + }, + { + "epoch": 0.62, + "objective/train/docs_used": 748325, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1241387128829956, + "objective/train/theoretical_loss": 3.453467506647412, + "objective/train/tokens_used": 2043771360, + "theoretical_loss": 3.453467506647412, + "tokens_seen": 2038169600 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003091514543630893, + "loss": 1.1241, + "theoretical_loss": 3.453467506647412, + "tokens_seen": 2038169600 + }, + { + "epoch": 0.62, + "learning_rate": 0.000309135406218656, + "loss": 1.2668, + "theoretical_loss": 3.4534583586481995, + "tokens_seen": 2038235136 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003091193580742227, + "loss": 1.2682, + "theoretical_loss": 3.453449211025476, + "tokens_seen": 2038300672 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003091033099297894, + "loss": 1.1772, + "theoretical_loss": 3.4534400637792153, + "tokens_seen": 2038366208 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003090872617853561, + "loss": 1.3311, + "theoretical_loss": 3.4534309169093884, + "tokens_seen": 2038431744 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003090712136409228, + "loss": 1.1842, + "theoretical_loss": 3.453421770415968, + "tokens_seen": 2038497280 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003090551654964895, + "loss": 1.2375, + "theoretical_loss": 3.453412624298927, + "tokens_seen": 2038562816 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003090391173520562, + "loss": 1.2438, + "theoretical_loss": 3.4534034785582373, + "tokens_seen": 2038628352 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003090230692076229, + "loss": 1.1157, + "theoretical_loss": 3.4533943331938715, + "tokens_seen": 2038693888 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003090070210631896, + "loss": 1.2457, + "theoretical_loss": 3.453385188205802, + "tokens_seen": 2038759424 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003089909729187563, + "loss": 1.1991, + "theoretical_loss": 3.4533760435940013, + "tokens_seen": 2038824960 + }, + { + "epoch": 0.62, + "learning_rate": 0.000308974924774323, + "loss": 1.181, + "theoretical_loss": 3.453366899358442, + "tokens_seen": 2038890496 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003089588766298897, + "loss": 1.2498, + "theoretical_loss": 3.4533577554990957, + "tokens_seen": 2038956032 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003089428284854564, + "loss": 1.1868, + "theoretical_loss": 3.453348612015936, + "tokens_seen": 2039021568 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003089267803410231, + "loss": 1.27, + "theoretical_loss": 3.453339468908934, + "tokens_seen": 2039087104 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003089107321965898, + "loss": 1.3075, + "theoretical_loss": 3.4533303261780635, + "tokens_seen": 2039152640 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003088946840521565, + "loss": 1.229, + "theoretical_loss": 3.453321183823296, + "tokens_seen": 2039218176 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003088786359077232, + "loss": 1.1911, + "theoretical_loss": 3.4533120418446046, + "tokens_seen": 2039283712 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003088625877632899, + "loss": 1.2546, + "theoretical_loss": 3.453302900241961, + "tokens_seen": 2039349248 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003088465396188566, + "loss": 1.2702, + "theoretical_loss": 3.453293759015338, + "tokens_seen": 2039414784 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003088304914744233, + "loss": 1.2516, + "theoretical_loss": 3.4532846181647088, + "tokens_seen": 2039480320 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030881444332999, + "loss": 1.256, + "theoretical_loss": 3.4532754776900445, + "tokens_seen": 2039545856 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003087983951855567, + "loss": 1.2774, + "theoretical_loss": 3.4532663375913186, + "tokens_seen": 2039611392 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003087823470411234, + "loss": 1.2115, + "theoretical_loss": 3.4532571978685027, + "tokens_seen": 2039676928 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003087662988966901, + "loss": 1.1991, + "theoretical_loss": 3.45324805852157, + "tokens_seen": 2039742464 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003087502507522568, + "loss": 1.2003, + "theoretical_loss": 3.4532389195504933, + "tokens_seen": 2039808000 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003087342026078235, + "loss": 1.1686, + "theoretical_loss": 3.453229780955244, + "tokens_seen": 2039873536 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003087181544633902, + "loss": 1.3441, + "theoretical_loss": 3.4532206427357948, + "tokens_seen": 2039939072 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003087021063189569, + "loss": 1.2169, + "theoretical_loss": 3.4532115048921184, + "tokens_seen": 2040004608 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003086860581745236, + "loss": 1.1412, + "theoretical_loss": 3.4532023674241876, + "tokens_seen": 2040070144 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003086700100300903, + "loss": 1.2713, + "theoretical_loss": 3.4531932303319746, + "tokens_seen": 2040135680 + }, + { + "epoch": 0.62, + "learning_rate": 0.000308653961885657, + "loss": 1.3524, + "theoretical_loss": 3.4531840936154516, + "tokens_seen": 2040201216 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003086379137412237, + "loss": 1.368, + "theoretical_loss": 3.4531749572745913, + "tokens_seen": 2040266752 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003086218655967904, + "loss": 1.2728, + "theoretical_loss": 3.4531658213093666, + "tokens_seen": 2040332288 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003086058174523571, + "loss": 1.3406, + "theoretical_loss": 3.4531566857197493, + "tokens_seen": 2040397824 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003085897693079238, + "loss": 1.3193, + "theoretical_loss": 3.4531475505057125, + "tokens_seen": 2040463360 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003085737211634905, + "loss": 1.2619, + "theoretical_loss": 3.453138415667228, + "tokens_seen": 2040528896 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003085576730190572, + "loss": 1.2346, + "theoretical_loss": 3.453129281204269, + "tokens_seen": 2040594432 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003085416248746239, + "loss": 1.2919, + "theoretical_loss": 3.4531201471168074, + "tokens_seen": 2040659968 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003085255767301906, + "loss": 1.293, + "theoretical_loss": 3.4531110134048166, + "tokens_seen": 2040725504 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003085095285857573, + "loss": 1.1982, + "theoretical_loss": 3.453101880068268, + "tokens_seen": 2040791040 + }, + { + "epoch": 0.62, + "learning_rate": 0.000308493480441324, + "loss": 1.1653, + "theoretical_loss": 3.4530927471071347, + "tokens_seen": 2040856576 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003084774322968907, + "loss": 1.1875, + "theoretical_loss": 3.4530836145213892, + "tokens_seen": 2040922112 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003084613841524574, + "loss": 1.2304, + "theoretical_loss": 3.453074482311004, + "tokens_seen": 2040987648 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003084453360080241, + "loss": 1.202, + "theoretical_loss": 3.453065350475952, + "tokens_seen": 2041053184 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030842928786359075, + "loss": 1.2468, + "theoretical_loss": 3.4530562190162044, + "tokens_seen": 2041118720 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030841323971915745, + "loss": 1.3015, + "theoretical_loss": 3.4530470879317354, + "tokens_seen": 2041184256 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030839719157472415, + "loss": 1.2849, + "theoretical_loss": 3.4530379572225165, + "tokens_seen": 2041249792 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030838114343029085, + "loss": 1.107, + "theoretical_loss": 3.4530288268885205, + "tokens_seen": 2041315328 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030836509528585755, + "loss": 1.2995, + "theoretical_loss": 3.45301969692972, + "tokens_seen": 2041380864 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030834904714142425, + "loss": 1.3211, + "theoretical_loss": 3.4530105673460874, + "tokens_seen": 2041446400 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030833299899699095, + "loss": 1.2781, + "theoretical_loss": 3.4530014381375955, + "tokens_seen": 2041511936 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030831695085255765, + "loss": 1.1947, + "theoretical_loss": 3.4529923093042165, + "tokens_seen": 2041577472 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030830090270812436, + "loss": 1.1554, + "theoretical_loss": 3.452983180845923, + "tokens_seen": 2041643008 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030828485456369106, + "loss": 1.2767, + "theoretical_loss": 3.4529740527626878, + "tokens_seen": 2041708544 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030826880641925776, + "loss": 1.3027, + "theoretical_loss": 3.4529649250544834, + "tokens_seen": 2041774080 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030825275827482446, + "loss": 1.288, + "theoretical_loss": 3.452955797721282, + "tokens_seen": 2041839616 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030823671013039116, + "loss": 1.187, + "theoretical_loss": 3.4529466707630565, + "tokens_seen": 2041905152 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030822066198595786, + "loss": 1.2596, + "theoretical_loss": 3.45293754417978, + "tokens_seen": 2041970688 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030820461384152456, + "loss": 1.3891, + "theoretical_loss": 3.4529284179714237, + "tokens_seen": 2042036224 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030818856569709126, + "loss": 1.3187, + "theoretical_loss": 3.4529192921379606, + "tokens_seen": 2042101760 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030817251755265796, + "loss": 1.2572, + "theoretical_loss": 3.4529101666793642, + "tokens_seen": 2042167296 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030815646940822466, + "loss": 1.1427, + "theoretical_loss": 3.4529010415956063, + "tokens_seen": 2042232832 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030814042126379136, + "loss": 1.2557, + "theoretical_loss": 3.4528919168866596, + "tokens_seen": 2042298368 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030812437311935806, + "loss": 1.2117, + "theoretical_loss": 3.4528827925524967, + "tokens_seen": 2042363904 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030810832497492476, + "loss": 1.2441, + "theoretical_loss": 3.4528736685930905, + "tokens_seen": 2042429440 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003080922768304915, + "loss": 1.2491, + "theoretical_loss": 3.4528645450084126, + "tokens_seen": 2042494976 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003080762286860582, + "loss": 1.3134, + "theoretical_loss": 3.4528554217984366, + "tokens_seen": 2042560512 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003080601805416249, + "loss": 1.1986, + "theoretical_loss": 3.4528462989631348, + "tokens_seen": 2042626048 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003080441323971916, + "loss": 1.3348, + "theoretical_loss": 3.45283717650248, + "tokens_seen": 2042691584 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003080280842527583, + "loss": 1.2635, + "theoretical_loss": 3.452828054416444, + "tokens_seen": 2042757120 + }, + { + "epoch": 0.62, + "learning_rate": 0.000308012036108325, + "loss": 1.191, + "theoretical_loss": 3.452818932705, + "tokens_seen": 2042822656 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003079959879638917, + "loss": 1.2672, + "theoretical_loss": 3.452809811368121, + "tokens_seen": 2042888192 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003079799398194584, + "loss": 1.2422, + "theoretical_loss": 3.4528006904057786, + "tokens_seen": 2042953728 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003079638916750251, + "loss": 1.3378, + "theoretical_loss": 3.452791569817946, + "tokens_seen": 2043019264 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003079478435305918, + "loss": 1.2394, + "theoretical_loss": 3.4527824496045962, + "tokens_seen": 2043084800 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003079317953861585, + "loss": 1.2435, + "theoretical_loss": 3.452773329765701, + "tokens_seen": 2043150336 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003079157472417252, + "loss": 1.1995, + "theoretical_loss": 3.4527642103012335, + "tokens_seen": 2043215872 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003078996990972919, + "loss": 1.1843, + "theoretical_loss": 3.452755091211166, + "tokens_seen": 2043281408 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003078836509528586, + "loss": 1.2643, + "theoretical_loss": 3.452745972495472, + "tokens_seen": 2043346944 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003078676028084253, + "loss": 1.2598, + "theoretical_loss": 3.4527368541541223, + "tokens_seen": 2043412480 + }, + { + "epoch": 0.62, + "learning_rate": 0.000307851554663992, + "loss": 1.2445, + "theoretical_loss": 3.452727736187091, + "tokens_seen": 2043478016 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030783550651955873, + "loss": 1.2605, + "theoretical_loss": 3.452718618594351, + "tokens_seen": 2043543552 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030781945837512543, + "loss": 1.1022, + "theoretical_loss": 3.4527095013758737, + "tokens_seen": 2043609088 + }, + { + "epoch": 0.62, + "learning_rate": 0.00030780341023069213, + "loss": 1.3002, + "theoretical_loss": 3.452700384531633, + "tokens_seen": 2043674624 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003077873620862588, + "loss": 1.3739, + "theoretical_loss": 3.4526912680616006, + "tokens_seen": 2043740160 + }, + { + "epoch": 0.62, + "learning_rate": 0.0003077713139418255, + "loss": 1.2652, + "theoretical_loss": 3.4526841460797444, + "tokens_seen": 2043791360 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003077552657973922, + "loss": 1.3636, + "theoretical_loss": 3.452675030276204, + "tokens_seen": 2043856896 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003077392176529589, + "loss": 1.3791, + "theoretical_loss": 3.4526659148467953, + "tokens_seen": 2043922432 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003077231695085256, + "loss": 1.3258, + "theoretical_loss": 3.452656799791492, + "tokens_seen": 2043987968 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003077071213640923, + "loss": 1.3285, + "theoretical_loss": 3.452647685110266, + "tokens_seen": 2044053504 + }, + { + "epoch": 1.0, + "learning_rate": 0.000307691073219659, + "loss": 1.3795, + "theoretical_loss": 3.452638570803091, + "tokens_seen": 2044119040 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003076750250752257, + "loss": 1.3329, + "theoretical_loss": 3.452629456869939, + "tokens_seen": 2044184576 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003076589769307924, + "loss": 1.2693, + "theoretical_loss": 3.452620343310783, + "tokens_seen": 2044250112 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003076429287863591, + "loss": 1.2607, + "theoretical_loss": 3.452611230125595, + "tokens_seen": 2044315648 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003076268806419258, + "loss": 1.3277, + "theoretical_loss": 3.452602117314348, + "tokens_seen": 2044381184 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003076108324974925, + "loss": 1.3629, + "theoretical_loss": 3.4525930048770155, + "tokens_seen": 2044446720 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003075947843530592, + "loss": 1.3053, + "theoretical_loss": 3.452583892813569, + "tokens_seen": 2044512256 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003075787362086259, + "loss": 1.3387, + "theoretical_loss": 3.4525747811239818, + "tokens_seen": 2044577792 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003075626880641926, + "loss": 1.3529, + "theoretical_loss": 3.4525656698082265, + "tokens_seen": 2044643328 + }, + { + "epoch": 1.0, + "objective/train/docs_used": 756598, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3842358589172363, + "objective/train/theoretical_loss": 3.4525565588662754, + "objective/train/tokens_used": 2065168864, + "theoretical_loss": 3.4525565588662754, + "tokens_seen": 2044708864 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003075466399197593, + "loss": 1.3842, + "theoretical_loss": 3.4525565588662754, + "tokens_seen": 2044708864 + }, + { + "epoch": 1.0, + "learning_rate": 0.000307530591775326, + "loss": 1.3774, + "theoretical_loss": 3.452547448298102, + "tokens_seen": 2044774400 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003075145436308927, + "loss": 1.374, + "theoretical_loss": 3.452538338103678, + "tokens_seen": 2044839936 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003074984954864594, + "loss": 1.2844, + "theoretical_loss": 3.4525292282829767, + "tokens_seen": 2044905472 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003074824473420261, + "loss": 1.2669, + "theoretical_loss": 3.452520118835971, + "tokens_seen": 2044971008 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003074663991975928, + "loss": 1.2207, + "theoretical_loss": 3.452511009762633, + "tokens_seen": 2045036544 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003074503510531595, + "loss": 1.235, + "theoretical_loss": 3.4525019010629356, + "tokens_seen": 2045102080 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003074343029087262, + "loss": 1.2368, + "theoretical_loss": 3.4524927927368516, + "tokens_seen": 2045167616 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003074182547642929, + "loss": 1.3598, + "theoretical_loss": 3.4524836847843536, + "tokens_seen": 2045233152 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003074022066198596, + "loss": 1.2719, + "theoretical_loss": 3.452474577205414, + "tokens_seen": 2045298688 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003073861584754263, + "loss": 1.3792, + "theoretical_loss": 3.4524654700000066, + "tokens_seen": 2045364224 + }, + { + "epoch": 1.0, + "learning_rate": 0.000307370110330993, + "loss": 1.3952, + "theoretical_loss": 3.452456363168103, + "tokens_seen": 2045429760 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003073540621865597, + "loss": 1.3701, + "theoretical_loss": 3.452447256709676, + "tokens_seen": 2045495296 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003073380140421264, + "loss": 1.2827, + "theoretical_loss": 3.4524381506246993, + "tokens_seen": 2045560832 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003073219658976931, + "loss": 1.3433, + "theoretical_loss": 3.4524290449131447, + "tokens_seen": 2045626368 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003073059177532598, + "loss": 1.2546, + "theoretical_loss": 3.452419939574985, + "tokens_seen": 2045691904 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003072898696088265, + "loss": 1.2816, + "theoretical_loss": 3.452410834610193, + "tokens_seen": 2045757440 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003072738214643932, + "loss": 1.3943, + "theoretical_loss": 3.4524017300187415, + "tokens_seen": 2045822976 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003072577733199599, + "loss": 1.3569, + "theoretical_loss": 3.4523926258006035, + "tokens_seen": 2045888512 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003072417251755266, + "loss": 1.4086, + "theoretical_loss": 3.4523835219557517, + "tokens_seen": 2045954048 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003072256770310933, + "loss": 1.3009, + "theoretical_loss": 3.4523744184841583, + "tokens_seen": 2046019584 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030720962888666, + "loss": 1.4432, + "theoretical_loss": 3.452365315385796, + "tokens_seen": 2046085120 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003071935807422267, + "loss": 1.2091, + "theoretical_loss": 3.4523562126606384, + "tokens_seen": 2046150656 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003071775325977934, + "loss": 1.3767, + "theoretical_loss": 3.452347110308658, + "tokens_seen": 2046216192 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003071614844533601, + "loss": 1.2923, + "theoretical_loss": 3.4523380083298267, + "tokens_seen": 2046281728 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003071454363089268, + "loss": 1.331, + "theoretical_loss": 3.452328906724118, + "tokens_seen": 2046347264 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003071293881644935, + "loss": 1.3227, + "theoretical_loss": 3.4523198054915047, + "tokens_seen": 2046412800 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003071133400200602, + "loss": 1.2953, + "theoretical_loss": 3.4523107046319597, + "tokens_seen": 2046478336 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003070972918756269, + "loss": 1.2694, + "theoretical_loss": 3.452301604145455, + "tokens_seen": 2046543872 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003070812437311936, + "loss": 1.268, + "theoretical_loss": 3.4522925040319636, + "tokens_seen": 2046609408 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003070651955867603, + "loss": 1.342, + "theoretical_loss": 3.4522834042914585, + "tokens_seen": 2046674944 + }, + { + "epoch": 1.0, + "learning_rate": 0.000307049147442327, + "loss": 1.3556, + "theoretical_loss": 3.452274304923913, + "tokens_seen": 2046740480 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003070330992978937, + "loss": 1.317, + "theoretical_loss": 3.452265205929299, + "tokens_seen": 2046806016 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030701705115346035, + "loss": 1.143, + "theoretical_loss": 3.4522561073075897, + "tokens_seen": 2046871552 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030700100300902705, + "loss": 1.2357, + "theoretical_loss": 3.452247009058757, + "tokens_seen": 2046937088 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030698495486459375, + "loss": 1.3138, + "theoretical_loss": 3.4522379111827757, + "tokens_seen": 2047002624 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030696890672016045, + "loss": 1.2281, + "theoretical_loss": 3.4522288136796164, + "tokens_seen": 2047068160 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030695285857572715, + "loss": 1.3695, + "theoretical_loss": 3.452219716549253, + "tokens_seen": 2047133696 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030693681043129385, + "loss": 1.3301, + "theoretical_loss": 3.4522106197916584, + "tokens_seen": 2047199232 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030692076228686056, + "loss": 1.2243, + "theoretical_loss": 3.4522015234068046, + "tokens_seen": 2047264768 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003069047141424273, + "loss": 1.294, + "theoretical_loss": 3.4521924273946656, + "tokens_seen": 2047330304 + }, + { + "epoch": 1.0, + "learning_rate": 0.000306888665997994, + "loss": 1.266, + "theoretical_loss": 3.4521833317552133, + "tokens_seen": 2047395840 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003068726178535607, + "loss": 1.3294, + "theoretical_loss": 3.4521742364884203, + "tokens_seen": 2047461376 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003068565697091274, + "loss": 1.2538, + "theoretical_loss": 3.45216514159426, + "tokens_seen": 2047526912 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003068405215646941, + "loss": 1.2587, + "theoretical_loss": 3.452156047072705, + "tokens_seen": 2047592448 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003068244734202608, + "loss": 1.3835, + "theoretical_loss": 3.4521469529237283, + "tokens_seen": 2047657984 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003068084252758275, + "loss": 1.3022, + "theoretical_loss": 3.452137859147302, + "tokens_seen": 2047723520 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003067923771313942, + "loss": 1.2952, + "theoretical_loss": 3.4521287657434, + "tokens_seen": 2047789056 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003067763289869609, + "loss": 1.3007, + "theoretical_loss": 3.4521196727119943, + "tokens_seen": 2047854592 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003067602808425276, + "loss": 1.3493, + "theoretical_loss": 3.4521105800530583, + "tokens_seen": 2047920128 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003067442326980943, + "loss": 1.3441, + "theoretical_loss": 3.452101487766564, + "tokens_seen": 2047985664 + }, + { + "epoch": 1.0, + "learning_rate": 0.000306728184553661, + "loss": 1.2664, + "theoretical_loss": 3.452092395852485, + "tokens_seen": 2048051200 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003067121364092277, + "loss": 1.2675, + "theoretical_loss": 3.452083304310794, + "tokens_seen": 2048116736 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003066960882647944, + "loss": 1.2923, + "theoretical_loss": 3.4520742131414632, + "tokens_seen": 2048182272 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003066800401203611, + "loss": 1.3661, + "theoretical_loss": 3.4520651223444663, + "tokens_seen": 2048247808 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003066639919759278, + "loss": 1.2891, + "theoretical_loss": 3.452056031919776, + "tokens_seen": 2048313344 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003066479438314945, + "loss": 1.3075, + "theoretical_loss": 3.4520469418673647, + "tokens_seen": 2048378880 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003066318956870612, + "loss": 1.3533, + "theoretical_loss": 3.452037852187205, + "tokens_seen": 2048444416 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003066158475426279, + "loss": 1.3376, + "theoretical_loss": 3.452028762879271, + "tokens_seen": 2048509952 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003065997993981946, + "loss": 1.3655, + "theoretical_loss": 3.4520196739435343, + "tokens_seen": 2048575488 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003065837512537613, + "loss": 1.366, + "theoretical_loss": 3.452010585379968, + "tokens_seen": 2048641024 + }, + { + "epoch": 1.0, + "learning_rate": 0.000306567703109328, + "loss": 1.3601, + "theoretical_loss": 3.4520014971885455, + "tokens_seen": 2048706560 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003065516549648947, + "loss": 1.2917, + "theoretical_loss": 3.4519924093692396, + "tokens_seen": 2048772096 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003065356068204614, + "loss": 1.2856, + "theoretical_loss": 3.4519833219220226, + "tokens_seen": 2048837632 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003065195586760281, + "loss": 1.3636, + "theoretical_loss": 3.451974234846867, + "tokens_seen": 2048903168 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003065035105315948, + "loss": 1.3677, + "theoretical_loss": 3.4519651481437474, + "tokens_seen": 2048968704 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003064874623871615, + "loss": 1.2848, + "theoretical_loss": 3.451956061812635, + "tokens_seen": 2049034240 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030647141424272823, + "loss": 1.3592, + "theoretical_loss": 3.4519469758535033, + "tokens_seen": 2049099776 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030645536609829493, + "loss": 1.247, + "theoretical_loss": 3.451937890266325, + "tokens_seen": 2049165312 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030643931795386163, + "loss": 1.3528, + "theoretical_loss": 3.4519288050510735, + "tokens_seen": 2049230848 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030642326980942833, + "loss": 1.3098, + "theoretical_loss": 3.451919720207721, + "tokens_seen": 2049296384 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030640722166499503, + "loss": 1.267, + "theoretical_loss": 3.451910635736241, + "tokens_seen": 2049361920 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003063911735205617, + "loss": 1.3207, + "theoretical_loss": 3.4519015516366056, + "tokens_seen": 2049427456 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003063751253761284, + "loss": 1.1811, + "theoretical_loss": 3.451892467908788, + "tokens_seen": 2049492992 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003063590772316951, + "loss": 1.2849, + "theoretical_loss": 3.4518833845527617, + "tokens_seen": 2049558528 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003063430290872618, + "loss": 1.2016, + "theoretical_loss": 3.451874301568499, + "tokens_seen": 2049624064 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003063269809428285, + "loss": 1.3435, + "theoretical_loss": 3.451865218955973, + "tokens_seen": 2049689600 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003063109327983952, + "loss": 1.2976, + "theoretical_loss": 3.451856136715157, + "tokens_seen": 2049755136 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003062948846539619, + "loss": 1.3341, + "theoretical_loss": 3.4518470548460227, + "tokens_seen": 2049820672 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003062788365095286, + "loss": 1.3758, + "theoretical_loss": 3.451837973348544, + "tokens_seen": 2049886208 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003062627883650953, + "loss": 1.2722, + "theoretical_loss": 3.4518288922226934, + "tokens_seen": 2049951744 + }, + { + "epoch": 1.0, + "learning_rate": 0.000306246740220662, + "loss": 1.2829, + "theoretical_loss": 3.451819811468444, + "tokens_seen": 2050017280 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003062306920762287, + "loss": 1.2423, + "theoretical_loss": 3.451810731085769, + "tokens_seen": 2050082816 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003062146439317954, + "loss": 1.3276, + "theoretical_loss": 3.451801651074641, + "tokens_seen": 2050148352 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003061985957873621, + "loss": 1.3529, + "theoretical_loss": 3.451792571435033, + "tokens_seen": 2050213888 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003061825476429288, + "loss": 1.2361, + "theoretical_loss": 3.451783492166917, + "tokens_seen": 2050279424 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003061664994984955, + "loss": 1.2705, + "theoretical_loss": 3.4517744132702672, + "tokens_seen": 2050344960 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003061504513540622, + "loss": 1.1705, + "theoretical_loss": 3.4517653347450565, + "tokens_seen": 2050410496 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003061344032096289, + "loss": 1.4853, + "theoretical_loss": 3.451756256591257, + "tokens_seen": 2050476032 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003061183550651956, + "loss": 1.2668, + "theoretical_loss": 3.4517471788088425, + "tokens_seen": 2050541568 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003061023069207623, + "loss": 1.3483, + "theoretical_loss": 3.451738101397785, + "tokens_seen": 2050607104 + }, + { + "epoch": 1.0, + "learning_rate": 0.000306086258776329, + "loss": 1.2782, + "theoretical_loss": 3.451729024358058, + "tokens_seen": 2050672640 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003060702106318957, + "loss": 1.2483, + "theoretical_loss": 3.451719947689635, + "tokens_seen": 2050738176 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003060541624874624, + "loss": 1.1989, + "theoretical_loss": 3.451710871392488, + "tokens_seen": 2050803712 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003060381143430291, + "loss": 1.4079, + "theoretical_loss": 3.45170179546659, + "tokens_seen": 2050869248 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003060220661985958, + "loss": 1.3339, + "theoretical_loss": 3.4516927199119145, + "tokens_seen": 2050934784 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003060060180541625, + "loss": 1.2475, + "theoretical_loss": 3.4516836447284343, + "tokens_seen": 2051000320 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003059899699097292, + "loss": 1.285, + "theoretical_loss": 3.451674569916122, + "tokens_seen": 2051065856 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003059739217652959, + "loss": 1.2591, + "theoretical_loss": 3.451665495474951, + "tokens_seen": 2051131392 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003059578736208626, + "loss": 1.32, + "theoretical_loss": 3.451656421404894, + "tokens_seen": 2051196928 + }, + { + "epoch": 1.0, + "objective/train/docs_used": 758821, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3230470418930054, + "objective/train/theoretical_loss": 3.451647347705924, + "objective/train/tokens_used": 2071722464, + "theoretical_loss": 3.451647347705924, + "tokens_seen": 2051262464 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003059418254764293, + "loss": 1.323, + "theoretical_loss": 3.451647347705924, + "tokens_seen": 2051262464 + }, + { + "epoch": 1.0, + "learning_rate": 0.000305925777331996, + "loss": 1.2665, + "theoretical_loss": 3.4516382743780136, + "tokens_seen": 2051328000 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003059097291875627, + "loss": 1.3776, + "theoretical_loss": 3.451629201421137, + "tokens_seen": 2051393536 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003058936810431294, + "loss": 1.2762, + "theoretical_loss": 3.451620128835266, + "tokens_seen": 2051459072 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003058776328986961, + "loss": 1.3836, + "theoretical_loss": 3.451611056620374, + "tokens_seen": 2051524608 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003058615847542628, + "loss": 1.2674, + "theoretical_loss": 3.451601984776434, + "tokens_seen": 2051590144 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003058455366098295, + "loss": 1.3696, + "theoretical_loss": 3.4515929133034184, + "tokens_seen": 2051655680 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003058294884653962, + "loss": 1.3149, + "theoretical_loss": 3.451583842201301, + "tokens_seen": 2051721216 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003058134403209629, + "loss": 1.2606, + "theoretical_loss": 3.451574771470055, + "tokens_seen": 2051786752 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003057973921765296, + "loss": 1.2122, + "theoretical_loss": 3.4515657011096517, + "tokens_seen": 2051852288 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003057813440320963, + "loss": 1.2974, + "theoretical_loss": 3.4515566311200665, + "tokens_seen": 2051917824 + }, + { + "epoch": 1.0, + "learning_rate": 0.000305765295887663, + "loss": 1.142, + "theoretical_loss": 3.45154756150127, + "tokens_seen": 2051983360 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003057492477432297, + "loss": 1.2471, + "theoretical_loss": 3.4515384922532375, + "tokens_seen": 2052048896 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003057331995987964, + "loss": 1.2724, + "theoretical_loss": 3.4515294233759404, + "tokens_seen": 2052114432 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003057171514543631, + "loss": 1.23, + "theoretical_loss": 3.451520354869352, + "tokens_seen": 2052179968 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003057011033099298, + "loss": 1.2236, + "theoretical_loss": 3.451511286733445, + "tokens_seen": 2052245504 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003056850551654965, + "loss": 1.273, + "theoretical_loss": 3.4515022189681934, + "tokens_seen": 2052311040 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003056690070210632, + "loss": 1.3188, + "theoretical_loss": 3.45149315157357, + "tokens_seen": 2052376576 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003056529588766299, + "loss": 1.4152, + "theoretical_loss": 3.4514840845495476, + "tokens_seen": 2052442112 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003056369107321966, + "loss": 1.371, + "theoretical_loss": 3.4514750178960982, + "tokens_seen": 2052507648 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003056208625877633, + "loss": 1.3268, + "theoretical_loss": 3.451465951613197, + "tokens_seen": 2052573184 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030560481444333, + "loss": 1.235, + "theoretical_loss": 3.4514568857008148, + "tokens_seen": 2052638720 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003055887662988967, + "loss": 1.2921, + "theoretical_loss": 3.451447820158926, + "tokens_seen": 2052704256 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003055727181544634, + "loss": 1.3176, + "theoretical_loss": 3.451438754987503, + "tokens_seen": 2052769792 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003055566700100301, + "loss": 1.2677, + "theoretical_loss": 3.4514296901865196, + "tokens_seen": 2052835328 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003055406218655968, + "loss": 1.3071, + "theoretical_loss": 3.451420625755948, + "tokens_seen": 2052900864 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003055245737211635, + "loss": 1.2965, + "theoretical_loss": 3.4514115616957612, + "tokens_seen": 2052966400 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003055085255767302, + "loss": 1.1755, + "theoretical_loss": 3.4514024980059332, + "tokens_seen": 2053031936 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003054924774322969, + "loss": 1.2779, + "theoretical_loss": 3.4513934346864366, + "tokens_seen": 2053097472 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003054764292878636, + "loss": 1.2678, + "theoretical_loss": 3.4513843717372437, + "tokens_seen": 2053163008 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003054603811434303, + "loss": 1.3041, + "theoretical_loss": 3.4513753091583284, + "tokens_seen": 2053228544 + }, + { + "epoch": 1.0, + "learning_rate": 0.000305444332998997, + "loss": 1.319, + "theoretical_loss": 3.4513662469496635, + "tokens_seen": 2053294080 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003054282848545637, + "loss": 1.2373, + "theoretical_loss": 3.451357185111222, + "tokens_seen": 2053359616 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003054122367101304, + "loss": 1.2817, + "theoretical_loss": 3.451348123642977, + "tokens_seen": 2053425152 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003053961885656971, + "loss": 1.2231, + "theoretical_loss": 3.4513390625449016, + "tokens_seen": 2053490688 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003053801404212638, + "loss": 1.2354, + "theoretical_loss": 3.4513300018169693, + "tokens_seen": 2053556224 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003053640922768305, + "loss": 1.2469, + "theoretical_loss": 3.451320941459152, + "tokens_seen": 2053621760 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003053480441323972, + "loss": 1.3382, + "theoretical_loss": 3.4513118814714234, + "tokens_seen": 2053687296 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003053319959879639, + "loss": 1.3123, + "theoretical_loss": 3.4513028218537576, + "tokens_seen": 2053752832 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003053159478435306, + "loss": 1.3076, + "theoretical_loss": 3.4512937626061255, + "tokens_seen": 2053818368 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003052998996990973, + "loss": 1.3189, + "theoretical_loss": 3.4512847037285024, + "tokens_seen": 2053883904 + }, + { + "epoch": 1.0, + "learning_rate": 0.000305283851554664, + "loss": 1.282, + "theoretical_loss": 3.4512756452208597, + "tokens_seen": 2053949440 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003052678034102307, + "loss": 1.2709, + "theoretical_loss": 3.451266587083172, + "tokens_seen": 2054014976 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003052517552657974, + "loss": 1.2707, + "theoretical_loss": 3.451257529315411, + "tokens_seen": 2054080512 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003052357071213641, + "loss": 1.2396, + "theoretical_loss": 3.4512484719175505, + "tokens_seen": 2054146048 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003052196589769308, + "loss": 1.3418, + "theoretical_loss": 3.4512394148895638, + "tokens_seen": 2054211584 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003052036108324975, + "loss": 1.2364, + "theoretical_loss": 3.451230358231423, + "tokens_seen": 2054277120 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003051875626880642, + "loss": 1.3218, + "theoretical_loss": 3.451221301943102, + "tokens_seen": 2054342656 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003051715145436309, + "loss": 1.2686, + "theoretical_loss": 3.451212246024574, + "tokens_seen": 2054408192 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003051554663991976, + "loss": 1.2854, + "theoretical_loss": 3.4512031904758116, + "tokens_seen": 2054473728 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003051394182547643, + "loss": 1.3185, + "theoretical_loss": 3.451194135296788, + "tokens_seen": 2054539264 + }, + { + "epoch": 1.0, + "learning_rate": 0.000305123370110331, + "loss": 1.271, + "theoretical_loss": 3.451185080487477, + "tokens_seen": 2054604800 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030510732196589773, + "loss": 1.2799, + "theoretical_loss": 3.4511760260478512, + "tokens_seen": 2054670336 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030509127382146443, + "loss": 1.2444, + "theoretical_loss": 3.451166971977883, + "tokens_seen": 2054735872 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030507522567703113, + "loss": 1.2887, + "theoretical_loss": 3.4511579182775467, + "tokens_seen": 2054801408 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030505917753259783, + "loss": 1.2643, + "theoretical_loss": 3.451148864946815, + "tokens_seen": 2054866944 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030504312938816453, + "loss": 1.2214, + "theoretical_loss": 3.451139811985661, + "tokens_seen": 2054932480 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030502708124373123, + "loss": 1.3424, + "theoretical_loss": 3.4511307593940574, + "tokens_seen": 2054998016 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030501103309929793, + "loss": 1.2589, + "theoretical_loss": 3.451121707171978, + "tokens_seen": 2055063552 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003049949849548646, + "loss": 1.2298, + "theoretical_loss": 3.4511126553193954, + "tokens_seen": 2055129088 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003049789368104313, + "loss": 1.3775, + "theoretical_loss": 3.4511036038362835, + "tokens_seen": 2055194624 + }, + { + "epoch": 1.0, + "learning_rate": 0.000304962888665998, + "loss": 1.2545, + "theoretical_loss": 3.4510945527226147, + "tokens_seen": 2055260160 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003049468405215647, + "loss": 1.2768, + "theoretical_loss": 3.451085501978362, + "tokens_seen": 2055325696 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003049307923771314, + "loss": 1.328, + "theoretical_loss": 3.4510764516034995, + "tokens_seen": 2055391232 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003049147442326981, + "loss": 1.327, + "theoretical_loss": 3.4510674015979994, + "tokens_seen": 2055456768 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003048986960882648, + "loss": 1.4702, + "theoretical_loss": 3.4510583519618354, + "tokens_seen": 2055522304 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003048826479438315, + "loss": 1.3023, + "theoretical_loss": 3.4510493026949804, + "tokens_seen": 2055587840 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003048665997993982, + "loss": 1.2652, + "theoretical_loss": 3.451040253797407, + "tokens_seen": 2055653376 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003048505516549649, + "loss": 1.2372, + "theoretical_loss": 3.45103120526909, + "tokens_seen": 2055718912 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003048345035105316, + "loss": 1.2619, + "theoretical_loss": 3.451022157110001, + "tokens_seen": 2055784448 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003048184553660983, + "loss": 1.3302, + "theoretical_loss": 3.4510131093201135, + "tokens_seen": 2055849984 + }, + { + "epoch": 1.0, + "learning_rate": 0.000304802407221665, + "loss": 1.1773, + "theoretical_loss": 3.4510040618994013, + "tokens_seen": 2055915520 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003047863590772317, + "loss": 1.2212, + "theoretical_loss": 3.450995014847837, + "tokens_seen": 2055981056 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003047703109327984, + "loss": 1.2964, + "theoretical_loss": 3.4509859681653943, + "tokens_seen": 2056046592 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003047542627883651, + "loss": 1.2497, + "theoretical_loss": 3.450976921852045, + "tokens_seen": 2056112128 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003047382146439318, + "loss": 1.222, + "theoretical_loss": 3.450967875907764, + "tokens_seen": 2056177664 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003047221664994985, + "loss": 1.2528, + "theoretical_loss": 3.4509588303325236, + "tokens_seen": 2056243200 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003047061183550652, + "loss": 1.1903, + "theoretical_loss": 3.450949785126297, + "tokens_seen": 2056308736 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003046900702106319, + "loss": 1.3019, + "theoretical_loss": 3.4509407402890577, + "tokens_seen": 2056374272 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003046740220661986, + "loss": 1.3195, + "theoretical_loss": 3.4509316958207785, + "tokens_seen": 2056439808 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003046579739217653, + "loss": 1.1704, + "theoretical_loss": 3.4509226517214326, + "tokens_seen": 2056505344 + }, + { + "epoch": 1.0, + "learning_rate": 0.000304641925777332, + "loss": 1.2396, + "theoretical_loss": 3.450913607990994, + "tokens_seen": 2056570880 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003046258776328987, + "loss": 1.3657, + "theoretical_loss": 3.450904564629435, + "tokens_seen": 2056636416 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003046098294884654, + "loss": 1.3116, + "theoretical_loss": 3.450895521636729, + "tokens_seen": 2056701952 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003045937813440321, + "loss": 1.3161, + "theoretical_loss": 3.4508864790128495, + "tokens_seen": 2056767488 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003045777331995988, + "loss": 1.2218, + "theoretical_loss": 3.450877436757769, + "tokens_seen": 2056833024 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003045616850551655, + "loss": 1.2741, + "theoretical_loss": 3.4508683948714616, + "tokens_seen": 2056898560 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003045456369107322, + "loss": 1.2858, + "theoretical_loss": 3.4508593533539003, + "tokens_seen": 2056964096 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003045295887662989, + "loss": 1.1919, + "theoretical_loss": 3.4508503122050573, + "tokens_seen": 2057029632 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003045135406218656, + "loss": 1.2129, + "theoretical_loss": 3.4508412714249075, + "tokens_seen": 2057095168 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030449749247743235, + "loss": 1.264, + "theoretical_loss": 3.4508322310134227, + "tokens_seen": 2057160704 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030448144433299905, + "loss": 1.3593, + "theoretical_loss": 3.450823190970577, + "tokens_seen": 2057226240 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030446539618856575, + "loss": 1.2766, + "theoretical_loss": 3.4508141512963433, + "tokens_seen": 2057291776 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030444934804413245, + "loss": 1.3261, + "theoretical_loss": 3.450805111990695, + "tokens_seen": 2057357312 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030443329989969915, + "loss": 1.3459, + "theoretical_loss": 3.450796073053605, + "tokens_seen": 2057422848 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030441725175526585, + "loss": 1.2245, + "theoretical_loss": 3.4507870344850464, + "tokens_seen": 2057488384 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030440120361083256, + "loss": 1.3037, + "theoretical_loss": 3.450777996284993, + "tokens_seen": 2057553920 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030438515546639926, + "loss": 1.2125, + "theoretical_loss": 3.450768958453418, + "tokens_seen": 2057619456 + }, + { + "epoch": 1.0, + "learning_rate": 0.00030436910732196596, + "loss": 1.4095, + "theoretical_loss": 3.450759920990294, + "tokens_seen": 2057684992 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003043530591775326, + "loss": 1.366, + "theoretical_loss": 3.4507508838955947, + "tokens_seen": 2057750528 + }, + { + "epoch": 1.0, + "objective/train/docs_used": 761152, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3860605955123901, + "objective/train/theoretical_loss": 3.4507418471692937, + "objective/train/tokens_used": 2078276064, + "theoretical_loss": 3.4507418471692937, + "tokens_seen": 2057816064 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003043370110330993, + "loss": 1.3861, + "theoretical_loss": 3.4507418471692937, + "tokens_seen": 2057816064 + }, + { + "epoch": 1.0, + "learning_rate": 0.000304320962888666, + "loss": 1.2297, + "theoretical_loss": 3.4507328108113633, + "tokens_seen": 2057881600 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003043049147442327, + "loss": 1.2184, + "theoretical_loss": 3.450723774821778, + "tokens_seen": 2057947136 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003042888665997994, + "loss": 1.3297, + "theoretical_loss": 3.4507147392005098, + "tokens_seen": 2058012672 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003042728184553661, + "loss": 1.2843, + "theoretical_loss": 3.4507057039475324, + "tokens_seen": 2058078208 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003042567703109328, + "loss": 1.3037, + "theoretical_loss": 3.45069666906282, + "tokens_seen": 2058143744 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003042407221664995, + "loss": 1.2377, + "theoretical_loss": 3.4506876345463446, + "tokens_seen": 2058209280 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003042246740220662, + "loss": 1.305, + "theoretical_loss": 3.4506786003980796, + "tokens_seen": 2058274816 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003042086258776329, + "loss": 1.3213, + "theoretical_loss": 3.450669566617999, + "tokens_seen": 2058340352 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003041925777331996, + "loss": 1.3646, + "theoretical_loss": 3.450660533206076, + "tokens_seen": 2058405888 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003041765295887663, + "loss": 1.3005, + "theoretical_loss": 3.450651500162283, + "tokens_seen": 2058471424 + }, + { + "epoch": 1.0, + "learning_rate": 0.000304160481444333, + "loss": 1.3498, + "theoretical_loss": 3.4506424674865936, + "tokens_seen": 2058536960 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003041444332998997, + "loss": 1.2793, + "theoretical_loss": 3.4506334351789816, + "tokens_seen": 2058602496 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003041283851554664, + "loss": 1.2774, + "theoretical_loss": 3.45062440323942, + "tokens_seen": 2058668032 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003041123370110331, + "loss": 1.3437, + "theoretical_loss": 3.4506153716678822, + "tokens_seen": 2058733568 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003040962888665998, + "loss": 1.3702, + "theoretical_loss": 3.4506063404643412, + "tokens_seen": 2058799104 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003040802407221665, + "loss": 1.3048, + "theoretical_loss": 3.4505973096287708, + "tokens_seen": 2058864640 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003040641925777332, + "loss": 1.3366, + "theoretical_loss": 3.4505882791611433, + "tokens_seen": 2058930176 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003040481444332999, + "loss": 1.1978, + "theoretical_loss": 3.450579249061433, + "tokens_seen": 2058995712 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003040320962888666, + "loss": 1.2589, + "theoretical_loss": 3.450570219329613, + "tokens_seen": 2059061248 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003040160481444333, + "loss": 1.3185, + "theoretical_loss": 3.450561189965656, + "tokens_seen": 2059126784 + }, + { + "epoch": 1.0, + "learning_rate": 0.000304, + "loss": 1.226, + "theoretical_loss": 3.4505521609695364, + "tokens_seen": 2059192320 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003039839518555667, + "loss": 1.2543, + "theoretical_loss": 3.4505431323412266, + "tokens_seen": 2059257856 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003039679037111334, + "loss": 1.287, + "theoretical_loss": 3.4505341040807, + "tokens_seen": 2059323392 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003039518555667001, + "loss": 1.2609, + "theoretical_loss": 3.45052507618793, + "tokens_seen": 2059388928 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003039358074222668, + "loss": 1.4211, + "theoretical_loss": 3.45051604866289, + "tokens_seen": 2059454464 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003039197592778335, + "loss": 1.2982, + "theoretical_loss": 3.4505070215055538, + "tokens_seen": 2059520000 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003039037111334002, + "loss": 1.3041, + "theoretical_loss": 3.450497994715894, + "tokens_seen": 2059585536 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003038876629889669, + "loss": 1.2131, + "theoretical_loss": 3.450488968293884, + "tokens_seen": 2059651072 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003038716148445336, + "loss": 1.1958, + "theoretical_loss": 3.4504799422394976, + "tokens_seen": 2059716608 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003038555667001003, + "loss": 1.2586, + "theoretical_loss": 3.4504709165527077, + "tokens_seen": 2059782144 + }, + { + "epoch": 1.0, + "learning_rate": 0.000303839518555667, + "loss": 1.3242, + "theoretical_loss": 3.4504618912334877, + "tokens_seen": 2059847680 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003038234704112337, + "loss": 1.3705, + "theoretical_loss": 3.450452866281811, + "tokens_seen": 2059913216 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003038074222668004, + "loss": 1.2628, + "theoretical_loss": 3.450443841697651, + "tokens_seen": 2059978752 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003037913741223671, + "loss": 1.2683, + "theoretical_loss": 3.450434817480981, + "tokens_seen": 2060044288 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003037753259779338, + "loss": 1.2395, + "theoretical_loss": 3.4504257936317746, + "tokens_seen": 2060109824 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003037592778335005, + "loss": 1.3596, + "theoretical_loss": 3.450416770150005, + "tokens_seen": 2060175360 + }, + { + "epoch": 1.0, + "learning_rate": 0.0003037432296890672, + "loss": 1.2205, + "theoretical_loss": 3.4504077470356447, + "tokens_seen": 2060240896 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030372718154463393, + "loss": 1.22, + "theoretical_loss": 3.450398724288668, + "tokens_seen": 2060306432 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030371113340020063, + "loss": 1.3339, + "theoretical_loss": 3.4503897019090486, + "tokens_seen": 2060371968 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030369508525576733, + "loss": 1.2763, + "theoretical_loss": 3.450380679896759, + "tokens_seen": 2060437504 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030367903711133403, + "loss": 1.3013, + "theoretical_loss": 3.4503716582517727, + "tokens_seen": 2060503040 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030366298896690073, + "loss": 1.2182, + "theoretical_loss": 3.450362636974063, + "tokens_seen": 2060568576 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030364694082246743, + "loss": 1.3233, + "theoretical_loss": 3.450353616063604, + "tokens_seen": 2060634112 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030363089267803413, + "loss": 1.3338, + "theoretical_loss": 3.4503445955203684, + "tokens_seen": 2060699648 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030361484453360083, + "loss": 1.3646, + "theoretical_loss": 3.4503355753443294, + "tokens_seen": 2060765184 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030359879638916753, + "loss": 1.2615, + "theoretical_loss": 3.4503265555354616, + "tokens_seen": 2060830720 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003035827482447342, + "loss": 1.1938, + "theoretical_loss": 3.4503175360937366, + "tokens_seen": 2060896256 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003035667001003009, + "loss": 1.2559, + "theoretical_loss": 3.4503085170191294, + "tokens_seen": 2060961792 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003035506519558676, + "loss": 1.3136, + "theoretical_loss": 3.450299498311612, + "tokens_seen": 2061027328 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003035346038114343, + "loss": 1.2596, + "theoretical_loss": 3.4502904799711587, + "tokens_seen": 2061092864 + }, + { + "epoch": 1.01, + "learning_rate": 0.000303518555667001, + "loss": 1.4179, + "theoretical_loss": 3.4502814619977427, + "tokens_seen": 2061158400 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003035025075225677, + "loss": 1.2672, + "theoretical_loss": 3.450272444391337, + "tokens_seen": 2061223936 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003034864593781344, + "loss": 1.1598, + "theoretical_loss": 3.450263427151916, + "tokens_seen": 2061289472 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003034704112337011, + "loss": 1.2862, + "theoretical_loss": 3.4502544102794523, + "tokens_seen": 2061355008 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003034543630892678, + "loss": 1.2578, + "theoretical_loss": 3.450245393773919, + "tokens_seen": 2061420544 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003034383149448345, + "loss": 1.2247, + "theoretical_loss": 3.4502363776352905, + "tokens_seen": 2061486080 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003034222668004012, + "loss": 1.2746, + "theoretical_loss": 3.450227361863539, + "tokens_seen": 2061551616 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003034062186559679, + "loss": 1.23, + "theoretical_loss": 3.450218346458639, + "tokens_seen": 2061617152 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003033901705115346, + "loss": 1.2772, + "theoretical_loss": 3.450209331420563, + "tokens_seen": 2061682688 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003033741223671013, + "loss": 1.2884, + "theoretical_loss": 3.450200316749285, + "tokens_seen": 2061748224 + }, + { + "epoch": 1.01, + "learning_rate": 0.000303358074222668, + "loss": 1.3017, + "theoretical_loss": 3.450191302444779, + "tokens_seen": 2061813760 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003033420260782347, + "loss": 1.2534, + "theoretical_loss": 3.450182288507017, + "tokens_seen": 2061879296 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003033259779338014, + "loss": 1.3238, + "theoretical_loss": 3.450173274935973, + "tokens_seen": 2061944832 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003033099297893681, + "loss": 1.2578, + "theoretical_loss": 3.450164261731621, + "tokens_seen": 2062010368 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030329388164493484, + "loss": 1.3755, + "theoretical_loss": 3.4501552488939335, + "tokens_seen": 2062075904 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030327783350050155, + "loss": 1.3099, + "theoretical_loss": 3.450146236422885, + "tokens_seen": 2062141440 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030326178535606825, + "loss": 1.3164, + "theoretical_loss": 3.450137224318448, + "tokens_seen": 2062206976 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030324573721163495, + "loss": 1.3624, + "theoretical_loss": 3.450128212580596, + "tokens_seen": 2062272512 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030322968906720165, + "loss": 1.3922, + "theoretical_loss": 3.4501192012093034, + "tokens_seen": 2062338048 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030321364092276835, + "loss": 1.2425, + "theoretical_loss": 3.4501101902045423, + "tokens_seen": 2062403584 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030319759277833505, + "loss": 1.2158, + "theoretical_loss": 3.4501011795662873, + "tokens_seen": 2062469120 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030318154463390175, + "loss": 1.3264, + "theoretical_loss": 3.4500921692945115, + "tokens_seen": 2062534656 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030316549648946845, + "loss": 1.3202, + "theoretical_loss": 3.450083159389188, + "tokens_seen": 2062600192 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030314944834503515, + "loss": 1.2557, + "theoretical_loss": 3.4500741498502903, + "tokens_seen": 2062665728 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030313340020060185, + "loss": 1.2661, + "theoretical_loss": 3.450065140677792, + "tokens_seen": 2062731264 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030311735205616855, + "loss": 1.1957, + "theoretical_loss": 3.450056131871667, + "tokens_seen": 2062796800 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030310130391173525, + "loss": 1.2565, + "theoretical_loss": 3.450047123431888, + "tokens_seen": 2062862336 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030308525576730195, + "loss": 1.2453, + "theoretical_loss": 3.4500381153584287, + "tokens_seen": 2062927872 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030306920762286865, + "loss": 1.2899, + "theoretical_loss": 3.450029107651263, + "tokens_seen": 2062993408 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030305315947843535, + "loss": 1.2863, + "theoretical_loss": 3.450020100310364, + "tokens_seen": 2063058944 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030303711133400205, + "loss": 1.2419, + "theoretical_loss": 3.450011093335705, + "tokens_seen": 2063124480 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030302106318956876, + "loss": 1.2998, + "theoretical_loss": 3.4500020867272596, + "tokens_seen": 2063190016 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030300501504513546, + "loss": 1.3434, + "theoretical_loss": 3.449993080485002, + "tokens_seen": 2063255552 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030298896690070216, + "loss": 1.3073, + "theoretical_loss": 3.4499840746089045, + "tokens_seen": 2063321088 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030297291875626886, + "loss": 1.2614, + "theoretical_loss": 3.449975069098941, + "tokens_seen": 2063386624 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003029568706118355, + "loss": 1.224, + "theoretical_loss": 3.4499660639550855, + "tokens_seen": 2063452160 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003029408224674022, + "loss": 1.3671, + "theoretical_loss": 3.449957059177311, + "tokens_seen": 2063517696 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003029247743229689, + "loss": 1.4427, + "theoretical_loss": 3.449948054765591, + "tokens_seen": 2063583232 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003029087261785356, + "loss": 1.2699, + "theoretical_loss": 3.449939050719899, + "tokens_seen": 2063648768 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003028926780341023, + "loss": 1.2529, + "theoretical_loss": 3.4499300470402092, + "tokens_seen": 2063714304 + }, + { + "epoch": 1.01, + "learning_rate": 0.000302876629889669, + "loss": 1.2825, + "theoretical_loss": 3.449921043726494, + "tokens_seen": 2063779840 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003028605817452357, + "loss": 1.2813, + "theoretical_loss": 3.4499120407787274, + "tokens_seen": 2063845376 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003028445336008024, + "loss": 1.3172, + "theoretical_loss": 3.449903038196883, + "tokens_seen": 2063910912 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003028284854563691, + "loss": 1.316, + "theoretical_loss": 3.4498940359809342, + "tokens_seen": 2063976448 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003028124373119358, + "loss": 1.3291, + "theoretical_loss": 3.4498850341308547, + "tokens_seen": 2064041984 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003027963891675025, + "loss": 1.288, + "theoretical_loss": 3.4498760326466176, + "tokens_seen": 2064107520 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003027803410230692, + "loss": 1.2474, + "theoretical_loss": 3.449867031528197, + "tokens_seen": 2064173056 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003027642928786359, + "loss": 1.2682, + "theoretical_loss": 3.449858030775566, + "tokens_seen": 2064238592 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003027482447342026, + "loss": 1.2998, + "theoretical_loss": 3.4498490303886977, + "tokens_seen": 2064304128 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 763515, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2607522010803223, + "objective/train/theoretical_loss": 3.4498400303675663, + "objective/train/tokens_used": 2084829664, + "theoretical_loss": 3.4498400303675663, + "tokens_seen": 2064369664 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003027321965897693, + "loss": 1.2608, + "theoretical_loss": 3.4498400303675663, + "tokens_seen": 2064369664 + }, + { + "epoch": 1.01, + "learning_rate": 0.000302716148445336, + "loss": 1.2759, + "theoretical_loss": 3.4498310307121454, + "tokens_seen": 2064435200 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003027001003009027, + "loss": 1.208, + "theoretical_loss": 3.449822031422408, + "tokens_seen": 2064500736 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003026840521564694, + "loss": 1.2646, + "theoretical_loss": 3.4498130324983283, + "tokens_seen": 2064566272 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003026680040120361, + "loss": 1.2026, + "theoretical_loss": 3.4498040339398797, + "tokens_seen": 2064631808 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003026519558676028, + "loss": 1.3383, + "theoretical_loss": 3.4497950357470346, + "tokens_seen": 2064697344 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003026359077231695, + "loss": 1.2266, + "theoretical_loss": 3.4497860379197682, + "tokens_seen": 2064762880 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003026198595787362, + "loss": 1.2752, + "theoretical_loss": 3.449777040458053, + "tokens_seen": 2064828416 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003026038114343029, + "loss": 1.3111, + "theoretical_loss": 3.4497680433618627, + "tokens_seen": 2064893952 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003025877632898696, + "loss": 1.282, + "theoretical_loss": 3.449759046631171, + "tokens_seen": 2064959488 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003025717151454363, + "loss": 1.3734, + "theoretical_loss": 3.4497500502659517, + "tokens_seen": 2065025024 + }, + { + "epoch": 1.01, + "learning_rate": 0.000302555667001003, + "loss": 1.2245, + "theoretical_loss": 3.4497410542661777, + "tokens_seen": 2065090560 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003025396188565697, + "loss": 1.3551, + "theoretical_loss": 3.449732058631823, + "tokens_seen": 2065156096 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003025235707121364, + "loss": 1.261, + "theoretical_loss": 3.4497230633628613, + "tokens_seen": 2065221632 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003025075225677031, + "loss": 1.2682, + "theoretical_loss": 3.449714068459266, + "tokens_seen": 2065287168 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003024914744232698, + "loss": 1.2473, + "theoretical_loss": 3.4497050739210104, + "tokens_seen": 2065352704 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003024754262788365, + "loss": 1.3309, + "theoretical_loss": 3.449696079748069, + "tokens_seen": 2065418240 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003024593781344032, + "loss": 1.3509, + "theoretical_loss": 3.449687085940414, + "tokens_seen": 2065483776 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003024433299899699, + "loss": 1.3266, + "theoretical_loss": 3.4496780924980195, + "tokens_seen": 2065549312 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003024272818455366, + "loss": 1.2944, + "theoretical_loss": 3.4496690994208596, + "tokens_seen": 2065614848 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003024112337011033, + "loss": 1.1423, + "theoretical_loss": 3.449660106708907, + "tokens_seen": 2065680384 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030239518555667, + "loss": 1.3363, + "theoretical_loss": 3.449651114362137, + "tokens_seen": 2065745920 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003023791374122367, + "loss": 1.3168, + "theoretical_loss": 3.4496421223805207, + "tokens_seen": 2065811456 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030236308926780343, + "loss": 1.2247, + "theoretical_loss": 3.4496331307640338, + "tokens_seen": 2065876992 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030234704112337013, + "loss": 1.266, + "theoretical_loss": 3.4496241395126486, + "tokens_seen": 2065942528 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030233099297893683, + "loss": 1.2464, + "theoretical_loss": 3.4496151486263393, + "tokens_seen": 2066008064 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030231494483450353, + "loss": 1.4622, + "theoretical_loss": 3.4496061581050794, + "tokens_seen": 2066073600 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030229889669007023, + "loss": 1.3103, + "theoretical_loss": 3.449597167948842, + "tokens_seen": 2066139136 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030228284854563693, + "loss": 1.2851, + "theoretical_loss": 3.449588178157602, + "tokens_seen": 2066204672 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030226680040120363, + "loss": 1.25, + "theoretical_loss": 3.449579188731331, + "tokens_seen": 2066270208 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030225075225677033, + "loss": 1.228, + "theoretical_loss": 3.4495701996700046, + "tokens_seen": 2066335744 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030223470411233703, + "loss": 1.2941, + "theoretical_loss": 3.4495612109735956, + "tokens_seen": 2066401280 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030221865596790373, + "loss": 1.3572, + "theoretical_loss": 3.4495522226420774, + "tokens_seen": 2066466816 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030220260782347043, + "loss": 1.387, + "theoretical_loss": 3.449543234675424, + "tokens_seen": 2066532352 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003021865596790371, + "loss": 1.2367, + "theoretical_loss": 3.4495342470736086, + "tokens_seen": 2066597888 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003021705115346038, + "loss": 1.197, + "theoretical_loss": 3.449525259836605, + "tokens_seen": 2066663424 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003021544633901705, + "loss": 1.2088, + "theoretical_loss": 3.4495162729643867, + "tokens_seen": 2066728960 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003021384152457372, + "loss": 1.2453, + "theoretical_loss": 3.4495072864569276, + "tokens_seen": 2066794496 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003021223671013039, + "loss": 1.3044, + "theoretical_loss": 3.4494983003142012, + "tokens_seen": 2066860032 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003021063189568706, + "loss": 1.2337, + "theoretical_loss": 3.449489314536182, + "tokens_seen": 2066925568 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030209027081243734, + "loss": 1.3918, + "theoretical_loss": 3.4494803291228413, + "tokens_seen": 2066991104 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030207422266800404, + "loss": 1.3907, + "theoretical_loss": 3.4494713440741553, + "tokens_seen": 2067056640 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030205817452357074, + "loss": 1.245, + "theoretical_loss": 3.4494623593900964, + "tokens_seen": 2067122176 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030204212637913744, + "loss": 1.2409, + "theoretical_loss": 3.449453375070638, + "tokens_seen": 2067187712 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030202607823470414, + "loss": 1.3496, + "theoretical_loss": 3.4494443911157546, + "tokens_seen": 2067253248 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030201003009027084, + "loss": 1.2856, + "theoretical_loss": 3.449435407525419, + "tokens_seen": 2067318784 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030199398194583754, + "loss": 1.1583, + "theoretical_loss": 3.449426424299606, + "tokens_seen": 2067384320 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030197793380140424, + "loss": 1.3076, + "theoretical_loss": 3.449417441438288, + "tokens_seen": 2067449856 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030196188565697094, + "loss": 1.2534, + "theoretical_loss": 3.449408458941439, + "tokens_seen": 2067515392 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030194583751253764, + "loss": 1.2947, + "theoretical_loss": 3.4493994768090332, + "tokens_seen": 2067580928 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030192978936810434, + "loss": 1.2935, + "theoretical_loss": 3.449390495041044, + "tokens_seen": 2067646464 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030191374122367105, + "loss": 1.316, + "theoretical_loss": 3.4493815136374444, + "tokens_seen": 2067712000 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030189769307923775, + "loss": 1.3754, + "theoretical_loss": 3.4493725325982085, + "tokens_seen": 2067777536 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030188164493480445, + "loss": 1.2541, + "theoretical_loss": 3.449363551923311, + "tokens_seen": 2067843072 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030186559679037115, + "loss": 1.2324, + "theoretical_loss": 3.449354571612724, + "tokens_seen": 2067908608 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030184954864593785, + "loss": 1.2494, + "theoretical_loss": 3.449345591666422, + "tokens_seen": 2067974144 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030183350050150455, + "loss": 1.3118, + "theoretical_loss": 3.4493366120843785, + "tokens_seen": 2068039680 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030181745235707125, + "loss": 1.2776, + "theoretical_loss": 3.4493276328665674, + "tokens_seen": 2068105216 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030180140421263795, + "loss": 1.2943, + "theoretical_loss": 3.449318654012962, + "tokens_seen": 2068170752 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030178535606820465, + "loss": 1.2556, + "theoretical_loss": 3.449309675523536, + "tokens_seen": 2068236288 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030176930792377135, + "loss": 1.2548, + "theoretical_loss": 3.4493006973982636, + "tokens_seen": 2068301824 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030175325977933805, + "loss": 1.2582, + "theoretical_loss": 3.4492917196371184, + "tokens_seen": 2068367360 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030173721163490475, + "loss": 1.3141, + "theoretical_loss": 3.4492827422400736, + "tokens_seen": 2068432896 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030172116349047145, + "loss": 1.1502, + "theoretical_loss": 3.449273765207103, + "tokens_seen": 2068498432 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030170511534603815, + "loss": 1.3135, + "theoretical_loss": 3.449264788538181, + "tokens_seen": 2068563968 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030168906720160485, + "loss": 1.264, + "theoretical_loss": 3.4492558122332797, + "tokens_seen": 2068629504 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030167301905717155, + "loss": 1.2372, + "theoretical_loss": 3.4492468362923745, + "tokens_seen": 2068695040 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030165697091273826, + "loss": 1.276, + "theoretical_loss": 3.4492378607154386, + "tokens_seen": 2068760576 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030164092276830496, + "loss": 1.3355, + "theoretical_loss": 3.4492288855024453, + "tokens_seen": 2068826112 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030162487462387166, + "loss": 1.2789, + "theoretical_loss": 3.449219910653369, + "tokens_seen": 2068891648 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030160882647943836, + "loss": 1.3203, + "theoretical_loss": 3.449210936168183, + "tokens_seen": 2068957184 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030159277833500506, + "loss": 1.2742, + "theoretical_loss": 3.4492019620468604, + "tokens_seen": 2069022720 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030157673019057176, + "loss": 1.191, + "theoretical_loss": 3.449192988289376, + "tokens_seen": 2069088256 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030156068204613846, + "loss": 1.2192, + "theoretical_loss": 3.449184014895703, + "tokens_seen": 2069153792 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003015446339017051, + "loss": 1.1325, + "theoretical_loss": 3.4491750418658156, + "tokens_seen": 2069219328 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003015285857572718, + "loss": 1.2279, + "theoretical_loss": 3.449166069199687, + "tokens_seen": 2069284864 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003015125376128385, + "loss": 1.2219, + "theoretical_loss": 3.4491570968972907, + "tokens_seen": 2069350400 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003014964894684052, + "loss": 1.3453, + "theoretical_loss": 3.449148124958601, + "tokens_seen": 2069415936 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003014804413239719, + "loss": 1.3409, + "theoretical_loss": 3.449139153383592, + "tokens_seen": 2069481472 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003014643931795386, + "loss": 1.2625, + "theoretical_loss": 3.4491301821722358, + "tokens_seen": 2069547008 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003014483450351053, + "loss": 1.3204, + "theoretical_loss": 3.449121211324508, + "tokens_seen": 2069612544 + }, + { + "epoch": 1.01, + "learning_rate": 0.000301432296890672, + "loss": 1.3694, + "theoretical_loss": 3.4491122408403814, + "tokens_seen": 2069678080 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003014162487462387, + "loss": 1.2568, + "theoretical_loss": 3.4491032707198297, + "tokens_seen": 2069743616 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003014002006018054, + "loss": 1.2757, + "theoretical_loss": 3.4490943009628277, + "tokens_seen": 2069809152 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003013841524573721, + "loss": 1.3085, + "theoretical_loss": 3.4490853315693473, + "tokens_seen": 2069874688 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003013681043129388, + "loss": 1.3146, + "theoretical_loss": 3.449076362539364, + "tokens_seen": 2069940224 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003013520561685055, + "loss": 1.3001, + "theoretical_loss": 3.4490673938728507, + "tokens_seen": 2070005760 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003013360080240722, + "loss": 1.2442, + "theoretical_loss": 3.449058425569781, + "tokens_seen": 2070071296 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003013199598796389, + "loss": 1.329, + "theoretical_loss": 3.44904945763013, + "tokens_seen": 2070136832 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003013039117352056, + "loss": 1.2639, + "theoretical_loss": 3.4490404900538696, + "tokens_seen": 2070202368 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003012878635907723, + "loss": 1.1844, + "theoretical_loss": 3.4490315228409747, + "tokens_seen": 2070267904 + }, + { + "epoch": 1.01, + "learning_rate": 0.000301271815446339, + "loss": 1.3952, + "theoretical_loss": 3.4490225559914185, + "tokens_seen": 2070333440 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003012557673019057, + "loss": 1.2302, + "theoretical_loss": 3.4490135895051752, + "tokens_seen": 2070398976 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003012397191574724, + "loss": 1.2328, + "theoretical_loss": 3.4490046233822187, + "tokens_seen": 2070464512 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003012236710130391, + "loss": 1.3281, + "theoretical_loss": 3.4489956576225227, + "tokens_seen": 2070530048 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003012076228686058, + "loss": 1.2394, + "theoretical_loss": 3.44898669222606, + "tokens_seen": 2070595584 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003011915747241725, + "loss": 1.2765, + "theoretical_loss": 3.448977727192806, + "tokens_seen": 2070661120 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003011755265797392, + "loss": 1.2943, + "theoretical_loss": 3.4489687625227337, + "tokens_seen": 2070726656 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003011594784353059, + "loss": 1.306, + "theoretical_loss": 3.4489597982158164, + "tokens_seen": 2070792192 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003011434302908726, + "loss": 1.2791, + "theoretical_loss": 3.448950834272029, + "tokens_seen": 2070857728 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 766006, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2282626628875732, + "objective/train/theoretical_loss": 3.4489418706913444, + "objective/train/tokens_used": 2091383264, + "theoretical_loss": 3.4489418706913444, + "tokens_seen": 2070923264 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003011273821464393, + "loss": 1.2283, + "theoretical_loss": 3.4489418706913444, + "tokens_seen": 2070923264 + }, + { + "epoch": 1.01, + "learning_rate": 0.000301111334002006, + "loss": 1.3769, + "theoretical_loss": 3.448932907473737, + "tokens_seen": 2070988800 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003010952858575727, + "loss": 1.2361, + "theoretical_loss": 3.44892394461918, + "tokens_seen": 2071054336 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003010792377131394, + "loss": 1.2497, + "theoretical_loss": 3.448914982127648, + "tokens_seen": 2071119872 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003010631895687061, + "loss": 1.4138, + "theoretical_loss": 3.4489060199991135, + "tokens_seen": 2071185408 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003010471414242728, + "loss": 1.3221, + "theoretical_loss": 3.448897058233552, + "tokens_seen": 2071250944 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003010310932798395, + "loss": 1.2807, + "theoretical_loss": 3.4488880968309354, + "tokens_seen": 2071316480 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003010150451354062, + "loss": 1.3318, + "theoretical_loss": 3.4488791357912394, + "tokens_seen": 2071382016 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003009989969909729, + "loss": 1.3106, + "theoretical_loss": 3.448870175114437, + "tokens_seen": 2071447552 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030098294884653963, + "loss": 1.3058, + "theoretical_loss": 3.448861214800502, + "tokens_seen": 2071513088 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030096690070210633, + "loss": 1.2514, + "theoretical_loss": 3.448852254849408, + "tokens_seen": 2071578624 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030095085255767303, + "loss": 1.2481, + "theoretical_loss": 3.448843295261129, + "tokens_seen": 2071644160 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030093480441323973, + "loss": 1.2809, + "theoretical_loss": 3.448834336035639, + "tokens_seen": 2071709696 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030091875626880643, + "loss": 1.3475, + "theoretical_loss": 3.448825377172912, + "tokens_seen": 2071775232 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030090270812437313, + "loss": 1.2217, + "theoretical_loss": 3.448816418672921, + "tokens_seen": 2071840768 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030088665997993983, + "loss": 1.1512, + "theoretical_loss": 3.448807460535641, + "tokens_seen": 2071906304 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030087061183550653, + "loss": 1.1832, + "theoretical_loss": 3.4487985027610453, + "tokens_seen": 2071971840 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030085456369107323, + "loss": 1.1953, + "theoretical_loss": 3.4487895453491073, + "tokens_seen": 2072037376 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030083851554663993, + "loss": 1.2693, + "theoretical_loss": 3.4487805882998015, + "tokens_seen": 2072102912 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030082246740220663, + "loss": 1.1932, + "theoretical_loss": 3.4487716316131016, + "tokens_seen": 2072168448 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030080641925777333, + "loss": 1.2554, + "theoretical_loss": 3.448762675288981, + "tokens_seen": 2072233984 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030079037111334004, + "loss": 1.2912, + "theoretical_loss": 3.4487537193274145, + "tokens_seen": 2072299520 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030077432296890674, + "loss": 1.2404, + "theoretical_loss": 3.448744763728375, + "tokens_seen": 2072365056 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030075827482447344, + "loss": 1.2625, + "theoretical_loss": 3.4487358084918363, + "tokens_seen": 2072430592 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030074222668004014, + "loss": 1.3629, + "theoretical_loss": 3.4487268536177735, + "tokens_seen": 2072496128 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030072617853560684, + "loss": 1.2593, + "theoretical_loss": 3.448717899106159, + "tokens_seen": 2072561664 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030071013039117354, + "loss": 1.2636, + "theoretical_loss": 3.4487089449569677, + "tokens_seen": 2072627200 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030069408224674024, + "loss": 1.2183, + "theoretical_loss": 3.448699991170173, + "tokens_seen": 2072692736 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030067803410230694, + "loss": 1.2718, + "theoretical_loss": 3.448691037745749, + "tokens_seen": 2072758272 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030066198595787364, + "loss": 1.3176, + "theoretical_loss": 3.448682084683669, + "tokens_seen": 2072823808 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030064593781344034, + "loss": 1.3806, + "theoretical_loss": 3.448673131983908, + "tokens_seen": 2072889344 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030062988966900704, + "loss": 1.27, + "theoretical_loss": 3.448664179646439, + "tokens_seen": 2072954880 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030061384152457374, + "loss": 1.2614, + "theoretical_loss": 3.448655227671236, + "tokens_seen": 2073020416 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030059779338014044, + "loss": 1.2966, + "theoretical_loss": 3.4486462760582732, + "tokens_seen": 2073085952 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030058174523570714, + "loss": 1.2435, + "theoretical_loss": 3.448637324807524, + "tokens_seen": 2073151488 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030056569709127384, + "loss": 1.3758, + "theoretical_loss": 3.4486283739189627, + "tokens_seen": 2073217024 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030054964894684054, + "loss": 1.3031, + "theoretical_loss": 3.448619423392563, + "tokens_seen": 2073282560 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030053360080240725, + "loss": 1.3042, + "theoretical_loss": 3.448610473228299, + "tokens_seen": 2073348096 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030051755265797395, + "loss": 1.3724, + "theoretical_loss": 3.4486015234261447, + "tokens_seen": 2073413632 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030050150451354065, + "loss": 1.2364, + "theoretical_loss": 3.4485925739860734, + "tokens_seen": 2073479168 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030048545636910735, + "loss": 1.2124, + "theoretical_loss": 3.4485836249080597, + "tokens_seen": 2073544704 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030046940822467405, + "loss": 1.3501, + "theoretical_loss": 3.4485746761920772, + "tokens_seen": 2073610240 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030045336008024075, + "loss": 1.3099, + "theoretical_loss": 3.4485657278380994, + "tokens_seen": 2073675776 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030043731193580745, + "loss": 1.2666, + "theoretical_loss": 3.4485567798461014, + "tokens_seen": 2073741312 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030042126379137415, + "loss": 1.3583, + "theoretical_loss": 3.4485478322160557, + "tokens_seen": 2073806848 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030040521564694085, + "loss": 1.2566, + "theoretical_loss": 3.4485388849479373, + "tokens_seen": 2073872384 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030038916750250755, + "loss": 1.2222, + "theoretical_loss": 3.448529938041719, + "tokens_seen": 2073937920 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030037311935807425, + "loss": 1.2691, + "theoretical_loss": 3.448520991497376, + "tokens_seen": 2074003456 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030035707121364095, + "loss": 1.272, + "theoretical_loss": 3.4485120453148816, + "tokens_seen": 2074068992 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030034102306920765, + "loss": 1.2341, + "theoretical_loss": 3.44850309949421, + "tokens_seen": 2074134528 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030032497492477435, + "loss": 1.1844, + "theoretical_loss": 3.448494154035334, + "tokens_seen": 2074200064 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030030892678034105, + "loss": 1.2596, + "theoretical_loss": 3.448485208938229, + "tokens_seen": 2074265600 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030029287863590775, + "loss": 1.3277, + "theoretical_loss": 3.448476264202869, + "tokens_seen": 2074331136 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030027683049147446, + "loss": 1.3081, + "theoretical_loss": 3.4484673198292266, + "tokens_seen": 2074396672 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030026078234704116, + "loss": 1.2699, + "theoretical_loss": 3.4484583758172764, + "tokens_seen": 2074462208 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030024473420260786, + "loss": 1.2799, + "theoretical_loss": 3.448449432166993, + "tokens_seen": 2074527744 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030022868605817456, + "loss": 1.1957, + "theoretical_loss": 3.44844048887835, + "tokens_seen": 2074593280 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030021263791374126, + "loss": 1.2484, + "theoretical_loss": 3.44843154595132, + "tokens_seen": 2074658816 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030019658976930796, + "loss": 1.2906, + "theoretical_loss": 3.448422603385879, + "tokens_seen": 2074724352 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030018054162487466, + "loss": 1.1545, + "theoretical_loss": 3.4484136611819998, + "tokens_seen": 2074789888 + }, + { + "epoch": 1.01, + "learning_rate": 0.00030016449348044136, + "loss": 1.2566, + "theoretical_loss": 3.4484047193396568, + "tokens_seen": 2074855424 + }, + { + "epoch": 1.01, + "learning_rate": 0.000300148445336008, + "loss": 1.2363, + "theoretical_loss": 3.4483957778588232, + "tokens_seen": 2074920960 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003001323971915747, + "loss": 1.2268, + "theoretical_loss": 3.4483868367394743, + "tokens_seen": 2074986496 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003001163490471414, + "loss": 1.2917, + "theoretical_loss": 3.448377895981583, + "tokens_seen": 2075052032 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003001003009027081, + "loss": 1.2917, + "theoretical_loss": 3.4483689555851234, + "tokens_seen": 2075117568 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003000842527582748, + "loss": 1.2139, + "theoretical_loss": 3.4483600155500698, + "tokens_seen": 2075183104 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003000682046138415, + "loss": 1.4576, + "theoretical_loss": 3.4483510758763964, + "tokens_seen": 2075248640 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003000521564694082, + "loss": 1.2514, + "theoretical_loss": 3.4483421365640763, + "tokens_seen": 2075314176 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003000361083249749, + "loss": 1.297, + "theoretical_loss": 3.448333197613084, + "tokens_seen": 2075379712 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003000200601805416, + "loss": 1.2099, + "theoretical_loss": 3.448324259023394, + "tokens_seen": 2075445248 + }, + { + "epoch": 1.01, + "learning_rate": 0.0003000040120361083, + "loss": 1.3013, + "theoretical_loss": 3.448315320794979, + "tokens_seen": 2075510784 + }, + { + "epoch": 1.01, + "learning_rate": 0.000299987963891675, + "loss": 1.2655, + "theoretical_loss": 3.4483063829278144, + "tokens_seen": 2075576320 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002999719157472417, + "loss": 1.3819, + "theoretical_loss": 3.4482974454218733, + "tokens_seen": 2075641856 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002999558676028084, + "loss": 1.2601, + "theoretical_loss": 3.4482885082771304, + "tokens_seen": 2075707392 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002999398194583751, + "loss": 1.2897, + "theoretical_loss": 3.4482795714935586, + "tokens_seen": 2075772928 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002999237713139418, + "loss": 1.2043, + "theoretical_loss": 3.448270635071133, + "tokens_seen": 2075838464 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002999077231695085, + "loss": 1.2674, + "theoretical_loss": 3.448261699009827, + "tokens_seen": 2075904000 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002998916750250752, + "loss": 1.3213, + "theoretical_loss": 3.4482527633096147, + "tokens_seen": 2075969536 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002998756268806419, + "loss": 1.279, + "theoretical_loss": 3.4482438279704706, + "tokens_seen": 2076035072 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002998595787362086, + "loss": 1.3362, + "theoretical_loss": 3.4482348929923674, + "tokens_seen": 2076100608 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002998435305917753, + "loss": 1.3858, + "theoretical_loss": 3.4482259583752812, + "tokens_seen": 2076166144 + }, + { + "epoch": 1.01, + "learning_rate": 0.000299827482447342, + "loss": 1.1828, + "theoretical_loss": 3.4482170241191836, + "tokens_seen": 2076231680 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002998114343029087, + "loss": 1.3066, + "theoretical_loss": 3.4482080902240506, + "tokens_seen": 2076297216 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002997953861584754, + "loss": 1.4215, + "theoretical_loss": 3.4481991566898555, + "tokens_seen": 2076362752 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002997793380140421, + "loss": 1.1793, + "theoretical_loss": 3.4481902235165722, + "tokens_seen": 2076428288 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002997632898696088, + "loss": 1.2106, + "theoretical_loss": 3.4481812907041745, + "tokens_seen": 2076493824 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002997472417251755, + "loss": 1.2407, + "theoretical_loss": 3.4481723582526373, + "tokens_seen": 2076559360 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002997311935807422, + "loss": 1.3242, + "theoretical_loss": 3.448163426161934, + "tokens_seen": 2076624896 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002997151454363089, + "loss": 1.2787, + "theoretical_loss": 3.448154494432038, + "tokens_seen": 2076690432 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002996990972918757, + "loss": 1.2653, + "theoretical_loss": 3.448145563062925, + "tokens_seen": 2076755968 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002996830491474424, + "loss": 1.3022, + "theoretical_loss": 3.4481366320545677, + "tokens_seen": 2076821504 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002996670010030091, + "loss": 1.248, + "theoretical_loss": 3.448127701406941, + "tokens_seen": 2076887040 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002996509528585758, + "loss": 1.2791, + "theoretical_loss": 3.4481187711200176, + "tokens_seen": 2076952576 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002996349047141425, + "loss": 1.2921, + "theoretical_loss": 3.448109841193773, + "tokens_seen": 2077018112 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002996188565697092, + "loss": 1.404, + "theoretical_loss": 3.448100911628181, + "tokens_seen": 2077083648 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002996028084252759, + "loss": 1.3368, + "theoretical_loss": 3.4480919824232146, + "tokens_seen": 2077149184 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002995867602808426, + "loss": 1.2519, + "theoretical_loss": 3.4480830535788494, + "tokens_seen": 2077214720 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002995707121364093, + "loss": 1.2025, + "theoretical_loss": 3.4480741250950584, + "tokens_seen": 2077280256 + }, + { + "epoch": 1.01, + "learning_rate": 0.000299554663991976, + "loss": 1.3039, + "theoretical_loss": 3.4480651969718155, + "tokens_seen": 2077345792 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002995386158475427, + "loss": 1.1718, + "theoretical_loss": 3.448056269209096, + "tokens_seen": 2077411328 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 768762, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2815393209457397, + "objective/train/theoretical_loss": 3.4480473418068724, + "objective/train/tokens_used": 2097936864, + "theoretical_loss": 3.4480473418068724, + "tokens_seen": 2077476864 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029952256770310933, + "loss": 1.2815, + "theoretical_loss": 3.4480473418068724, + "tokens_seen": 2077476864 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029950651955867603, + "loss": 1.2611, + "theoretical_loss": 3.4480384147651204, + "tokens_seen": 2077542400 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029949047141424273, + "loss": 1.2132, + "theoretical_loss": 3.4480294880838125, + "tokens_seen": 2077607936 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029947442326980943, + "loss": 1.3123, + "theoretical_loss": 3.448020561762924, + "tokens_seen": 2077673472 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029945837512537613, + "loss": 1.3809, + "theoretical_loss": 3.4480116358024286, + "tokens_seen": 2077739008 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029944232698094283, + "loss": 1.2532, + "theoretical_loss": 3.4480027102022994, + "tokens_seen": 2077804544 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029942627883650953, + "loss": 1.2412, + "theoretical_loss": 3.447993784962512, + "tokens_seen": 2077870080 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029941023069207624, + "loss": 1.255, + "theoretical_loss": 3.4479848600830403, + "tokens_seen": 2077935616 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029939418254764294, + "loss": 1.1664, + "theoretical_loss": 3.447975935563857, + "tokens_seen": 2078001152 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029937813440320964, + "loss": 1.3252, + "theoretical_loss": 3.4479670114049377, + "tokens_seen": 2078066688 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029936208625877634, + "loss": 1.3261, + "theoretical_loss": 3.4479580876062554, + "tokens_seen": 2078132224 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029934603811434304, + "loss": 1.2769, + "theoretical_loss": 3.4479491641677855, + "tokens_seen": 2078197760 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029932998996990974, + "loss": 1.2672, + "theoretical_loss": 3.447940241089501, + "tokens_seen": 2078263296 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029931394182547644, + "loss": 1.3312, + "theoretical_loss": 3.447931318371376, + "tokens_seen": 2078328832 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029929789368104314, + "loss": 1.2742, + "theoretical_loss": 3.447922396013385, + "tokens_seen": 2078394368 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029928184553660984, + "loss": 1.2544, + "theoretical_loss": 3.447913474015502, + "tokens_seen": 2078459904 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029926579739217654, + "loss": 1.268, + "theoretical_loss": 3.4479045523777017, + "tokens_seen": 2078525440 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029924974924774324, + "loss": 1.2849, + "theoretical_loss": 3.447895631099957, + "tokens_seen": 2078590976 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029923370110330994, + "loss": 1.2969, + "theoretical_loss": 3.4478867101822432, + "tokens_seen": 2078656512 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029921765295887664, + "loss": 1.245, + "theoretical_loss": 3.447877789624534, + "tokens_seen": 2078722048 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029920160481444334, + "loss": 1.3092, + "theoretical_loss": 3.447868869426803, + "tokens_seen": 2078787584 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029918555667001004, + "loss": 1.3374, + "theoretical_loss": 3.447859949589025, + "tokens_seen": 2078853120 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029916950852557675, + "loss": 1.3133, + "theoretical_loss": 3.447851030111174, + "tokens_seen": 2078918656 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029915346038114345, + "loss": 1.3068, + "theoretical_loss": 3.4478421109932236, + "tokens_seen": 2078984192 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029913741223671015, + "loss": 1.2992, + "theoretical_loss": 3.447833192235149, + "tokens_seen": 2079049728 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029912136409227685, + "loss": 1.2598, + "theoretical_loss": 3.4478242738369227, + "tokens_seen": 2079115264 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029910531594784355, + "loss": 1.2858, + "theoretical_loss": 3.4478153557985207, + "tokens_seen": 2079180800 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029908926780341025, + "loss": 1.2775, + "theoretical_loss": 3.447806438119916, + "tokens_seen": 2079246336 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029907321965897695, + "loss": 1.3361, + "theoretical_loss": 3.4477975208010827, + "tokens_seen": 2079311872 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029905717151454365, + "loss": 1.2247, + "theoretical_loss": 3.4477886038419956, + "tokens_seen": 2079377408 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029904112337011035, + "loss": 1.3316, + "theoretical_loss": 3.4477796872426287, + "tokens_seen": 2079442944 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029902507522567705, + "loss": 1.2232, + "theoretical_loss": 3.4477707710029555, + "tokens_seen": 2079508480 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029900902708124375, + "loss": 1.3225, + "theoretical_loss": 3.447761855122951, + "tokens_seen": 2079574016 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029899297893681045, + "loss": 1.2128, + "theoretical_loss": 3.4477529396025886, + "tokens_seen": 2079639552 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029897693079237715, + "loss": 1.2684, + "theoretical_loss": 3.447744024441843, + "tokens_seen": 2079705088 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029896088264794385, + "loss": 1.2804, + "theoretical_loss": 3.4477351096406883, + "tokens_seen": 2079770624 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029894483450351055, + "loss": 1.2757, + "theoretical_loss": 3.447726195199098, + "tokens_seen": 2079836160 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029892878635907725, + "loss": 1.2745, + "theoretical_loss": 3.447717281117048, + "tokens_seen": 2079901696 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029891273821464396, + "loss": 1.2674, + "theoretical_loss": 3.4477083673945104, + "tokens_seen": 2079967232 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029889669007021066, + "loss": 1.368, + "theoretical_loss": 3.4476994540314605, + "tokens_seen": 2080032768 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029888064192577736, + "loss": 1.1645, + "theoretical_loss": 3.447690541027872, + "tokens_seen": 2080098304 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029886459378134406, + "loss": 1.2773, + "theoretical_loss": 3.4476816283837195, + "tokens_seen": 2080163840 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029884854563691076, + "loss": 1.3029, + "theoretical_loss": 3.4476727160989773, + "tokens_seen": 2080229376 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029883249749247746, + "loss": 1.2386, + "theoretical_loss": 3.4476638041736187, + "tokens_seen": 2080294912 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029881644934804416, + "loss": 1.209, + "theoretical_loss": 3.447654892607619, + "tokens_seen": 2080360448 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029880040120361086, + "loss": 1.2025, + "theoretical_loss": 3.4476459814009517, + "tokens_seen": 2080425984 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029878435305917756, + "loss": 1.278, + "theoretical_loss": 3.447637070553591, + "tokens_seen": 2080491520 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029876830491474426, + "loss": 1.354, + "theoretical_loss": 3.447628160065512, + "tokens_seen": 2080557056 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002987522567703109, + "loss": 1.1872, + "theoretical_loss": 3.447619249936687, + "tokens_seen": 2080622592 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002987362086258776, + "loss": 1.2538, + "theoretical_loss": 3.447610340167092, + "tokens_seen": 2080688128 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002987201604814443, + "loss": 1.2557, + "theoretical_loss": 3.4476014307567007, + "tokens_seen": 2080753664 + }, + { + "epoch": 1.01, + "learning_rate": 0.000298704112337011, + "loss": 1.3134, + "theoretical_loss": 3.447592521705487, + "tokens_seen": 2080819200 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002986880641925777, + "loss": 1.2788, + "theoretical_loss": 3.4475836130134248, + "tokens_seen": 2080884736 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002986720160481444, + "loss": 1.2474, + "theoretical_loss": 3.4475747046804894, + "tokens_seen": 2080950272 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002986559679037111, + "loss": 1.2174, + "theoretical_loss": 3.4475657967066544, + "tokens_seen": 2081015808 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002986399197592778, + "loss": 1.3089, + "theoretical_loss": 3.447556889091894, + "tokens_seen": 2081081344 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002986238716148445, + "loss": 1.298, + "theoretical_loss": 3.447547981836182, + "tokens_seen": 2081146880 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002986078234704112, + "loss": 1.3035, + "theoretical_loss": 3.447539074939493, + "tokens_seen": 2081212416 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002985917753259779, + "loss": 1.195, + "theoretical_loss": 3.447530168401802, + "tokens_seen": 2081277952 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002985757271815446, + "loss": 1.3527, + "theoretical_loss": 3.4475212622230815, + "tokens_seen": 2081343488 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002985596790371113, + "loss": 1.2682, + "theoretical_loss": 3.4475123564033074, + "tokens_seen": 2081409024 + }, + { + "epoch": 1.01, + "learning_rate": 0.000298543630892678, + "loss": 1.2308, + "theoretical_loss": 3.4475034509424534, + "tokens_seen": 2081474560 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002985275827482447, + "loss": 1.293, + "theoretical_loss": 3.4474945458404935, + "tokens_seen": 2081540096 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002985115346038114, + "loss": 1.3364, + "theoretical_loss": 3.4474856410974017, + "tokens_seen": 2081605632 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029849548645937817, + "loss": 1.1865, + "theoretical_loss": 3.4474767367131527, + "tokens_seen": 2081671168 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029847943831494487, + "loss": 1.2302, + "theoretical_loss": 3.447467832687721, + "tokens_seen": 2081736704 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002984633901705116, + "loss": 1.3152, + "theoretical_loss": 3.44745892902108, + "tokens_seen": 2081802240 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002984473420260783, + "loss": 1.2003, + "theoretical_loss": 3.447450025713205, + "tokens_seen": 2081867776 + }, + { + "epoch": 1.01, + "learning_rate": 0.000298431293881645, + "loss": 1.3066, + "theoretical_loss": 3.4474411227640687, + "tokens_seen": 2081933312 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002984152457372117, + "loss": 1.2574, + "theoretical_loss": 3.4474322201736474, + "tokens_seen": 2081998848 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002983991975927784, + "loss": 1.3331, + "theoretical_loss": 3.4474233179419134, + "tokens_seen": 2082064384 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002983831494483451, + "loss": 1.3304, + "theoretical_loss": 3.4474144160688427, + "tokens_seen": 2082129920 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002983671013039118, + "loss": 1.285, + "theoretical_loss": 3.447405514554408, + "tokens_seen": 2082195456 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002983510531594785, + "loss": 1.2811, + "theoretical_loss": 3.4473966133985843, + "tokens_seen": 2082260992 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002983350050150452, + "loss": 1.2147, + "theoretical_loss": 3.447387712601346, + "tokens_seen": 2082326528 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002983189568706119, + "loss": 1.2056, + "theoretical_loss": 3.4473788121626674, + "tokens_seen": 2082392064 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002983029087261786, + "loss": 1.3467, + "theoretical_loss": 3.447369912082522, + "tokens_seen": 2082457600 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002982868605817453, + "loss": 1.211, + "theoretical_loss": 3.447361012360885, + "tokens_seen": 2082523136 + }, + { + "epoch": 1.01, + "learning_rate": 0.000298270812437312, + "loss": 1.1733, + "theoretical_loss": 3.4473521129977303, + "tokens_seen": 2082588672 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002982547642928787, + "loss": 1.2673, + "theoretical_loss": 3.4473432139930322, + "tokens_seen": 2082654208 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002982387161484454, + "loss": 1.348, + "theoretical_loss": 3.4473343153467653, + "tokens_seen": 2082719744 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002982226680040121, + "loss": 1.3087, + "theoretical_loss": 3.447325417058903, + "tokens_seen": 2082785280 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002982066198595788, + "loss": 1.261, + "theoretical_loss": 3.44731651912942, + "tokens_seen": 2082850816 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002981905717151455, + "loss": 1.2229, + "theoretical_loss": 3.4473076215582914, + "tokens_seen": 2082916352 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002981745235707122, + "loss": 1.2393, + "theoretical_loss": 3.4472987243454907, + "tokens_seen": 2082981888 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002981584754262789, + "loss": 1.2169, + "theoretical_loss": 3.447289827490992, + "tokens_seen": 2083047424 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002981424272818456, + "loss": 1.2684, + "theoretical_loss": 3.4472809309947703, + "tokens_seen": 2083112960 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002981263791374123, + "loss": 1.2777, + "theoretical_loss": 3.4472720348567996, + "tokens_seen": 2083178496 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029811033099297893, + "loss": 1.2061, + "theoretical_loss": 3.447263139077054, + "tokens_seen": 2083244032 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029809428284854563, + "loss": 1.3259, + "theoretical_loss": 3.447254243655508, + "tokens_seen": 2083309568 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029807823470411233, + "loss": 1.2946, + "theoretical_loss": 3.4472453485921353, + "tokens_seen": 2083375104 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029806218655967903, + "loss": 1.2417, + "theoretical_loss": 3.4472364538869114, + "tokens_seen": 2083440640 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029804613841524574, + "loss": 1.1513, + "theoretical_loss": 3.44722755953981, + "tokens_seen": 2083506176 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029803009027081244, + "loss": 1.355, + "theoretical_loss": 3.447218665550805, + "tokens_seen": 2083571712 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029801404212637914, + "loss": 1.2977, + "theoretical_loss": 3.447209771919871, + "tokens_seen": 2083637248 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029799799398194584, + "loss": 1.1771, + "theoretical_loss": 3.447200878646983, + "tokens_seen": 2083702784 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029798194583751254, + "loss": 1.2513, + "theoretical_loss": 3.4471919857321147, + "tokens_seen": 2083768320 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029796589769307924, + "loss": 1.2902, + "theoretical_loss": 3.4471830931752403, + "tokens_seen": 2083833856 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029794984954864594, + "loss": 1.2962, + "theoretical_loss": 3.447174200976334, + "tokens_seen": 2083899392 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029793380140421264, + "loss": 1.3257, + "theoretical_loss": 3.447165309135371, + "tokens_seen": 2083964928 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 771222, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2231221199035645, + "objective/train/theoretical_loss": 3.447156417652325, + "objective/train/tokens_used": 2104490464, + "theoretical_loss": 3.447156417652325, + "tokens_seen": 2084030464 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029791775325977934, + "loss": 1.2231, + "theoretical_loss": 3.447156417652325, + "tokens_seen": 2084030464 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029790170511534604, + "loss": 1.2521, + "theoretical_loss": 3.4471475265271705, + "tokens_seen": 2084096000 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029788565697091274, + "loss": 1.264, + "theoretical_loss": 3.4471386357598814, + "tokens_seen": 2084161536 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029786960882647944, + "loss": 1.3121, + "theoretical_loss": 3.4471297453504324, + "tokens_seen": 2084227072 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029785356068204614, + "loss": 1.2201, + "theoretical_loss": 3.4471208552987984, + "tokens_seen": 2084292608 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029783751253761284, + "loss": 1.294, + "theoretical_loss": 3.447111965604953, + "tokens_seen": 2084358144 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029782146439317954, + "loss": 1.4133, + "theoretical_loss": 3.4471030762688706, + "tokens_seen": 2084423680 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029780541624874624, + "loss": 1.3105, + "theoretical_loss": 3.4470941872905256, + "tokens_seen": 2084489216 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029778936810431295, + "loss": 1.2141, + "theoretical_loss": 3.4470852986698928, + "tokens_seen": 2084554752 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029777331995987965, + "loss": 1.3292, + "theoretical_loss": 3.447076410406946, + "tokens_seen": 2084620288 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029775727181544635, + "loss": 1.2606, + "theoretical_loss": 3.44706752250166, + "tokens_seen": 2084685824 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029774122367101305, + "loss": 1.3221, + "theoretical_loss": 3.4470586349540087, + "tokens_seen": 2084751360 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029772517552657975, + "loss": 1.3309, + "theoretical_loss": 3.447049747763967, + "tokens_seen": 2084816896 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029770912738214645, + "loss": 1.2669, + "theoretical_loss": 3.4470408609315086, + "tokens_seen": 2084882432 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029769307923771315, + "loss": 1.2806, + "theoretical_loss": 3.4470319744566087, + "tokens_seen": 2084947968 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029767703109327985, + "loss": 1.3787, + "theoretical_loss": 3.447023088339241, + "tokens_seen": 2085013504 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029766098294884655, + "loss": 1.2581, + "theoretical_loss": 3.44701420257938, + "tokens_seen": 2085079040 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029764493480441325, + "loss": 1.1297, + "theoretical_loss": 3.4470053171770005, + "tokens_seen": 2085144576 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029762888665997995, + "loss": 1.2371, + "theoretical_loss": 3.4469964321320767, + "tokens_seen": 2085210112 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029761283851554665, + "loss": 1.2672, + "theoretical_loss": 3.4469875474445826, + "tokens_seen": 2085275648 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029759679037111335, + "loss": 1.2715, + "theoretical_loss": 3.4469786631144927, + "tokens_seen": 2085341184 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029758074222668005, + "loss": 1.272, + "theoretical_loss": 3.4469697791417824, + "tokens_seen": 2085406720 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029756469408224675, + "loss": 1.2487, + "theoretical_loss": 3.4469608955264244, + "tokens_seen": 2085472256 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029754864593781345, + "loss": 1.3476, + "theoretical_loss": 3.446952012268394, + "tokens_seen": 2085537792 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029753259779338016, + "loss": 1.2053, + "theoretical_loss": 3.446943129367666, + "tokens_seen": 2085603328 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029751654964894686, + "loss": 1.3284, + "theoretical_loss": 3.446934246824214, + "tokens_seen": 2085668864 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029750050150451356, + "loss": 1.1756, + "theoretical_loss": 3.4469253646380125, + "tokens_seen": 2085734400 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029748445336008026, + "loss": 1.327, + "theoretical_loss": 3.446916482809037, + "tokens_seen": 2085799936 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029746840521564696, + "loss": 1.2451, + "theoretical_loss": 3.4469076013372604, + "tokens_seen": 2085865472 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029745235707121366, + "loss": 1.2258, + "theoretical_loss": 3.4468987202226575, + "tokens_seen": 2085931008 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029743630892678036, + "loss": 1.3, + "theoretical_loss": 3.4468898394652037, + "tokens_seen": 2085996544 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029742026078234706, + "loss": 1.2222, + "theoretical_loss": 3.446880959064872, + "tokens_seen": 2086062080 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029740421263791376, + "loss": 1.2546, + "theoretical_loss": 3.4468720790216385, + "tokens_seen": 2086127616 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029738816449348046, + "loss": 1.3098, + "theoretical_loss": 3.446863199335476, + "tokens_seen": 2086193152 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029737211634904716, + "loss": 1.2463, + "theoretical_loss": 3.44685432000636, + "tokens_seen": 2086258688 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029735606820461386, + "loss": 1.2035, + "theoretical_loss": 3.446845441034264, + "tokens_seen": 2086324224 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002973400200601805, + "loss": 1.2718, + "theoretical_loss": 3.446836562419163, + "tokens_seen": 2086389760 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002973239719157472, + "loss": 1.298, + "theoretical_loss": 3.4468276841610317, + "tokens_seen": 2086455296 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002973079237713139, + "loss": 1.2676, + "theoretical_loss": 3.446818806259844, + "tokens_seen": 2086520832 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029729187562688067, + "loss": 1.2859, + "theoretical_loss": 3.4468099287155747, + "tokens_seen": 2086586368 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029727582748244737, + "loss": 1.3958, + "theoretical_loss": 3.446801051528198, + "tokens_seen": 2086651904 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029725977933801407, + "loss": 1.3192, + "theoretical_loss": 3.446792174697688, + "tokens_seen": 2086717440 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029724373119358077, + "loss": 1.3833, + "theoretical_loss": 3.4467832982240205, + "tokens_seen": 2086782976 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029722768304914747, + "loss": 1.252, + "theoretical_loss": 3.4467744221071683, + "tokens_seen": 2086848512 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029721163490471417, + "loss": 1.3107, + "theoretical_loss": 3.4467655463471067, + "tokens_seen": 2086914048 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029719558676028087, + "loss": 1.3156, + "theoretical_loss": 3.4467566709438104, + "tokens_seen": 2086979584 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029717953861584757, + "loss": 1.257, + "theoretical_loss": 3.446747795897253, + "tokens_seen": 2087045120 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029716349047141427, + "loss": 1.2328, + "theoretical_loss": 3.4467389212074098, + "tokens_seen": 2087110656 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029714744232698097, + "loss": 1.3028, + "theoretical_loss": 3.4467300468742548, + "tokens_seen": 2087176192 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029713139418254767, + "loss": 1.1826, + "theoretical_loss": 3.4467211728977625, + "tokens_seen": 2087241728 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029711534603811437, + "loss": 1.229, + "theoretical_loss": 3.4467122992779076, + "tokens_seen": 2087307264 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029709929789368107, + "loss": 1.2701, + "theoretical_loss": 3.4467034260146643, + "tokens_seen": 2087372800 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002970832497492478, + "loss": 1.2533, + "theoretical_loss": 3.4466945531080073, + "tokens_seen": 2087438336 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002970672016048145, + "loss": 1.3146, + "theoretical_loss": 3.446685680557911, + "tokens_seen": 2087503872 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002970511534603812, + "loss": 1.1903, + "theoretical_loss": 3.4466768083643498, + "tokens_seen": 2087569408 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002970351053159479, + "loss": 1.2918, + "theoretical_loss": 3.446667936527298, + "tokens_seen": 2087634944 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002970190571715146, + "loss": 1.2864, + "theoretical_loss": 3.44665906504673, + "tokens_seen": 2087700480 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002970030090270813, + "loss": 1.2615, + "theoretical_loss": 3.446650193922621, + "tokens_seen": 2087766016 + }, + { + "epoch": 1.01, + "learning_rate": 0.000296986960882648, + "loss": 1.2319, + "theoretical_loss": 3.4466413231549455, + "tokens_seen": 2087831552 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002969709127382147, + "loss": 1.2646, + "theoretical_loss": 3.4466324527436774, + "tokens_seen": 2087897088 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002969548645937814, + "loss": 1.3014, + "theoretical_loss": 3.446623582688791, + "tokens_seen": 2087962624 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002969388164493481, + "loss": 1.2273, + "theoretical_loss": 3.4466147129902613, + "tokens_seen": 2088028160 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002969227683049148, + "loss": 1.2765, + "theoretical_loss": 3.4466058436480624, + "tokens_seen": 2088093696 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002969067201604815, + "loss": 1.28, + "theoretical_loss": 3.44659697466217, + "tokens_seen": 2088159232 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002968906720160482, + "loss": 1.352, + "theoretical_loss": 3.446588106032557, + "tokens_seen": 2088224768 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002968746238716149, + "loss": 1.3525, + "theoretical_loss": 3.4465792377591984, + "tokens_seen": 2088290304 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002968585757271816, + "loss": 1.2118, + "theoretical_loss": 3.4465703698420693, + "tokens_seen": 2088355840 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002968425275827483, + "loss": 1.3016, + "theoretical_loss": 3.4465615022811438, + "tokens_seen": 2088421376 + }, + { + "epoch": 1.01, + "learning_rate": 0.000296826479438315, + "loss": 1.3825, + "theoretical_loss": 3.446552635076396, + "tokens_seen": 2088486912 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002968104312938817, + "loss": 1.2083, + "theoretical_loss": 3.4465437682278006, + "tokens_seen": 2088552448 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002967943831494484, + "loss": 1.2773, + "theoretical_loss": 3.4465349017353333, + "tokens_seen": 2088617984 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002967783350050151, + "loss": 1.2677, + "theoretical_loss": 3.446526035598967, + "tokens_seen": 2088683520 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002967622868605818, + "loss": 1.3568, + "theoretical_loss": 3.446517169818677, + "tokens_seen": 2088749056 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002967462387161485, + "loss": 1.2222, + "theoretical_loss": 3.4465083043944382, + "tokens_seen": 2088814592 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002967301905717152, + "loss": 1.1441, + "theoretical_loss": 3.4464994393262236, + "tokens_seen": 2088880128 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029671414242728183, + "loss": 1.3651, + "theoretical_loss": 3.4464905746140095, + "tokens_seen": 2088945664 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029669809428284853, + "loss": 1.2525, + "theoretical_loss": 3.44648171025777, + "tokens_seen": 2089011200 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029668204613841524, + "loss": 1.1959, + "theoretical_loss": 3.446472846257479, + "tokens_seen": 2089076736 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029666599799398194, + "loss": 1.3863, + "theoretical_loss": 3.4464639826131114, + "tokens_seen": 2089142272 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029664994984954864, + "loss": 1.2908, + "theoretical_loss": 3.446455119324642, + "tokens_seen": 2089207808 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029663390170511534, + "loss": 1.2394, + "theoretical_loss": 3.4464462563920444, + "tokens_seen": 2089273344 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029661785356068204, + "loss": 1.2831, + "theoretical_loss": 3.4464373938152946, + "tokens_seen": 2089338880 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029660180541624874, + "loss": 1.1943, + "theoretical_loss": 3.446428531594366, + "tokens_seen": 2089404416 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029658575727181544, + "loss": 1.2776, + "theoretical_loss": 3.446419669729234, + "tokens_seen": 2089469952 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029656970912738214, + "loss": 1.2099, + "theoretical_loss": 3.4464108082198726, + "tokens_seen": 2089535488 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029655366098294884, + "loss": 1.3856, + "theoretical_loss": 3.4464019470662564, + "tokens_seen": 2089601024 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029653761283851554, + "loss": 1.2792, + "theoretical_loss": 3.4463930862683596, + "tokens_seen": 2089666560 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029652156469408224, + "loss": 1.1873, + "theoretical_loss": 3.4463842258261574, + "tokens_seen": 2089732096 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029650551654964894, + "loss": 1.2244, + "theoretical_loss": 3.446375365739624, + "tokens_seen": 2089797632 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029648946840521564, + "loss": 1.1946, + "theoretical_loss": 3.4463665060087347, + "tokens_seen": 2089863168 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029647342026078234, + "loss": 1.2023, + "theoretical_loss": 3.446357646633463, + "tokens_seen": 2089928704 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029645737211634904, + "loss": 1.2197, + "theoretical_loss": 3.4463487876137844, + "tokens_seen": 2089994240 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029644132397191574, + "loss": 1.2434, + "theoretical_loss": 3.446339928949673, + "tokens_seen": 2090059776 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029642527582748245, + "loss": 1.3783, + "theoretical_loss": 3.446331070641103, + "tokens_seen": 2090125312 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029640922768304915, + "loss": 1.401, + "theoretical_loss": 3.4463222126880497, + "tokens_seen": 2090190848 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029639317953861585, + "loss": 1.3192, + "theoretical_loss": 3.4463133550904876, + "tokens_seen": 2090256384 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029637713139418255, + "loss": 1.2468, + "theoretical_loss": 3.4463044978483905, + "tokens_seen": 2090321920 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029636108324974925, + "loss": 1.1854, + "theoretical_loss": 3.4462956409617345, + "tokens_seen": 2090387456 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029634503510531595, + "loss": 1.2686, + "theoretical_loss": 3.4462867844304927, + "tokens_seen": 2090452992 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029632898696088265, + "loss": 1.3235, + "theoretical_loss": 3.44627792825464, + "tokens_seen": 2090518528 + }, + { + "epoch": 1.01, + "objective/train/docs_used": 773595, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3018739223480225, + "objective/train/theoretical_loss": 3.446269072434152, + "objective/train/tokens_used": 2111044064, + "theoretical_loss": 3.446269072434152, + "tokens_seen": 2090584064 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029631293881644935, + "loss": 1.3019, + "theoretical_loss": 3.446269072434152, + "tokens_seen": 2090584064 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029629689067201605, + "loss": 1.2532, + "theoretical_loss": 3.446260216969002, + "tokens_seen": 2090649600 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029628084252758275, + "loss": 1.3137, + "theoretical_loss": 3.4462513618591655, + "tokens_seen": 2090715136 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029626479438314945, + "loss": 1.3738, + "theoretical_loss": 3.4462425071046168, + "tokens_seen": 2090780672 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029624874623871615, + "loss": 1.2088, + "theoretical_loss": 3.44623365270533, + "tokens_seen": 2090846208 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029623269809428285, + "loss": 1.2429, + "theoretical_loss": 3.44622479866128, + "tokens_seen": 2090911744 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029621664994984955, + "loss": 1.2897, + "theoretical_loss": 3.446215944972443, + "tokens_seen": 2090977280 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029620060180541625, + "loss": 1.2624, + "theoretical_loss": 3.446207091638791, + "tokens_seen": 2091042816 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029618455366098295, + "loss": 1.2294, + "theoretical_loss": 3.4461982386603003, + "tokens_seen": 2091108352 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029616850551654966, + "loss": 1.2608, + "theoretical_loss": 3.446189386036945, + "tokens_seen": 2091173888 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029615245737211636, + "loss": 1.3092, + "theoretical_loss": 3.4461805337687, + "tokens_seen": 2091239424 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029613640922768306, + "loss": 1.1803, + "theoretical_loss": 3.4461716818555397, + "tokens_seen": 2091304960 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029612036108324976, + "loss": 1.1723, + "theoretical_loss": 3.4461628302974385, + "tokens_seen": 2091370496 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029610431293881646, + "loss": 1.2177, + "theoretical_loss": 3.4461539790943716, + "tokens_seen": 2091436032 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029608826479438316, + "loss": 1.3024, + "theoretical_loss": 3.4461451282463127, + "tokens_seen": 2091501568 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029607221664994986, + "loss": 1.1964, + "theoretical_loss": 3.446136277753238, + "tokens_seen": 2091567104 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029605616850551656, + "loss": 1.2404, + "theoretical_loss": 3.446127427615121, + "tokens_seen": 2091632640 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029604012036108326, + "loss": 1.3571, + "theoretical_loss": 3.446118577831936, + "tokens_seen": 2091698176 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029602407221664996, + "loss": 1.2836, + "theoretical_loss": 3.446109728403658, + "tokens_seen": 2091763712 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029600802407221666, + "loss": 1.322, + "theoretical_loss": 3.4461008793302623, + "tokens_seen": 2091829248 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029599197592778336, + "loss": 1.2232, + "theoretical_loss": 3.4460920306117235, + "tokens_seen": 2091894784 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029597592778335006, + "loss": 1.295, + "theoretical_loss": 3.4460831822480156, + "tokens_seen": 2091960320 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029595987963891676, + "loss": 1.273, + "theoretical_loss": 3.446074334239113, + "tokens_seen": 2092025856 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029594383149448346, + "loss": 1.3442, + "theoretical_loss": 3.446065486584992, + "tokens_seen": 2092091392 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029592778335005016, + "loss": 1.2422, + "theoretical_loss": 3.446056639285625, + "tokens_seen": 2092156928 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029591173520561687, + "loss": 1.3137, + "theoretical_loss": 3.446047792340988, + "tokens_seen": 2092222464 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029589568706118357, + "loss": 1.2182, + "theoretical_loss": 3.4460389457510554, + "tokens_seen": 2092288000 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029587963891675027, + "loss": 1.2858, + "theoretical_loss": 3.446030099515802, + "tokens_seen": 2092353536 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029586359077231697, + "loss": 1.2669, + "theoretical_loss": 3.4460212536352026, + "tokens_seen": 2092419072 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029584754262788367, + "loss": 1.3639, + "theoretical_loss": 3.446012408109232, + "tokens_seen": 2092484608 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029583149448345037, + "loss": 1.1743, + "theoretical_loss": 3.446003562937864, + "tokens_seen": 2092550144 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029581544633901707, + "loss": 1.3278, + "theoretical_loss": 3.445994718121074, + "tokens_seen": 2092615680 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029579939819458377, + "loss": 1.2294, + "theoretical_loss": 3.4459858736588362, + "tokens_seen": 2092681216 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029578335005015047, + "loss": 1.1915, + "theoretical_loss": 3.445977029551126, + "tokens_seen": 2092746752 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029576730190571717, + "loss": 1.1816, + "theoretical_loss": 3.4459681857979176, + "tokens_seen": 2092812288 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029575125376128387, + "loss": 1.2191, + "theoretical_loss": 3.4459593423991857, + "tokens_seen": 2092877824 + }, + { + "epoch": 1.01, + "learning_rate": 0.00029573520561685057, + "loss": 1.3073, + "theoretical_loss": 3.445950499354905, + "tokens_seen": 2092943360 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002957191574724173, + "loss": 1.2686, + "theoretical_loss": 3.4459416566650507, + "tokens_seen": 2093008896 + }, + { + "epoch": 1.01, + "learning_rate": 0.000295703109327984, + "loss": 1.208, + "theoretical_loss": 3.4459328143295966, + "tokens_seen": 2093074432 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002956870611835507, + "loss": 1.2469, + "theoretical_loss": 3.4459239723485178, + "tokens_seen": 2093139968 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002956710130391174, + "loss": 1.3019, + "theoretical_loss": 3.4459151307217892, + "tokens_seen": 2093205504 + }, + { + "epoch": 1.01, + "learning_rate": 0.0002956549648946841, + "loss": 1.2846, + "theoretical_loss": 3.4459062894493853, + "tokens_seen": 2093271040 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002956389167502508, + "loss": 1.3505, + "theoretical_loss": 3.445897448531281, + "tokens_seen": 2093336576 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002956228686058175, + "loss": 1.2782, + "theoretical_loss": 3.4458886079674507, + "tokens_seen": 2093402112 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002956068204613842, + "loss": 1.3715, + "theoretical_loss": 3.4458797677578694, + "tokens_seen": 2093467648 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002955907723169509, + "loss": 1.2634, + "theoretical_loss": 3.445870927902512, + "tokens_seen": 2093533184 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002955747241725176, + "loss": 1.3504, + "theoretical_loss": 3.4458620884013524, + "tokens_seen": 2093598720 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002955586760280843, + "loss": 1.2246, + "theoretical_loss": 3.445853249254366, + "tokens_seen": 2093664256 + }, + { + "epoch": 1.02, + "learning_rate": 0.000295542627883651, + "loss": 1.2533, + "theoretical_loss": 3.445844410461528, + "tokens_seen": 2093729792 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002955265797392177, + "loss": 1.2454, + "theoretical_loss": 3.445835572022812, + "tokens_seen": 2093795328 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002955105315947844, + "loss": 1.2938, + "theoretical_loss": 3.445826733938193, + "tokens_seen": 2093860864 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002954944834503511, + "loss": 1.2857, + "theoretical_loss": 3.445817896207646, + "tokens_seen": 2093926400 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002954784353059178, + "loss": 1.1867, + "theoretical_loss": 3.445809058831146, + "tokens_seen": 2093991936 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002954623871614845, + "loss": 1.1818, + "theoretical_loss": 3.4458002218086676, + "tokens_seen": 2094057472 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002954463390170512, + "loss": 1.2227, + "theoretical_loss": 3.4457913851401845, + "tokens_seen": 2094123008 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002954302908726179, + "loss": 1.2271, + "theoretical_loss": 3.4457825488256733, + "tokens_seen": 2094188544 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002954142427281846, + "loss": 1.1664, + "theoretical_loss": 3.4457737128651074, + "tokens_seen": 2094254080 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002953981945837513, + "loss": 1.2818, + "theoretical_loss": 3.445764877258462, + "tokens_seen": 2094319616 + }, + { + "epoch": 1.02, + "learning_rate": 0.000295382146439318, + "loss": 1.4085, + "theoretical_loss": 3.445756042005711, + "tokens_seen": 2094385152 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002953660982948847, + "loss": 1.2848, + "theoretical_loss": 3.445747207106831, + "tokens_seen": 2094450688 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002953500501504514, + "loss": 1.1838, + "theoretical_loss": 3.445738372561795, + "tokens_seen": 2094516224 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002953340020060181, + "loss": 1.3488, + "theoretical_loss": 3.4457295383705784, + "tokens_seen": 2094581760 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029531795386158473, + "loss": 1.2794, + "theoretical_loss": 3.4457207045331564, + "tokens_seen": 2094647296 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029530190571715144, + "loss": 1.2196, + "theoretical_loss": 3.445711871049503, + "tokens_seen": 2094712832 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029528585757271814, + "loss": 1.3113, + "theoretical_loss": 3.4457030379195936, + "tokens_seen": 2094778368 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029526980942828484, + "loss": 1.2806, + "theoretical_loss": 3.4456942051434023, + "tokens_seen": 2094843904 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029525376128385154, + "loss": 1.3627, + "theoretical_loss": 3.4456853727209045, + "tokens_seen": 2094909440 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029523771313941824, + "loss": 1.3518, + "theoretical_loss": 3.445676540652075, + "tokens_seen": 2094974976 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029522166499498494, + "loss": 1.2479, + "theoretical_loss": 3.4456677089368872, + "tokens_seen": 2095040512 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029520561685055164, + "loss": 1.2185, + "theoretical_loss": 3.445658877575318, + "tokens_seen": 2095106048 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029518956870611834, + "loss": 1.2966, + "theoretical_loss": 3.4456500465673408, + "tokens_seen": 2095171584 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029517352056168504, + "loss": 1.3079, + "theoretical_loss": 3.4456412159129313, + "tokens_seen": 2095237120 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029515747241725174, + "loss": 1.2458, + "theoretical_loss": 3.445632385612063, + "tokens_seen": 2095302656 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029514142427281844, + "loss": 1.2243, + "theoretical_loss": 3.4456235556647115, + "tokens_seen": 2095368192 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029512537612838514, + "loss": 1.2785, + "theoretical_loss": 3.4456147260708514, + "tokens_seen": 2095433728 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029510932798395184, + "loss": 1.2077, + "theoretical_loss": 3.445605896830458, + "tokens_seen": 2095499264 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029509327983951854, + "loss": 1.2714, + "theoretical_loss": 3.4455970679435053, + "tokens_seen": 2095564800 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029507723169508524, + "loss": 1.2579, + "theoretical_loss": 3.445588239409969, + "tokens_seen": 2095630336 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029506118355065194, + "loss": 1.2433, + "theoretical_loss": 3.445579411229823, + "tokens_seen": 2095695872 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029504513540621865, + "loss": 1.3991, + "theoretical_loss": 3.4455705834030423, + "tokens_seen": 2095761408 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029502908726178535, + "loss": 1.3178, + "theoretical_loss": 3.445561755929602, + "tokens_seen": 2095826944 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029501303911735205, + "loss": 1.2252, + "theoretical_loss": 3.4455529288094775, + "tokens_seen": 2095892480 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029499699097291875, + "loss": 1.3102, + "theoretical_loss": 3.4455441020426423, + "tokens_seen": 2095958016 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029498094282848545, + "loss": 1.1512, + "theoretical_loss": 3.445535275629072, + "tokens_seen": 2096023552 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029496489468405215, + "loss": 1.2656, + "theoretical_loss": 3.445526449568741, + "tokens_seen": 2096089088 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029494884653961885, + "loss": 1.2773, + "theoretical_loss": 3.4455176238616247, + "tokens_seen": 2096154624 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029493279839518555, + "loss": 1.3353, + "theoretical_loss": 3.4455087985076975, + "tokens_seen": 2096220160 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029491675025075225, + "loss": 1.3043, + "theoretical_loss": 3.445499973506934, + "tokens_seen": 2096285696 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029490070210631895, + "loss": 1.287, + "theoretical_loss": 3.44549114885931, + "tokens_seen": 2096351232 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002948846539618857, + "loss": 1.2183, + "theoretical_loss": 3.445482324564799, + "tokens_seen": 2096416768 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002948686058174524, + "loss": 1.2583, + "theoretical_loss": 3.4454735006233768, + "tokens_seen": 2096482304 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002948525576730191, + "loss": 1.2695, + "theoretical_loss": 3.445464677035018, + "tokens_seen": 2096547840 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002948365095285858, + "loss": 1.1979, + "theoretical_loss": 3.4454558537996975, + "tokens_seen": 2096613376 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002948204613841525, + "loss": 1.3329, + "theoretical_loss": 3.4454470309173897, + "tokens_seen": 2096678912 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002948044132397192, + "loss": 1.3072, + "theoretical_loss": 3.4454382083880697, + "tokens_seen": 2096744448 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002947883650952859, + "loss": 1.1574, + "theoretical_loss": 3.4454293862117127, + "tokens_seen": 2096809984 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002947723169508526, + "loss": 1.1818, + "theoretical_loss": 3.4454205643882934, + "tokens_seen": 2096875520 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002947562688064193, + "loss": 1.2445, + "theoretical_loss": 3.445411742917786, + "tokens_seen": 2096941056 + }, + { + "epoch": 1.02, + "learning_rate": 0.000294740220661986, + "loss": 1.2411, + "theoretical_loss": 3.445402921800166, + "tokens_seen": 2097006592 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002947241725175527, + "loss": 1.3001, + "theoretical_loss": 3.4453941010354083, + "tokens_seen": 2097072128 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 775802, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.267109990119934, + "objective/train/theoretical_loss": 3.445385280623488, + "objective/train/tokens_used": 2117597664, + "theoretical_loss": 3.445385280623488, + "tokens_seen": 2097137664 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002947081243731194, + "loss": 1.2671, + "theoretical_loss": 3.445385280623488, + "tokens_seen": 2097137664 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002946920762286861, + "loss": 1.2022, + "theoretical_loss": 3.4453764605643786, + "tokens_seen": 2097203200 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029467602808425276, + "loss": 1.2647, + "theoretical_loss": 3.4453676408580565, + "tokens_seen": 2097268736 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029465997993981946, + "loss": 1.3054, + "theoretical_loss": 3.4453588215044957, + "tokens_seen": 2097334272 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029464393179538616, + "loss": 1.2622, + "theoretical_loss": 3.4453500025036714, + "tokens_seen": 2097399808 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029462788365095286, + "loss": 1.2602, + "theoretical_loss": 3.4453411838555583, + "tokens_seen": 2097465344 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029461183550651956, + "loss": 1.1873, + "theoretical_loss": 3.4453323655601316, + "tokens_seen": 2097530880 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029459578736208626, + "loss": 1.1801, + "theoretical_loss": 3.445323547617366, + "tokens_seen": 2097596416 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029457973921765296, + "loss": 1.3488, + "theoretical_loss": 3.4453147300272358, + "tokens_seen": 2097661952 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029456369107321966, + "loss": 1.251, + "theoretical_loss": 3.445305912789717, + "tokens_seen": 2097727488 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029454764292878637, + "loss": 1.2012, + "theoretical_loss": 3.4452970959047837, + "tokens_seen": 2097793024 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029453159478435307, + "loss": 1.2203, + "theoretical_loss": 3.445288279372411, + "tokens_seen": 2097858560 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029451554663991977, + "loss": 1.1713, + "theoretical_loss": 3.445279463192574, + "tokens_seen": 2097924096 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029449949849548647, + "loss": 1.1437, + "theoretical_loss": 3.4452706473652466, + "tokens_seen": 2097989632 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029448345035105317, + "loss": 1.1952, + "theoretical_loss": 3.445261831890405, + "tokens_seen": 2098055168 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029446740220661987, + "loss": 1.2282, + "theoretical_loss": 3.4452530167680235, + "tokens_seen": 2098120704 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029445135406218657, + "loss": 1.2049, + "theoretical_loss": 3.445244201998077, + "tokens_seen": 2098186240 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029443530591775327, + "loss": 1.2891, + "theoretical_loss": 3.445235387580541, + "tokens_seen": 2098251776 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029441925777331997, + "loss": 1.2658, + "theoretical_loss": 3.445226573515389, + "tokens_seen": 2098317312 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029440320962888667, + "loss": 1.2857, + "theoretical_loss": 3.4452177598025973, + "tokens_seen": 2098382848 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029438716148445337, + "loss": 1.201, + "theoretical_loss": 3.44520894644214, + "tokens_seen": 2098448384 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029437111334002007, + "loss": 1.1875, + "theoretical_loss": 3.4452001334339926, + "tokens_seen": 2098513920 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029435506519558677, + "loss": 1.2128, + "theoretical_loss": 3.4451913207781293, + "tokens_seen": 2098579456 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002943390170511535, + "loss": 1.3471, + "theoretical_loss": 3.4451825084745256, + "tokens_seen": 2098644992 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002943229689067202, + "loss": 1.2874, + "theoretical_loss": 3.445173696523156, + "tokens_seen": 2098710528 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002943069207622869, + "loss": 1.1971, + "theoretical_loss": 3.4451648849239964, + "tokens_seen": 2098776064 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002942908726178536, + "loss": 1.2942, + "theoretical_loss": 3.4451560736770204, + "tokens_seen": 2098841600 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002942748244734203, + "loss": 1.3116, + "theoretical_loss": 3.4451472627822035, + "tokens_seen": 2098907136 + }, + { + "epoch": 1.02, + "learning_rate": 0.000294258776328987, + "loss": 1.2725, + "theoretical_loss": 3.445138452239521, + "tokens_seen": 2098972672 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002942427281845537, + "loss": 1.2635, + "theoretical_loss": 3.445129642048947, + "tokens_seen": 2099038208 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002942266800401204, + "loss": 1.167, + "theoretical_loss": 3.445120832210457, + "tokens_seen": 2099103744 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002942106318956871, + "loss": 1.3631, + "theoretical_loss": 3.445112022724026, + "tokens_seen": 2099169280 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002941945837512538, + "loss": 1.2902, + "theoretical_loss": 3.445103213589629, + "tokens_seen": 2099234816 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002941785356068205, + "loss": 1.2097, + "theoretical_loss": 3.4450944048072403, + "tokens_seen": 2099300352 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002941624874623872, + "loss": 1.2573, + "theoretical_loss": 3.4450855963768356, + "tokens_seen": 2099365888 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002941464393179539, + "loss": 1.2108, + "theoretical_loss": 3.4450767882983895, + "tokens_seen": 2099431424 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002941303911735206, + "loss": 1.2219, + "theoretical_loss": 3.445067980571877, + "tokens_seen": 2099496960 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002941143430290873, + "loss": 1.3765, + "theoretical_loss": 3.445059173197273, + "tokens_seen": 2099562496 + }, + { + "epoch": 1.02, + "learning_rate": 0.000294098294884654, + "loss": 1.2555, + "theoretical_loss": 3.445050366174552, + "tokens_seen": 2099628032 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002940822467402207, + "loss": 1.1623, + "theoretical_loss": 3.4450415595036903, + "tokens_seen": 2099693568 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002940661985957874, + "loss": 1.2671, + "theoretical_loss": 3.4450327531846616, + "tokens_seen": 2099759104 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002940501504513541, + "loss": 1.2395, + "theoretical_loss": 3.4450239472174413, + "tokens_seen": 2099824640 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002940341023069208, + "loss": 1.1715, + "theoretical_loss": 3.4450151416020045, + "tokens_seen": 2099890176 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002940180541624875, + "loss": 1.1989, + "theoretical_loss": 3.4450063363383254, + "tokens_seen": 2099955712 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002940020060180542, + "loss": 1.2222, + "theoretical_loss": 3.44499753142638, + "tokens_seen": 2100021248 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002939859578736209, + "loss": 1.2014, + "theoretical_loss": 3.4449887268661428, + "tokens_seen": 2100086784 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002939699097291876, + "loss": 1.1814, + "theoretical_loss": 3.4449799226575886, + "tokens_seen": 2100152320 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002939538615847543, + "loss": 1.2716, + "theoretical_loss": 3.444971118800693, + "tokens_seen": 2100217856 + }, + { + "epoch": 1.02, + "learning_rate": 0.000293937813440321, + "loss": 1.3661, + "theoretical_loss": 3.4449623152954305, + "tokens_seen": 2100283392 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002939217652958877, + "loss": 1.224, + "theoretical_loss": 3.444953512141776, + "tokens_seen": 2100348928 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029390571715145434, + "loss": 1.2533, + "theoretical_loss": 3.4449447093397048, + "tokens_seen": 2100414464 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029388966900702104, + "loss": 1.2896, + "theoretical_loss": 3.444935906889192, + "tokens_seen": 2100480000 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029387362086258774, + "loss": 1.264, + "theoretical_loss": 3.444927104790212, + "tokens_seen": 2100545536 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029385757271815444, + "loss": 1.2941, + "theoretical_loss": 3.4449183030427397, + "tokens_seen": 2100611072 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029384152457372114, + "loss": 1.3146, + "theoretical_loss": 3.4449095016467512, + "tokens_seen": 2100676608 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029382547642928784, + "loss": 1.2157, + "theoretical_loss": 3.4449007006022208, + "tokens_seen": 2100742144 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029380942828485454, + "loss": 1.1998, + "theoretical_loss": 3.4448918999091234, + "tokens_seen": 2100807680 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029379338014042124, + "loss": 1.2746, + "theoretical_loss": 3.4448830995674338, + "tokens_seen": 2100873216 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029377733199598794, + "loss": 1.2319, + "theoretical_loss": 3.444874299577128, + "tokens_seen": 2100938752 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029376128385155464, + "loss": 1.1091, + "theoretical_loss": 3.44486549993818, + "tokens_seen": 2101004288 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029374523570712134, + "loss": 1.2685, + "theoretical_loss": 3.444856700650565, + "tokens_seen": 2101069824 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029372918756268804, + "loss": 1.377, + "theoretical_loss": 3.444847901714258, + "tokens_seen": 2101135360 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029371313941825474, + "loss": 1.2719, + "theoretical_loss": 3.444839103129235, + "tokens_seen": 2101200896 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029369709127382144, + "loss": 1.1978, + "theoretical_loss": 3.4448303048954694, + "tokens_seen": 2101266432 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002936810431293882, + "loss": 1.2605, + "theoretical_loss": 3.4448215070129375, + "tokens_seen": 2101331968 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002936649949849549, + "loss": 1.2878, + "theoretical_loss": 3.4448127094816137, + "tokens_seen": 2101397504 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002936489468405216, + "loss": 1.3351, + "theoretical_loss": 3.444803912301473, + "tokens_seen": 2101463040 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002936328986960883, + "loss": 1.3647, + "theoretical_loss": 3.444795115472491, + "tokens_seen": 2101528576 + }, + { + "epoch": 1.02, + "learning_rate": 0.000293616850551655, + "loss": 1.1914, + "theoretical_loss": 3.444786318994642, + "tokens_seen": 2101594112 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002936008024072217, + "loss": 1.349, + "theoretical_loss": 3.4447775228679016, + "tokens_seen": 2101659648 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002935847542627884, + "loss": 1.3305, + "theoretical_loss": 3.444768727092245, + "tokens_seen": 2101725184 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002935687061183551, + "loss": 1.2804, + "theoretical_loss": 3.4447599316676465, + "tokens_seen": 2101790720 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002935526579739218, + "loss": 1.2558, + "theoretical_loss": 3.444751136594081, + "tokens_seen": 2101856256 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002935366098294885, + "loss": 1.2154, + "theoretical_loss": 3.4447423418715246, + "tokens_seen": 2101921792 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002935205616850552, + "loss": 1.2277, + "theoretical_loss": 3.444733547499952, + "tokens_seen": 2101987328 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002935045135406219, + "loss": 1.3379, + "theoretical_loss": 3.4447247534793375, + "tokens_seen": 2102052864 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002934884653961886, + "loss": 1.327, + "theoretical_loss": 3.444715959809657, + "tokens_seen": 2102118400 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002934724172517553, + "loss": 1.2551, + "theoretical_loss": 3.4447071664908853, + "tokens_seen": 2102183936 + }, + { + "epoch": 1.02, + "learning_rate": 0.000293456369107322, + "loss": 1.3411, + "theoretical_loss": 3.4446983735229972, + "tokens_seen": 2102249472 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002934403209628887, + "loss": 1.303, + "theoretical_loss": 3.444689580905968, + "tokens_seen": 2102315008 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002934242728184554, + "loss": 1.3305, + "theoretical_loss": 3.444680788639773, + "tokens_seen": 2102380544 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002934082246740221, + "loss": 1.2691, + "theoretical_loss": 3.444671996724387, + "tokens_seen": 2102446080 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002933921765295888, + "loss": 1.3196, + "theoretical_loss": 3.4446632051597845, + "tokens_seen": 2102511616 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002933761283851555, + "loss": 1.2, + "theoretical_loss": 3.4446544139459423, + "tokens_seen": 2102577152 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002933600802407222, + "loss": 1.2971, + "theoretical_loss": 3.4446456230828333, + "tokens_seen": 2102642688 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002933440320962889, + "loss": 1.2641, + "theoretical_loss": 3.444636832570434, + "tokens_seen": 2102708224 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002933279839518556, + "loss": 1.2713, + "theoretical_loss": 3.444628042408719, + "tokens_seen": 2102773760 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002933119358074223, + "loss": 1.3441, + "theoretical_loss": 3.444619252597663, + "tokens_seen": 2102839296 + }, + { + "epoch": 1.02, + "learning_rate": 0.000293295887662989, + "loss": 1.3163, + "theoretical_loss": 3.4446104631372423, + "tokens_seen": 2102904832 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029327983951855566, + "loss": 1.3095, + "theoretical_loss": 3.444601674027431, + "tokens_seen": 2102970368 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029326379137412236, + "loss": 1.3071, + "theoretical_loss": 3.444592885268204, + "tokens_seen": 2103035904 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029324774322968906, + "loss": 1.1679, + "theoretical_loss": 3.444584096859537, + "tokens_seen": 2103101440 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029323169508525576, + "loss": 1.2771, + "theoretical_loss": 3.4445753088014053, + "tokens_seen": 2103166976 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029321564694082246, + "loss": 1.3309, + "theoretical_loss": 3.4445665210937833, + "tokens_seen": 2103232512 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029319959879638916, + "loss": 1.3397, + "theoretical_loss": 3.4445577337366466, + "tokens_seen": 2103298048 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029318355065195586, + "loss": 1.2116, + "theoretical_loss": 3.44454894672997, + "tokens_seen": 2103363584 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029316750250752257, + "loss": 1.3333, + "theoretical_loss": 3.4445401600737284, + "tokens_seen": 2103429120 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029315145436308927, + "loss": 1.3547, + "theoretical_loss": 3.4445313737678975, + "tokens_seen": 2103494656 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029313540621865597, + "loss": 1.297, + "theoretical_loss": 3.444522587812452, + "tokens_seen": 2103560192 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029311935807422267, + "loss": 1.2832, + "theoretical_loss": 3.444513802207367, + "tokens_seen": 2103625728 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 778141, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1972675323486328, + "objective/train/theoretical_loss": 3.4445050169526183, + "objective/train/tokens_used": 2124151264, + "theoretical_loss": 3.4445050169526183, + "tokens_seen": 2103691264 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029310330992978937, + "loss": 1.1973, + "theoretical_loss": 3.4445050169526183, + "tokens_seen": 2103691264 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029308726178535607, + "loss": 1.2487, + "theoretical_loss": 3.44449623204818, + "tokens_seen": 2103756800 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029307121364092277, + "loss": 1.2787, + "theoretical_loss": 3.4444874474940272, + "tokens_seen": 2103822336 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029305516549648947, + "loss": 1.3281, + "theoretical_loss": 3.4444786632901363, + "tokens_seen": 2103887872 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029303911735205617, + "loss": 1.2607, + "theoretical_loss": 3.4444698794364816, + "tokens_seen": 2103953408 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029302306920762287, + "loss": 1.2368, + "theoretical_loss": 3.444461095933038, + "tokens_seen": 2104018944 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029300702106318957, + "loss": 1.2198, + "theoretical_loss": 3.444452312779781, + "tokens_seen": 2104084480 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029299097291875627, + "loss": 1.2514, + "theoretical_loss": 3.4444435299766853, + "tokens_seen": 2104150016 + }, + { + "epoch": 1.02, + "learning_rate": 0.000292974924774323, + "loss": 1.3973, + "theoretical_loss": 3.4444347475237267, + "tokens_seen": 2104215552 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002929588766298897, + "loss": 1.2227, + "theoretical_loss": 3.4444259654208795, + "tokens_seen": 2104281088 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002929428284854564, + "loss": 1.2904, + "theoretical_loss": 3.4444171836681194, + "tokens_seen": 2104346624 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002929267803410231, + "loss": 1.1922, + "theoretical_loss": 3.444408402265422, + "tokens_seen": 2104412160 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002929107321965898, + "loss": 1.1775, + "theoretical_loss": 3.444399621212762, + "tokens_seen": 2104477696 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002928946840521565, + "loss": 1.1278, + "theoretical_loss": 3.4443908405101133, + "tokens_seen": 2104543232 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002928786359077232, + "loss": 1.3168, + "theoretical_loss": 3.4443820601574533, + "tokens_seen": 2104608768 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002928625877632899, + "loss": 1.2253, + "theoretical_loss": 3.444373280154756, + "tokens_seen": 2104674304 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002928465396188566, + "loss": 1.246, + "theoretical_loss": 3.444364500501996, + "tokens_seen": 2104739840 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002928304914744233, + "loss": 1.2613, + "theoretical_loss": 3.4443557211991496, + "tokens_seen": 2104805376 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029281444332999, + "loss": 1.3147, + "theoretical_loss": 3.444346942246191, + "tokens_seen": 2104870912 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002927983951855567, + "loss": 1.1887, + "theoretical_loss": 3.444338163643096, + "tokens_seen": 2104936448 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002927823470411234, + "loss": 1.2509, + "theoretical_loss": 3.44432938538984, + "tokens_seen": 2105001984 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002927662988966901, + "loss": 1.1802, + "theoretical_loss": 3.444320607486397, + "tokens_seen": 2105067520 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002927502507522568, + "loss": 1.3659, + "theoretical_loss": 3.444311829932743, + "tokens_seen": 2105133056 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002927342026078235, + "loss": 1.1108, + "theoretical_loss": 3.444303052728854, + "tokens_seen": 2105198592 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002927181544633902, + "loss": 1.2198, + "theoretical_loss": 3.444294275874703, + "tokens_seen": 2105264128 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002927021063189569, + "loss": 1.2241, + "theoretical_loss": 3.4442854993702667, + "tokens_seen": 2105329664 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002926860581745236, + "loss": 1.2874, + "theoretical_loss": 3.4442767232155207, + "tokens_seen": 2105395200 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002926700100300903, + "loss": 1.3583, + "theoretical_loss": 3.444267947410439, + "tokens_seen": 2105460736 + }, + { + "epoch": 1.02, + "learning_rate": 0.000292653961885657, + "loss": 1.2613, + "theoretical_loss": 3.444259171954997, + "tokens_seen": 2105526272 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002926379137412237, + "loss": 1.2397, + "theoretical_loss": 3.4442503968491707, + "tokens_seen": 2105591808 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002926218655967904, + "loss": 1.2498, + "theoretical_loss": 3.4442416220929344, + "tokens_seen": 2105657344 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002926058174523571, + "loss": 1.2809, + "theoretical_loss": 3.444232847686264, + "tokens_seen": 2105722880 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002925897693079238, + "loss": 1.2286, + "theoretical_loss": 3.4442240736291336, + "tokens_seen": 2105788416 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002925737211634905, + "loss": 1.1816, + "theoretical_loss": 3.4442152999215194, + "tokens_seen": 2105853952 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002925576730190572, + "loss": 1.4047, + "theoretical_loss": 3.4442065265633968, + "tokens_seen": 2105919488 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002925416248746239, + "loss": 1.1835, + "theoretical_loss": 3.4441977535547403, + "tokens_seen": 2105985024 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002925255767301906, + "loss": 1.1238, + "theoretical_loss": 3.4441889808955253, + "tokens_seen": 2106050560 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029250952858575724, + "loss": 1.204, + "theoretical_loss": 3.4441802085857267, + "tokens_seen": 2106116096 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029249348044132394, + "loss": 1.2045, + "theoretical_loss": 3.4441714366253207, + "tokens_seen": 2106181632 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002924774322968907, + "loss": 1.2842, + "theoretical_loss": 3.4441626650142814, + "tokens_seen": 2106247168 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002924613841524574, + "loss": 1.2307, + "theoretical_loss": 3.444153893752585, + "tokens_seen": 2106312704 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002924453360080241, + "loss": 1.2385, + "theoretical_loss": 3.4441451228402054, + "tokens_seen": 2106378240 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002924292878635908, + "loss": 1.2185, + "theoretical_loss": 3.4441363522771193, + "tokens_seen": 2106443776 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002924132397191575, + "loss": 1.353, + "theoretical_loss": 3.444127582063301, + "tokens_seen": 2106509312 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002923971915747242, + "loss": 1.249, + "theoretical_loss": 3.4441188121987256, + "tokens_seen": 2106574848 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002923811434302909, + "loss": 1.3077, + "theoretical_loss": 3.4441100426833695, + "tokens_seen": 2106640384 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002923650952858576, + "loss": 1.3165, + "theoretical_loss": 3.444101273517207, + "tokens_seen": 2106705920 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002923490471414243, + "loss": 1.3798, + "theoretical_loss": 3.4440925047002127, + "tokens_seen": 2106771456 + }, + { + "epoch": 1.02, + "learning_rate": 0.000292332998996991, + "loss": 1.1609, + "theoretical_loss": 3.444083736232363, + "tokens_seen": 2106836992 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002923169508525577, + "loss": 1.3035, + "theoretical_loss": 3.4440749681136333, + "tokens_seen": 2106902528 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002923009027081244, + "loss": 1.2862, + "theoretical_loss": 3.4440662003439977, + "tokens_seen": 2106968064 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002922848545636911, + "loss": 1.3212, + "theoretical_loss": 3.444057432923432, + "tokens_seen": 2107033600 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002922688064192578, + "loss": 1.1666, + "theoretical_loss": 3.444048665851912, + "tokens_seen": 2107099136 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002922527582748245, + "loss": 1.2198, + "theoretical_loss": 3.444039899129412, + "tokens_seen": 2107164672 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002922367101303912, + "loss": 1.2303, + "theoretical_loss": 3.4440311327559074, + "tokens_seen": 2107230208 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002922206619859579, + "loss": 1.3045, + "theoretical_loss": 3.444022366731374, + "tokens_seen": 2107295744 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002922046138415246, + "loss": 1.1988, + "theoretical_loss": 3.444013601055787, + "tokens_seen": 2107361280 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002921885656970913, + "loss": 1.3715, + "theoretical_loss": 3.444004835729121, + "tokens_seen": 2107426816 + }, + { + "epoch": 1.02, + "learning_rate": 0.000292172517552658, + "loss": 1.3063, + "theoretical_loss": 3.443996070751352, + "tokens_seen": 2107492352 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002921564694082247, + "loss": 1.1607, + "theoretical_loss": 3.4439873061224553, + "tokens_seen": 2107557888 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002921404212637914, + "loss": 1.2929, + "theoretical_loss": 3.443978541842405, + "tokens_seen": 2107623424 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002921243731193581, + "loss": 1.2763, + "theoretical_loss": 3.4439697779111778, + "tokens_seen": 2107688960 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002921083249749248, + "loss": 1.3053, + "theoretical_loss": 3.4439610143287482, + "tokens_seen": 2107754496 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002920922768304915, + "loss": 1.2972, + "theoretical_loss": 3.4439522510950917, + "tokens_seen": 2107820032 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002920762286860582, + "loss": 1.1799, + "theoretical_loss": 3.4439434882101834, + "tokens_seen": 2107885568 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002920601805416249, + "loss": 1.2718, + "theoretical_loss": 3.4439347256739987, + "tokens_seen": 2107951104 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002920441323971916, + "loss": 1.1989, + "theoretical_loss": 3.443925963486513, + "tokens_seen": 2108016640 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002920280842527583, + "loss": 1.3149, + "theoretical_loss": 3.4439172016477015, + "tokens_seen": 2108082176 + }, + { + "epoch": 1.02, + "learning_rate": 0.000292012036108325, + "loss": 1.2636, + "theoretical_loss": 3.4439084401575393, + "tokens_seen": 2108147712 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002919959879638917, + "loss": 1.2224, + "theoretical_loss": 3.4438996790160017, + "tokens_seen": 2108213248 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002919799398194584, + "loss": 1.2229, + "theoretical_loss": 3.4438909182230644, + "tokens_seen": 2108278784 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002919638916750251, + "loss": 1.2542, + "theoretical_loss": 3.4438821577787024, + "tokens_seen": 2108344320 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002919478435305918, + "loss": 1.1848, + "theoretical_loss": 3.443873397682891, + "tokens_seen": 2108409856 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002919317953861585, + "loss": 1.2945, + "theoretical_loss": 3.4438646379356053, + "tokens_seen": 2108475392 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002919157472417252, + "loss": 1.2176, + "theoretical_loss": 3.443855878536821, + "tokens_seen": 2108540928 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002918996990972919, + "loss": 1.2367, + "theoretical_loss": 3.443847119486514, + "tokens_seen": 2108606464 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002918836509528586, + "loss": 1.3342, + "theoretical_loss": 3.4438383607846577, + "tokens_seen": 2108672000 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029186760280842526, + "loss": 1.2821, + "theoretical_loss": 3.4438296024312285, + "tokens_seen": 2108737536 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029185155466399196, + "loss": 1.2523, + "theoretical_loss": 3.4438208444262024, + "tokens_seen": 2108803072 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029183550651955866, + "loss": 1.2667, + "theoretical_loss": 3.4438120867695536, + "tokens_seen": 2108868608 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029181945837512536, + "loss": 1.1976, + "theoretical_loss": 3.4438033294612582, + "tokens_seen": 2108934144 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029180341023069207, + "loss": 1.2828, + "theoretical_loss": 3.4437945725012913, + "tokens_seen": 2108999680 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029178736208625877, + "loss": 1.2717, + "theoretical_loss": 3.443785815889628, + "tokens_seen": 2109065216 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029177131394182547, + "loss": 1.2634, + "theoretical_loss": 3.4437770596262434, + "tokens_seen": 2109130752 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029175526579739217, + "loss": 1.2126, + "theoretical_loss": 3.4437683037111135, + "tokens_seen": 2109196288 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029173921765295887, + "loss": 1.3361, + "theoretical_loss": 3.443759548144213, + "tokens_seen": 2109261824 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029172316950852557, + "loss": 1.213, + "theoretical_loss": 3.443750792925518, + "tokens_seen": 2109327360 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029170712136409227, + "loss": 1.2479, + "theoretical_loss": 3.443742038055003, + "tokens_seen": 2109392896 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029169107321965897, + "loss": 1.1985, + "theoretical_loss": 3.443733283532644, + "tokens_seen": 2109458432 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029167502507522567, + "loss": 1.2196, + "theoretical_loss": 3.4437245293584158, + "tokens_seen": 2109523968 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029165897693079237, + "loss": 1.3443, + "theoretical_loss": 3.443715775532294, + "tokens_seen": 2109589504 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029164292878635907, + "loss": 1.1668, + "theoretical_loss": 3.4437070220542543, + "tokens_seen": 2109655040 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029162688064192577, + "loss": 1.2578, + "theoretical_loss": 3.443698268924271, + "tokens_seen": 2109720576 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002916108324974925, + "loss": 1.2426, + "theoretical_loss": 3.443689516142321, + "tokens_seen": 2109786112 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002915947843530592, + "loss": 1.1877, + "theoretical_loss": 3.443680763708378, + "tokens_seen": 2109851648 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002915787362086259, + "loss": 1.2549, + "theoretical_loss": 3.443672011622418, + "tokens_seen": 2109917184 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002915626880641926, + "loss": 1.2792, + "theoretical_loss": 3.443663259884417, + "tokens_seen": 2109982720 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002915466399197593, + "loss": 1.3534, + "theoretical_loss": 3.4436545084943497, + "tokens_seen": 2110048256 + }, + { + "epoch": 1.02, + "learning_rate": 0.000291530591775326, + "loss": 1.2046, + "theoretical_loss": 3.4436457574521917, + "tokens_seen": 2110113792 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002915145436308927, + "loss": 1.367, + "theoretical_loss": 3.4436370067579185, + "tokens_seen": 2110179328 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 780862, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1962933540344238, + "objective/train/theoretical_loss": 3.443628256411505, + "objective/train/tokens_used": 2130704864, + "theoretical_loss": 3.443628256411505, + "tokens_seen": 2110244864 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002914984954864594, + "loss": 1.1963, + "theoretical_loss": 3.443628256411505, + "tokens_seen": 2110244864 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002914824473420261, + "loss": 1.1991, + "theoretical_loss": 3.4436195064129262, + "tokens_seen": 2110310400 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002914663991975928, + "loss": 1.2477, + "theoretical_loss": 3.4436107567621588, + "tokens_seen": 2110375936 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002914503510531595, + "loss": 1.1367, + "theoretical_loss": 3.4436020074591775, + "tokens_seen": 2110441472 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002914343029087262, + "loss": 1.382, + "theoretical_loss": 3.443593258503957, + "tokens_seen": 2110507008 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002914182547642929, + "loss": 1.3138, + "theoretical_loss": 3.443584509896474, + "tokens_seen": 2110572544 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002914022066198596, + "loss": 1.278, + "theoretical_loss": 3.443575761636703, + "tokens_seen": 2110638080 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002913861584754263, + "loss": 1.3375, + "theoretical_loss": 3.4435670137246195, + "tokens_seen": 2110703616 + }, + { + "epoch": 1.02, + "learning_rate": 0.000291370110330993, + "loss": 1.2624, + "theoretical_loss": 3.4435582661601987, + "tokens_seen": 2110769152 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002913540621865597, + "loss": 1.2571, + "theoretical_loss": 3.443549518943416, + "tokens_seen": 2110834688 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002913380140421264, + "loss": 1.2769, + "theoretical_loss": 3.4435407720742477, + "tokens_seen": 2110900224 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002913219658976931, + "loss": 1.2774, + "theoretical_loss": 3.4435320255526687, + "tokens_seen": 2110965760 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002913059177532598, + "loss": 1.2796, + "theoretical_loss": 3.443523279378654, + "tokens_seen": 2111031296 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029128986960882654, + "loss": 1.2415, + "theoretical_loss": 3.443514533552179, + "tokens_seen": 2111096832 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029127382146439324, + "loss": 1.266, + "theoretical_loss": 3.443505788073219, + "tokens_seen": 2111162368 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029125777331995994, + "loss": 1.2376, + "theoretical_loss": 3.4434970429417504, + "tokens_seen": 2111227904 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002912417251755266, + "loss": 1.3241, + "theoretical_loss": 3.4434882981577477, + "tokens_seen": 2111293440 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002912256770310933, + "loss": 1.3218, + "theoretical_loss": 3.443479553721187, + "tokens_seen": 2111358976 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029120962888666, + "loss": 1.2471, + "theoretical_loss": 3.4434708096320428, + "tokens_seen": 2111424512 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002911935807422267, + "loss": 1.2568, + "theoretical_loss": 3.443462065890291, + "tokens_seen": 2111490048 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002911775325977934, + "loss": 1.1577, + "theoretical_loss": 3.443453322495907, + "tokens_seen": 2111555584 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002911614844533601, + "loss": 1.2099, + "theoretical_loss": 3.443444579448866, + "tokens_seen": 2111621120 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002911454363089268, + "loss": 1.3542, + "theoretical_loss": 3.443435836749144, + "tokens_seen": 2111686656 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002911293881644935, + "loss": 1.1775, + "theoretical_loss": 3.443427094396716, + "tokens_seen": 2111752192 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002911133400200602, + "loss": 1.2547, + "theoretical_loss": 3.443418352391558, + "tokens_seen": 2111817728 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002910972918756269, + "loss": 1.2054, + "theoretical_loss": 3.443409610733644, + "tokens_seen": 2111883264 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002910812437311936, + "loss": 1.2984, + "theoretical_loss": 3.443400869422951, + "tokens_seen": 2111948800 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002910651955867603, + "loss": 1.2858, + "theoretical_loss": 3.443392128459453, + "tokens_seen": 2112014336 + }, + { + "epoch": 1.02, + "learning_rate": 0.000291049147442327, + "loss": 1.1551, + "theoretical_loss": 3.4433833878431273, + "tokens_seen": 2112079872 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002910330992978937, + "loss": 1.1795, + "theoretical_loss": 3.4433746475739477, + "tokens_seen": 2112145408 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002910170511534604, + "loss": 1.3087, + "theoretical_loss": 3.44336590765189, + "tokens_seen": 2112210944 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002910010030090271, + "loss": 1.2923, + "theoretical_loss": 3.44335716807693, + "tokens_seen": 2112276480 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002909849548645938, + "loss": 1.2765, + "theoretical_loss": 3.4433484288490432, + "tokens_seen": 2112342016 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002909689067201605, + "loss": 1.1198, + "theoretical_loss": 3.4433396899682047, + "tokens_seen": 2112407552 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002909528585757272, + "loss": 1.1451, + "theoretical_loss": 3.4433309514343904, + "tokens_seen": 2112473088 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002909368104312939, + "loss": 1.2928, + "theoretical_loss": 3.443322213247575, + "tokens_seen": 2112538624 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002909207622868606, + "loss": 1.1581, + "theoretical_loss": 3.4433134754077344, + "tokens_seen": 2112604160 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002909047141424273, + "loss": 1.2931, + "theoretical_loss": 3.4433047379148443, + "tokens_seen": 2112669696 + }, + { + "epoch": 1.02, + "learning_rate": 0.000290888665997994, + "loss": 1.3144, + "theoretical_loss": 3.44329600076888, + "tokens_seen": 2112735232 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002908726178535607, + "loss": 1.348, + "theoretical_loss": 3.4432872639698164, + "tokens_seen": 2112800768 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002908565697091274, + "loss": 1.1251, + "theoretical_loss": 3.44327852751763, + "tokens_seen": 2112866304 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002908405215646941, + "loss": 1.2912, + "theoretical_loss": 3.4432697914122956, + "tokens_seen": 2112931840 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002908244734202608, + "loss": 1.3106, + "theoretical_loss": 3.443261055653789, + "tokens_seen": 2112997376 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002908084252758275, + "loss": 1.3115, + "theoretical_loss": 3.4432523202420855, + "tokens_seen": 2113062912 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002907923771313942, + "loss": 1.3393, + "theoretical_loss": 3.44324358517716, + "tokens_seen": 2113128448 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002907763289869609, + "loss": 1.2552, + "theoretical_loss": 3.443234850458989, + "tokens_seen": 2113193984 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002907602808425276, + "loss": 1.2076, + "theoretical_loss": 3.443226116087547, + "tokens_seen": 2113259520 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002907442326980943, + "loss": 1.3285, + "theoretical_loss": 3.4432173820628105, + "tokens_seen": 2113325056 + }, + { + "epoch": 1.02, + "learning_rate": 0.000290728184553661, + "loss": 1.3055, + "theoretical_loss": 3.443208648384754, + "tokens_seen": 2113390592 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002907121364092277, + "loss": 1.1829, + "theoretical_loss": 3.443199915053354, + "tokens_seen": 2113456128 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002906960882647944, + "loss": 1.3525, + "theoretical_loss": 3.443191182068585, + "tokens_seen": 2113521664 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002906800401203611, + "loss": 1.3038, + "theoretical_loss": 3.4431824494304237, + "tokens_seen": 2113587200 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002906639919759278, + "loss": 1.3202, + "theoretical_loss": 3.443173717138844, + "tokens_seen": 2113652736 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002906479438314945, + "loss": 1.3343, + "theoretical_loss": 3.443164985193823, + "tokens_seen": 2113718272 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002906318956870612, + "loss": 1.2645, + "theoretical_loss": 3.443156253595335, + "tokens_seen": 2113783808 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002906158475426279, + "loss": 1.2848, + "theoretical_loss": 3.443147522343356, + "tokens_seen": 2113849344 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002905997993981946, + "loss": 1.3131, + "theoretical_loss": 3.4431387914378613, + "tokens_seen": 2113914880 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002905837512537613, + "loss": 1.2555, + "theoretical_loss": 3.4431300608788273, + "tokens_seen": 2113980416 + }, + { + "epoch": 1.02, + "learning_rate": 0.000290567703109328, + "loss": 1.2419, + "theoretical_loss": 3.4431213306662283, + "tokens_seen": 2114045952 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002905516549648947, + "loss": 1.2662, + "theoretical_loss": 3.44311260080004, + "tokens_seen": 2114111488 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002905356068204614, + "loss": 1.3044, + "theoretical_loss": 3.443103871280239, + "tokens_seen": 2114177024 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002905195586760281, + "loss": 1.1785, + "theoretical_loss": 3.4430951421067992, + "tokens_seen": 2114242560 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002905035105315948, + "loss": 1.2129, + "theoretical_loss": 3.4430864132796972, + "tokens_seen": 2114308096 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002904874623871615, + "loss": 1.3455, + "theoretical_loss": 3.443077684798909, + "tokens_seen": 2114373632 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029047141424272816, + "loss": 1.3331, + "theoretical_loss": 3.4430689566644084, + "tokens_seen": 2114439168 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029045536609829486, + "loss": 1.2118, + "theoretical_loss": 3.4430602288761722, + "tokens_seen": 2114504704 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029043931795386157, + "loss": 1.2777, + "theoretical_loss": 3.443051501434176, + "tokens_seen": 2114570240 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029042326980942827, + "loss": 1.2514, + "theoretical_loss": 3.443042774338395, + "tokens_seen": 2114635776 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029040722166499497, + "loss": 1.2753, + "theoretical_loss": 3.4430340475888044, + "tokens_seen": 2114701312 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029039117352056167, + "loss": 1.1925, + "theoretical_loss": 3.44302532118538, + "tokens_seen": 2114766848 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029037512537612837, + "loss": 1.197, + "theoretical_loss": 3.443016595128098, + "tokens_seen": 2114832384 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029035907723169507, + "loss": 1.3504, + "theoretical_loss": 3.4430078694169333, + "tokens_seen": 2114897920 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029034302908726177, + "loss": 1.2977, + "theoretical_loss": 3.4429991440518615, + "tokens_seen": 2114963456 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029032698094282847, + "loss": 1.2179, + "theoretical_loss": 3.4429904190328577, + "tokens_seen": 2115028992 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029031093279839517, + "loss": 1.2791, + "theoretical_loss": 3.442981694359898, + "tokens_seen": 2115094528 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029029488465396187, + "loss": 1.2691, + "theoretical_loss": 3.442972970032958, + "tokens_seen": 2115160064 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029027883650952857, + "loss": 1.1494, + "theoretical_loss": 3.4429642460520133, + "tokens_seen": 2115225600 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029026278836509527, + "loss": 1.2076, + "theoretical_loss": 3.442955522417039, + "tokens_seen": 2115291136 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029024674022066197, + "loss": 1.2759, + "theoretical_loss": 3.4429467991280114, + "tokens_seen": 2115356672 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002902306920762287, + "loss": 1.2948, + "theoretical_loss": 3.4429380761849053, + "tokens_seen": 2115422208 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002902146439317954, + "loss": 1.2735, + "theoretical_loss": 3.4429293535876964, + "tokens_seen": 2115487744 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002901985957873621, + "loss": 1.1781, + "theoretical_loss": 3.442920631336361, + "tokens_seen": 2115553280 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002901825476429288, + "loss": 1.1698, + "theoretical_loss": 3.4429119094308733, + "tokens_seen": 2115618816 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002901664994984955, + "loss": 1.1742, + "theoretical_loss": 3.4429031878712104, + "tokens_seen": 2115684352 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002901504513540622, + "loss": 1.2372, + "theoretical_loss": 3.4428944666573473, + "tokens_seen": 2115749888 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002901344032096289, + "loss": 1.1359, + "theoretical_loss": 3.4428857457892588, + "tokens_seen": 2115815424 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002901183550651956, + "loss": 1.1444, + "theoretical_loss": 3.442877025266921, + "tokens_seen": 2115880960 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002901023069207623, + "loss": 1.3026, + "theoretical_loss": 3.44286830509031, + "tokens_seen": 2115946496 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029008625877632903, + "loss": 1.1901, + "theoretical_loss": 3.4428595852594013, + "tokens_seen": 2116012032 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029007021063189573, + "loss": 1.3754, + "theoretical_loss": 3.44285086577417, + "tokens_seen": 2116077568 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029005416248746243, + "loss": 1.2692, + "theoretical_loss": 3.4428421466345918, + "tokens_seen": 2116143104 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029003811434302914, + "loss": 1.2918, + "theoretical_loss": 3.4428334278406423, + "tokens_seen": 2116208640 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029002206619859584, + "loss": 1.2789, + "theoretical_loss": 3.4428247093922972, + "tokens_seen": 2116274176 + }, + { + "epoch": 1.02, + "learning_rate": 0.00029000601805416254, + "loss": 1.1723, + "theoretical_loss": 3.442815991289532, + "tokens_seen": 2116339712 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028998996990972924, + "loss": 1.1823, + "theoretical_loss": 3.4428072735323227, + "tokens_seen": 2116405248 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028997392176529594, + "loss": 1.1356, + "theoretical_loss": 3.4427985561206444, + "tokens_seen": 2116470784 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028995787362086264, + "loss": 1.2173, + "theoretical_loss": 3.442789839054473, + "tokens_seen": 2116536320 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028994182547642934, + "loss": 1.2218, + "theoretical_loss": 3.442781122333783, + "tokens_seen": 2116601856 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028992577733199604, + "loss": 1.2987, + "theoretical_loss": 3.4427724059585523, + "tokens_seen": 2116667392 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028990972918756274, + "loss": 1.2694, + "theoretical_loss": 3.442763689928755, + "tokens_seen": 2116732928 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 783438, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2316176891326904, + "objective/train/theoretical_loss": 3.4427549742443664, + "objective/train/tokens_used": 2137258464, + "theoretical_loss": 3.4427549742443664, + "tokens_seen": 2116798464 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028989368104312944, + "loss": 1.2316, + "theoretical_loss": 3.4427549742443664, + "tokens_seen": 2116798464 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028987763289869614, + "loss": 1.3563, + "theoretical_loss": 3.442746258905363, + "tokens_seen": 2116864000 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028986158475426284, + "loss": 1.3638, + "theoretical_loss": 3.44273754391172, + "tokens_seen": 2116929536 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002898455366098295, + "loss": 1.2402, + "theoretical_loss": 3.442728829263413, + "tokens_seen": 2116995072 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002898294884653962, + "loss": 1.2006, + "theoretical_loss": 3.4427201149604176, + "tokens_seen": 2117060608 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002898134403209629, + "loss": 1.2286, + "theoretical_loss": 3.44271140100271, + "tokens_seen": 2117126144 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002897973921765296, + "loss": 1.2803, + "theoretical_loss": 3.442702687390265, + "tokens_seen": 2117191680 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002897813440320963, + "loss": 1.1941, + "theoretical_loss": 3.4426939741230584, + "tokens_seen": 2117257216 + }, + { + "epoch": 1.02, + "learning_rate": 0.000289765295887663, + "loss": 1.3159, + "theoretical_loss": 3.4426852612010665, + "tokens_seen": 2117322752 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002897492477432297, + "loss": 1.2482, + "theoretical_loss": 3.4426765486242643, + "tokens_seen": 2117388288 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002897331995987964, + "loss": 1.335, + "theoretical_loss": 3.4426678363926277, + "tokens_seen": 2117453824 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002897171514543631, + "loss": 1.3081, + "theoretical_loss": 3.4426591245061324, + "tokens_seen": 2117519360 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002897011033099298, + "loss": 1.1895, + "theoretical_loss": 3.4426504129647535, + "tokens_seen": 2117584896 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002896850551654965, + "loss": 1.2115, + "theoretical_loss": 3.4426417017684674, + "tokens_seen": 2117650432 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002896690070210632, + "loss": 1.2812, + "theoretical_loss": 3.4426329909172493, + "tokens_seen": 2117715968 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002896529588766299, + "loss": 1.2234, + "theoretical_loss": 3.4426242804110747, + "tokens_seen": 2117781504 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002896369107321966, + "loss": 1.2476, + "theoretical_loss": 3.44261557024992, + "tokens_seen": 2117847040 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002896208625877633, + "loss": 1.2683, + "theoretical_loss": 3.44260686043376, + "tokens_seen": 2117912576 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028960481444333, + "loss": 1.2563, + "theoretical_loss": 3.4425981509625707, + "tokens_seen": 2117978112 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002895887662988967, + "loss": 1.4257, + "theoretical_loss": 3.4425894418363283, + "tokens_seen": 2118043648 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002895727181544634, + "loss": 1.2902, + "theoretical_loss": 3.4425807330550073, + "tokens_seen": 2118109184 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002895566700100301, + "loss": 1.3576, + "theoretical_loss": 3.4425720246185847, + "tokens_seen": 2118174720 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002895406218655968, + "loss": 1.3128, + "theoretical_loss": 3.442563316527035, + "tokens_seen": 2118240256 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002895245737211635, + "loss": 1.2385, + "theoretical_loss": 3.442554608780335, + "tokens_seen": 2118305792 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002895085255767302, + "loss": 1.2966, + "theoretical_loss": 3.442545901378459, + "tokens_seen": 2118371328 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002894924774322969, + "loss": 1.2952, + "theoretical_loss": 3.4425371943213836, + "tokens_seen": 2118436864 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002894764292878636, + "loss": 1.2157, + "theoretical_loss": 3.442528487609085, + "tokens_seen": 2118502400 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002894603811434303, + "loss": 1.2645, + "theoretical_loss": 3.442519781241537, + "tokens_seen": 2118567936 + }, + { + "epoch": 1.02, + "learning_rate": 0.000289444332998997, + "loss": 1.2964, + "theoretical_loss": 3.4425110752187176, + "tokens_seen": 2118633472 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002894282848545637, + "loss": 1.2486, + "theoretical_loss": 3.442502369540601, + "tokens_seen": 2118699008 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002894122367101304, + "loss": 1.2155, + "theoretical_loss": 3.442493664207163, + "tokens_seen": 2118764544 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002893961885656971, + "loss": 1.227, + "theoretical_loss": 3.4424849592183797, + "tokens_seen": 2118830080 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002893801404212638, + "loss": 1.322, + "theoretical_loss": 3.4424762545742267, + "tokens_seen": 2118895616 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002893640922768305, + "loss": 1.2169, + "theoretical_loss": 3.4424675502746798, + "tokens_seen": 2118961152 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002893480441323972, + "loss": 1.2762, + "theoretical_loss": 3.442458846319714, + "tokens_seen": 2119026688 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002893319959879639, + "loss": 1.2845, + "theoretical_loss": 3.442450142709306, + "tokens_seen": 2119092224 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002893159478435306, + "loss": 1.1847, + "theoretical_loss": 3.442441439443431, + "tokens_seen": 2119157760 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002892998996990973, + "loss": 1.328, + "theoretical_loss": 3.442432736522065, + "tokens_seen": 2119223296 + }, + { + "epoch": 1.02, + "learning_rate": 0.000289283851554664, + "loss": 1.2479, + "theoretical_loss": 3.4424240339451826, + "tokens_seen": 2119288832 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002892678034102307, + "loss": 1.2978, + "theoretical_loss": 3.4424153317127613, + "tokens_seen": 2119354368 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002892517552657974, + "loss": 1.3098, + "theoretical_loss": 3.4424066298247755, + "tokens_seen": 2119419904 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002892357071213641, + "loss": 1.3215, + "theoretical_loss": 3.4423979282812014, + "tokens_seen": 2119485440 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002892196589769308, + "loss": 1.1718, + "theoretical_loss": 3.442389227082015, + "tokens_seen": 2119550976 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002892036108324975, + "loss": 1.2258, + "theoretical_loss": 3.4423805262271907, + "tokens_seen": 2119616512 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002891875626880642, + "loss": 1.2475, + "theoretical_loss": 3.442371825716706, + "tokens_seen": 2119682048 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002891715145436309, + "loss": 1.2839, + "theoretical_loss": 3.4423631255505356, + "tokens_seen": 2119747584 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002891554663991976, + "loss": 1.195, + "theoretical_loss": 3.442354425728655, + "tokens_seen": 2119813120 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002891394182547643, + "loss": 1.1982, + "theoretical_loss": 3.442345726251041, + "tokens_seen": 2119878656 + }, + { + "epoch": 1.02, + "learning_rate": 0.000289123370110331, + "loss": 1.3776, + "theoretical_loss": 3.4423370271176683, + "tokens_seen": 2119944192 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002891073219658977, + "loss": 1.3086, + "theoretical_loss": 3.4423283283285135, + "tokens_seen": 2120009728 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002890912738214644, + "loss": 1.2555, + "theoretical_loss": 3.4423196298835514, + "tokens_seen": 2120075264 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028907522567703106, + "loss": 1.2717, + "theoretical_loss": 3.4423109317827585, + "tokens_seen": 2120140800 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028905917753259777, + "loss": 1.2699, + "theoretical_loss": 3.4423022340261102, + "tokens_seen": 2120206336 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028904312938816447, + "loss": 1.2881, + "theoretical_loss": 3.4422935366135823, + "tokens_seen": 2120271872 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028902708124373117, + "loss": 1.2639, + "theoretical_loss": 3.44228483954515, + "tokens_seen": 2120337408 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028901103309929787, + "loss": 1.1563, + "theoretical_loss": 3.4422761428207904, + "tokens_seen": 2120402944 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028899498495486457, + "loss": 1.206, + "theoretical_loss": 3.442267446440478, + "tokens_seen": 2120468480 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028897893681043127, + "loss": 1.2287, + "theoretical_loss": 3.4422587504041893, + "tokens_seen": 2120534016 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028896288866599797, + "loss": 1.3392, + "theoretical_loss": 3.4422500547119, + "tokens_seen": 2120599552 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028894684052156467, + "loss": 1.2587, + "theoretical_loss": 3.442241359363585, + "tokens_seen": 2120665088 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028893079237713137, + "loss": 1.3074, + "theoretical_loss": 3.4422326643592216, + "tokens_seen": 2120730624 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028891474423269807, + "loss": 1.2131, + "theoretical_loss": 3.4422239696987837, + "tokens_seen": 2120796160 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028889869608826477, + "loss": 1.2796, + "theoretical_loss": 3.4422152753822486, + "tokens_seen": 2120861696 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002888826479438315, + "loss": 1.2692, + "theoretical_loss": 3.4422065814095912, + "tokens_seen": 2120927232 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028886659979939823, + "loss": 1.3003, + "theoretical_loss": 3.4421978877807877, + "tokens_seen": 2120992768 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028885055165496493, + "loss": 1.2886, + "theoretical_loss": 3.442189194495814, + "tokens_seen": 2121058304 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028883450351053163, + "loss": 1.137, + "theoretical_loss": 3.4421805015546454, + "tokens_seen": 2121123840 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028881845536609833, + "loss": 1.3412, + "theoretical_loss": 3.442171808957258, + "tokens_seen": 2121189376 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028880240722166503, + "loss": 1.2918, + "theoretical_loss": 3.442163116703628, + "tokens_seen": 2121254912 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028878635907723173, + "loss": 1.2093, + "theoretical_loss": 3.44215442479373, + "tokens_seen": 2121320448 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028877031093279843, + "loss": 1.2831, + "theoretical_loss": 3.442145733227541, + "tokens_seen": 2121385984 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028875426278836513, + "loss": 1.2542, + "theoretical_loss": 3.4421370420050357, + "tokens_seen": 2121451520 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028873821464393183, + "loss": 1.3141, + "theoretical_loss": 3.4421283511261906, + "tokens_seen": 2121517056 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028872216649949853, + "loss": 1.3045, + "theoretical_loss": 3.442119660590982, + "tokens_seen": 2121582592 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028870611835506523, + "loss": 1.2588, + "theoretical_loss": 3.4421109703993844, + "tokens_seen": 2121648128 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028869007021063193, + "loss": 1.2957, + "theoretical_loss": 3.4421022805513743, + "tokens_seen": 2121713664 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028867402206619863, + "loss": 1.3744, + "theoretical_loss": 3.4420935910469272, + "tokens_seen": 2121779200 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028865797392176534, + "loss": 1.2291, + "theoretical_loss": 3.4420849018860196, + "tokens_seen": 2121844736 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028864192577733204, + "loss": 1.3344, + "theoretical_loss": 3.442076213068627, + "tokens_seen": 2121910272 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028862587763289874, + "loss": 1.2332, + "theoretical_loss": 3.4420675245947248, + "tokens_seen": 2121975808 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028860982948846544, + "loss": 1.3274, + "theoretical_loss": 3.4420588364642892, + "tokens_seen": 2122041344 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028859378134403214, + "loss": 1.3012, + "theoretical_loss": 3.4420501486772963, + "tokens_seen": 2122106880 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028857773319959884, + "loss": 1.3143, + "theoretical_loss": 3.442041461233721, + "tokens_seen": 2122172416 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028856168505516554, + "loss": 1.2937, + "theoretical_loss": 3.4420327741335397, + "tokens_seen": 2122237952 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028854563691073224, + "loss": 1.3382, + "theoretical_loss": 3.4420240873767285, + "tokens_seen": 2122303488 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028852958876629894, + "loss": 1.285, + "theoretical_loss": 3.442015400963263, + "tokens_seen": 2122369024 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028851354062186564, + "loss": 1.17, + "theoretical_loss": 3.442006714893118, + "tokens_seen": 2122434560 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028849749247743234, + "loss": 1.3022, + "theoretical_loss": 3.441998029166271, + "tokens_seen": 2122500096 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028848144433299904, + "loss": 1.269, + "theoretical_loss": 3.4419893437826974, + "tokens_seen": 2122565632 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028846539618856574, + "loss": 1.1484, + "theoretical_loss": 3.4419806587423727, + "tokens_seen": 2122631168 + }, + { + "epoch": 1.02, + "learning_rate": 0.00028844934804413244, + "loss": 1.2344, + "theoretical_loss": 3.441971974045272, + "tokens_seen": 2122696704 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002884332998996991, + "loss": 1.2973, + "theoretical_loss": 3.4419632896913726, + "tokens_seen": 2122762240 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002884172517552658, + "loss": 1.2596, + "theoretical_loss": 3.441954605680649, + "tokens_seen": 2122827776 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002884012036108325, + "loss": 1.2756, + "theoretical_loss": 3.441945922013078, + "tokens_seen": 2122893312 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002883851554663992, + "loss": 1.2486, + "theoretical_loss": 3.4419372386886353, + "tokens_seen": 2122958848 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002883691073219659, + "loss": 1.2029, + "theoretical_loss": 3.441928555707297, + "tokens_seen": 2123024384 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002883530591775326, + "loss": 1.2747, + "theoretical_loss": 3.441919873069038, + "tokens_seen": 2123089920 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002883370110330993, + "loss": 1.2256, + "theoretical_loss": 3.4419111907738342, + "tokens_seen": 2123155456 + }, + { + "epoch": 1.02, + "learning_rate": 0.000288320962888666, + "loss": 1.251, + "theoretical_loss": 3.441902508821663, + "tokens_seen": 2123220992 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002883049147442327, + "loss": 1.2718, + "theoretical_loss": 3.441893827212499, + "tokens_seen": 2123286528 + }, + { + "epoch": 1.02, + "objective/train/docs_used": 785673, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1697146892547607, + "objective/train/theoretical_loss": 3.4418851459463173, + "objective/train/tokens_used": 2143812064, + "theoretical_loss": 3.4418851459463173, + "tokens_seen": 2123352064 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002882888665997994, + "loss": 1.1697, + "theoretical_loss": 3.4418851459463173, + "tokens_seen": 2123352064 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002882728184553661, + "loss": 1.3276, + "theoretical_loss": 3.4418764650230957, + "tokens_seen": 2123417600 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002882567703109328, + "loss": 1.3095, + "theoretical_loss": 3.4418677844428087, + "tokens_seen": 2123483136 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002882407221664995, + "loss": 1.3378, + "theoretical_loss": 3.4418591042054327, + "tokens_seen": 2123548672 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002882246740220662, + "loss": 1.2572, + "theoretical_loss": 3.441850424310944, + "tokens_seen": 2123614208 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002882086258776329, + "loss": 1.2891, + "theoretical_loss": 3.441841744759317, + "tokens_seen": 2123679744 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002881925777331996, + "loss": 1.0922, + "theoretical_loss": 3.4418330655505294, + "tokens_seen": 2123745280 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002881765295887663, + "loss": 1.1793, + "theoretical_loss": 3.4418243866845555, + "tokens_seen": 2123810816 + }, + { + "epoch": 1.02, + "learning_rate": 0.000288160481444333, + "loss": 1.2613, + "theoretical_loss": 3.4418157081613723, + "tokens_seen": 2123876352 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002881444332998997, + "loss": 1.2735, + "theoretical_loss": 3.4418070299809544, + "tokens_seen": 2123941888 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002881283851554664, + "loss": 1.2717, + "theoretical_loss": 3.4417983521432793, + "tokens_seen": 2124007424 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002881123370110331, + "loss": 1.2695, + "theoretical_loss": 3.441789674648322, + "tokens_seen": 2124072960 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002880962888665998, + "loss": 1.2779, + "theoretical_loss": 3.4417809974960587, + "tokens_seen": 2124138496 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002880802407221665, + "loss": 1.3353, + "theoretical_loss": 3.4417723206864648, + "tokens_seen": 2124204032 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002880641925777332, + "loss": 1.302, + "theoretical_loss": 3.4417636442195167, + "tokens_seen": 2124269568 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002880481444332999, + "loss": 1.3181, + "theoretical_loss": 3.44175496809519, + "tokens_seen": 2124335104 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002880320962888666, + "loss": 1.3468, + "theoretical_loss": 3.4417462923134607, + "tokens_seen": 2124400640 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002880160481444333, + "loss": 1.2777, + "theoretical_loss": 3.4417376168743044, + "tokens_seen": 2124466176 + }, + { + "epoch": 1.02, + "learning_rate": 0.000288, + "loss": 1.2607, + "theoretical_loss": 3.441728941777698, + "tokens_seen": 2124531712 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002879839518555667, + "loss": 1.2478, + "theoretical_loss": 3.441720267023616, + "tokens_seen": 2124597248 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002879679037111334, + "loss": 1.1862, + "theoretical_loss": 3.4417115926120356, + "tokens_seen": 2124662784 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002879518555667001, + "loss": 1.312, + "theoretical_loss": 3.4417029185429318, + "tokens_seen": 2124728320 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002879358074222668, + "loss": 1.324, + "theoretical_loss": 3.441694244816281, + "tokens_seen": 2124793856 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002879197592778335, + "loss": 1.2585, + "theoretical_loss": 3.4416855714320587, + "tokens_seen": 2124859392 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002879037111334002, + "loss": 1.2616, + "theoretical_loss": 3.4416768983902415, + "tokens_seen": 2124924928 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002878876629889669, + "loss": 1.26, + "theoretical_loss": 3.441668225690805, + "tokens_seen": 2124990464 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002878716148445336, + "loss": 1.1904, + "theoretical_loss": 3.441659553333725, + "tokens_seen": 2125056000 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002878555667001003, + "loss": 1.2546, + "theoretical_loss": 3.441650881318977, + "tokens_seen": 2125121536 + }, + { + "epoch": 1.02, + "learning_rate": 0.000287839518555667, + "loss": 1.2901, + "theoretical_loss": 3.4416422096465378, + "tokens_seen": 2125187072 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002878234704112337, + "loss": 1.2867, + "theoretical_loss": 3.441633538316383, + "tokens_seen": 2125252608 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002878074222668004, + "loss": 1.3371, + "theoretical_loss": 3.441624867328488, + "tokens_seen": 2125318144 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002877913741223671, + "loss": 1.2105, + "theoretical_loss": 3.44161619668283, + "tokens_seen": 2125383680 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002877753259779338, + "loss": 1.1714, + "theoretical_loss": 3.4416075263793835, + "tokens_seen": 2125449216 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002877592778335005, + "loss": 1.299, + "theoretical_loss": 3.4415988564181252, + "tokens_seen": 2125514752 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002877432296890672, + "loss": 1.3105, + "theoretical_loss": 3.441590186799031, + "tokens_seen": 2125580288 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002877271815446339, + "loss": 1.2422, + "theoretical_loss": 3.441581517522077, + "tokens_seen": 2125645824 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002877111334002006, + "loss": 1.2209, + "theoretical_loss": 3.4415728485872386, + "tokens_seen": 2125711360 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002876950852557673, + "loss": 1.2572, + "theoretical_loss": 3.4415641799944923, + "tokens_seen": 2125776896 + }, + { + "epoch": 1.02, + "learning_rate": 0.000287679037111334, + "loss": 1.2356, + "theoretical_loss": 3.4415555117438137, + "tokens_seen": 2125842432 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002876629889669007, + "loss": 1.3269, + "theoretical_loss": 3.4415468438351793, + "tokens_seen": 2125907968 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002876469408224674, + "loss": 1.3146, + "theoretical_loss": 3.4415381762685637, + "tokens_seen": 2125973504 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002876308926780341, + "loss": 1.2438, + "theoretical_loss": 3.441529509043945, + "tokens_seen": 2126039040 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002876148445336008, + "loss": 1.2072, + "theoretical_loss": 3.4415208421612973, + "tokens_seen": 2126104576 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002875987963891675, + "loss": 1.2776, + "theoretical_loss": 3.4415121756205975, + "tokens_seen": 2126170112 + }, + { + "epoch": 1.02, + "learning_rate": 0.0002875827482447342, + "loss": 1.2315, + "theoretical_loss": 3.441503509421821, + "tokens_seen": 2126235648 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002875667001003009, + "loss": 1.1382, + "theoretical_loss": 3.441494843564944, + "tokens_seen": 2126301184 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002875506519558676, + "loss": 1.2821, + "theoretical_loss": 3.441486178049943, + "tokens_seen": 2126366720 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002875346038114343, + "loss": 1.22, + "theoretical_loss": 3.4414775128767934, + "tokens_seen": 2126432256 + }, + { + "epoch": 1.03, + "learning_rate": 0.000287518555667001, + "loss": 1.255, + "theoretical_loss": 3.4414688480454716, + "tokens_seen": 2126497792 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002875025075225677, + "loss": 1.2321, + "theoretical_loss": 3.441460183555953, + "tokens_seen": 2126563328 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028748645937813443, + "loss": 1.2764, + "theoretical_loss": 3.441451519408214, + "tokens_seen": 2126628864 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028747041123370113, + "loss": 1.2421, + "theoretical_loss": 3.4414428556022303, + "tokens_seen": 2126694400 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028745436308926783, + "loss": 1.2035, + "theoretical_loss": 3.4414341921379785, + "tokens_seen": 2126759936 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028743831494483453, + "loss": 1.2507, + "theoretical_loss": 3.441425529015434, + "tokens_seen": 2126825472 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028742226680040123, + "loss": 1.2449, + "theoretical_loss": 3.4414168662345723, + "tokens_seen": 2126891008 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028740621865596793, + "loss": 1.3051, + "theoretical_loss": 3.441408203795371, + "tokens_seen": 2126956544 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028739017051153463, + "loss": 1.2661, + "theoretical_loss": 3.441399541697804, + "tokens_seen": 2127022080 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028737412236710133, + "loss": 1.268, + "theoretical_loss": 3.4413908799418493, + "tokens_seen": 2127087616 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028735807422266803, + "loss": 1.2034, + "theoretical_loss": 3.441382218527482, + "tokens_seen": 2127153152 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028734202607823473, + "loss": 1.2361, + "theoretical_loss": 3.441373557454678, + "tokens_seen": 2127218688 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028732597793380143, + "loss": 1.3594, + "theoretical_loss": 3.4413648967234134, + "tokens_seen": 2127284224 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028730992978936813, + "loss": 1.19, + "theoretical_loss": 3.441356236333664, + "tokens_seen": 2127349760 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028729388164493484, + "loss": 1.2993, + "theoretical_loss": 3.4413475762854064, + "tokens_seen": 2127415296 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028727783350050154, + "loss": 1.3207, + "theoretical_loss": 3.4413389165786166, + "tokens_seen": 2127480832 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028726178535606824, + "loss": 1.2737, + "theoretical_loss": 3.44133025721327, + "tokens_seen": 2127546368 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028724573721163494, + "loss": 1.2951, + "theoretical_loss": 3.441321598189343, + "tokens_seen": 2127611904 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028722968906720164, + "loss": 1.2577, + "theoretical_loss": 3.441312939506812, + "tokens_seen": 2127677440 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028721364092276834, + "loss": 1.176, + "theoretical_loss": 3.4413042811656513, + "tokens_seen": 2127742976 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028719759277833504, + "loss": 1.3552, + "theoretical_loss": 3.441295623165839, + "tokens_seen": 2127808512 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028718154463390174, + "loss": 1.263, + "theoretical_loss": 3.4412869655073504, + "tokens_seen": 2127874048 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028716549648946844, + "loss": 1.2788, + "theoretical_loss": 3.441278308190162, + "tokens_seen": 2127939584 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028714944834503514, + "loss": 1.2718, + "theoretical_loss": 3.4412696512142484, + "tokens_seen": 2128005120 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028713340020060184, + "loss": 1.2207, + "theoretical_loss": 3.441260994579587, + "tokens_seen": 2128070656 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028711735205616854, + "loss": 1.2887, + "theoretical_loss": 3.4412523382861533, + "tokens_seen": 2128136192 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028710130391173524, + "loss": 1.2482, + "theoretical_loss": 3.441243682333923, + "tokens_seen": 2128201728 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028708525576730194, + "loss": 1.3197, + "theoretical_loss": 3.4412350267228735, + "tokens_seen": 2128267264 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028706920762286864, + "loss": 1.2172, + "theoretical_loss": 3.441226371452979, + "tokens_seen": 2128332800 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028705315947843534, + "loss": 1.171, + "theoretical_loss": 3.441217716524217, + "tokens_seen": 2128398336 + }, + { + "epoch": 1.03, + "learning_rate": 0.000287037111334002, + "loss": 1.2301, + "theoretical_loss": 3.441209061936563, + "tokens_seen": 2128463872 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002870210631895687, + "loss": 1.151, + "theoretical_loss": 3.4412004076899927, + "tokens_seen": 2128529408 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002870050150451354, + "loss": 1.2974, + "theoretical_loss": 3.4411917537844827, + "tokens_seen": 2128594944 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002869889669007021, + "loss": 1.1813, + "theoretical_loss": 3.441183100220009, + "tokens_seen": 2128660480 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002869729187562688, + "loss": 1.2569, + "theoretical_loss": 3.4411744469965475, + "tokens_seen": 2128726016 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002869568706118355, + "loss": 1.2845, + "theoretical_loss": 3.4411657941140747, + "tokens_seen": 2128791552 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002869408224674022, + "loss": 1.2339, + "theoretical_loss": 3.4411571415725657, + "tokens_seen": 2128857088 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002869247743229689, + "loss": 1.3704, + "theoretical_loss": 3.4411484893719972, + "tokens_seen": 2128922624 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002869087261785356, + "loss": 1.2118, + "theoretical_loss": 3.4411398375123454, + "tokens_seen": 2128988160 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002868926780341023, + "loss": 1.3136, + "theoretical_loss": 3.4411311859935863, + "tokens_seen": 2129053696 + }, + { + "epoch": 1.03, + "learning_rate": 0.000286876629889669, + "loss": 1.1475, + "theoretical_loss": 3.4411225348156957, + "tokens_seen": 2129119232 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002868605817452357, + "loss": 1.1717, + "theoretical_loss": 3.44111388397865, + "tokens_seen": 2129184768 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002868445336008024, + "loss": 1.3213, + "theoretical_loss": 3.441105233482425, + "tokens_seen": 2129250304 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002868284854563691, + "loss": 1.2234, + "theoretical_loss": 3.4410965833269964, + "tokens_seen": 2129315840 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002868124373119358, + "loss": 1.2598, + "theoretical_loss": 3.4410879335123417, + "tokens_seen": 2129381376 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002867963891675025, + "loss": 1.3333, + "theoretical_loss": 3.4410792840384357, + "tokens_seen": 2129446912 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002867803410230692, + "loss": 1.2792, + "theoretical_loss": 3.441070634905255, + "tokens_seen": 2129512448 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002867642928786359, + "loss": 1.2653, + "theoretical_loss": 3.441061986112775, + "tokens_seen": 2129577984 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002867482447342026, + "loss": 1.3255, + "theoretical_loss": 3.441053337660973, + "tokens_seen": 2129643520 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002867321965897693, + "loss": 1.2849, + "theoretical_loss": 3.441044689549824, + "tokens_seen": 2129709056 + }, + { + "epoch": 1.03, + "learning_rate": 0.000286716148445336, + "loss": 1.2734, + "theoretical_loss": 3.441036041779305, + "tokens_seen": 2129774592 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002867001003009027, + "loss": 1.369, + "theoretical_loss": 3.4410273943493914, + "tokens_seen": 2129840128 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 787951, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1792333126068115, + "objective/train/theoretical_loss": 3.4410187472600597, + "objective/train/tokens_used": 2150365664, + "theoretical_loss": 3.4410187472600597, + "tokens_seen": 2129905664 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002866840521564694, + "loss": 1.1792, + "theoretical_loss": 3.4410187472600597, + "tokens_seen": 2129905664 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002866680040120361, + "loss": 1.2098, + "theoretical_loss": 3.441010100511286, + "tokens_seen": 2129971200 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002866519558676028, + "loss": 1.1604, + "theoretical_loss": 3.441001454103046, + "tokens_seen": 2130036736 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002866359077231695, + "loss": 1.2384, + "theoretical_loss": 3.4409928080353165, + "tokens_seen": 2130102272 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002866198595787362, + "loss": 1.2256, + "theoretical_loss": 3.4409841623080726, + "tokens_seen": 2130167808 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002866038114343029, + "loss": 1.3296, + "theoretical_loss": 3.4409755169212914, + "tokens_seen": 2130233344 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002865877632898696, + "loss": 1.322, + "theoretical_loss": 3.4409668718749487, + "tokens_seen": 2130298880 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002865717151454363, + "loss": 1.2623, + "theoretical_loss": 3.4409582271690202, + "tokens_seen": 2130364416 + }, + { + "epoch": 1.03, + "learning_rate": 0.000286555667001003, + "loss": 1.1604, + "theoretical_loss": 3.4409495828034826, + "tokens_seen": 2130429952 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002865396188565697, + "loss": 1.3087, + "theoretical_loss": 3.4409409387783123, + "tokens_seen": 2130495488 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002865235707121364, + "loss": 1.2623, + "theoretical_loss": 3.440932295093484, + "tokens_seen": 2130561024 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002865075225677031, + "loss": 1.2342, + "theoretical_loss": 3.4409236517489754, + "tokens_seen": 2130626560 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002864914744232698, + "loss": 1.2399, + "theoretical_loss": 3.4409150087447617, + "tokens_seen": 2130692096 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028647542627883657, + "loss": 1.2497, + "theoretical_loss": 3.44090636608082, + "tokens_seen": 2130757632 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028645937813440327, + "loss": 1.4154, + "theoretical_loss": 3.440897723757125, + "tokens_seen": 2130823168 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028644332998996997, + "loss": 1.295, + "theoretical_loss": 3.440889081773654, + "tokens_seen": 2130888704 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028642728184553667, + "loss": 1.2996, + "theoretical_loss": 3.4408804401303827, + "tokens_seen": 2130954240 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028641123370110337, + "loss": 1.3005, + "theoretical_loss": 3.4408717988272874, + "tokens_seen": 2131019776 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028639518555667, + "loss": 1.3134, + "theoretical_loss": 3.440863157864344, + "tokens_seen": 2131085312 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002863791374122367, + "loss": 1.253, + "theoretical_loss": 3.440854517241529, + "tokens_seen": 2131150848 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002863630892678034, + "loss": 1.3697, + "theoretical_loss": 3.440845876958818, + "tokens_seen": 2131216384 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002863470411233701, + "loss": 1.229, + "theoretical_loss": 3.440837237016188, + "tokens_seen": 2131281920 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002863309929789368, + "loss": 1.1556, + "theoretical_loss": 3.4408285974136144, + "tokens_seen": 2131347456 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002863149448345035, + "loss": 1.2194, + "theoretical_loss": 3.440819958151074, + "tokens_seen": 2131412992 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002862988966900702, + "loss": 1.3028, + "theoretical_loss": 3.4408113192285423, + "tokens_seen": 2131478528 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002862828485456369, + "loss": 1.2433, + "theoretical_loss": 3.4408026806459957, + "tokens_seen": 2131544064 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002862668004012036, + "loss": 1.2649, + "theoretical_loss": 3.4407940424034105, + "tokens_seen": 2131609600 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002862507522567703, + "loss": 1.2741, + "theoretical_loss": 3.4407854045007626, + "tokens_seen": 2131675136 + }, + { + "epoch": 1.03, + "learning_rate": 0.000286234704112337, + "loss": 1.2584, + "theoretical_loss": 3.4407767669380287, + "tokens_seen": 2131740672 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002862186559679037, + "loss": 1.2811, + "theoretical_loss": 3.4407681297151846, + "tokens_seen": 2131806208 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002862026078234704, + "loss": 1.2903, + "theoretical_loss": 3.4407594928322065, + "tokens_seen": 2131871744 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002861865596790371, + "loss": 1.2285, + "theoretical_loss": 3.4407508562890707, + "tokens_seen": 2131937280 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002861705115346038, + "loss": 1.148, + "theoretical_loss": 3.4407422200857534, + "tokens_seen": 2132002816 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002861544633901705, + "loss": 1.2426, + "theoretical_loss": 3.4407335842222304, + "tokens_seen": 2132068352 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002861384152457372, + "loss": 1.3023, + "theoretical_loss": 3.4407249486984783, + "tokens_seen": 2132133888 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028612236710130393, + "loss": 1.1782, + "theoretical_loss": 3.440716313514473, + "tokens_seen": 2132199424 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028610631895687063, + "loss": 1.1975, + "theoretical_loss": 3.440707678670191, + "tokens_seen": 2132264960 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028609027081243733, + "loss": 1.2826, + "theoretical_loss": 3.440699044165608, + "tokens_seen": 2132330496 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028607422266800403, + "loss": 1.237, + "theoretical_loss": 3.440690410000701, + "tokens_seen": 2132396032 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028605817452357073, + "loss": 1.1768, + "theoretical_loss": 3.4406817761754453, + "tokens_seen": 2132461568 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028604212637913743, + "loss": 1.2942, + "theoretical_loss": 3.4406731426898176, + "tokens_seen": 2132527104 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028602607823470413, + "loss": 1.2839, + "theoretical_loss": 3.4406645095437947, + "tokens_seen": 2132592640 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028601003009027083, + "loss": 1.3414, + "theoretical_loss": 3.4406558767373516, + "tokens_seen": 2132658176 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028599398194583753, + "loss": 1.2021, + "theoretical_loss": 3.4406472442704645, + "tokens_seen": 2132723712 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028597793380140423, + "loss": 1.3231, + "theoretical_loss": 3.440638612143111, + "tokens_seen": 2132789248 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028596188565697093, + "loss": 1.1887, + "theoretical_loss": 3.4406299803552662, + "tokens_seen": 2132854784 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028594583751253763, + "loss": 1.3289, + "theoretical_loss": 3.4406213489069066, + "tokens_seen": 2132920320 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028592978936810433, + "loss": 1.2566, + "theoretical_loss": 3.4406127177980084, + "tokens_seen": 2132985856 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028591374122367104, + "loss": 1.277, + "theoretical_loss": 3.440604087028548, + "tokens_seen": 2133051392 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028589769307923774, + "loss": 1.3596, + "theoretical_loss": 3.4405954565985013, + "tokens_seen": 2133116928 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028588164493480444, + "loss": 1.2187, + "theoretical_loss": 3.4405868265078445, + "tokens_seen": 2133182464 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028586559679037114, + "loss": 1.2415, + "theoretical_loss": 3.4405781967565545, + "tokens_seen": 2133248000 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028584954864593784, + "loss": 1.2142, + "theoretical_loss": 3.4405695673446064, + "tokens_seen": 2133313536 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028583350050150454, + "loss": 1.3057, + "theoretical_loss": 3.440560938271977, + "tokens_seen": 2133379072 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028581745235707124, + "loss": 1.2638, + "theoretical_loss": 3.4405523095386434, + "tokens_seen": 2133444608 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028580140421263794, + "loss": 1.2346, + "theoretical_loss": 3.4405436811445806, + "tokens_seen": 2133510144 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028578535606820464, + "loss": 1.2987, + "theoretical_loss": 3.440535053089765, + "tokens_seen": 2133575680 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028576930792377134, + "loss": 1.2382, + "theoretical_loss": 3.4405264253741734, + "tokens_seen": 2133641216 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028575325977933804, + "loss": 1.3102, + "theoretical_loss": 3.440517797997782, + "tokens_seen": 2133706752 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028573721163490474, + "loss": 1.3332, + "theoretical_loss": 3.440509170960566, + "tokens_seen": 2133772288 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028572116349047144, + "loss": 1.3365, + "theoretical_loss": 3.4405005442625036, + "tokens_seen": 2133837824 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028570511534603814, + "loss": 1.2436, + "theoretical_loss": 3.440491917903569, + "tokens_seen": 2133903360 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028568906720160484, + "loss": 1.2063, + "theoretical_loss": 3.44048329188374, + "tokens_seen": 2133968896 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028567301905717155, + "loss": 1.2373, + "theoretical_loss": 3.4404746662029915, + "tokens_seen": 2134034432 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028565697091273825, + "loss": 1.3501, + "theoretical_loss": 3.440466040861301, + "tokens_seen": 2134099968 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028564092276830495, + "loss": 1.1879, + "theoretical_loss": 3.440457415858644, + "tokens_seen": 2134165504 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002856248746238716, + "loss": 1.2986, + "theoretical_loss": 3.440448791194997, + "tokens_seen": 2134231040 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002856088264794383, + "loss": 1.2796, + "theoretical_loss": 3.4404401668703364, + "tokens_seen": 2134296576 + }, + { + "epoch": 1.03, + "learning_rate": 0.000285592778335005, + "loss": 1.2124, + "theoretical_loss": 3.440431542884638, + "tokens_seen": 2134362112 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002855767301905717, + "loss": 1.2048, + "theoretical_loss": 3.440422919237879, + "tokens_seen": 2134427648 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002855606820461384, + "loss": 1.2421, + "theoretical_loss": 3.440414295930035, + "tokens_seen": 2134493184 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002855446339017051, + "loss": 1.2351, + "theoretical_loss": 3.4404056729610817, + "tokens_seen": 2134558720 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002855285857572718, + "loss": 1.2294, + "theoretical_loss": 3.440397050330996, + "tokens_seen": 2134624256 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002855125376128385, + "loss": 1.3022, + "theoretical_loss": 3.440388428039755, + "tokens_seen": 2134689792 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002854964894684052, + "loss": 1.2889, + "theoretical_loss": 3.440379806087334, + "tokens_seen": 2134755328 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002854804413239719, + "loss": 1.3133, + "theoretical_loss": 3.440371184473709, + "tokens_seen": 2134820864 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002854643931795386, + "loss": 1.2295, + "theoretical_loss": 3.4403625631988572, + "tokens_seen": 2134886400 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002854483450351053, + "loss": 1.2423, + "theoretical_loss": 3.440353942262754, + "tokens_seen": 2134951936 + }, + { + "epoch": 1.03, + "learning_rate": 0.000285432296890672, + "loss": 1.3674, + "theoretical_loss": 3.4403453216653768, + "tokens_seen": 2135017472 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002854162487462387, + "loss": 1.1961, + "theoretical_loss": 3.440336701406701, + "tokens_seen": 2135083008 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002854002006018054, + "loss": 1.2451, + "theoretical_loss": 3.4403280814867028, + "tokens_seen": 2135148544 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002853841524573721, + "loss": 1.3711, + "theoretical_loss": 3.4403194619053594, + "tokens_seen": 2135214080 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002853681043129388, + "loss": 1.1968, + "theoretical_loss": 3.440310842662646, + "tokens_seen": 2135279616 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002853520561685055, + "loss": 1.2988, + "theoretical_loss": 3.4403022237585397, + "tokens_seen": 2135345152 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002853360080240722, + "loss": 1.1587, + "theoretical_loss": 3.440293605193016, + "tokens_seen": 2135410688 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002853199598796389, + "loss": 1.2715, + "theoretical_loss": 3.440284986966053, + "tokens_seen": 2135476224 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002853039117352056, + "loss": 1.2113, + "theoretical_loss": 3.440276369077625, + "tokens_seen": 2135541760 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002852878635907723, + "loss": 1.1418, + "theoretical_loss": 3.440267751527709, + "tokens_seen": 2135607296 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028527181544633906, + "loss": 1.2431, + "theoretical_loss": 3.4402591343162814, + "tokens_seen": 2135672832 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028525576730190576, + "loss": 1.2827, + "theoretical_loss": 3.4402505174433182, + "tokens_seen": 2135738368 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028523971915747246, + "loss": 1.299, + "theoretical_loss": 3.440241900908797, + "tokens_seen": 2135803904 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028522367101303916, + "loss": 1.3204, + "theoretical_loss": 3.4402332847126926, + "tokens_seen": 2135869440 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028520762286860586, + "loss": 1.2143, + "theoretical_loss": 3.440224668854982, + "tokens_seen": 2135934976 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028519157472417256, + "loss": 1.2633, + "theoretical_loss": 3.4402160533356407, + "tokens_seen": 2136000512 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028517552657973926, + "loss": 1.2485, + "theoretical_loss": 3.4402074381546464, + "tokens_seen": 2136066048 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028515947843530597, + "loss": 1.2869, + "theoretical_loss": 3.4401988233119747, + "tokens_seen": 2136131584 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028514343029087267, + "loss": 1.1333, + "theoretical_loss": 3.4401902088076017, + "tokens_seen": 2136197120 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028512738214643937, + "loss": 1.2179, + "theoretical_loss": 3.440181594641505, + "tokens_seen": 2136262656 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028511133400200607, + "loss": 1.1608, + "theoretical_loss": 3.440172980813659, + "tokens_seen": 2136328192 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028509528585757277, + "loss": 1.3305, + "theoretical_loss": 3.440164367324041, + "tokens_seen": 2136393728 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 790226, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.18367600440979, + "objective/train/theoretical_loss": 3.440155754172628, + "objective/train/tokens_used": 2156919264, + "theoretical_loss": 3.440155754172628, + "tokens_seen": 2136459264 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028507923771313947, + "loss": 1.1837, + "theoretical_loss": 3.440155754172628, + "tokens_seen": 2136459264 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028506318956870617, + "loss": 1.2764, + "theoretical_loss": 3.4401471413593954, + "tokens_seen": 2136524800 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028504714142427287, + "loss": 1.2353, + "theoretical_loss": 3.4401385288843196, + "tokens_seen": 2136590336 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028503109327983957, + "loss": 1.2556, + "theoretical_loss": 3.440129916747378, + "tokens_seen": 2136655872 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028501504513540627, + "loss": 1.2328, + "theoretical_loss": 3.4401213049485455, + "tokens_seen": 2136721408 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002849989969909729, + "loss": 1.3936, + "theoretical_loss": 3.4401126934877992, + "tokens_seen": 2136786944 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002849829488465396, + "loss": 1.2888, + "theoretical_loss": 3.4401040823651154, + "tokens_seen": 2136852480 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002849669007021063, + "loss": 1.2776, + "theoretical_loss": 3.4400954715804706, + "tokens_seen": 2136918016 + }, + { + "epoch": 1.03, + "learning_rate": 0.000284950852557673, + "loss": 1.2605, + "theoretical_loss": 3.440086861133841, + "tokens_seen": 2136983552 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002849348044132397, + "loss": 1.2902, + "theoretical_loss": 3.440078251025203, + "tokens_seen": 2137049088 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002849187562688064, + "loss": 1.2535, + "theoretical_loss": 3.440069641254533, + "tokens_seen": 2137114624 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002849027081243731, + "loss": 1.2664, + "theoretical_loss": 3.440061031821807, + "tokens_seen": 2137180160 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002848866599799398, + "loss": 1.2526, + "theoretical_loss": 3.4400524227270015, + "tokens_seen": 2137245696 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002848706118355065, + "loss": 1.3039, + "theoretical_loss": 3.4400438139700937, + "tokens_seen": 2137311232 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002848545636910732, + "loss": 1.2723, + "theoretical_loss": 3.4400352055510592, + "tokens_seen": 2137376768 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002848385155466399, + "loss": 1.2328, + "theoretical_loss": 3.440026597469874, + "tokens_seen": 2137442304 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002848224674022066, + "loss": 1.2627, + "theoretical_loss": 3.440017989726516, + "tokens_seen": 2137507840 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002848064192577733, + "loss": 1.2378, + "theoretical_loss": 3.4400093823209597, + "tokens_seen": 2137573376 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028479037111334, + "loss": 1.2918, + "theoretical_loss": 3.440000775253183, + "tokens_seen": 2137638912 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002847743229689067, + "loss": 1.3541, + "theoretical_loss": 3.439992168523161, + "tokens_seen": 2137704448 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002847582748244734, + "loss": 1.2894, + "theoretical_loss": 3.439983562130871, + "tokens_seen": 2137769984 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028474222668004013, + "loss": 1.362, + "theoretical_loss": 3.4399749560762896, + "tokens_seen": 2137835520 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028472617853560683, + "loss": 1.2873, + "theoretical_loss": 3.4399663503593922, + "tokens_seen": 2137901056 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028471013039117353, + "loss": 1.2977, + "theoretical_loss": 3.4399577449801555, + "tokens_seen": 2137966592 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028469408224674023, + "loss": 1.228, + "theoretical_loss": 3.439949139938557, + "tokens_seen": 2138032128 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028467803410230693, + "loss": 1.3216, + "theoretical_loss": 3.4399405352345718, + "tokens_seen": 2138097664 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028466198595787363, + "loss": 1.2787, + "theoretical_loss": 3.439931930868177, + "tokens_seen": 2138163200 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028464593781344033, + "loss": 1.3232, + "theoretical_loss": 3.4399233268393483, + "tokens_seen": 2138228736 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028462988966900703, + "loss": 1.2461, + "theoretical_loss": 3.4399147231480627, + "tokens_seen": 2138294272 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028461384152457373, + "loss": 1.2022, + "theoretical_loss": 3.439906119794297, + "tokens_seen": 2138359808 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028459779338014043, + "loss": 1.2035, + "theoretical_loss": 3.4398975167780264, + "tokens_seen": 2138425344 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028458174523570713, + "loss": 1.2667, + "theoretical_loss": 3.439888914099228, + "tokens_seen": 2138490880 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028456569709127383, + "loss": 1.2029, + "theoretical_loss": 3.4398803117578787, + "tokens_seen": 2138556416 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028454964894684054, + "loss": 1.2451, + "theoretical_loss": 3.4398717097539544, + "tokens_seen": 2138621952 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028453360080240724, + "loss": 1.2056, + "theoretical_loss": 3.4398631080874313, + "tokens_seen": 2138687488 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028451755265797394, + "loss": 1.2552, + "theoretical_loss": 3.439854506758287, + "tokens_seen": 2138753024 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028450150451354064, + "loss": 1.3338, + "theoretical_loss": 3.439845905766496, + "tokens_seen": 2138818560 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028448545636910734, + "loss": 1.2222, + "theoretical_loss": 3.4398373051120363, + "tokens_seen": 2138884096 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028446940822467404, + "loss": 1.2761, + "theoretical_loss": 3.439828704794884, + "tokens_seen": 2138949632 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028445336008024074, + "loss": 1.2159, + "theoretical_loss": 3.4398201048150145, + "tokens_seen": 2139015168 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028443731193580744, + "loss": 1.2447, + "theoretical_loss": 3.439811505172406, + "tokens_seen": 2139080704 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028442126379137414, + "loss": 1.1945, + "theoretical_loss": 3.4398029058670336, + "tokens_seen": 2139146240 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028440521564694084, + "loss": 1.2768, + "theoretical_loss": 3.4397943068988743, + "tokens_seen": 2139211776 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028438916750250754, + "loss": 1.2051, + "theoretical_loss": 3.4397857082679044, + "tokens_seen": 2139277312 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028437311935807424, + "loss": 1.2239, + "theoretical_loss": 3.4397771099741004, + "tokens_seen": 2139342848 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028435707121364094, + "loss": 1.239, + "theoretical_loss": 3.439768512017439, + "tokens_seen": 2139408384 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028434102306920764, + "loss": 1.2376, + "theoretical_loss": 3.4397599143978956, + "tokens_seen": 2139473920 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028432497492477434, + "loss": 1.2382, + "theoretical_loss": 3.439751317115448, + "tokens_seen": 2139539456 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028430892678034104, + "loss": 1.3603, + "theoretical_loss": 3.439742720170072, + "tokens_seen": 2139604992 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028429287863590775, + "loss": 1.2929, + "theoretical_loss": 3.4397341235617445, + "tokens_seen": 2139670528 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028427683049147445, + "loss": 1.2474, + "theoretical_loss": 3.439725527290441, + "tokens_seen": 2139736064 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028426078234704115, + "loss": 1.2743, + "theoretical_loss": 3.4397169313561387, + "tokens_seen": 2139801600 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028424473420260785, + "loss": 1.3337, + "theoretical_loss": 3.439708335758814, + "tokens_seen": 2139867136 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002842286860581745, + "loss": 1.2797, + "theoretical_loss": 3.439699740498444, + "tokens_seen": 2139932672 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002842126379137412, + "loss": 1.2376, + "theoretical_loss": 3.4396911455750034, + "tokens_seen": 2139998208 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002841965897693079, + "loss": 1.3348, + "theoretical_loss": 3.4396825509884703, + "tokens_seen": 2140063744 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002841805416248746, + "loss": 1.2778, + "theoretical_loss": 3.4396739567388206, + "tokens_seen": 2140129280 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002841644934804413, + "loss": 1.2563, + "theoretical_loss": 3.439665362826031, + "tokens_seen": 2140194816 + }, + { + "epoch": 1.03, + "learning_rate": 0.000284148445336008, + "loss": 1.0793, + "theoretical_loss": 3.4396567692500777, + "tokens_seen": 2140260352 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002841323971915747, + "loss": 1.3263, + "theoretical_loss": 3.4396481760109374, + "tokens_seen": 2140325888 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002841163490471414, + "loss": 1.2452, + "theoretical_loss": 3.439639583108586, + "tokens_seen": 2140391424 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002841003009027081, + "loss": 1.2645, + "theoretical_loss": 3.4396309905430007, + "tokens_seen": 2140456960 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002840842527582748, + "loss": 1.1838, + "theoretical_loss": 3.439622398314158, + "tokens_seen": 2140522496 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028406820461384155, + "loss": 1.1849, + "theoretical_loss": 3.439613806422034, + "tokens_seen": 2140588032 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028405215646940825, + "loss": 1.248, + "theoretical_loss": 3.4396052148666056, + "tokens_seen": 2140653568 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028403610832497496, + "loss": 1.2506, + "theoretical_loss": 3.439596623647849, + "tokens_seen": 2140719104 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028402006018054166, + "loss": 1.2726, + "theoretical_loss": 3.43958803276574, + "tokens_seen": 2140784640 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028400401203610836, + "loss": 1.2825, + "theoretical_loss": 3.4395794422202566, + "tokens_seen": 2140850176 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028398796389167506, + "loss": 1.2415, + "theoretical_loss": 3.439570852011374, + "tokens_seen": 2140915712 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028397191574724176, + "loss": 1.2355, + "theoretical_loss": 3.43956226213907, + "tokens_seen": 2140981248 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028395586760280846, + "loss": 1.2427, + "theoretical_loss": 3.43955367260332, + "tokens_seen": 2141046784 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028393981945837516, + "loss": 1.2935, + "theoretical_loss": 3.4395450834041004, + "tokens_seen": 2141112320 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028392377131394186, + "loss": 1.2531, + "theoretical_loss": 3.4395364945413887, + "tokens_seen": 2141177856 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028390772316950856, + "loss": 1.2464, + "theoretical_loss": 3.4395279060151607, + "tokens_seen": 2141243392 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028389167502507526, + "loss": 1.2097, + "theoretical_loss": 3.4395193178253933, + "tokens_seen": 2141308928 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028387562688064196, + "loss": 1.3593, + "theoretical_loss": 3.439510729972063, + "tokens_seen": 2141374464 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028385957873620866, + "loss": 1.1216, + "theoretical_loss": 3.4395021424551464, + "tokens_seen": 2141440000 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028384353059177536, + "loss": 1.193, + "theoretical_loss": 3.439493555274619, + "tokens_seen": 2141505536 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028382748244734206, + "loss": 1.2856, + "theoretical_loss": 3.4394849684304587, + "tokens_seen": 2141571072 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028381143430290876, + "loss": 1.2898, + "theoretical_loss": 3.4394763819226415, + "tokens_seen": 2141636608 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028379538615847547, + "loss": 1.274, + "theoretical_loss": 3.4394677957511437, + "tokens_seen": 2141702144 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028377933801404217, + "loss": 1.2615, + "theoretical_loss": 3.439459209915942, + "tokens_seen": 2141767680 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028376328986960887, + "loss": 1.2738, + "theoretical_loss": 3.439450624417013, + "tokens_seen": 2141833216 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028374724172517557, + "loss": 1.3524, + "theoretical_loss": 3.4394420392543337, + "tokens_seen": 2141898752 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028373119358074227, + "loss": 1.2962, + "theoretical_loss": 3.4394334544278795, + "tokens_seen": 2141964288 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028371514543630897, + "loss": 1.3578, + "theoretical_loss": 3.439424869937628, + "tokens_seen": 2142029824 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028369909729187567, + "loss": 1.2598, + "theoretical_loss": 3.439416285783555, + "tokens_seen": 2142095360 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028368304914744237, + "loss": 1.3946, + "theoretical_loss": 3.4394077019656377, + "tokens_seen": 2142160896 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028366700100300907, + "loss": 1.2468, + "theoretical_loss": 3.4393991184838524, + "tokens_seen": 2142226432 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028365095285857577, + "loss": 1.3335, + "theoretical_loss": 3.439390535338175, + "tokens_seen": 2142291968 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028363490471414247, + "loss": 1.2442, + "theoretical_loss": 3.4393819525285836, + "tokens_seen": 2142357504 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028361885656970917, + "loss": 1.2794, + "theoretical_loss": 3.439373370055053, + "tokens_seen": 2142423040 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002836028084252758, + "loss": 1.2376, + "theoretical_loss": 3.4393647879175613, + "tokens_seen": 2142488576 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002835867602808425, + "loss": 1.1318, + "theoretical_loss": 3.4393562061160843, + "tokens_seen": 2142554112 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002835707121364092, + "loss": 1.274, + "theoretical_loss": 3.4393476246505985, + "tokens_seen": 2142619648 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002835546639919759, + "loss": 1.2793, + "theoretical_loss": 3.4393390435210804, + "tokens_seen": 2142685184 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002835386158475426, + "loss": 1.2434, + "theoretical_loss": 3.4393304627275065, + "tokens_seen": 2142750720 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002835225677031093, + "loss": 1.3113, + "theoretical_loss": 3.439321882269854, + "tokens_seen": 2142816256 + }, + { + "epoch": 1.03, + "learning_rate": 0.000283506519558676, + "loss": 1.2791, + "theoretical_loss": 3.439313302148099, + "tokens_seen": 2142881792 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002834904714142427, + "loss": 1.2177, + "theoretical_loss": 3.4393047223622184, + "tokens_seen": 2142947328 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 792514, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2533661127090454, + "objective/train/theoretical_loss": 3.4392961429121884, + "objective/train/tokens_used": 2163472864, + "theoretical_loss": 3.4392961429121884, + "tokens_seen": 2143012864 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002834744232698094, + "loss": 1.2534, + "theoretical_loss": 3.4392961429121884, + "tokens_seen": 2143012864 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002834583751253761, + "loss": 1.3038, + "theoretical_loss": 3.439287563797986, + "tokens_seen": 2143078400 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002834423269809428, + "loss": 1.2434, + "theoretical_loss": 3.4392789850195875, + "tokens_seen": 2143143936 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002834262788365095, + "loss": 1.3376, + "theoretical_loss": 3.439270406576969, + "tokens_seen": 2143209472 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002834102306920762, + "loss": 1.3508, + "theoretical_loss": 3.439261828470108, + "tokens_seen": 2143275008 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002833941825476429, + "loss": 1.3389, + "theoretical_loss": 3.439253250698981, + "tokens_seen": 2143340544 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028337813440320963, + "loss": 1.2522, + "theoretical_loss": 3.4392446732635635, + "tokens_seen": 2143406080 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028336208625877633, + "loss": 1.326, + "theoretical_loss": 3.4392360961638335, + "tokens_seen": 2143471616 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028334603811434303, + "loss": 1.2123, + "theoretical_loss": 3.4392275193997675, + "tokens_seen": 2143537152 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028332998996990973, + "loss": 1.1836, + "theoretical_loss": 3.4392189429713405, + "tokens_seen": 2143602688 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028331394182547643, + "loss": 1.3077, + "theoretical_loss": 3.439210366878531, + "tokens_seen": 2143668224 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028329789368104313, + "loss": 1.2807, + "theoretical_loss": 3.439201791121315, + "tokens_seen": 2143733760 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028328184553660983, + "loss": 1.3644, + "theoretical_loss": 3.4391932156996683, + "tokens_seen": 2143799296 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028326579739217653, + "loss": 1.2691, + "theoretical_loss": 3.4391846406135684, + "tokens_seen": 2143864832 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028324974924774323, + "loss": 1.202, + "theoretical_loss": 3.4391760658629913, + "tokens_seen": 2143930368 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028323370110330993, + "loss": 1.2859, + "theoretical_loss": 3.4391674914479142, + "tokens_seen": 2143995904 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028321765295887663, + "loss": 1.2475, + "theoretical_loss": 3.4391589173683137, + "tokens_seen": 2144061440 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028320160481444333, + "loss": 1.3279, + "theoretical_loss": 3.439150343624166, + "tokens_seen": 2144126976 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028318555667001004, + "loss": 1.2359, + "theoretical_loss": 3.439141770215448, + "tokens_seen": 2144192512 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028316950852557674, + "loss": 1.2133, + "theoretical_loss": 3.4391331971421364, + "tokens_seen": 2144258048 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028315346038114344, + "loss": 1.2535, + "theoretical_loss": 3.439124624404207, + "tokens_seen": 2144323584 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028313741223671014, + "loss": 1.2689, + "theoretical_loss": 3.439116052001638, + "tokens_seen": 2144389120 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028312136409227684, + "loss": 1.2242, + "theoretical_loss": 3.439107479934405, + "tokens_seen": 2144454656 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028310531594784354, + "loss": 1.3546, + "theoretical_loss": 3.4390989082024843, + "tokens_seen": 2144520192 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028308926780341024, + "loss": 1.2783, + "theoretical_loss": 3.439090336805853, + "tokens_seen": 2144585728 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028307321965897694, + "loss": 1.4174, + "theoretical_loss": 3.439081765744488, + "tokens_seen": 2144651264 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028305717151454364, + "loss": 1.2734, + "theoretical_loss": 3.439073195018366, + "tokens_seen": 2144716800 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028304112337011034, + "loss": 1.2177, + "theoretical_loss": 3.439064624627463, + "tokens_seen": 2144782336 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028302507522567704, + "loss": 1.3012, + "theoretical_loss": 3.439056054571756, + "tokens_seen": 2144847872 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028300902708124374, + "loss": 1.2811, + "theoretical_loss": 3.439047484851222, + "tokens_seen": 2144913408 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028299297893681044, + "loss": 1.21, + "theoretical_loss": 3.439038915465837, + "tokens_seen": 2144978944 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028297693079237714, + "loss": 1.3, + "theoretical_loss": 3.439030346415578, + "tokens_seen": 2145044480 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028296088264794384, + "loss": 1.2625, + "theoretical_loss": 3.4390217777004213, + "tokens_seen": 2145110016 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028294483450351054, + "loss": 1.1899, + "theoretical_loss": 3.4390132093203443, + "tokens_seen": 2145175552 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028292878635907725, + "loss": 1.3404, + "theoretical_loss": 3.439004641275323, + "tokens_seen": 2145241088 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028291273821464395, + "loss": 1.207, + "theoretical_loss": 3.438996073565334, + "tokens_seen": 2145306624 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028289669007021065, + "loss": 1.2295, + "theoretical_loss": 3.4389875061903545, + "tokens_seen": 2145372160 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028288064192577735, + "loss": 1.3415, + "theoretical_loss": 3.438978939150361, + "tokens_seen": 2145437696 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028286459378134405, + "loss": 1.2746, + "theoretical_loss": 3.43897037244533, + "tokens_seen": 2145503232 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028284854563691075, + "loss": 1.2869, + "theoretical_loss": 3.4389618060752385, + "tokens_seen": 2145568768 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028283249749247745, + "loss": 1.2722, + "theoretical_loss": 3.4389532400400626, + "tokens_seen": 2145634304 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028281644934804415, + "loss": 1.3137, + "theoretical_loss": 3.43894467433978, + "tokens_seen": 2145699840 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028280040120361085, + "loss": 1.2234, + "theoretical_loss": 3.4389361089743655, + "tokens_seen": 2145765376 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028278435305917755, + "loss": 1.2563, + "theoretical_loss": 3.438927543943798, + "tokens_seen": 2145830912 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028276830491474425, + "loss": 1.1827, + "theoretical_loss": 3.4389189792480526, + "tokens_seen": 2145896448 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028275225677031095, + "loss": 1.2375, + "theoretical_loss": 3.438910414887107, + "tokens_seen": 2145961984 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028273620862587765, + "loss": 1.3172, + "theoretical_loss": 3.438901850860937, + "tokens_seen": 2146027520 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028272016048144435, + "loss": 1.2811, + "theoretical_loss": 3.4388932871695195, + "tokens_seen": 2146093056 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028270411233701105, + "loss": 1.2656, + "theoretical_loss": 3.4388847238128317, + "tokens_seen": 2146158592 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028268806419257775, + "loss": 1.3792, + "theoretical_loss": 3.43887616079085, + "tokens_seen": 2146224128 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028267201604814446, + "loss": 1.2495, + "theoretical_loss": 3.438867598103551, + "tokens_seen": 2146289664 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028265596790371116, + "loss": 1.1777, + "theoretical_loss": 3.4388590357509115, + "tokens_seen": 2146355200 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028263991975927786, + "loss": 1.167, + "theoretical_loss": 3.4388504737329084, + "tokens_seen": 2146420736 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028262387161484456, + "loss": 1.2984, + "theoretical_loss": 3.4388419120495186, + "tokens_seen": 2146486272 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028260782347041126, + "loss": 1.2753, + "theoretical_loss": 3.438833350700718, + "tokens_seen": 2146551808 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028259177532597796, + "loss": 1.1856, + "theoretical_loss": 3.4388247896864836, + "tokens_seen": 2146617344 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028257572718154466, + "loss": 1.2712, + "theoretical_loss": 3.4388162290067923, + "tokens_seen": 2146682880 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028255967903711136, + "loss": 1.2586, + "theoretical_loss": 3.438807668661621, + "tokens_seen": 2146748416 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028254363089267806, + "loss": 1.214, + "theoretical_loss": 3.4387991086509455, + "tokens_seen": 2146813952 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028252758274824476, + "loss": 1.2644, + "theoretical_loss": 3.438790548974744, + "tokens_seen": 2146879488 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028251153460381146, + "loss": 1.2947, + "theoretical_loss": 3.438781989632992, + "tokens_seen": 2146945024 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028249548645937816, + "loss": 1.2755, + "theoretical_loss": 3.4387734306256665, + "tokens_seen": 2147010560 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028247943831494486, + "loss": 1.4147, + "theoretical_loss": 3.4387648719527446, + "tokens_seen": 2147076096 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028246339017051156, + "loss": 1.2925, + "theoretical_loss": 3.4387563136142028, + "tokens_seen": 2147141632 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028244734202607826, + "loss": 1.2631, + "theoretical_loss": 3.438747755610018, + "tokens_seen": 2147207168 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028243129388164496, + "loss": 1.1795, + "theoretical_loss": 3.4387391979401665, + "tokens_seen": 2147272704 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028241524573721167, + "loss": 1.2583, + "theoretical_loss": 3.438730640604625, + "tokens_seen": 2147338240 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028239919759277837, + "loss": 1.2753, + "theoretical_loss": 3.4387220836033707, + "tokens_seen": 2147403776 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028238314944834507, + "loss": 1.2491, + "theoretical_loss": 3.4387135269363807, + "tokens_seen": 2147469312 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028236710130391177, + "loss": 1.1169, + "theoretical_loss": 3.438704970603631, + "tokens_seen": 2147534848 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028235105315947847, + "loss": 1.2617, + "theoretical_loss": 3.438696414605098, + "tokens_seen": 2147600384 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028233500501504517, + "loss": 1.3089, + "theoretical_loss": 3.4386878589407597, + "tokens_seen": 2147665920 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028231895687061187, + "loss": 1.2861, + "theoretical_loss": 3.4386793036105914, + "tokens_seen": 2147731456 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028230290872617857, + "loss": 1.3224, + "theoretical_loss": 3.4386707486145713, + "tokens_seen": 2147796992 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028228686058174527, + "loss": 1.2214, + "theoretical_loss": 3.438662193952675, + "tokens_seen": 2147862528 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028227081243731197, + "loss": 1.2369, + "theoretical_loss": 3.4386536396248797, + "tokens_seen": 2147928064 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028225476429287867, + "loss": 1.2334, + "theoretical_loss": 3.438645085631163, + "tokens_seen": 2147993600 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028223871614844537, + "loss": 1.3165, + "theoretical_loss": 3.4386365319715, + "tokens_seen": 2148059136 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002822226680040121, + "loss": 1.2208, + "theoretical_loss": 3.4386279786458687, + "tokens_seen": 2148124672 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002822066198595788, + "loss": 1.2671, + "theoretical_loss": 3.438619425654245, + "tokens_seen": 2148190208 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002821905717151454, + "loss": 1.2644, + "theoretical_loss": 3.4386108729966063, + "tokens_seen": 2148255744 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002821745235707121, + "loss": 1.4316, + "theoretical_loss": 3.438602320672929, + "tokens_seen": 2148321280 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002821584754262788, + "loss": 1.3899, + "theoretical_loss": 3.438593768683191, + "tokens_seen": 2148386816 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002821424272818455, + "loss": 1.2957, + "theoretical_loss": 3.438585217027367, + "tokens_seen": 2148452352 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002821263791374122, + "loss": 1.3007, + "theoretical_loss": 3.438576665705435, + "tokens_seen": 2148517888 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002821103309929789, + "loss": 1.3143, + "theoretical_loss": 3.4385681147173726, + "tokens_seen": 2148583424 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002820942828485456, + "loss": 1.3, + "theoretical_loss": 3.438559564063155, + "tokens_seen": 2148648960 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002820782347041123, + "loss": 1.272, + "theoretical_loss": 3.43855101374276, + "tokens_seen": 2148714496 + }, + { + "epoch": 1.03, + "learning_rate": 0.000282062186559679, + "loss": 1.2815, + "theoretical_loss": 3.4385424637561637, + "tokens_seen": 2148780032 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002820461384152457, + "loss": 1.1861, + "theoretical_loss": 3.4385339141033433, + "tokens_seen": 2148845568 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002820300902708124, + "loss": 1.2987, + "theoretical_loss": 3.438525364784276, + "tokens_seen": 2148911104 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028201404212637913, + "loss": 1.3082, + "theoretical_loss": 3.4385168157989376, + "tokens_seen": 2148976640 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028199799398194583, + "loss": 1.2179, + "theoretical_loss": 3.4385082671473057, + "tokens_seen": 2149042176 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028198194583751253, + "loss": 1.3122, + "theoretical_loss": 3.4384997188293562, + "tokens_seen": 2149107712 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028196589769307923, + "loss": 1.355, + "theoretical_loss": 3.4384911708450674, + "tokens_seen": 2149173248 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028194984954864593, + "loss": 1.3922, + "theoretical_loss": 3.4384826231944148, + "tokens_seen": 2149238784 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028193380140421263, + "loss": 1.389, + "theoretical_loss": 3.4384740758773757, + "tokens_seen": 2149304320 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028191775325977933, + "loss": 1.2894, + "theoretical_loss": 3.4384655288939268, + "tokens_seen": 2149369856 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028190170511534603, + "loss": 1.1809, + "theoretical_loss": 3.4384569822440447, + "tokens_seen": 2149435392 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028188565697091273, + "loss": 1.1659, + "theoretical_loss": 3.438448435927707, + "tokens_seen": 2149500928 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 795149, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2580324411392212, + "objective/train/theoretical_loss": 3.4384398899448896, + "objective/train/tokens_used": 2170026464, + "theoretical_loss": 3.4384398899448896, + "tokens_seen": 2149566464 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028186960882647943, + "loss": 1.258, + "theoretical_loss": 3.4384398899448896, + "tokens_seen": 2149566464 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028186960882647943, + "loss": 1.4046, + "theoretical_loss": 3.43843134429557, + "tokens_seen": 2149632000 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028185356068204613, + "loss": 1.2808, + "theoretical_loss": 3.4384227989797242, + "tokens_seen": 2149697536 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028183751253761283, + "loss": 1.3161, + "theoretical_loss": 3.4384142539973297, + "tokens_seen": 2149763072 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028182146439317953, + "loss": 1.2693, + "theoretical_loss": 3.4384057093483635, + "tokens_seen": 2149828608 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028180541624874624, + "loss": 1.3139, + "theoretical_loss": 3.438397165032802, + "tokens_seen": 2149894144 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028178936810431294, + "loss": 1.2595, + "theoretical_loss": 3.4383886210506214, + "tokens_seen": 2149959680 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028177331995987964, + "loss": 1.1852, + "theoretical_loss": 3.4383800774018, + "tokens_seen": 2150025216 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028175727181544634, + "loss": 1.3669, + "theoretical_loss": 3.438371534086314, + "tokens_seen": 2150090752 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028174122367101304, + "loss": 1.3529, + "theoretical_loss": 3.4383629911041393, + "tokens_seen": 2150156288 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028172517552657974, + "loss": 1.2366, + "theoretical_loss": 3.4383544484552537, + "tokens_seen": 2150221824 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028170912738214644, + "loss": 1.2429, + "theoretical_loss": 3.4383459061396344, + "tokens_seen": 2150287360 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028169307923771314, + "loss": 1.2261, + "theoretical_loss": 3.4383373641572574, + "tokens_seen": 2150352896 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002816770310932799, + "loss": 1.1974, + "theoretical_loss": 3.4383288225081, + "tokens_seen": 2150418432 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002816609829488466, + "loss": 1.3104, + "theoretical_loss": 3.438320281192139, + "tokens_seen": 2150483968 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002816449348044133, + "loss": 1.245, + "theoretical_loss": 3.438311740209351, + "tokens_seen": 2150549504 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028162888665998, + "loss": 1.2441, + "theoretical_loss": 3.438303199559713, + "tokens_seen": 2150615040 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002816128385155467, + "loss": 1.2224, + "theoretical_loss": 3.4382946592432013, + "tokens_seen": 2150680576 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002815967903711134, + "loss": 1.3283, + "theoretical_loss": 3.438286119259794, + "tokens_seen": 2150746112 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002815807422266801, + "loss": 1.1843, + "theoretical_loss": 3.4382775796094673, + "tokens_seen": 2150811648 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028156469408224674, + "loss": 1.2384, + "theoretical_loss": 3.438269040292198, + "tokens_seen": 2150877184 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028154864593781345, + "loss": 1.2911, + "theoretical_loss": 3.438260501307963, + "tokens_seen": 2150942720 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028153259779338015, + "loss": 1.3058, + "theoretical_loss": 3.438251962656739, + "tokens_seen": 2151008256 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028151654964894685, + "loss": 1.278, + "theoretical_loss": 3.438243424338503, + "tokens_seen": 2151073792 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028150050150451355, + "loss": 1.2687, + "theoretical_loss": 3.438234886353232, + "tokens_seen": 2151139328 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028148445336008025, + "loss": 1.3025, + "theoretical_loss": 3.438226348700903, + "tokens_seen": 2151204864 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028146840521564695, + "loss": 1.2222, + "theoretical_loss": 3.4382178113814925, + "tokens_seen": 2151270400 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028145235707121365, + "loss": 1.2484, + "theoretical_loss": 3.4382092743949775, + "tokens_seen": 2151335936 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028143630892678035, + "loss": 1.211, + "theoretical_loss": 3.438200737741335, + "tokens_seen": 2151401472 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028142026078234705, + "loss": 1.2958, + "theoretical_loss": 3.4381922014205415, + "tokens_seen": 2151467008 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028140421263791375, + "loss": 1.1743, + "theoretical_loss": 3.4381836654325744, + "tokens_seen": 2151532544 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028138816449348045, + "loss": 1.2127, + "theoretical_loss": 3.4381751297774104, + "tokens_seen": 2151598080 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028137211634904715, + "loss": 1.252, + "theoretical_loss": 3.438166594455026, + "tokens_seen": 2151663616 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028135606820461385, + "loss": 1.2839, + "theoretical_loss": 3.4381580594653984, + "tokens_seen": 2151729152 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028134002006018055, + "loss": 1.2811, + "theoretical_loss": 3.438149524808505, + "tokens_seen": 2151794688 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028132397191574725, + "loss": 1.2325, + "theoretical_loss": 3.438140990484322, + "tokens_seen": 2151860224 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028130792377131396, + "loss": 1.2382, + "theoretical_loss": 3.4381324564928266, + "tokens_seen": 2151925760 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028129187562688066, + "loss": 1.325, + "theoretical_loss": 3.438123922833996, + "tokens_seen": 2151991296 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028127582748244736, + "loss": 1.3801, + "theoretical_loss": 3.4381153895078063, + "tokens_seen": 2152056832 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028125977933801406, + "loss": 1.1853, + "theoretical_loss": 3.4381068565142345, + "tokens_seen": 2152122368 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028124373119358076, + "loss": 1.2015, + "theoretical_loss": 3.438098323853258, + "tokens_seen": 2152187904 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028122768304914746, + "loss": 1.1395, + "theoretical_loss": 3.438089791524854, + "tokens_seen": 2152253440 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028121163490471416, + "loss": 1.2126, + "theoretical_loss": 3.4380812595289987, + "tokens_seen": 2152318976 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028119558676028086, + "loss": 1.1703, + "theoretical_loss": 3.438072727865669, + "tokens_seen": 2152384512 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028117953861584756, + "loss": 1.3423, + "theoretical_loss": 3.4380641965348424, + "tokens_seen": 2152450048 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028116349047141426, + "loss": 1.2376, + "theoretical_loss": 3.4380556655364956, + "tokens_seen": 2152515584 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028114744232698096, + "loss": 1.2764, + "theoretical_loss": 3.4380471348706054, + "tokens_seen": 2152581120 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028113139418254766, + "loss": 1.2406, + "theoretical_loss": 3.438038604537148, + "tokens_seen": 2152646656 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028111534603811436, + "loss": 1.2657, + "theoretical_loss": 3.438030074536102, + "tokens_seen": 2152712192 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028109929789368106, + "loss": 1.2169, + "theoretical_loss": 3.438021544867443, + "tokens_seen": 2152777728 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028108324974924776, + "loss": 1.2255, + "theoretical_loss": 3.4380130155311486, + "tokens_seen": 2152843264 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028106720160481446, + "loss": 1.2418, + "theoretical_loss": 3.4380044865271953, + "tokens_seen": 2152908800 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028105115346038117, + "loss": 1.2705, + "theoretical_loss": 3.43799595785556, + "tokens_seen": 2152974336 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028103510531594787, + "loss": 1.3154, + "theoretical_loss": 3.4379874295162205, + "tokens_seen": 2153039872 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028101905717151457, + "loss": 1.1856, + "theoretical_loss": 3.437978901509152, + "tokens_seen": 2153105408 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028100300902708127, + "loss": 1.3003, + "theoretical_loss": 3.4379703738343332, + "tokens_seen": 2153170944 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028098696088264797, + "loss": 1.2663, + "theoretical_loss": 3.4379618464917403, + "tokens_seen": 2153236480 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028097091273821467, + "loss": 1.2422, + "theoretical_loss": 3.43795331948135, + "tokens_seen": 2153302016 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028095486459378137, + "loss": 1.2853, + "theoretical_loss": 3.4379447928031404, + "tokens_seen": 2153367552 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028093881644934807, + "loss": 1.255, + "theoretical_loss": 3.4379362664570867, + "tokens_seen": 2153433088 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028092276830491477, + "loss": 1.262, + "theoretical_loss": 3.437927740443167, + "tokens_seen": 2153498624 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028090672016048147, + "loss": 1.2283, + "theoretical_loss": 3.437919214761358, + "tokens_seen": 2153564160 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028089067201604817, + "loss": 1.2823, + "theoretical_loss": 3.4379106894116367, + "tokens_seen": 2153629696 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028087462387161487, + "loss": 1.274, + "theoretical_loss": 3.4379021643939804, + "tokens_seen": 2153695232 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028085857572718157, + "loss": 1.196, + "theoretical_loss": 3.4378936397083653, + "tokens_seen": 2153760768 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002808425275827483, + "loss": 1.2179, + "theoretical_loss": 3.437885115354769, + "tokens_seen": 2153826304 + }, + { + "epoch": 1.03, + "learning_rate": 0.000280826479438315, + "loss": 1.2349, + "theoretical_loss": 3.4378765913331675, + "tokens_seen": 2153891840 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002808104312938817, + "loss": 1.2711, + "theoretical_loss": 3.437868067643539, + "tokens_seen": 2153957376 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002807943831494483, + "loss": 1.3124, + "theoretical_loss": 3.4378595442858604, + "tokens_seen": 2154022912 + }, + { + "epoch": 1.03, + "learning_rate": 0.000280778335005015, + "loss": 1.1967, + "theoretical_loss": 3.4378510212601077, + "tokens_seen": 2154088448 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002807622868605817, + "loss": 1.2663, + "theoretical_loss": 3.4378424985662583, + "tokens_seen": 2154153984 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002807462387161484, + "loss": 1.1956, + "theoretical_loss": 3.4378339762042893, + "tokens_seen": 2154219520 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002807301905717151, + "loss": 1.2661, + "theoretical_loss": 3.4378254541741784, + "tokens_seen": 2154285056 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002807141424272818, + "loss": 1.2127, + "theoretical_loss": 3.4378169324759007, + "tokens_seen": 2154350592 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002806980942828485, + "loss": 1.2565, + "theoretical_loss": 3.4378084111094354, + "tokens_seen": 2154416128 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002806820461384152, + "loss": 1.4033, + "theoretical_loss": 3.437799890074758, + "tokens_seen": 2154481664 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002806659979939819, + "loss": 1.2986, + "theoretical_loss": 3.437791369371846, + "tokens_seen": 2154547200 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002806499498495486, + "loss": 1.2925, + "theoretical_loss": 3.437782849000676, + "tokens_seen": 2154612736 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028063390170511533, + "loss": 1.2472, + "theoretical_loss": 3.4377743289612255, + "tokens_seen": 2154678272 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028061785356068203, + "loss": 1.3049, + "theoretical_loss": 3.437765809253471, + "tokens_seen": 2154743808 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028060180541624873, + "loss": 1.287, + "theoretical_loss": 3.43775728987739, + "tokens_seen": 2154809344 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028058575727181543, + "loss": 1.2491, + "theoretical_loss": 3.4377487708329597, + "tokens_seen": 2154874880 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028056970912738213, + "loss": 1.2042, + "theoretical_loss": 3.4377402521201565, + "tokens_seen": 2154940416 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028055366098294883, + "loss": 1.2635, + "theoretical_loss": 3.437731733738957, + "tokens_seen": 2155005952 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028053761283851553, + "loss": 1.2472, + "theoretical_loss": 3.4377232156893394, + "tokens_seen": 2155071488 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028052156469408223, + "loss": 1.2254, + "theoretical_loss": 3.4377146979712805, + "tokens_seen": 2155137024 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028050551654964893, + "loss": 1.2479, + "theoretical_loss": 3.4377061805847564, + "tokens_seen": 2155202560 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028048946840521563, + "loss": 1.318, + "theoretical_loss": 3.4376976635297445, + "tokens_seen": 2155268096 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002804734202607824, + "loss": 1.3165, + "theoretical_loss": 3.437689146806222, + "tokens_seen": 2155333632 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002804573721163491, + "loss": 1.294, + "theoretical_loss": 3.437680630414166, + "tokens_seen": 2155399168 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002804413239719158, + "loss": 1.3445, + "theoretical_loss": 3.4376721143535534, + "tokens_seen": 2155464704 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002804252758274825, + "loss": 1.2265, + "theoretical_loss": 3.4376635986243613, + "tokens_seen": 2155530240 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002804092276830492, + "loss": 1.3327, + "theoretical_loss": 3.437655083226567, + "tokens_seen": 2155595776 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002803931795386159, + "loss": 1.2719, + "theoretical_loss": 3.4376465681601465, + "tokens_seen": 2155661312 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002803771313941826, + "loss": 1.2232, + "theoretical_loss": 3.437638053425078, + "tokens_seen": 2155726848 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002803610832497493, + "loss": 1.1786, + "theoretical_loss": 3.437629539021338, + "tokens_seen": 2155792384 + }, + { + "epoch": 1.03, + "learning_rate": 0.000280345035105316, + "loss": 1.2414, + "theoretical_loss": 3.4376210249489034, + "tokens_seen": 2155857920 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002803289869608827, + "loss": 1.4051, + "theoretical_loss": 3.4376125112077514, + "tokens_seen": 2155923456 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002803129388164494, + "loss": 1.219, + "theoretical_loss": 3.437603997797859, + "tokens_seen": 2155988992 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002802968906720161, + "loss": 1.1114, + "theoretical_loss": 3.4375954847192034, + "tokens_seen": 2156054528 + }, + { + "epoch": 1.03, + "objective/train/docs_used": 797348, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.284947395324707, + "objective/train/theoretical_loss": 3.4375869719717618, + "objective/train/tokens_used": 2176580064, + "theoretical_loss": 3.4375869719717618, + "tokens_seen": 2156120064 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002802808425275828, + "loss": 1.2849, + "theoretical_loss": 3.4375869719717618, + "tokens_seen": 2156120064 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002802647943831495, + "loss": 1.1486, + "theoretical_loss": 3.437578459555511, + "tokens_seen": 2156185600 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002802487462387162, + "loss": 1.2312, + "theoretical_loss": 3.4375699474704273, + "tokens_seen": 2156251136 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002802326980942829, + "loss": 1.2227, + "theoretical_loss": 3.437561435716489, + "tokens_seen": 2156316672 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002802166499498496, + "loss": 1.2835, + "theoretical_loss": 3.437552924293673, + "tokens_seen": 2156382208 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002802006018054163, + "loss": 1.2655, + "theoretical_loss": 3.437544413201956, + "tokens_seen": 2156447744 + }, + { + "epoch": 1.03, + "learning_rate": 0.000280184553660983, + "loss": 1.2825, + "theoretical_loss": 3.4375359024413146, + "tokens_seen": 2156513280 + }, + { + "epoch": 1.03, + "learning_rate": 0.0002801685055165497, + "loss": 1.3292, + "theoretical_loss": 3.4375273920117264, + "tokens_seen": 2156578816 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028015245737211635, + "loss": 1.2458, + "theoretical_loss": 3.4375188819131686, + "tokens_seen": 2156644352 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028013640922768305, + "loss": 1.2628, + "theoretical_loss": 3.437510372145618, + "tokens_seen": 2156709888 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028012036108324975, + "loss": 1.2267, + "theoretical_loss": 3.437501862709052, + "tokens_seen": 2156775424 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028010431293881645, + "loss": 1.3227, + "theoretical_loss": 3.437493353603447, + "tokens_seen": 2156840960 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028008826479438315, + "loss": 1.1991, + "theoretical_loss": 3.4374848448287803, + "tokens_seen": 2156906496 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028007221664994985, + "loss": 1.1823, + "theoretical_loss": 3.4374763363850294, + "tokens_seen": 2156972032 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028005616850551655, + "loss": 1.3857, + "theoretical_loss": 3.4374678282721707, + "tokens_seen": 2157037568 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028004012036108325, + "loss": 1.2437, + "theoretical_loss": 3.4374593204901824, + "tokens_seen": 2157103104 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028002407221664995, + "loss": 1.2613, + "theoretical_loss": 3.4374508130390407, + "tokens_seen": 2157168640 + }, + { + "epoch": 1.03, + "learning_rate": 0.00028000802407221665, + "loss": 1.2721, + "theoretical_loss": 3.4374423059187222, + "tokens_seen": 2157234176 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027999197592778335, + "loss": 1.2768, + "theoretical_loss": 3.4374337991292054, + "tokens_seen": 2157299712 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027997592778335005, + "loss": 1.3209, + "theoretical_loss": 3.4374252926704663, + "tokens_seen": 2157365248 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027995987963891675, + "loss": 1.2884, + "theoretical_loss": 3.4374167865424825, + "tokens_seen": 2157430784 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027994383149448345, + "loss": 1.2448, + "theoretical_loss": 3.437408280745231, + "tokens_seen": 2157496320 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027992778335005016, + "loss": 1.2566, + "theoretical_loss": 3.4373997752786885, + "tokens_seen": 2157561856 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027991173520561686, + "loss": 1.2368, + "theoretical_loss": 3.4373912701428324, + "tokens_seen": 2157627392 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027989568706118356, + "loss": 1.3852, + "theoretical_loss": 3.43738276533764, + "tokens_seen": 2157692928 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027987963891675026, + "loss": 1.2154, + "theoretical_loss": 3.437374260863088, + "tokens_seen": 2157758464 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027986359077231696, + "loss": 1.3041, + "theoretical_loss": 3.437365756719154, + "tokens_seen": 2157824000 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027984754262788366, + "loss": 1.2561, + "theoretical_loss": 3.4373572529058145, + "tokens_seen": 2157889536 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027983149448345036, + "loss": 1.2498, + "theoretical_loss": 3.4373487494230472, + "tokens_seen": 2157955072 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027981544633901706, + "loss": 1.2454, + "theoretical_loss": 3.437340246270829, + "tokens_seen": 2158020608 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027979939819458376, + "loss": 1.3149, + "theoretical_loss": 3.4373317434491364, + "tokens_seen": 2158086144 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027978335005015046, + "loss": 1.2345, + "theoretical_loss": 3.437323240957948, + "tokens_seen": 2158151680 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027976730190571716, + "loss": 1.2439, + "theoretical_loss": 3.437314738797239, + "tokens_seen": 2158217216 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027975125376128386, + "loss": 1.2846, + "theoretical_loss": 3.4373062369669882, + "tokens_seen": 2158282752 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027973520561685056, + "loss": 1.213, + "theoretical_loss": 3.4372977354671717, + "tokens_seen": 2158348288 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027971915747241726, + "loss": 1.2499, + "theoretical_loss": 3.437289234297767, + "tokens_seen": 2158413824 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027970310932798396, + "loss": 1.1985, + "theoretical_loss": 3.437280733458751, + "tokens_seen": 2158479360 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027968706118355066, + "loss": 1.1484, + "theoretical_loss": 3.4372722329501015, + "tokens_seen": 2158544896 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027967101303911737, + "loss": 1.2262, + "theoretical_loss": 3.4372637327717945, + "tokens_seen": 2158610432 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027965496489468407, + "loss": 1.3535, + "theoretical_loss": 3.437255232923808, + "tokens_seen": 2158675968 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027963891675025077, + "loss": 1.2617, + "theoretical_loss": 3.437246733406119, + "tokens_seen": 2158741504 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027962286860581747, + "loss": 1.2813, + "theoretical_loss": 3.4372382342187047, + "tokens_seen": 2158807040 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027960682046138417, + "loss": 1.3243, + "theoretical_loss": 3.437229735361542, + "tokens_seen": 2158872576 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027959077231695087, + "loss": 1.2801, + "theoretical_loss": 3.437221236834608, + "tokens_seen": 2158938112 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027957472417251757, + "loss": 1.3763, + "theoretical_loss": 3.4372127386378803, + "tokens_seen": 2159003648 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027955867602808427, + "loss": 1.298, + "theoretical_loss": 3.4372042407713352, + "tokens_seen": 2159069184 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027954262788365097, + "loss": 1.288, + "theoretical_loss": 3.437195743234951, + "tokens_seen": 2159134720 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027952657973921767, + "loss": 1.3549, + "theoretical_loss": 3.437187246028704, + "tokens_seen": 2159200256 + }, + { + "epoch": 1.03, + "learning_rate": 0.00027951053159478437, + "loss": 1.1581, + "theoretical_loss": 3.437178749152571, + "tokens_seen": 2159265792 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027949448345035107, + "loss": 1.2678, + "theoretical_loss": 3.4371702526065304, + "tokens_seen": 2159331328 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002794784353059178, + "loss": 1.3699, + "theoretical_loss": 3.4371617563905583, + "tokens_seen": 2159396864 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002794623871614845, + "loss": 1.2342, + "theoretical_loss": 3.4371532605046324, + "tokens_seen": 2159462400 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002794463390170512, + "loss": 1.3455, + "theoretical_loss": 3.4371447649487297, + "tokens_seen": 2159527936 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002794302908726179, + "loss": 1.2174, + "theoretical_loss": 3.4371362697228274, + "tokens_seen": 2159593472 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002794142427281846, + "loss": 1.2218, + "theoretical_loss": 3.4371277748269025, + "tokens_seen": 2159659008 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002793981945837513, + "loss": 1.2165, + "theoretical_loss": 3.4371192802609327, + "tokens_seen": 2159724544 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002793821464393179, + "loss": 1.2263, + "theoretical_loss": 3.4371107860248946, + "tokens_seen": 2159790080 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002793660982948846, + "loss": 1.3194, + "theoretical_loss": 3.4371022921187655, + "tokens_seen": 2159855616 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002793500501504513, + "loss": 1.2115, + "theoretical_loss": 3.4370937985425227, + "tokens_seen": 2159921152 + }, + { + "epoch": 1.04, + "learning_rate": 0.000279334002006018, + "loss": 1.2099, + "theoretical_loss": 3.437085305296143, + "tokens_seen": 2159986688 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002793179538615847, + "loss": 1.2321, + "theoretical_loss": 3.4370768123796043, + "tokens_seen": 2160052224 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002793019057171514, + "loss": 1.1699, + "theoretical_loss": 3.4370683197928833, + "tokens_seen": 2160117760 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002792858575727181, + "loss": 1.267, + "theoretical_loss": 3.437059827535957, + "tokens_seen": 2160183296 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002792698094282849, + "loss": 1.2665, + "theoretical_loss": 3.4370513356088033, + "tokens_seen": 2160248832 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002792537612838516, + "loss": 1.2466, + "theoretical_loss": 3.437042844011399, + "tokens_seen": 2160314368 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002792377131394183, + "loss": 1.1645, + "theoretical_loss": 3.437034352743721, + "tokens_seen": 2160379904 + }, + { + "epoch": 1.04, + "learning_rate": 0.000279221664994985, + "loss": 1.2303, + "theoretical_loss": 3.4370258618057465, + "tokens_seen": 2160445440 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002792056168505517, + "loss": 1.2309, + "theoretical_loss": 3.437017371197453, + "tokens_seen": 2160510976 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002791895687061184, + "loss": 1.295, + "theoretical_loss": 3.437008880918818, + "tokens_seen": 2160576512 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002791735205616851, + "loss": 1.2775, + "theoretical_loss": 3.437000390969818, + "tokens_seen": 2160642048 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002791574724172518, + "loss": 1.2266, + "theoretical_loss": 3.4369919013504306, + "tokens_seen": 2160707584 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002791414242728185, + "loss": 1.1788, + "theoretical_loss": 3.4369834120606333, + "tokens_seen": 2160773120 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002791253761283852, + "loss": 1.3036, + "theoretical_loss": 3.436974923100403, + "tokens_seen": 2160838656 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002791093279839519, + "loss": 1.2392, + "theoretical_loss": 3.436966434469716, + "tokens_seen": 2160904192 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002790932798395186, + "loss": 1.1797, + "theoretical_loss": 3.436957946168551, + "tokens_seen": 2160969728 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002790772316950853, + "loss": 1.3113, + "theoretical_loss": 3.4369494581968847, + "tokens_seen": 2161035264 + }, + { + "epoch": 1.04, + "learning_rate": 0.000279061183550652, + "loss": 1.2835, + "theoretical_loss": 3.4369409705546943, + "tokens_seen": 2161100800 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002790451354062187, + "loss": 1.3009, + "theoretical_loss": 3.4369324832419563, + "tokens_seen": 2161166336 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002790290872617854, + "loss": 1.3174, + "theoretical_loss": 3.436923996258649, + "tokens_seen": 2161231872 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002790130391173521, + "loss": 1.3818, + "theoretical_loss": 3.4369155096047495, + "tokens_seen": 2161297408 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002789969909729188, + "loss": 1.3645, + "theoretical_loss": 3.4369070232802343, + "tokens_seen": 2161362944 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002789809428284855, + "loss": 1.2516, + "theoretical_loss": 3.436898537285081, + "tokens_seen": 2161428480 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002789648946840522, + "loss": 1.2228, + "theoretical_loss": 3.436890051619267, + "tokens_seen": 2161494016 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002789488465396189, + "loss": 1.233, + "theoretical_loss": 3.436881566282769, + "tokens_seen": 2161559552 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002789327983951856, + "loss": 1.27, + "theoretical_loss": 3.4368730812755652, + "tokens_seen": 2161625088 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002789167502507523, + "loss": 1.2573, + "theoretical_loss": 3.436864596597632, + "tokens_seen": 2161690624 + }, + { + "epoch": 1.04, + "learning_rate": 0.000278900702106319, + "loss": 1.2861, + "theoretical_loss": 3.436856112248947, + "tokens_seen": 2161756160 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002788846539618857, + "loss": 1.2333, + "theoretical_loss": 3.4368476282294873, + "tokens_seen": 2161821696 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002788686058174524, + "loss": 1.2209, + "theoretical_loss": 3.4368391445392303, + "tokens_seen": 2161887232 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002788525576730191, + "loss": 1.2053, + "theoretical_loss": 3.4368306611781527, + "tokens_seen": 2161952768 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002788365095285858, + "loss": 1.2882, + "theoretical_loss": 3.4368221781462327, + "tokens_seen": 2162018304 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002788204613841525, + "loss": 1.1818, + "theoretical_loss": 3.4368136954434467, + "tokens_seen": 2162083840 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002788044132397192, + "loss": 1.1961, + "theoretical_loss": 3.436805213069773, + "tokens_seen": 2162149376 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002787883650952859, + "loss": 1.3311, + "theoretical_loss": 3.4367967310251872, + "tokens_seen": 2162214912 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002787723169508526, + "loss": 1.19, + "theoretical_loss": 3.436788249309668, + "tokens_seen": 2162280448 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027875626880641925, + "loss": 1.2837, + "theoretical_loss": 3.436779767923192, + "tokens_seen": 2162345984 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027874022066198595, + "loss": 1.1928, + "theoretical_loss": 3.436771286865737, + "tokens_seen": 2162411520 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027872417251755265, + "loss": 1.1772, + "theoretical_loss": 3.43676280613728, + "tokens_seen": 2162477056 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027870812437311935, + "loss": 1.3245, + "theoretical_loss": 3.436754325737798, + "tokens_seen": 2162542592 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027869207622868605, + "loss": 1.3062, + "theoretical_loss": 3.436745845667268, + "tokens_seen": 2162608128 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 799671, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.084551215171814, + "objective/train/theoretical_loss": 3.4367373659256684, + "objective/train/tokens_used": 2183133664, + "theoretical_loss": 3.4367373659256684, + "tokens_seen": 2162673664 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027867602808425275, + "loss": 1.0846, + "theoretical_loss": 3.4367373659256684, + "tokens_seen": 2162673664 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027865997993981945, + "loss": 1.2812, + "theoretical_loss": 3.4367288865129755, + "tokens_seen": 2162739200 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027864393179538615, + "loss": 1.1621, + "theoretical_loss": 3.4367204074291666, + "tokens_seen": 2162804736 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027862788365095285, + "loss": 1.302, + "theoretical_loss": 3.4367119286742196, + "tokens_seen": 2162870272 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027861183550651955, + "loss": 1.2079, + "theoretical_loss": 3.4367034502481113, + "tokens_seen": 2162935808 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027859578736208625, + "loss": 1.2337, + "theoretical_loss": 3.436694972150819, + "tokens_seen": 2163001344 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027857973921765295, + "loss": 1.3903, + "theoretical_loss": 3.4366864943823203, + "tokens_seen": 2163066880 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027856369107321966, + "loss": 1.2698, + "theoretical_loss": 3.436678016942592, + "tokens_seen": 2163132416 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027854764292878636, + "loss": 1.3306, + "theoretical_loss": 3.436669539831612, + "tokens_seen": 2163197952 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027853159478435306, + "loss": 1.234, + "theoretical_loss": 3.436661063049357, + "tokens_seen": 2163263488 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027851554663991976, + "loss": 1.2773, + "theoretical_loss": 3.436652586595805, + "tokens_seen": 2163329024 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027849949849548646, + "loss": 1.3155, + "theoretical_loss": 3.436644110470932, + "tokens_seen": 2163394560 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027848345035105316, + "loss": 1.2608, + "theoretical_loss": 3.436635634674717, + "tokens_seen": 2163460096 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027846740220661986, + "loss": 1.2084, + "theoretical_loss": 3.436627159207136, + "tokens_seen": 2163525632 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027845135406218656, + "loss": 1.1204, + "theoretical_loss": 3.436618684068167, + "tokens_seen": 2163591168 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027843530591775326, + "loss": 1.1977, + "theoretical_loss": 3.436610209257787, + "tokens_seen": 2163656704 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027841925777331996, + "loss": 1.3055, + "theoretical_loss": 3.4366017347759734, + "tokens_seen": 2163722240 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027840320962888666, + "loss": 1.2572, + "theoretical_loss": 3.436593260622703, + "tokens_seen": 2163787776 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027838716148445336, + "loss": 1.3394, + "theoretical_loss": 3.4365847867979546, + "tokens_seen": 2163853312 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027837111334002006, + "loss": 1.1457, + "theoretical_loss": 3.4365763133017038, + "tokens_seen": 2163918848 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027835506519558676, + "loss": 1.2442, + "theoretical_loss": 3.436567840133929, + "tokens_seen": 2163984384 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027833901705115346, + "loss": 1.2734, + "theoretical_loss": 3.4365593672946066, + "tokens_seen": 2164049920 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027832296890672016, + "loss": 1.2531, + "theoretical_loss": 3.436550894783715, + "tokens_seen": 2164115456 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027830692076228687, + "loss": 1.2737, + "theoretical_loss": 3.4365424226012307, + "tokens_seen": 2164180992 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027829087261785357, + "loss": 1.2518, + "theoretical_loss": 3.4365339507471315, + "tokens_seen": 2164246528 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027827482447342027, + "loss": 1.1961, + "theoretical_loss": 3.4365254792213946, + "tokens_seen": 2164312064 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027825877632898697, + "loss": 1.3204, + "theoretical_loss": 3.436517008023997, + "tokens_seen": 2164377600 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027824272818455367, + "loss": 1.2401, + "theoretical_loss": 3.4365085371549164, + "tokens_seen": 2164443136 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027822668004012037, + "loss": 1.195, + "theoretical_loss": 3.4365000666141303, + "tokens_seen": 2164508672 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027821063189568707, + "loss": 1.2433, + "theoretical_loss": 3.4364915964016154, + "tokens_seen": 2164574208 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027819458375125377, + "loss": 1.2667, + "theoretical_loss": 3.43648312651735, + "tokens_seen": 2164639744 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027817853560682047, + "loss": 1.339, + "theoretical_loss": 3.4364746569613103, + "tokens_seen": 2164705280 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027816248746238717, + "loss": 1.262, + "theoretical_loss": 3.4364661877334743, + "tokens_seen": 2164770816 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027814643931795387, + "loss": 1.313, + "theoretical_loss": 3.4364577188338195, + "tokens_seen": 2164836352 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027813039117352057, + "loss": 1.2589, + "theoretical_loss": 3.4364492502623225, + "tokens_seen": 2164901888 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002781143430290873, + "loss": 1.3072, + "theoretical_loss": 3.4364407820189617, + "tokens_seen": 2164967424 + }, + { + "epoch": 1.04, + "learning_rate": 0.000278098294884654, + "loss": 1.1594, + "theoretical_loss": 3.4364323141037136, + "tokens_seen": 2165032960 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002780822467402207, + "loss": 1.3667, + "theoretical_loss": 3.436423846516556, + "tokens_seen": 2165098496 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002780661985957874, + "loss": 1.1994, + "theoretical_loss": 3.436415379257466, + "tokens_seen": 2165164032 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002780501504513541, + "loss": 1.214, + "theoretical_loss": 3.4364069123264214, + "tokens_seen": 2165229568 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002780341023069208, + "loss": 1.2361, + "theoretical_loss": 3.436398445723399, + "tokens_seen": 2165295104 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002780180541624875, + "loss": 1.2289, + "theoretical_loss": 3.4363899794483768, + "tokens_seen": 2165360640 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002780020060180542, + "loss": 1.3724, + "theoretical_loss": 3.436381513501331, + "tokens_seen": 2165426176 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002779859578736209, + "loss": 1.2836, + "theoretical_loss": 3.4363730478822405, + "tokens_seen": 2165491712 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002779699097291876, + "loss": 1.2448, + "theoretical_loss": 3.4363645825910814, + "tokens_seen": 2165557248 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002779538615847543, + "loss": 1.1871, + "theoretical_loss": 3.436356117627832, + "tokens_seen": 2165622784 + }, + { + "epoch": 1.04, + "learning_rate": 0.000277937813440321, + "loss": 1.1939, + "theoretical_loss": 3.436347652992469, + "tokens_seen": 2165688320 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002779217652958877, + "loss": 1.2445, + "theoretical_loss": 3.43633918868497, + "tokens_seen": 2165753856 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002779057171514544, + "loss": 1.3971, + "theoretical_loss": 3.4363307247053125, + "tokens_seen": 2165819392 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002778896690070211, + "loss": 1.2909, + "theoretical_loss": 3.4363222610534736, + "tokens_seen": 2165884928 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002778736208625878, + "loss": 1.2755, + "theoretical_loss": 3.436313797729431, + "tokens_seen": 2165950464 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002778575727181545, + "loss": 1.3724, + "theoretical_loss": 3.4363053347331625, + "tokens_seen": 2166016000 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002778415245737212, + "loss": 1.2796, + "theoretical_loss": 3.4362968720646445, + "tokens_seen": 2166081536 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002778254764292879, + "loss": 1.2999, + "theoretical_loss": 3.436288409723855, + "tokens_seen": 2166147072 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002778094282848546, + "loss": 1.2726, + "theoretical_loss": 3.436279947710771, + "tokens_seen": 2166212608 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002777933801404213, + "loss": 1.1915, + "theoretical_loss": 3.4362714860253702, + "tokens_seen": 2166278144 + }, + { + "epoch": 1.04, + "learning_rate": 0.000277777331995988, + "loss": 1.2201, + "theoretical_loss": 3.43626302466763, + "tokens_seen": 2166343680 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002777612838515547, + "loss": 1.2706, + "theoretical_loss": 3.436254563637528, + "tokens_seen": 2166409216 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002777452357071214, + "loss": 1.218, + "theoretical_loss": 3.436246102935041, + "tokens_seen": 2166474752 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002777291875626881, + "loss": 1.2874, + "theoretical_loss": 3.436237642560147, + "tokens_seen": 2166540288 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002777131394182548, + "loss": 1.2519, + "theoretical_loss": 3.436229182512823, + "tokens_seen": 2166605824 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002776970912738215, + "loss": 1.3142, + "theoretical_loss": 3.436220722793047, + "tokens_seen": 2166671360 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002776810431293882, + "loss": 1.2898, + "theoretical_loss": 3.436212263400795, + "tokens_seen": 2166736896 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002776649949849549, + "loss": 1.319, + "theoretical_loss": 3.436203804336046, + "tokens_seen": 2166802432 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002776489468405216, + "loss": 1.2792, + "theoretical_loss": 3.436195345598777, + "tokens_seen": 2166867968 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002776328986960883, + "loss": 1.3053, + "theoretical_loss": 3.436186887188965, + "tokens_seen": 2166933504 + }, + { + "epoch": 1.04, + "learning_rate": 0.000277616850551655, + "loss": 1.3902, + "theoretical_loss": 3.4361784291065876, + "tokens_seen": 2166999040 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002776008024072217, + "loss": 1.3258, + "theoretical_loss": 3.4361699713516227, + "tokens_seen": 2167064576 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002775847542627884, + "loss": 1.328, + "theoretical_loss": 3.4361615139240467, + "tokens_seen": 2167130112 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002775687061183551, + "loss": 1.2625, + "theoretical_loss": 3.436153056823838, + "tokens_seen": 2167195648 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002775526579739218, + "loss": 1.2275, + "theoretical_loss": 3.4361446000509734, + "tokens_seen": 2167261184 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002775366098294885, + "loss": 1.3307, + "theoretical_loss": 3.436136143605431, + "tokens_seen": 2167326720 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002775205616850552, + "loss": 1.3627, + "theoretical_loss": 3.436127687487187, + "tokens_seen": 2167392256 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002775045135406219, + "loss": 1.264, + "theoretical_loss": 3.4361192316962206, + "tokens_seen": 2167457792 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002774884653961886, + "loss": 1.2985, + "theoretical_loss": 3.436110776232508, + "tokens_seen": 2167523328 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002774724172517553, + "loss": 1.2207, + "theoretical_loss": 3.4361023210960266, + "tokens_seen": 2167588864 + }, + { + "epoch": 1.04, + "learning_rate": 0.000277456369107322, + "loss": 1.3019, + "theoretical_loss": 3.436093866286754, + "tokens_seen": 2167654400 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002774403209628887, + "loss": 1.3036, + "theoretical_loss": 3.4360854118046684, + "tokens_seen": 2167719936 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002774242728184554, + "loss": 1.2071, + "theoretical_loss": 3.4360769576497465, + "tokens_seen": 2167785472 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002774082246740221, + "loss": 1.3436, + "theoretical_loss": 3.436068503821966, + "tokens_seen": 2167851008 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002773921765295888, + "loss": 1.3485, + "theoretical_loss": 3.436060050321304, + "tokens_seen": 2167916544 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002773761283851555, + "loss": 1.1956, + "theoretical_loss": 3.4360515971477383, + "tokens_seen": 2167982080 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027736008024072215, + "loss": 1.3198, + "theoretical_loss": 3.4360431443012462, + "tokens_seen": 2168047616 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027734403209628885, + "loss": 1.2226, + "theoretical_loss": 3.4360346917818054, + "tokens_seen": 2168113152 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027732798395185555, + "loss": 1.191, + "theoretical_loss": 3.4360262395893932, + "tokens_seen": 2168178688 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027731193580742225, + "loss": 1.2343, + "theoretical_loss": 3.436017787723987, + "tokens_seen": 2168244224 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027729588766298895, + "loss": 1.3128, + "theoretical_loss": 3.4360093361855646, + "tokens_seen": 2168309760 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027727983951855565, + "loss": 1.2338, + "theoretical_loss": 3.436000884974103, + "tokens_seen": 2168375296 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027726379137412235, + "loss": 1.2588, + "theoretical_loss": 3.4359924340895796, + "tokens_seen": 2168440832 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027724774322968905, + "loss": 1.2883, + "theoretical_loss": 3.4359839835319725, + "tokens_seen": 2168506368 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027723169508525575, + "loss": 1.1739, + "theoretical_loss": 3.4359755333012587, + "tokens_seen": 2168571904 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027721564694082245, + "loss": 1.1961, + "theoretical_loss": 3.435967083397416, + "tokens_seen": 2168637440 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027719959879638915, + "loss": 1.2484, + "theoretical_loss": 3.4359586338204213, + "tokens_seen": 2168702976 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027718355065195586, + "loss": 1.2504, + "theoretical_loss": 3.4359501845702525, + "tokens_seen": 2168768512 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027716750250752256, + "loss": 1.267, + "theoretical_loss": 3.435941735646887, + "tokens_seen": 2168834048 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027715145436308926, + "loss": 1.3608, + "theoretical_loss": 3.4359332870503025, + "tokens_seen": 2168899584 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027713540621865596, + "loss": 1.3219, + "theoretical_loss": 3.435924838780476, + "tokens_seen": 2168965120 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027711935807422266, + "loss": 1.3151, + "theoretical_loss": 3.4359163908373858, + "tokens_seen": 2169030656 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027710330992978936, + "loss": 1.2409, + "theoretical_loss": 3.4359079432210082, + "tokens_seen": 2169096192 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027708726178535606, + "loss": 1.2738, + "theoretical_loss": 3.435899495931322, + "tokens_seen": 2169161728 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 802022, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2531790733337402, + "objective/train/theoretical_loss": 3.435891048968304, + "objective/train/tokens_used": 2189687264, + "theoretical_loss": 3.435891048968304, + "tokens_seen": 2169227264 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027707121364092276, + "loss": 1.2532, + "theoretical_loss": 3.435891048968304, + "tokens_seen": 2169227264 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027705516549648946, + "loss": 1.2903, + "theoretical_loss": 3.4358826023319313, + "tokens_seen": 2169292800 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027703911735205616, + "loss": 1.2919, + "theoretical_loss": 3.4358741560221824, + "tokens_seen": 2169358336 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027702306920762286, + "loss": 1.3329, + "theoretical_loss": 3.435865710039034, + "tokens_seen": 2169423872 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027700702106318956, + "loss": 1.2205, + "theoretical_loss": 3.435857264382464, + "tokens_seen": 2169489408 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027699097291875626, + "loss": 1.2366, + "theoretical_loss": 3.43584881905245, + "tokens_seen": 2169554944 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027697492477432296, + "loss": 1.2617, + "theoretical_loss": 3.4358403740489685, + "tokens_seen": 2169620480 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027695887662988966, + "loss": 1.2159, + "theoretical_loss": 3.4358319293719983, + "tokens_seen": 2169686016 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027694282848545637, + "loss": 1.2922, + "theoretical_loss": 3.435823485021517, + "tokens_seen": 2169751552 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027692678034102307, + "loss": 1.2736, + "theoretical_loss": 3.435815040997501, + "tokens_seen": 2169817088 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027691073219658977, + "loss": 1.3235, + "theoretical_loss": 3.435806597299928, + "tokens_seen": 2169882624 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027689468405215647, + "loss": 1.2254, + "theoretical_loss": 3.4357981539287765, + "tokens_seen": 2169948160 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027687863590772317, + "loss": 1.3736, + "theoretical_loss": 3.435789710884023, + "tokens_seen": 2170013696 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002768625877632899, + "loss": 1.2283, + "theoretical_loss": 3.4357812681656457, + "tokens_seen": 2170079232 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002768465396188566, + "loss": 1.322, + "theoretical_loss": 3.4357728257736215, + "tokens_seen": 2170144768 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002768304914744233, + "loss": 1.1863, + "theoretical_loss": 3.435764383707929, + "tokens_seen": 2170210304 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027681444332999, + "loss": 1.1763, + "theoretical_loss": 3.4357559419685444, + "tokens_seen": 2170275840 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002767983951855567, + "loss": 1.1777, + "theoretical_loss": 3.4357475005554456, + "tokens_seen": 2170341376 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002767823470411234, + "loss": 1.2416, + "theoretical_loss": 3.435739059468611, + "tokens_seen": 2170406912 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002767662988966901, + "loss": 1.2538, + "theoretical_loss": 3.435730618708017, + "tokens_seen": 2170472448 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002767502507522568, + "loss": 1.2198, + "theoretical_loss": 3.4357221782736422, + "tokens_seen": 2170537984 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027673420260782353, + "loss": 1.3137, + "theoretical_loss": 3.4357137381654637, + "tokens_seen": 2170603520 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002767181544633902, + "loss": 1.3088, + "theoretical_loss": 3.4357052983834584, + "tokens_seen": 2170669056 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002767021063189569, + "loss": 1.2955, + "theoretical_loss": 3.4356968589276047, + "tokens_seen": 2170734592 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002766860581745236, + "loss": 1.285, + "theoretical_loss": 3.43568841979788, + "tokens_seen": 2170800128 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002766700100300903, + "loss": 1.184, + "theoretical_loss": 3.4356799809942613, + "tokens_seen": 2170865664 + }, + { + "epoch": 1.04, + "learning_rate": 0.000276653961885657, + "loss": 1.2631, + "theoretical_loss": 3.435671542516727, + "tokens_seen": 2170931200 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002766379137412237, + "loss": 1.298, + "theoretical_loss": 3.435663104365254, + "tokens_seen": 2170996736 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002766218655967904, + "loss": 1.2671, + "theoretical_loss": 3.43565466653982, + "tokens_seen": 2171062272 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002766058174523571, + "loss": 1.3114, + "theoretical_loss": 3.435646229040403, + "tokens_seen": 2171127808 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002765897693079238, + "loss": 1.3296, + "theoretical_loss": 3.43563779186698, + "tokens_seen": 2171193344 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002765737211634905, + "loss": 1.3501, + "theoretical_loss": 3.4356293550195285, + "tokens_seen": 2171258880 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002765576730190572, + "loss": 1.2444, + "theoretical_loss": 3.4356209184980266, + "tokens_seen": 2171324416 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002765416248746239, + "loss": 1.1466, + "theoretical_loss": 3.435612482302451, + "tokens_seen": 2171389952 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002765255767301906, + "loss": 1.2089, + "theoretical_loss": 3.435604046432781, + "tokens_seen": 2171455488 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002765095285857573, + "loss": 1.2319, + "theoretical_loss": 3.4355956108889925, + "tokens_seen": 2171521024 + }, + { + "epoch": 1.04, + "learning_rate": 0.000276493480441324, + "loss": 1.2744, + "theoretical_loss": 3.4355871756710634, + "tokens_seen": 2171586560 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002764774322968907, + "loss": 1.2562, + "theoretical_loss": 3.435578740778972, + "tokens_seen": 2171652096 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002764613841524574, + "loss": 1.2766, + "theoretical_loss": 3.435570306212695, + "tokens_seen": 2171717632 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002764453360080241, + "loss": 1.1954, + "theoretical_loss": 3.4355618719722103, + "tokens_seen": 2171783168 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002764292878635908, + "loss": 1.2592, + "theoretical_loss": 3.4355534380574957, + "tokens_seen": 2171848704 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002764132397191575, + "loss": 1.2824, + "theoretical_loss": 3.4355450044685285, + "tokens_seen": 2171914240 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002763971915747242, + "loss": 1.36, + "theoretical_loss": 3.435536571205287, + "tokens_seen": 2171979776 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002763811434302909, + "loss": 1.3501, + "theoretical_loss": 3.4355281382677476, + "tokens_seen": 2172045312 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002763650952858576, + "loss": 1.2835, + "theoretical_loss": 3.4355197056558886, + "tokens_seen": 2172110848 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002763490471414243, + "loss": 1.321, + "theoretical_loss": 3.4355112733696878, + "tokens_seen": 2172176384 + }, + { + "epoch": 1.04, + "learning_rate": 0.000276332998996991, + "loss": 1.2641, + "theoretical_loss": 3.4355028414091224, + "tokens_seen": 2172241920 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002763169508525577, + "loss": 1.2195, + "theoretical_loss": 3.43549440977417, + "tokens_seen": 2172307456 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002763009027081244, + "loss": 1.3159, + "theoretical_loss": 3.4354859784648086, + "tokens_seen": 2172372992 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002762848545636911, + "loss": 1.2665, + "theoretical_loss": 3.4354775474810153, + "tokens_seen": 2172438528 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002762688064192578, + "loss": 1.2751, + "theoretical_loss": 3.435469116822768, + "tokens_seen": 2172504064 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002762527582748245, + "loss": 1.2856, + "theoretical_loss": 3.4354606864900443, + "tokens_seen": 2172569600 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002762367101303912, + "loss": 1.1973, + "theoretical_loss": 3.4354522564828214, + "tokens_seen": 2172635136 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002762206619859579, + "loss": 1.2235, + "theoretical_loss": 3.4354438268010776, + "tokens_seen": 2172700672 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002762046138415246, + "loss": 1.2931, + "theoretical_loss": 3.4354353974447904, + "tokens_seen": 2172766208 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002761885656970913, + "loss": 1.2018, + "theoretical_loss": 3.4354269684139367, + "tokens_seen": 2172831744 + }, + { + "epoch": 1.04, + "learning_rate": 0.000276172517552658, + "loss": 1.36, + "theoretical_loss": 3.435418539708495, + "tokens_seen": 2172897280 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002761564694082247, + "loss": 1.2925, + "theoretical_loss": 3.4354101113284425, + "tokens_seen": 2172962816 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002761404212637914, + "loss": 1.2768, + "theoretical_loss": 3.4354016832737564, + "tokens_seen": 2173028352 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002761243731193581, + "loss": 1.2747, + "theoretical_loss": 3.4353932555444153, + "tokens_seen": 2173093888 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002761083249749248, + "loss": 1.4329, + "theoretical_loss": 3.4353848281403963, + "tokens_seen": 2173159424 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002760922768304915, + "loss": 1.3249, + "theoretical_loss": 3.435376401061677, + "tokens_seen": 2173224960 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002760762286860582, + "loss": 1.1902, + "theoretical_loss": 3.4353679743082353, + "tokens_seen": 2173290496 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002760601805416249, + "loss": 1.233, + "theoretical_loss": 3.435359547880048, + "tokens_seen": 2173356032 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002760441323971916, + "loss": 1.23, + "theoretical_loss": 3.435351121777094, + "tokens_seen": 2173421568 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002760280842527583, + "loss": 1.326, + "theoretical_loss": 3.43534269599935, + "tokens_seen": 2173487104 + }, + { + "epoch": 1.04, + "learning_rate": 0.000276012036108325, + "loss": 1.3139, + "theoretical_loss": 3.435334270546794, + "tokens_seen": 2173552640 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002759959879638917, + "loss": 1.3033, + "theoretical_loss": 3.4353258454194036, + "tokens_seen": 2173618176 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002759799398194584, + "loss": 1.2658, + "theoretical_loss": 3.4353174206171566, + "tokens_seen": 2173683712 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002759638916750251, + "loss": 1.297, + "theoretical_loss": 3.4353089961400305, + "tokens_seen": 2173749248 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027594784353059175, + "loss": 1.2438, + "theoretical_loss": 3.4353005719880025, + "tokens_seen": 2173814784 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027593179538615845, + "loss": 1.2991, + "theoretical_loss": 3.435292148161051, + "tokens_seen": 2173880320 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027591574724172515, + "loss": 1.308, + "theoretical_loss": 3.4352837246591537, + "tokens_seen": 2173945856 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027589969909729185, + "loss": 1.243, + "theoretical_loss": 3.4352753014822874, + "tokens_seen": 2174011392 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027588365095285855, + "loss": 1.2734, + "theoretical_loss": 3.4352668786304306, + "tokens_seen": 2174076928 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027586760280842525, + "loss": 1.1868, + "theoretical_loss": 3.4352584561035604, + "tokens_seen": 2174142464 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027585155466399195, + "loss": 1.143, + "theoretical_loss": 3.435250033901655, + "tokens_seen": 2174208000 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027583550651955865, + "loss": 1.1367, + "theoretical_loss": 3.435241612024692, + "tokens_seen": 2174273536 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027581945837512536, + "loss": 1.3204, + "theoretical_loss": 3.435233190472648, + "tokens_seen": 2174339072 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027580341023069206, + "loss": 1.2917, + "theoretical_loss": 3.4352247692455022, + "tokens_seen": 2174404608 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027578736208625876, + "loss": 1.2879, + "theoretical_loss": 3.435216348343231, + "tokens_seen": 2174470144 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027577131394182546, + "loss": 1.2743, + "theoretical_loss": 3.435207927765813, + "tokens_seen": 2174535680 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027575526579739216, + "loss": 1.2461, + "theoretical_loss": 3.435199507513226, + "tokens_seen": 2174601216 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027573921765295886, + "loss": 1.1971, + "theoretical_loss": 3.4351910875854466, + "tokens_seen": 2174666752 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027572316950852556, + "loss": 1.2515, + "theoretical_loss": 3.4351826679824535, + "tokens_seen": 2174732288 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027570712136409226, + "loss": 1.241, + "theoretical_loss": 3.4351742487042234, + "tokens_seen": 2174797824 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027569107321965896, + "loss": 1.2286, + "theoretical_loss": 3.4351658297507353, + "tokens_seen": 2174863360 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002756750250752257, + "loss": 1.255, + "theoretical_loss": 3.4351574111219656, + "tokens_seen": 2174928896 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002756589769307924, + "loss": 1.3194, + "theoretical_loss": 3.435148992817893, + "tokens_seen": 2174994432 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002756429287863591, + "loss": 1.4093, + "theoretical_loss": 3.4351405748384947, + "tokens_seen": 2175059968 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002756268806419258, + "loss": 1.3304, + "theoretical_loss": 3.435132157183748, + "tokens_seen": 2175125504 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002756108324974925, + "loss": 1.328, + "theoretical_loss": 3.4351237398536316, + "tokens_seen": 2175191040 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002755947843530592, + "loss": 1.2608, + "theoretical_loss": 3.435115322848122, + "tokens_seen": 2175256576 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002755787362086259, + "loss": 1.2502, + "theoretical_loss": 3.4351069061671984, + "tokens_seen": 2175322112 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002755626880641926, + "loss": 1.1841, + "theoretical_loss": 3.435098489810837, + "tokens_seen": 2175387648 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002755466399197593, + "loss": 1.2933, + "theoretical_loss": 3.4350900737790164, + "tokens_seen": 2175453184 + }, + { + "epoch": 1.04, + "learning_rate": 0.000275530591775326, + "loss": 1.3249, + "theoretical_loss": 3.435081658071714, + "tokens_seen": 2175518720 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002755145436308927, + "loss": 1.213, + "theoretical_loss": 3.4350732426889077, + "tokens_seen": 2175584256 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002754984954864594, + "loss": 1.3261, + "theoretical_loss": 3.435064827630575, + "tokens_seen": 2175649792 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002754824473420261, + "loss": 1.2698, + "theoretical_loss": 3.4350564128966936, + "tokens_seen": 2175715328 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 804359, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.194758415222168, + "objective/train/theoretical_loss": 3.4350479984872413, + "objective/train/tokens_used": 2196240864, + "theoretical_loss": 3.4350479984872413, + "tokens_seen": 2175780864 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002754663991975928, + "loss": 1.1948, + "theoretical_loss": 3.4350479984872413, + "tokens_seen": 2175780864 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002754503510531595, + "loss": 1.2708, + "theoretical_loss": 3.435039584402196, + "tokens_seen": 2175846400 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002754343029087262, + "loss": 1.3583, + "theoretical_loss": 3.4350311706415355, + "tokens_seen": 2175911936 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002754182547642929, + "loss": 1.3223, + "theoretical_loss": 3.435022757205237, + "tokens_seen": 2175977472 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002754022066198596, + "loss": 1.1329, + "theoretical_loss": 3.435014344093279, + "tokens_seen": 2176043008 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002753861584754263, + "loss": 1.2578, + "theoretical_loss": 3.4350059313056382, + "tokens_seen": 2176108544 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027537011033099303, + "loss": 1.3483, + "theoretical_loss": 3.4349975188422928, + "tokens_seen": 2176174080 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027535406218655973, + "loss": 1.1695, + "theoretical_loss": 3.434989106703221, + "tokens_seen": 2176239616 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027533801404212643, + "loss": 1.205, + "theoretical_loss": 3.4349806948884, + "tokens_seen": 2176305152 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002753219658976931, + "loss": 1.1768, + "theoretical_loss": 3.4349722833978076, + "tokens_seen": 2176370688 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002753059177532598, + "loss": 1.2659, + "theoretical_loss": 3.434963872231422, + "tokens_seen": 2176436224 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002752898696088265, + "loss": 1.1922, + "theoretical_loss": 3.43495546138922, + "tokens_seen": 2176501760 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002752738214643932, + "loss": 1.2457, + "theoretical_loss": 3.4349470508711804, + "tokens_seen": 2176567296 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002752577733199599, + "loss": 1.2889, + "theoretical_loss": 3.4349386406772804, + "tokens_seen": 2176632832 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002752417251755266, + "loss": 1.2858, + "theoretical_loss": 3.4349302308074976, + "tokens_seen": 2176698368 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002752256770310933, + "loss": 1.2298, + "theoretical_loss": 3.4349218212618102, + "tokens_seen": 2176763904 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027520962888666, + "loss": 1.3266, + "theoretical_loss": 3.4349134120401956, + "tokens_seen": 2176829440 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002751935807422267, + "loss": 1.4047, + "theoretical_loss": 3.4349050031426316, + "tokens_seen": 2176894976 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002751775325977934, + "loss": 1.2796, + "theoretical_loss": 3.4348965945690964, + "tokens_seen": 2176960512 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002751614844533601, + "loss": 1.2034, + "theoretical_loss": 3.434888186319567, + "tokens_seen": 2177026048 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002751454363089268, + "loss": 1.1283, + "theoretical_loss": 3.4348797783940213, + "tokens_seen": 2177091584 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002751293881644935, + "loss": 1.3307, + "theoretical_loss": 3.434871370792438, + "tokens_seen": 2177157120 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002751133400200602, + "loss": 1.3926, + "theoretical_loss": 3.4348629635147936, + "tokens_seen": 2177222656 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002750972918756269, + "loss": 1.2888, + "theoretical_loss": 3.4348545565610666, + "tokens_seen": 2177288192 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002750812437311936, + "loss": 1.3087, + "theoretical_loss": 3.434846149931235, + "tokens_seen": 2177353728 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002750651955867603, + "loss": 1.2734, + "theoretical_loss": 3.434837743625276, + "tokens_seen": 2177419264 + }, + { + "epoch": 1.04, + "learning_rate": 0.000275049147442327, + "loss": 1.2361, + "theoretical_loss": 3.4348293376431673, + "tokens_seen": 2177484800 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002750330992978937, + "loss": 1.2035, + "theoretical_loss": 3.434820931984887, + "tokens_seen": 2177550336 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002750170511534604, + "loss": 1.3352, + "theoretical_loss": 3.434812526650413, + "tokens_seen": 2177615872 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002750010030090271, + "loss": 1.2719, + "theoretical_loss": 3.434804121639723, + "tokens_seen": 2177681408 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002749849548645938, + "loss": 1.2508, + "theoretical_loss": 3.4347957169527943, + "tokens_seen": 2177746944 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002749689067201605, + "loss": 1.2363, + "theoretical_loss": 3.4347873125896053, + "tokens_seen": 2177812480 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002749528585757272, + "loss": 1.2328, + "theoretical_loss": 3.434778908550134, + "tokens_seen": 2177878016 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002749368104312939, + "loss": 1.1768, + "theoretical_loss": 3.4347705048343573, + "tokens_seen": 2177943552 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002749207622868606, + "loss": 1.4072, + "theoretical_loss": 3.4347621014422534, + "tokens_seen": 2178009088 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002749047141424273, + "loss": 1.3625, + "theoretical_loss": 3.4347536983738003, + "tokens_seen": 2178074624 + }, + { + "epoch": 1.04, + "learning_rate": 0.000274888665997994, + "loss": 1.294, + "theoretical_loss": 3.4347452956289755, + "tokens_seen": 2178140160 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002748726178535607, + "loss": 1.245, + "theoretical_loss": 3.434736893207757, + "tokens_seen": 2178205696 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002748565697091274, + "loss": 1.267, + "theoretical_loss": 3.4347284911101226, + "tokens_seen": 2178271232 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002748405215646941, + "loss": 1.2482, + "theoretical_loss": 3.43472008933605, + "tokens_seen": 2178336768 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002748244734202608, + "loss": 1.2873, + "theoretical_loss": 3.434711687885517, + "tokens_seen": 2178402304 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002748084252758275, + "loss": 1.3406, + "theoretical_loss": 3.4347032867585017, + "tokens_seen": 2178467840 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002747923771313942, + "loss": 1.2877, + "theoretical_loss": 3.4346948859549817, + "tokens_seen": 2178533376 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002747763289869609, + "loss": 1.2345, + "theoretical_loss": 3.4346864854749346, + "tokens_seen": 2178598912 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002747602808425276, + "loss": 1.3014, + "theoretical_loss": 3.434678085318338, + "tokens_seen": 2178664448 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002747442326980943, + "loss": 1.2982, + "theoretical_loss": 3.434669685485171, + "tokens_seen": 2178729984 + }, + { + "epoch": 1.04, + "learning_rate": 0.000274728184553661, + "loss": 1.3699, + "theoretical_loss": 3.4346612859754098, + "tokens_seen": 2178795520 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002747121364092277, + "loss": 1.2706, + "theoretical_loss": 3.434652886789033, + "tokens_seen": 2178861056 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002746960882647944, + "loss": 1.3221, + "theoretical_loss": 3.4346444879260187, + "tokens_seen": 2178926592 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002746800401203611, + "loss": 1.3396, + "theoretical_loss": 3.4346360893863443, + "tokens_seen": 2178992128 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002746639919759278, + "loss": 1.2422, + "theoretical_loss": 3.434627691169988, + "tokens_seen": 2179057664 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002746479438314945, + "loss": 1.3265, + "theoretical_loss": 3.434619293276927, + "tokens_seen": 2179123200 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002746318956870612, + "loss": 1.2508, + "theoretical_loss": 3.4346108957071393, + "tokens_seen": 2179188736 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002746158475426279, + "loss": 1.275, + "theoretical_loss": 3.4346024984606034, + "tokens_seen": 2179254272 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002745997993981946, + "loss": 1.2287, + "theoretical_loss": 3.4345941015372965, + "tokens_seen": 2179319808 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002745837512537613, + "loss": 1.2559, + "theoretical_loss": 3.4345857049371964, + "tokens_seen": 2179385344 + }, + { + "epoch": 1.04, + "learning_rate": 0.000274567703109328, + "loss": 1.0945, + "theoretical_loss": 3.4345773086602813, + "tokens_seen": 2179450880 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027455165496489465, + "loss": 1.3035, + "theoretical_loss": 3.434568912706529, + "tokens_seen": 2179516416 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027453560682046135, + "loss": 1.2616, + "theoretical_loss": 3.4345605170759175, + "tokens_seen": 2179581952 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027451955867602805, + "loss": 1.1944, + "theoretical_loss": 3.434552121768424, + "tokens_seen": 2179647488 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027450351053159475, + "loss": 1.2189, + "theoretical_loss": 3.4345437267840264, + "tokens_seen": 2179713024 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027448746238716145, + "loss": 1.2388, + "theoretical_loss": 3.434535332122703, + "tokens_seen": 2179778560 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002744714142427282, + "loss": 1.2377, + "theoretical_loss": 3.4345269377844323, + "tokens_seen": 2179844096 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002744553660982949, + "loss": 1.2146, + "theoretical_loss": 3.4345185437691903, + "tokens_seen": 2179909632 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002744393179538616, + "loss": 1.1899, + "theoretical_loss": 3.4345101500769566, + "tokens_seen": 2179975168 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002744232698094283, + "loss": 1.2518, + "theoretical_loss": 3.4345017567077085, + "tokens_seen": 2180040704 + }, + { + "epoch": 1.04, + "learning_rate": 0.000274407221664995, + "loss": 1.1167, + "theoretical_loss": 3.4344933636614234, + "tokens_seen": 2180106240 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002743911735205617, + "loss": 1.2347, + "theoretical_loss": 3.4344849709380796, + "tokens_seen": 2180171776 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002743751253761284, + "loss": 1.264, + "theoretical_loss": 3.434476578537655, + "tokens_seen": 2180237312 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002743590772316951, + "loss": 1.2301, + "theoretical_loss": 3.4344681864601276, + "tokens_seen": 2180302848 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002743430290872618, + "loss": 1.3604, + "theoretical_loss": 3.434459794705475, + "tokens_seen": 2180368384 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002743269809428285, + "loss": 1.217, + "theoretical_loss": 3.434451403273675, + "tokens_seen": 2180433920 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002743109327983952, + "loss": 1.2457, + "theoretical_loss": 3.4344430121647056, + "tokens_seen": 2180499456 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002742948846539619, + "loss": 1.2871, + "theoretical_loss": 3.4344346213785446, + "tokens_seen": 2180564992 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002742788365095286, + "loss": 1.2622, + "theoretical_loss": 3.43442623091517, + "tokens_seen": 2180630528 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002742627883650953, + "loss": 1.2345, + "theoretical_loss": 3.4344178407745596, + "tokens_seen": 2180696064 + }, + { + "epoch": 1.04, + "learning_rate": 0.000274246740220662, + "loss": 1.2888, + "theoretical_loss": 3.434409450956691, + "tokens_seen": 2180761600 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002742306920762287, + "loss": 1.2813, + "theoretical_loss": 3.434401061461543, + "tokens_seen": 2180827136 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002742146439317954, + "loss": 1.2144, + "theoretical_loss": 3.4343926722890927, + "tokens_seen": 2180892672 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002741985957873621, + "loss": 1.2472, + "theoretical_loss": 3.434384283439318, + "tokens_seen": 2180958208 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002741825476429288, + "loss": 1.1586, + "theoretical_loss": 3.434375894912197, + "tokens_seen": 2181023744 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002741664994984955, + "loss": 1.1836, + "theoretical_loss": 3.4343675067077077, + "tokens_seen": 2181089280 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002741504513540622, + "loss": 1.2593, + "theoretical_loss": 3.434359118825828, + "tokens_seen": 2181154816 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002741344032096289, + "loss": 1.2898, + "theoretical_loss": 3.4343507312665356, + "tokens_seen": 2181220352 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002741183550651956, + "loss": 1.2128, + "theoretical_loss": 3.434342344029808, + "tokens_seen": 2181285888 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002741023069207623, + "loss": 1.1957, + "theoretical_loss": 3.434333957115624, + "tokens_seen": 2181351424 + }, + { + "epoch": 1.04, + "learning_rate": 0.000274086258776329, + "loss": 1.2861, + "theoretical_loss": 3.4343255705239613, + "tokens_seen": 2181416960 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002740702106318957, + "loss": 1.2552, + "theoretical_loss": 3.4343171842547973, + "tokens_seen": 2181482496 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002740541624874624, + "loss": 1.3097, + "theoretical_loss": 3.4343087983081104, + "tokens_seen": 2181548032 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002740381143430291, + "loss": 1.244, + "theoretical_loss": 3.434300412683878, + "tokens_seen": 2181613568 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002740220661985958, + "loss": 1.2325, + "theoretical_loss": 3.4342920273820785, + "tokens_seen": 2181679104 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002740060180541625, + "loss": 1.3334, + "theoretical_loss": 3.43428364240269, + "tokens_seen": 2181744640 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027398996990972923, + "loss": 1.3208, + "theoretical_loss": 3.4342752577456896, + "tokens_seen": 2181810176 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027397392176529593, + "loss": 1.1553, + "theoretical_loss": 3.4342668734110555, + "tokens_seen": 2181875712 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027395787362086263, + "loss": 1.241, + "theoretical_loss": 3.434258489398766, + "tokens_seen": 2181941248 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027394182547642933, + "loss": 1.2751, + "theoretical_loss": 3.4342501057087995, + "tokens_seen": 2182006784 + }, + { + "epoch": 1.04, + "learning_rate": 0.000273925777331996, + "loss": 1.2683, + "theoretical_loss": 3.434241722341133, + "tokens_seen": 2182072320 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002739097291875627, + "loss": 1.2347, + "theoretical_loss": 3.434233339295744, + "tokens_seen": 2182137856 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002738936810431294, + "loss": 1.1536, + "theoretical_loss": 3.4342249565726117, + "tokens_seen": 2182203392 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002738776328986961, + "loss": 1.208, + "theoretical_loss": 3.434216574171713, + "tokens_seen": 2182268928 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 806840, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1982872486114502, + "objective/train/theoretical_loss": 3.4342081920930267, + "objective/train/tokens_used": 2202794464, + "theoretical_loss": 3.4342081920930267, + "tokens_seen": 2182334464 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002738615847542628, + "loss": 1.1983, + "theoretical_loss": 3.4342081920930267, + "tokens_seen": 2182334464 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002738455366098295, + "loss": 1.2959, + "theoretical_loss": 3.4341998103365303, + "tokens_seen": 2182400000 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002738294884653962, + "loss": 1.2269, + "theoretical_loss": 3.434191428902202, + "tokens_seen": 2182465536 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002738134403209629, + "loss": 1.1248, + "theoretical_loss": 3.434183047790019, + "tokens_seen": 2182531072 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002737973921765296, + "loss": 1.3456, + "theoretical_loss": 3.43417466699996, + "tokens_seen": 2182596608 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002737813440320963, + "loss": 1.3385, + "theoretical_loss": 3.434166286532003, + "tokens_seen": 2182662144 + }, + { + "epoch": 1.04, + "learning_rate": 0.000273765295887663, + "loss": 1.1689, + "theoretical_loss": 3.434157906386125, + "tokens_seen": 2182727680 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002737492477432297, + "loss": 1.2359, + "theoretical_loss": 3.4341495265623054, + "tokens_seen": 2182793216 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002737331995987964, + "loss": 1.2597, + "theoretical_loss": 3.434141147060521, + "tokens_seen": 2182858752 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002737171514543631, + "loss": 1.2161, + "theoretical_loss": 3.43413276788075, + "tokens_seen": 2182924288 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002737011033099298, + "loss": 1.4159, + "theoretical_loss": 3.434124389022971, + "tokens_seen": 2182989824 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002736850551654965, + "loss": 1.3342, + "theoretical_loss": 3.434116010487161, + "tokens_seen": 2183055360 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002736690070210632, + "loss": 1.37, + "theoretical_loss": 3.434107632273298, + "tokens_seen": 2183120896 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002736529588766299, + "loss": 1.2867, + "theoretical_loss": 3.434099254381361, + "tokens_seen": 2183186432 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002736369107321966, + "loss": 1.4143, + "theoretical_loss": 3.4340908768113274, + "tokens_seen": 2183251968 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002736208625877633, + "loss": 1.2467, + "theoretical_loss": 3.434082499563175, + "tokens_seen": 2183317504 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027360481444333, + "loss": 1.274, + "theoretical_loss": 3.434074122636882, + "tokens_seen": 2183383040 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002735887662988967, + "loss": 1.1863, + "theoretical_loss": 3.4340657460324255, + "tokens_seen": 2183448576 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002735727181544634, + "loss": 1.2598, + "theoretical_loss": 3.4340573697497847, + "tokens_seen": 2183514112 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002735566700100301, + "loss": 1.192, + "theoretical_loss": 3.4340489937889376, + "tokens_seen": 2183579648 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002735406218655968, + "loss": 1.2985, + "theoretical_loss": 3.434040618149861, + "tokens_seen": 2183645184 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002735245737211635, + "loss": 1.2113, + "theoretical_loss": 3.434032242832534, + "tokens_seen": 2183710720 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002735085255767302, + "loss": 1.2062, + "theoretical_loss": 3.434023867836934, + "tokens_seen": 2183776256 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002734924774322969, + "loss": 1.2168, + "theoretical_loss": 3.4340154931630393, + "tokens_seen": 2183841792 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002734764292878636, + "loss": 1.254, + "theoretical_loss": 3.4340071188108277, + "tokens_seen": 2183907328 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002734603811434303, + "loss": 1.2987, + "theoretical_loss": 3.433998744780277, + "tokens_seen": 2183972864 + }, + { + "epoch": 1.04, + "learning_rate": 0.000273444332998997, + "loss": 1.3978, + "theoretical_loss": 3.4339903710713657, + "tokens_seen": 2184038400 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002734282848545637, + "loss": 1.2826, + "theoretical_loss": 3.433981997684071, + "tokens_seen": 2184103936 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002734122367101304, + "loss": 1.15, + "theoretical_loss": 3.433973624618372, + "tokens_seen": 2184169472 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002733961885656971, + "loss": 1.3099, + "theoretical_loss": 3.433965251874246, + "tokens_seen": 2184235008 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002733801404212638, + "loss": 1.3398, + "theoretical_loss": 3.433956879451671, + "tokens_seen": 2184300544 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002733640922768305, + "loss": 1.3341, + "theoretical_loss": 3.433948507350625, + "tokens_seen": 2184366080 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002733480441323972, + "loss": 1.2551, + "theoretical_loss": 3.4339401355710866, + "tokens_seen": 2184431616 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002733319959879639, + "loss": 1.2679, + "theoretical_loss": 3.4339317641130327, + "tokens_seen": 2184497152 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002733159478435306, + "loss": 1.3143, + "theoretical_loss": 3.433923392976442, + "tokens_seen": 2184562688 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002732998996990973, + "loss": 1.319, + "theoretical_loss": 3.433915022161293, + "tokens_seen": 2184628224 + }, + { + "epoch": 1.04, + "learning_rate": 0.000273283851554664, + "loss": 1.2507, + "theoretical_loss": 3.433906651667563, + "tokens_seen": 2184693760 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002732678034102307, + "loss": 1.2733, + "theoretical_loss": 3.43389828149523, + "tokens_seen": 2184759296 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002732517552657974, + "loss": 1.3397, + "theoretical_loss": 3.433889911644272, + "tokens_seen": 2184824832 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002732357071213641, + "loss": 1.2609, + "theoretical_loss": 3.4338815421146673, + "tokens_seen": 2184890368 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002732196589769308, + "loss": 1.2299, + "theoretical_loss": 3.4338731729063943, + "tokens_seen": 2184955904 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002732036108324975, + "loss": 1.2321, + "theoretical_loss": 3.4338648040194304, + "tokens_seen": 2185021440 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002731875626880642, + "loss": 1.3212, + "theoretical_loss": 3.4338564354537535, + "tokens_seen": 2185086976 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002731715145436309, + "loss": 1.1713, + "theoretical_loss": 3.433848067209342, + "tokens_seen": 2185152512 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002731554663991976, + "loss": 1.2917, + "theoretical_loss": 3.4338396992861737, + "tokens_seen": 2185218048 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002731394182547643, + "loss": 1.4137, + "theoretical_loss": 3.4338313316842273, + "tokens_seen": 2185283584 + }, + { + "epoch": 1.04, + "learning_rate": 0.000273123370110331, + "loss": 1.2437, + "theoretical_loss": 3.4338229644034803, + "tokens_seen": 2185349120 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002731073219658977, + "loss": 1.3654, + "theoretical_loss": 3.4338145974439103, + "tokens_seen": 2185414656 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002730912738214644, + "loss": 1.1835, + "theoretical_loss": 3.433806230805496, + "tokens_seen": 2185480192 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002730752256770311, + "loss": 1.2009, + "theoretical_loss": 3.4337978644882154, + "tokens_seen": 2185545728 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002730591775325978, + "loss": 1.3187, + "theoretical_loss": 3.4337894984920467, + "tokens_seen": 2185611264 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002730431293881645, + "loss": 1.447, + "theoretical_loss": 3.433781132816967, + "tokens_seen": 2185676800 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002730270812437312, + "loss": 1.3579, + "theoretical_loss": 3.4337727674629557, + "tokens_seen": 2185742336 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002730110330992979, + "loss": 1.2983, + "theoretical_loss": 3.43376440242999, + "tokens_seen": 2185807872 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002729949849548646, + "loss": 1.3389, + "theoretical_loss": 3.433756037718047, + "tokens_seen": 2185873408 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002729789368104313, + "loss": 1.2748, + "theoretical_loss": 3.4337476733271073, + "tokens_seen": 2185938944 + }, + { + "epoch": 1.04, + "learning_rate": 0.000272962888665998, + "loss": 1.162, + "theoretical_loss": 3.433739309257147, + "tokens_seen": 2186004480 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002729468405215647, + "loss": 1.253, + "theoretical_loss": 3.4337309455081444, + "tokens_seen": 2186070016 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002729307923771314, + "loss": 1.2376, + "theoretical_loss": 3.4337225820800783, + "tokens_seen": 2186135552 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002729147442326981, + "loss": 1.2895, + "theoretical_loss": 3.433714218972926, + "tokens_seen": 2186201088 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002728986960882648, + "loss": 1.2926, + "theoretical_loss": 3.4337058561866662, + "tokens_seen": 2186266624 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002728826479438315, + "loss": 1.2467, + "theoretical_loss": 3.4336974937212763, + "tokens_seen": 2186332160 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002728665997993982, + "loss": 1.3204, + "theoretical_loss": 3.4336891315767346, + "tokens_seen": 2186397696 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002728505516549649, + "loss": 1.2423, + "theoretical_loss": 3.4336807697530194, + "tokens_seen": 2186463232 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002728345035105316, + "loss": 1.2463, + "theoretical_loss": 3.433672408250109, + "tokens_seen": 2186528768 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002728184553660983, + "loss": 1.1424, + "theoretical_loss": 3.433664047067981, + "tokens_seen": 2186594304 + }, + { + "epoch": 1.04, + "learning_rate": 0.000272802407221665, + "loss": 1.322, + "theoretical_loss": 3.433655686206613, + "tokens_seen": 2186659840 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002727863590772317, + "loss": 1.3362, + "theoretical_loss": 3.4336473256659845, + "tokens_seen": 2186725376 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002727703109327984, + "loss": 1.3884, + "theoretical_loss": 3.4336389654460726, + "tokens_seen": 2186790912 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002727542627883651, + "loss": 1.2272, + "theoretical_loss": 3.4336306055468553, + "tokens_seen": 2186856448 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002727382146439318, + "loss": 1.3435, + "theoretical_loss": 3.4336222459683112, + "tokens_seen": 2186921984 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002727221664994985, + "loss": 1.2619, + "theoretical_loss": 3.4336138867104182, + "tokens_seen": 2186987520 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002727061183550652, + "loss": 1.3967, + "theoretical_loss": 3.433605527773154, + "tokens_seen": 2187053056 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002726900702106319, + "loss": 1.2485, + "theoretical_loss": 3.4335971691564975, + "tokens_seen": 2187118592 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002726740220661986, + "loss": 1.2547, + "theoretical_loss": 3.4335888108604258, + "tokens_seen": 2187184128 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002726579739217653, + "loss": 1.2471, + "theoretical_loss": 3.4335804528849176, + "tokens_seen": 2187249664 + }, + { + "epoch": 1.04, + "learning_rate": 0.000272641925777332, + "loss": 1.28, + "theoretical_loss": 3.4335720952299513, + "tokens_seen": 2187315200 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027262587763289873, + "loss": 1.2664, + "theoretical_loss": 3.4335637378955046, + "tokens_seen": 2187380736 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027260982948846543, + "loss": 1.2213, + "theoretical_loss": 3.4335553808815553, + "tokens_seen": 2187446272 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027259378134403213, + "loss": 1.3092, + "theoretical_loss": 3.433547024188082, + "tokens_seen": 2187511808 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027257773319959883, + "loss": 1.3212, + "theoretical_loss": 3.433538667815063, + "tokens_seen": 2187577344 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027256168505516553, + "loss": 1.2078, + "theoretical_loss": 3.4335303117624756, + "tokens_seen": 2187642880 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027254563691073223, + "loss": 1.3183, + "theoretical_loss": 3.4335219560302987, + "tokens_seen": 2187708416 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027252958876629893, + "loss": 1.3814, + "theoretical_loss": 3.43351360061851, + "tokens_seen": 2187773952 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002725135406218656, + "loss": 1.108, + "theoretical_loss": 3.433505245527088, + "tokens_seen": 2187839488 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002724974924774323, + "loss": 1.2623, + "theoretical_loss": 3.4334968907560097, + "tokens_seen": 2187905024 + }, + { + "epoch": 1.04, + "learning_rate": 0.000272481444332999, + "loss": 1.2935, + "theoretical_loss": 3.4334885363052545, + "tokens_seen": 2187970560 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002724653961885657, + "loss": 1.3516, + "theoretical_loss": 3.4334801821748004, + "tokens_seen": 2188036096 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002724493480441324, + "loss": 1.2546, + "theoretical_loss": 3.433471828364625, + "tokens_seen": 2188101632 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002724332998996991, + "loss": 1.2558, + "theoretical_loss": 3.4334634748747064, + "tokens_seen": 2188167168 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002724172517552658, + "loss": 1.4398, + "theoretical_loss": 3.4334551217050233, + "tokens_seen": 2188232704 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002724012036108325, + "loss": 1.2189, + "theoretical_loss": 3.4334467688555534, + "tokens_seen": 2188298240 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002723851554663992, + "loss": 1.4047, + "theoretical_loss": 3.433438416326275, + "tokens_seen": 2188363776 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002723691073219659, + "loss": 1.2266, + "theoretical_loss": 3.4334300641171662, + "tokens_seen": 2188429312 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002723530591775326, + "loss": 1.3981, + "theoretical_loss": 3.433421712228205, + "tokens_seen": 2188494848 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002723370110330993, + "loss": 1.3479, + "theoretical_loss": 3.4334133606593698, + "tokens_seen": 2188560384 + }, + { + "epoch": 1.04, + "learning_rate": 0.000272320962888666, + "loss": 1.1207, + "theoretical_loss": 3.4334050094106385, + "tokens_seen": 2188625920 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002723049147442327, + "loss": 1.3589, + "theoretical_loss": 3.433396658481989, + "tokens_seen": 2188691456 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002722888665997994, + "loss": 1.3157, + "theoretical_loss": 3.4333883078734004, + "tokens_seen": 2188756992 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002722728184553661, + "loss": 1.2462, + "theoretical_loss": 3.43337995758485, + "tokens_seen": 2188822528 + }, + { + "epoch": 1.04, + "objective/train/docs_used": 809770, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3143064975738525, + "objective/train/theoretical_loss": 3.4333716076163165, + "objective/train/tokens_used": 2209348064, + "theoretical_loss": 3.4333716076163165, + "tokens_seen": 2188888064 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002722567703109328, + "loss": 1.3143, + "theoretical_loss": 3.4333716076163165, + "tokens_seen": 2188888064 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002722407221664995, + "loss": 1.2922, + "theoretical_loss": 3.4333632579677773, + "tokens_seen": 2188953600 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002722246740220662, + "loss": 1.2483, + "theoretical_loss": 3.433354908639211, + "tokens_seen": 2189019136 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002722086258776329, + "loss": 1.3043, + "theoretical_loss": 3.4333465596305963, + "tokens_seen": 2189084672 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002721925777331996, + "loss": 1.2614, + "theoretical_loss": 3.43333821094191, + "tokens_seen": 2189150208 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002721765295887663, + "loss": 1.406, + "theoretical_loss": 3.433329862573132, + "tokens_seen": 2189215744 + }, + { + "epoch": 1.04, + "learning_rate": 0.000272160481444333, + "loss": 1.1879, + "theoretical_loss": 3.433321514524239, + "tokens_seen": 2189281280 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002721444332998997, + "loss": 1.3067, + "theoretical_loss": 3.43331316679521, + "tokens_seen": 2189346816 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002721283851554664, + "loss": 1.3046, + "theoretical_loss": 3.4333048193860227, + "tokens_seen": 2189412352 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002721123370110331, + "loss": 1.2562, + "theoretical_loss": 3.4332964722966555, + "tokens_seen": 2189477888 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002720962888665998, + "loss": 1.242, + "theoretical_loss": 3.4332881255270866, + "tokens_seen": 2189543424 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002720802407221665, + "loss": 1.3123, + "theoretical_loss": 3.4332797790772944, + "tokens_seen": 2189608960 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027206419257773325, + "loss": 1.3052, + "theoretical_loss": 3.4332714329472562, + "tokens_seen": 2189674496 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027204814443329995, + "loss": 1.3192, + "theoretical_loss": 3.4332630871369516, + "tokens_seen": 2189740032 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027203209628886665, + "loss": 1.2834, + "theoretical_loss": 3.4332547416463575, + "tokens_seen": 2189805568 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027201604814443335, + "loss": 1.2773, + "theoretical_loss": 3.4332463964754525, + "tokens_seen": 2189871104 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027200000000000005, + "loss": 1.3047, + "theoretical_loss": 3.433238051624215, + "tokens_seen": 2189936640 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027198395185556675, + "loss": 1.2177, + "theoretical_loss": 3.4332297070926225, + "tokens_seen": 2190002176 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027196790371113345, + "loss": 1.2147, + "theoretical_loss": 3.4332213628806545, + "tokens_seen": 2190067712 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027195185556670015, + "loss": 1.3113, + "theoretical_loss": 3.4332130189882877, + "tokens_seen": 2190133248 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027193580742226685, + "loss": 1.4101, + "theoretical_loss": 3.4332046754155017, + "tokens_seen": 2190198784 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027191975927783356, + "loss": 1.3139, + "theoretical_loss": 3.4331963321622734, + "tokens_seen": 2190264320 + }, + { + "epoch": 1.04, + "learning_rate": 0.00027190371113340026, + "loss": 1.1739, + "theoretical_loss": 3.433187989228582, + "tokens_seen": 2190329856 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002718876629889669, + "loss": 1.2873, + "theoretical_loss": 3.433179646614405, + "tokens_seen": 2190395392 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002718716148445336, + "loss": 1.182, + "theoretical_loss": 3.433171304319721, + "tokens_seen": 2190460928 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002718555667001003, + "loss": 1.3564, + "theoretical_loss": 3.4331629623445084, + "tokens_seen": 2190526464 + }, + { + "epoch": 1.04, + "learning_rate": 0.000271839518555667, + "loss": 1.2374, + "theoretical_loss": 3.433154620688745, + "tokens_seen": 2190592000 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002718234704112337, + "loss": 1.2807, + "theoretical_loss": 3.433146279352409, + "tokens_seen": 2190657536 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002718074222668004, + "loss": 1.2934, + "theoretical_loss": 3.433137938335479, + "tokens_seen": 2190723072 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002717913741223671, + "loss": 1.355, + "theoretical_loss": 3.4331295976379326, + "tokens_seen": 2190788608 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002717753259779338, + "loss": 1.2692, + "theoretical_loss": 3.4331212572597485, + "tokens_seen": 2190854144 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002717592778335005, + "loss": 1.2745, + "theoretical_loss": 3.433112917200905, + "tokens_seen": 2190919680 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002717432296890672, + "loss": 1.2113, + "theoretical_loss": 3.43310457746138, + "tokens_seen": 2190985216 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002717271815446339, + "loss": 1.3526, + "theoretical_loss": 3.4330962380411516, + "tokens_seen": 2191050752 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002717111334002006, + "loss": 1.2232, + "theoretical_loss": 3.4330878989401987, + "tokens_seen": 2191116288 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002716950852557673, + "loss": 1.3001, + "theoretical_loss": 3.433079560158499, + "tokens_seen": 2191181824 + }, + { + "epoch": 1.04, + "learning_rate": 0.000271679037111334, + "loss": 1.2848, + "theoretical_loss": 3.4330712216960304, + "tokens_seen": 2191247360 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002716629889669007, + "loss": 1.2047, + "theoretical_loss": 3.433062883552772, + "tokens_seen": 2191312896 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002716469408224674, + "loss": 1.2365, + "theoretical_loss": 3.4330545457287016, + "tokens_seen": 2191378432 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002716308926780341, + "loss": 1.223, + "theoretical_loss": 3.4330462082237974, + "tokens_seen": 2191443968 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002716148445336008, + "loss": 1.2492, + "theoretical_loss": 3.4330378710380374, + "tokens_seen": 2191509504 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002715987963891675, + "loss": 1.3922, + "theoretical_loss": 3.4330295341714003, + "tokens_seen": 2191575040 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002715827482447342, + "loss": 1.2998, + "theoretical_loss": 3.433021197623864, + "tokens_seen": 2191640576 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002715667001003009, + "loss": 1.2572, + "theoretical_loss": 3.433012861395407, + "tokens_seen": 2191706112 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002715506519558676, + "loss": 1.301, + "theoretical_loss": 3.4330045254860075, + "tokens_seen": 2191771648 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002715346038114343, + "loss": 1.2425, + "theoretical_loss": 3.4329961898956434, + "tokens_seen": 2191837184 + }, + { + "epoch": 1.04, + "learning_rate": 0.000271518555667001, + "loss": 1.3101, + "theoretical_loss": 3.432987854624294, + "tokens_seen": 2191902720 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002715025075225677, + "loss": 1.2794, + "theoretical_loss": 3.432979519671936, + "tokens_seen": 2191968256 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002714864593781344, + "loss": 1.3493, + "theoretical_loss": 3.4329711850385487, + "tokens_seen": 2192033792 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002714704112337011, + "loss": 1.3297, + "theoretical_loss": 3.43296285072411, + "tokens_seen": 2192099328 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002714543630892678, + "loss": 1.3419, + "theoretical_loss": 3.4329545167285986, + "tokens_seen": 2192164864 + }, + { + "epoch": 1.04, + "learning_rate": 0.0002714383149448345, + "loss": 1.2401, + "theoretical_loss": 3.432946183051992, + "tokens_seen": 2192230400 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002714222668004012, + "loss": 1.321, + "theoretical_loss": 3.4329378496942695, + "tokens_seen": 2192295936 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002714062186559679, + "loss": 1.2014, + "theoretical_loss": 3.4329295166554084, + "tokens_seen": 2192361472 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002713901705115346, + "loss": 1.2809, + "theoretical_loss": 3.4329211839353873, + "tokens_seen": 2192427008 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002713741223671013, + "loss": 1.2516, + "theoretical_loss": 3.4329128515341845, + "tokens_seen": 2192492544 + }, + { + "epoch": 1.05, + "learning_rate": 0.000271358074222668, + "loss": 1.2771, + "theoretical_loss": 3.432904519451778, + "tokens_seen": 2192558080 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002713420260782347, + "loss": 1.2884, + "theoretical_loss": 3.4328961876881467, + "tokens_seen": 2192623616 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002713259779338014, + "loss": 1.2017, + "theoretical_loss": 3.4328878562432683, + "tokens_seen": 2192689152 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002713099297893681, + "loss": 1.3146, + "theoretical_loss": 3.4328795251171216, + "tokens_seen": 2192754688 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002712938816449348, + "loss": 1.3374, + "theoretical_loss": 3.4328711943096843, + "tokens_seen": 2192820224 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002712778335005015, + "loss": 1.2293, + "theoretical_loss": 3.4328628638209353, + "tokens_seen": 2192885760 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002712617853560682, + "loss": 1.2554, + "theoretical_loss": 3.4328545336508522, + "tokens_seen": 2192951296 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027124573721163493, + "loss": 1.2242, + "theoretical_loss": 3.432846203799414, + "tokens_seen": 2193016832 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027122968906720163, + "loss": 1.3682, + "theoretical_loss": 3.4328378742665984, + "tokens_seen": 2193082368 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027121364092276833, + "loss": 1.2867, + "theoretical_loss": 3.4328295450523836, + "tokens_seen": 2193147904 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027119759277833503, + "loss": 1.3237, + "theoretical_loss": 3.4328212161567486, + "tokens_seen": 2193213440 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027118154463390173, + "loss": 1.2947, + "theoretical_loss": 3.4328128875796713, + "tokens_seen": 2193278976 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027116549648946843, + "loss": 1.2712, + "theoretical_loss": 3.43280455932113, + "tokens_seen": 2193344512 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027114944834503513, + "loss": 1.3021, + "theoretical_loss": 3.432796231381103, + "tokens_seen": 2193410048 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027113340020060183, + "loss": 1.3202, + "theoretical_loss": 3.4327879037595688, + "tokens_seen": 2193475584 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002711173520561685, + "loss": 1.2626, + "theoretical_loss": 3.432779576456505, + "tokens_seen": 2193541120 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002711013039117352, + "loss": 1.2377, + "theoretical_loss": 3.432771249471891, + "tokens_seen": 2193606656 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002710852557673019, + "loss": 1.2793, + "theoretical_loss": 3.432762922805704, + "tokens_seen": 2193672192 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002710692076228686, + "loss": 1.3694, + "theoretical_loss": 3.432754596457923, + "tokens_seen": 2193737728 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002710531594784353, + "loss": 1.3011, + "theoretical_loss": 3.432746270428526, + "tokens_seen": 2193803264 + }, + { + "epoch": 1.05, + "learning_rate": 0.000271037111334002, + "loss": 1.2809, + "theoretical_loss": 3.4327379447174917, + "tokens_seen": 2193868800 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002710210631895687, + "loss": 1.2133, + "theoretical_loss": 3.4327296193247983, + "tokens_seen": 2193934336 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002710050150451354, + "loss": 1.3681, + "theoretical_loss": 3.432721294250424, + "tokens_seen": 2193999872 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002709889669007021, + "loss": 1.3495, + "theoretical_loss": 3.4327129694943466, + "tokens_seen": 2194065408 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002709729187562688, + "loss": 1.3355, + "theoretical_loss": 3.4327046450565453, + "tokens_seen": 2194130944 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002709568706118355, + "loss": 1.2852, + "theoretical_loss": 3.4326963209369983, + "tokens_seen": 2194196480 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002709408224674022, + "loss": 1.3827, + "theoretical_loss": 3.4326879971356834, + "tokens_seen": 2194262016 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002709247743229689, + "loss": 1.2718, + "theoretical_loss": 3.4326796736525793, + "tokens_seen": 2194327552 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002709087261785356, + "loss": 1.2558, + "theoretical_loss": 3.4326713504876647, + "tokens_seen": 2194393088 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002708926780341023, + "loss": 1.3346, + "theoretical_loss": 3.432663027640917, + "tokens_seen": 2194458624 + }, + { + "epoch": 1.05, + "learning_rate": 0.000270876629889669, + "loss": 1.2342, + "theoretical_loss": 3.4326547051123146, + "tokens_seen": 2194524160 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027086058174523574, + "loss": 1.286, + "theoretical_loss": 3.432646382901837, + "tokens_seen": 2194589696 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027084453360080244, + "loss": 1.2388, + "theoretical_loss": 3.432638061009462, + "tokens_seen": 2194655232 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027082848545636914, + "loss": 1.2094, + "theoretical_loss": 3.432629739435167, + "tokens_seen": 2194720768 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027081243731193584, + "loss": 1.2519, + "theoretical_loss": 3.4326214181789316, + "tokens_seen": 2194786304 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027079638916750255, + "loss": 1.2496, + "theoretical_loss": 3.4326130972407336, + "tokens_seen": 2194851840 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027078034102306925, + "loss": 1.3013, + "theoretical_loss": 3.432604776620551, + "tokens_seen": 2194917376 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027076429287863595, + "loss": 1.2146, + "theoretical_loss": 3.432596456318363, + "tokens_seen": 2194982912 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027074824473420265, + "loss": 1.1487, + "theoretical_loss": 3.432588136334147, + "tokens_seen": 2195048448 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027073219658976935, + "loss": 1.2759, + "theoretical_loss": 3.4325798166678823, + "tokens_seen": 2195113984 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027071614844533605, + "loss": 1.3595, + "theoretical_loss": 3.4325714973195467, + "tokens_seen": 2195179520 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027070010030090275, + "loss": 1.3643, + "theoretical_loss": 3.4325631782891186, + "tokens_seen": 2195245056 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027068405215646945, + "loss": 1.1677, + "theoretical_loss": 3.4325548595765767, + "tokens_seen": 2195310592 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027066800401203615, + "loss": 1.2403, + "theoretical_loss": 3.4325465411818987, + "tokens_seen": 2195376128 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 812068, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.279381513595581, + "objective/train/theoretical_loss": 3.4325382231050634, + "objective/train/tokens_used": 2215901664, + "theoretical_loss": 3.4325382231050634, + "tokens_seen": 2195441664 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027065195586760285, + "loss": 1.2794, + "theoretical_loss": 3.4325382231050634, + "tokens_seen": 2195441664 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027063590772316955, + "loss": 1.2606, + "theoretical_loss": 3.4325299053460494, + "tokens_seen": 2195507200 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027061985957873625, + "loss": 1.2573, + "theoretical_loss": 3.4325215879048345, + "tokens_seen": 2195572736 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027060381143430295, + "loss": 1.1708, + "theoretical_loss": 3.4325132707813975, + "tokens_seen": 2195638272 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027058776328986965, + "loss": 1.333, + "theoretical_loss": 3.432504953975717, + "tokens_seen": 2195703808 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027057171514543635, + "loss": 1.3299, + "theoretical_loss": 3.4324966374877706, + "tokens_seen": 2195769344 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027055566700100305, + "loss": 1.2903, + "theoretical_loss": 3.4324883213175372, + "tokens_seen": 2195834880 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027053961885656976, + "loss": 1.2796, + "theoretical_loss": 3.432480005464995, + "tokens_seen": 2195900416 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027052357071213646, + "loss": 1.3795, + "theoretical_loss": 3.4324716899301224, + "tokens_seen": 2195965952 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027050752256770316, + "loss": 1.29, + "theoretical_loss": 3.4324633747128983, + "tokens_seen": 2196031488 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027049147442326986, + "loss": 1.2889, + "theoretical_loss": 3.4324550598133, + "tokens_seen": 2196097024 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002704754262788365, + "loss": 1.1936, + "theoretical_loss": 3.4324467452313074, + "tokens_seen": 2196162560 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002704593781344032, + "loss": 1.2675, + "theoretical_loss": 3.432438430966897, + "tokens_seen": 2196228096 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002704433299899699, + "loss": 1.235, + "theoretical_loss": 3.432430117020049, + "tokens_seen": 2196293632 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002704272818455366, + "loss": 1.4513, + "theoretical_loss": 3.4324218033907408, + "tokens_seen": 2196359168 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002704112337011033, + "loss": 1.2465, + "theoretical_loss": 3.4324134900789507, + "tokens_seen": 2196424704 + }, + { + "epoch": 1.05, + "learning_rate": 0.00027039518555667, + "loss": 1.3193, + "theoretical_loss": 3.4324051770846578, + "tokens_seen": 2196490240 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002703791374122367, + "loss": 1.2637, + "theoretical_loss": 3.4323968644078398, + "tokens_seen": 2196555776 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002703630892678034, + "loss": 1.2241, + "theoretical_loss": 3.4323885520484754, + "tokens_seen": 2196621312 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002703470411233701, + "loss": 1.2314, + "theoretical_loss": 3.4323802400065433, + "tokens_seen": 2196686848 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002703309929789368, + "loss": 1.2741, + "theoretical_loss": 3.4323719282820213, + "tokens_seen": 2196752384 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002703149448345035, + "loss": 1.2748, + "theoretical_loss": 3.432363616874888, + "tokens_seen": 2196817920 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002702988966900702, + "loss": 1.2492, + "theoretical_loss": 3.4323553057851224, + "tokens_seen": 2196883456 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002702828485456369, + "loss": 1.3002, + "theoretical_loss": 3.4323469950127024, + "tokens_seen": 2196948992 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002702668004012036, + "loss": 1.3182, + "theoretical_loss": 3.4323386845576063, + "tokens_seen": 2197014528 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002702507522567703, + "loss": 1.3649, + "theoretical_loss": 3.432330374419813, + "tokens_seen": 2197080064 + }, + { + "epoch": 1.05, + "learning_rate": 0.000270234704112337, + "loss": 1.3047, + "theoretical_loss": 3.4323220645993002, + "tokens_seen": 2197145600 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002702186559679037, + "loss": 1.2561, + "theoretical_loss": 3.4323137550960467, + "tokens_seen": 2197211136 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002702026078234704, + "loss": 1.3212, + "theoretical_loss": 3.4323054459100315, + "tokens_seen": 2197276672 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002701865596790371, + "loss": 1.3401, + "theoretical_loss": 3.432297137041232, + "tokens_seen": 2197342208 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002701705115346038, + "loss": 1.1938, + "theoretical_loss": 3.4322888284896274, + "tokens_seen": 2197407744 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002701544633901705, + "loss": 1.4107, + "theoretical_loss": 3.432280520255196, + "tokens_seen": 2197473280 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002701384152457372, + "loss": 1.3858, + "theoretical_loss": 3.4322722123379155, + "tokens_seen": 2197538816 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002701223671013039, + "loss": 1.2738, + "theoretical_loss": 3.4322639047377654, + "tokens_seen": 2197604352 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002701063189568706, + "loss": 1.2436, + "theoretical_loss": 3.4322555974547235, + "tokens_seen": 2197669888 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002700902708124373, + "loss": 1.2864, + "theoretical_loss": 3.4322472904887684, + "tokens_seen": 2197735424 + }, + { + "epoch": 1.05, + "learning_rate": 0.000270074222668004, + "loss": 1.2782, + "theoretical_loss": 3.4322389838398784, + "tokens_seen": 2197800960 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002700581745235707, + "loss": 1.2142, + "theoretical_loss": 3.4322306775080325, + "tokens_seen": 2197866496 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002700421263791374, + "loss": 1.2993, + "theoretical_loss": 3.432222371493208, + "tokens_seen": 2197932032 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002700260782347041, + "loss": 1.4003, + "theoretical_loss": 3.432214065795385, + "tokens_seen": 2197997568 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002700100300902708, + "loss": 1.4194, + "theoretical_loss": 3.4322057604145404, + "tokens_seen": 2198063104 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002699939819458375, + "loss": 1.202, + "theoretical_loss": 3.4321974553506536, + "tokens_seen": 2198128640 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002699779338014042, + "loss": 1.3284, + "theoretical_loss": 3.4321891506037026, + "tokens_seen": 2198194176 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002699618856569709, + "loss": 1.3748, + "theoretical_loss": 3.432180846173666, + "tokens_seen": 2198259712 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002699458375125376, + "loss": 1.356, + "theoretical_loss": 3.4321725420605222, + "tokens_seen": 2198325248 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002699297893681043, + "loss": 1.3124, + "theoretical_loss": 3.4321642382642494, + "tokens_seen": 2198390784 + }, + { + "epoch": 1.05, + "learning_rate": 0.000269913741223671, + "loss": 1.2911, + "theoretical_loss": 3.432155934784827, + "tokens_seen": 2198456320 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002698976930792377, + "loss": 1.3533, + "theoretical_loss": 3.4321476316222324, + "tokens_seen": 2198521856 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026988164493480443, + "loss": 1.2298, + "theoretical_loss": 3.432139328776445, + "tokens_seen": 2198587392 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026986559679037113, + "loss": 1.2299, + "theoretical_loss": 3.4321310262474425, + "tokens_seen": 2198652928 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026984954864593783, + "loss": 1.2351, + "theoretical_loss": 3.4321227240352035, + "tokens_seen": 2198718464 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026983350050150453, + "loss": 1.3313, + "theoretical_loss": 3.4321144221397066, + "tokens_seen": 2198784000 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026981745235707123, + "loss": 1.3169, + "theoretical_loss": 3.4321061205609307, + "tokens_seen": 2198849536 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026980140421263793, + "loss": 1.3717, + "theoretical_loss": 3.4320978192988534, + "tokens_seen": 2198915072 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026978535606820463, + "loss": 1.3572, + "theoretical_loss": 3.432089518353454, + "tokens_seen": 2198980608 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026976930792377133, + "loss": 1.2731, + "theoretical_loss": 3.4320812177247104, + "tokens_seen": 2199046144 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026975325977933803, + "loss": 1.2481, + "theoretical_loss": 3.4320729174126017, + "tokens_seen": 2199111680 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026973721163490473, + "loss": 1.2509, + "theoretical_loss": 3.4320646174171054, + "tokens_seen": 2199177216 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026972116349047143, + "loss": 1.2388, + "theoretical_loss": 3.432056317738201, + "tokens_seen": 2199242752 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002697051153460381, + "loss": 1.2877, + "theoretical_loss": 3.4320480183758666, + "tokens_seen": 2199308288 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002696890672016048, + "loss": 1.345, + "theoretical_loss": 3.4320397193300805, + "tokens_seen": 2199373824 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002696730190571715, + "loss": 1.4045, + "theoretical_loss": 3.4320314206008216, + "tokens_seen": 2199439360 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026965697091273824, + "loss": 1.2432, + "theoretical_loss": 3.4320231221880677, + "tokens_seen": 2199504896 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026964092276830494, + "loss": 1.2623, + "theoretical_loss": 3.4320148240917985, + "tokens_seen": 2199570432 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026962487462387164, + "loss": 1.2832, + "theoretical_loss": 3.432006526311991, + "tokens_seen": 2199635968 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026960882647943834, + "loss": 1.3662, + "theoretical_loss": 3.431998228848625, + "tokens_seen": 2199701504 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026959277833500504, + "loss": 1.2248, + "theoretical_loss": 3.4319899317016787, + "tokens_seen": 2199767040 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026957673019057174, + "loss": 1.2984, + "theoretical_loss": 3.43198163487113, + "tokens_seen": 2199832576 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026956068204613844, + "loss": 1.3553, + "theoretical_loss": 3.4319733383569577, + "tokens_seen": 2199898112 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026954463390170514, + "loss": 1.3593, + "theoretical_loss": 3.4319650421591406, + "tokens_seen": 2199963648 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026952858575727184, + "loss": 1.3387, + "theoretical_loss": 3.431956746277657, + "tokens_seen": 2200029184 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026951253761283854, + "loss": 1.2224, + "theoretical_loss": 3.4319484507124853, + "tokens_seen": 2200094720 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026949648946840524, + "loss": 1.3175, + "theoretical_loss": 3.431940155463604, + "tokens_seen": 2200160256 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026948044132397194, + "loss": 1.1766, + "theoretical_loss": 3.4319318605309923, + "tokens_seen": 2200225792 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026946439317953864, + "loss": 1.2799, + "theoretical_loss": 3.431923565914628, + "tokens_seen": 2200291328 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026944834503510534, + "loss": 1.2814, + "theoretical_loss": 3.4319152716144896, + "tokens_seen": 2200356864 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026943229689067205, + "loss": 1.2225, + "theoretical_loss": 3.4319069776305557, + "tokens_seen": 2200422400 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026941624874623875, + "loss": 1.3122, + "theoretical_loss": 3.4318986839628054, + "tokens_seen": 2200487936 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026940020060180545, + "loss": 1.358, + "theoretical_loss": 3.431890390611217, + "tokens_seen": 2200553472 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026938415245737215, + "loss": 1.3694, + "theoretical_loss": 3.4318820975757682, + "tokens_seen": 2200619008 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026936810431293885, + "loss": 1.3525, + "theoretical_loss": 3.431873804856439, + "tokens_seen": 2200684544 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026935205616850555, + "loss": 1.2897, + "theoretical_loss": 3.4318655124532063, + "tokens_seen": 2200750080 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026933600802407225, + "loss": 1.1937, + "theoretical_loss": 3.43185722036605, + "tokens_seen": 2200815616 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026931995987963895, + "loss": 1.2286, + "theoretical_loss": 3.431848928594947, + "tokens_seen": 2200881152 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026930391173520565, + "loss": 1.3975, + "theoretical_loss": 3.431840637139878, + "tokens_seen": 2200946688 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026928786359077235, + "loss": 1.2049, + "theoretical_loss": 3.43183234600082, + "tokens_seen": 2201012224 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026927181544633905, + "loss": 1.4276, + "theoretical_loss": 3.4318240551777524, + "tokens_seen": 2201077760 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026925576730190575, + "loss": 1.2819, + "theoretical_loss": 3.4318157646706533, + "tokens_seen": 2201143296 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026923971915747245, + "loss": 1.2884, + "theoretical_loss": 3.431807474479501, + "tokens_seen": 2201208832 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026922367101303915, + "loss": 1.2741, + "theoretical_loss": 3.431799184604275, + "tokens_seen": 2201274368 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026920762286860585, + "loss": 1.2728, + "theoretical_loss": 3.4317908950449527, + "tokens_seen": 2201339904 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026919157472417255, + "loss": 1.321, + "theoretical_loss": 3.431782605801513, + "tokens_seen": 2201405440 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026917552657973926, + "loss": 1.2116, + "theoretical_loss": 3.431774316873935, + "tokens_seen": 2201470976 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026915947843530596, + "loss": 1.2858, + "theoretical_loss": 3.431766028262197, + "tokens_seen": 2201536512 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026914343029087266, + "loss": 1.3256, + "theoretical_loss": 3.4317577399662778, + "tokens_seen": 2201602048 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026912738214643936, + "loss": 1.2498, + "theoretical_loss": 3.431749451986155, + "tokens_seen": 2201667584 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026911133400200606, + "loss": 1.238, + "theoretical_loss": 3.431741164321808, + "tokens_seen": 2201733120 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026909528585757276, + "loss": 1.3533, + "theoretical_loss": 3.431732876973215, + "tokens_seen": 2201798656 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002690792377131394, + "loss": 1.2356, + "theoretical_loss": 3.4317245899403552, + "tokens_seen": 2201864192 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002690631895687061, + "loss": 1.324, + "theoretical_loss": 3.4317163032232063, + "tokens_seen": 2201929728 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 814453, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2705497741699219, + "objective/train/theoretical_loss": 3.431708016821747, + "objective/train/tokens_used": 2222455264, + "theoretical_loss": 3.431708016821747, + "tokens_seen": 2201995264 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002690471414242728, + "loss": 1.2705, + "theoretical_loss": 3.431708016821747, + "tokens_seen": 2201995264 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002690310932798395, + "loss": 1.2451, + "theoretical_loss": 3.431699730735957, + "tokens_seen": 2202060800 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002690150451354062, + "loss": 1.2432, + "theoretical_loss": 3.431691444965814, + "tokens_seen": 2202126336 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002689989969909729, + "loss": 1.3592, + "theoretical_loss": 3.431683159511296, + "tokens_seen": 2202191872 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002689829488465396, + "loss": 1.1328, + "theoretical_loss": 3.4316748743723826, + "tokens_seen": 2202257408 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002689669007021063, + "loss": 1.2892, + "theoretical_loss": 3.431666589549052, + "tokens_seen": 2202322944 + }, + { + "epoch": 1.05, + "learning_rate": 0.000268950852557673, + "loss": 1.2749, + "theoretical_loss": 3.431658305041283, + "tokens_seen": 2202388480 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002689348044132397, + "loss": 1.1474, + "theoretical_loss": 3.4316500208490535, + "tokens_seen": 2202454016 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002689187562688064, + "loss": 1.2519, + "theoretical_loss": 3.4316417369723426, + "tokens_seen": 2202519552 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002689027081243731, + "loss": 1.2357, + "theoretical_loss": 3.4316334534111297, + "tokens_seen": 2202585088 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002688866599799398, + "loss": 1.1418, + "theoretical_loss": 3.431625170165392, + "tokens_seen": 2202650624 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002688706118355065, + "loss": 1.3427, + "theoretical_loss": 3.4316168872351085, + "tokens_seen": 2202716160 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002688545636910732, + "loss": 1.2129, + "theoretical_loss": 3.431608604620258, + "tokens_seen": 2202781696 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002688385155466399, + "loss": 1.232, + "theoretical_loss": 3.4316003223208194, + "tokens_seen": 2202847232 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002688224674022066, + "loss": 1.3742, + "theoretical_loss": 3.431592040336771, + "tokens_seen": 2202912768 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002688064192577733, + "loss": 1.2773, + "theoretical_loss": 3.431583758668091, + "tokens_seen": 2202978304 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026879037111334, + "loss": 1.2015, + "theoretical_loss": 3.431575477314759, + "tokens_seen": 2203043840 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002687743229689067, + "loss": 1.3003, + "theoretical_loss": 3.4315671962767524, + "tokens_seen": 2203109376 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002687582748244734, + "loss": 1.3738, + "theoretical_loss": 3.4315589155540507, + "tokens_seen": 2203174912 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002687422266800401, + "loss": 1.2639, + "theoretical_loss": 3.4315506351466323, + "tokens_seen": 2203240448 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002687261785356068, + "loss": 1.233, + "theoretical_loss": 3.431542355054476, + "tokens_seen": 2203305984 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002687101303911735, + "loss": 1.3563, + "theoretical_loss": 3.4315340752775594, + "tokens_seen": 2203371520 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002686940822467402, + "loss": 1.2916, + "theoretical_loss": 3.4315257958158627, + "tokens_seen": 2203437056 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002686780341023069, + "loss": 1.186, + "theoretical_loss": 3.431517516669363, + "tokens_seen": 2203502592 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002686619859578736, + "loss": 1.235, + "theoretical_loss": 3.4315092378380405, + "tokens_seen": 2203568128 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002686459378134403, + "loss": 1.3629, + "theoretical_loss": 3.4315009593218724, + "tokens_seen": 2203633664 + }, + { + "epoch": 1.05, + "learning_rate": 0.000268629889669007, + "loss": 1.2258, + "theoretical_loss": 3.4314926811208384, + "tokens_seen": 2203699200 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002686138415245737, + "loss": 1.2587, + "theoretical_loss": 3.4314844032349163, + "tokens_seen": 2203764736 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002685977933801404, + "loss": 1.3764, + "theoretical_loss": 3.4314761256640853, + "tokens_seen": 2203830272 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002685817452357071, + "loss": 1.3729, + "theoretical_loss": 3.4314678484083236, + "tokens_seen": 2203895808 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002685656970912738, + "loss": 1.2116, + "theoretical_loss": 3.43145957146761, + "tokens_seen": 2203961344 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002685496489468405, + "loss": 1.349, + "theoretical_loss": 3.4314512948419233, + "tokens_seen": 2204026880 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002685336008024072, + "loss": 1.2796, + "theoretical_loss": 3.431443018531242, + "tokens_seen": 2204092416 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026851755265797393, + "loss": 1.2719, + "theoretical_loss": 3.4314347425355454, + "tokens_seen": 2204157952 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026850150451354063, + "loss": 1.2714, + "theoretical_loss": 3.431426466854811, + "tokens_seen": 2204223488 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026848545636910733, + "loss": 1.1202, + "theoretical_loss": 3.431418191489018, + "tokens_seen": 2204289024 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026846940822467403, + "loss": 1.3012, + "theoretical_loss": 3.431409916438145, + "tokens_seen": 2204354560 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026845336008024073, + "loss": 1.3333, + "theoretical_loss": 3.4314016417021707, + "tokens_seen": 2204420096 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026843731193580743, + "loss": 1.3081, + "theoretical_loss": 3.4313933672810735, + "tokens_seen": 2204485632 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026842126379137413, + "loss": 1.2876, + "theoretical_loss": 3.431385093174833, + "tokens_seen": 2204551168 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026840521564694083, + "loss": 1.2486, + "theoretical_loss": 3.431376819383426, + "tokens_seen": 2204616704 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026838916750250753, + "loss": 1.226, + "theoretical_loss": 3.4313685459068335, + "tokens_seen": 2204682240 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026837311935807423, + "loss": 1.2547, + "theoretical_loss": 3.4313602727450325, + "tokens_seen": 2204747776 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026835707121364093, + "loss": 1.2989, + "theoretical_loss": 3.4313519998980024, + "tokens_seen": 2204813312 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026834102306920763, + "loss": 1.2282, + "theoretical_loss": 3.4313437273657215, + "tokens_seen": 2204878848 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026832497492477433, + "loss": 1.2508, + "theoretical_loss": 3.4313354551481687, + "tokens_seen": 2204944384 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026830892678034104, + "loss": 1.428, + "theoretical_loss": 3.4313271832453225, + "tokens_seen": 2205009920 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026829287863590774, + "loss": 1.2191, + "theoretical_loss": 3.4313189116571614, + "tokens_seen": 2205075456 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026827683049147444, + "loss": 1.2142, + "theoretical_loss": 3.4313106403836646, + "tokens_seen": 2205140992 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026826078234704114, + "loss": 1.3723, + "theoretical_loss": 3.4313023694248104, + "tokens_seen": 2205206528 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026824473420260784, + "loss": 1.3024, + "theoretical_loss": 3.4312940987805773, + "tokens_seen": 2205272064 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026822868605817454, + "loss": 1.3386, + "theoretical_loss": 3.4312858284509447, + "tokens_seen": 2205337600 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026821263791374124, + "loss": 1.3955, + "theoretical_loss": 3.4312775584358906, + "tokens_seen": 2205403136 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026819658976930794, + "loss": 1.2339, + "theoretical_loss": 3.431269288735394, + "tokens_seen": 2205468672 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026818054162487464, + "loss": 1.2531, + "theoretical_loss": 3.4312610193494337, + "tokens_seen": 2205534208 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026816449348044134, + "loss": 1.3112, + "theoretical_loss": 3.431252750277988, + "tokens_seen": 2205599744 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026814844533600804, + "loss": 1.2757, + "theoretical_loss": 3.431244481521036, + "tokens_seen": 2205665280 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026813239719157474, + "loss": 1.3816, + "theoretical_loss": 3.431236213078556, + "tokens_seen": 2205730816 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026811634904714144, + "loss": 1.3076, + "theoretical_loss": 3.431227944950527, + "tokens_seen": 2205796352 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026810030090270814, + "loss": 1.3032, + "theoretical_loss": 3.4312196771369274, + "tokens_seen": 2205861888 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026808425275827484, + "loss": 1.3151, + "theoretical_loss": 3.4312114096377364, + "tokens_seen": 2205927424 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026806820461384154, + "loss": 1.1672, + "theoretical_loss": 3.431203142452932, + "tokens_seen": 2205992960 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026805215646940825, + "loss": 1.2912, + "theoretical_loss": 3.431194875582494, + "tokens_seen": 2206058496 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026803610832497495, + "loss": 1.2342, + "theoretical_loss": 3.4311866090264, + "tokens_seen": 2206124032 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026802006018054165, + "loss": 1.3094, + "theoretical_loss": 3.431178342784629, + "tokens_seen": 2206189568 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026800401203610835, + "loss": 1.3719, + "theoretical_loss": 3.43117007685716, + "tokens_seen": 2206255104 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026798796389167505, + "loss": 1.205, + "theoretical_loss": 3.4311618112439715, + "tokens_seen": 2206320640 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026797191574724175, + "loss": 1.3001, + "theoretical_loss": 3.431153545945042, + "tokens_seen": 2206386176 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026795586760280845, + "loss": 1.3638, + "theoretical_loss": 3.431145280960351, + "tokens_seen": 2206451712 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026793981945837515, + "loss": 1.3978, + "theoretical_loss": 3.4311370162898767, + "tokens_seen": 2206517248 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026792377131394185, + "loss": 1.2946, + "theoretical_loss": 3.4311287519335973, + "tokens_seen": 2206582784 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026790772316950855, + "loss": 1.2531, + "theoretical_loss": 3.4311204878914925, + "tokens_seen": 2206648320 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026789167502507525, + "loss": 1.1701, + "theoretical_loss": 3.4311122241635403, + "tokens_seen": 2206713856 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026787562688064195, + "loss": 1.2619, + "theoretical_loss": 3.43110396074972, + "tokens_seen": 2206779392 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026785957873620865, + "loss": 1.3504, + "theoretical_loss": 3.4310956976500098, + "tokens_seen": 2206844928 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026784353059177535, + "loss": 1.3689, + "theoretical_loss": 3.4310874348643887, + "tokens_seen": 2206910464 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026782748244734205, + "loss": 1.2827, + "theoretical_loss": 3.4310791723928356, + "tokens_seen": 2206976000 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026781143430290876, + "loss": 1.2287, + "theoretical_loss": 3.431070910235329, + "tokens_seen": 2207041536 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026779538615847546, + "loss": 1.2324, + "theoretical_loss": 3.4310626483918476, + "tokens_seen": 2207107072 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026777933801404216, + "loss": 1.4274, + "theoretical_loss": 3.43105438686237, + "tokens_seen": 2207172608 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026776328986960886, + "loss": 1.1961, + "theoretical_loss": 3.4310461256468754, + "tokens_seen": 2207238144 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026774724172517556, + "loss": 1.1989, + "theoretical_loss": 3.431037864745342, + "tokens_seen": 2207303680 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026773119358074226, + "loss": 1.4161, + "theoretical_loss": 3.4310296041577493, + "tokens_seen": 2207369216 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026771514543630896, + "loss": 1.2091, + "theoretical_loss": 3.4310213438840753, + "tokens_seen": 2207434752 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026769909729187566, + "loss": 1.3321, + "theoretical_loss": 3.4310130839242996, + "tokens_seen": 2207500288 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002676830491474423, + "loss": 1.1933, + "theoretical_loss": 3.4310048242784, + "tokens_seen": 2207565824 + }, + { + "epoch": 1.05, + "learning_rate": 0.000267667001003009, + "loss": 1.2446, + "theoretical_loss": 3.4309965649463554, + "tokens_seen": 2207631360 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002676509528585757, + "loss": 1.235, + "theoretical_loss": 3.430988305928145, + "tokens_seen": 2207696896 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002676349047141424, + "loss": 1.2539, + "theoretical_loss": 3.4309800472237475, + "tokens_seen": 2207762432 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002676188565697091, + "loss": 1.2474, + "theoretical_loss": 3.4309717888331415, + "tokens_seen": 2207827968 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002676028084252758, + "loss": 1.3703, + "theoretical_loss": 3.430963530756306, + "tokens_seen": 2207893504 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002675867602808425, + "loss": 1.1774, + "theoretical_loss": 3.430955272993219, + "tokens_seen": 2207959040 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002675707121364092, + "loss": 1.2114, + "theoretical_loss": 3.43094701554386, + "tokens_seen": 2208024576 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002675546639919759, + "loss": 1.223, + "theoretical_loss": 3.430938758408208, + "tokens_seen": 2208090112 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002675386158475426, + "loss": 1.3041, + "theoretical_loss": 3.430930501586241, + "tokens_seen": 2208155648 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002675225677031093, + "loss": 1.4648, + "theoretical_loss": 3.4309222450779386, + "tokens_seen": 2208221184 + }, + { + "epoch": 1.05, + "learning_rate": 0.000267506519558676, + "loss": 1.3499, + "theoretical_loss": 3.4309139888832787, + "tokens_seen": 2208286720 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002674904714142427, + "loss": 1.2697, + "theoretical_loss": 3.4309057330022408, + "tokens_seen": 2208352256 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002674744232698094, + "loss": 1.2839, + "theoretical_loss": 3.430897477434803, + "tokens_seen": 2208417792 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002674583751253761, + "loss": 1.1941, + "theoretical_loss": 3.4308892221809453, + "tokens_seen": 2208483328 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 816733, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.251988410949707, + "objective/train/theoretical_loss": 3.430880967240645, + "objective/train/tokens_used": 2229008864, + "theoretical_loss": 3.430880967240645, + "tokens_seen": 2208548864 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002674423269809428, + "loss": 1.252, + "theoretical_loss": 3.430880967240645, + "tokens_seen": 2208548864 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002674262788365095, + "loss": 1.2799, + "theoretical_loss": 3.4308727126138816, + "tokens_seen": 2208614400 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002674102306920762, + "loss": 1.3051, + "theoretical_loss": 3.4308644583006336, + "tokens_seen": 2208679936 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002673941825476429, + "loss": 1.2787, + "theoretical_loss": 3.4308562043008806, + "tokens_seen": 2208745472 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002673781344032096, + "loss": 1.3176, + "theoretical_loss": 3.4308479506146004, + "tokens_seen": 2208811008 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002673620862587763, + "loss": 1.442, + "theoretical_loss": 3.430839697241772, + "tokens_seen": 2208876544 + }, + { + "epoch": 1.05, + "learning_rate": 0.000267346038114343, + "loss": 1.32, + "theoretical_loss": 3.430831444182375, + "tokens_seen": 2208942080 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002673299899699097, + "loss": 1.2555, + "theoretical_loss": 3.430823191436387, + "tokens_seen": 2209007616 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002673139418254764, + "loss": 1.3256, + "theoretical_loss": 3.430814939003788, + "tokens_seen": 2209073152 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002672978936810431, + "loss": 1.2654, + "theoretical_loss": 3.430806686884556, + "tokens_seen": 2209138688 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002672818455366098, + "loss": 1.3137, + "theoretical_loss": 3.4307984350786698, + "tokens_seen": 2209204224 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002672657973921766, + "loss": 1.2299, + "theoretical_loss": 3.4307901835861085, + "tokens_seen": 2209269760 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002672497492477433, + "loss": 1.2046, + "theoretical_loss": 3.430781932406851, + "tokens_seen": 2209335296 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026723370110331, + "loss": 1.3545, + "theoretical_loss": 3.4307736815408756, + "tokens_seen": 2209400832 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002672176529588767, + "loss": 1.2651, + "theoretical_loss": 3.4307654309881617, + "tokens_seen": 2209466368 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002672016048144434, + "loss": 1.205, + "theoretical_loss": 3.430757180748688, + "tokens_seen": 2209531904 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002671855566700101, + "loss": 1.2716, + "theoretical_loss": 3.430748930822433, + "tokens_seen": 2209597440 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002671695085255768, + "loss": 1.2809, + "theoretical_loss": 3.4307406812093753, + "tokens_seen": 2209662976 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002671534603811435, + "loss": 1.3081, + "theoretical_loss": 3.4307324319094947, + "tokens_seen": 2209728512 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002671374122367102, + "loss": 1.2378, + "theoretical_loss": 3.430724182922769, + "tokens_seen": 2209794048 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002671213640922769, + "loss": 1.2329, + "theoretical_loss": 3.4307159342491778, + "tokens_seen": 2209859584 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002671053159478436, + "loss": 1.3439, + "theoretical_loss": 3.4307076858886996, + "tokens_seen": 2209925120 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002670892678034103, + "loss": 1.2827, + "theoretical_loss": 3.430699437841313, + "tokens_seen": 2209990656 + }, + { + "epoch": 1.05, + "learning_rate": 0.000267073219658977, + "loss": 1.2262, + "theoretical_loss": 3.430691190106997, + "tokens_seen": 2210056192 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002670571715145437, + "loss": 1.1805, + "theoretical_loss": 3.430682942685731, + "tokens_seen": 2210121728 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026704112337011033, + "loss": 1.3012, + "theoretical_loss": 3.430674695577493, + "tokens_seen": 2210187264 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026702507522567703, + "loss": 1.2895, + "theoretical_loss": 3.430666448782262, + "tokens_seen": 2210252800 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026700902708124373, + "loss": 1.132, + "theoretical_loss": 3.4306582023000174, + "tokens_seen": 2210318336 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026699297893681043, + "loss": 1.2152, + "theoretical_loss": 3.4306499561307375, + "tokens_seen": 2210383872 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026697693079237713, + "loss": 1.3216, + "theoretical_loss": 3.430641710274401, + "tokens_seen": 2210449408 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026696088264794383, + "loss": 1.2738, + "theoretical_loss": 3.4306334647309873, + "tokens_seen": 2210514944 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026694483450351054, + "loss": 1.1886, + "theoretical_loss": 3.430625219500475, + "tokens_seen": 2210580480 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026692878635907724, + "loss": 1.3825, + "theoretical_loss": 3.430616974582843, + "tokens_seen": 2210646016 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026691273821464394, + "loss": 1.3775, + "theoretical_loss": 3.43060872997807, + "tokens_seen": 2210711552 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026689669007021064, + "loss": 1.1692, + "theoretical_loss": 3.430600485686135, + "tokens_seen": 2210777088 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026688064192577734, + "loss": 1.3103, + "theoretical_loss": 3.430592241707017, + "tokens_seen": 2210842624 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026686459378134404, + "loss": 1.3334, + "theoretical_loss": 3.430583998040694, + "tokens_seen": 2210908160 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026684854563691074, + "loss": 1.249, + "theoretical_loss": 3.430575754687146, + "tokens_seen": 2210973696 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026683249749247744, + "loss": 1.3171, + "theoretical_loss": 3.430567511646351, + "tokens_seen": 2211039232 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026681644934804414, + "loss": 1.2021, + "theoretical_loss": 3.430559268918289, + "tokens_seen": 2211104768 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026680040120361084, + "loss": 1.2405, + "theoretical_loss": 3.4305510265029375, + "tokens_seen": 2211170304 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026678435305917754, + "loss": 1.3059, + "theoretical_loss": 3.4305427844002763, + "tokens_seen": 2211235840 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026676830491474424, + "loss": 1.3664, + "theoretical_loss": 3.4305345426102836, + "tokens_seen": 2211301376 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026675225677031094, + "loss": 1.2761, + "theoretical_loss": 3.4305263011329385, + "tokens_seen": 2211366912 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026673620862587764, + "loss": 1.2714, + "theoretical_loss": 3.4305180599682203, + "tokens_seen": 2211432448 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026672016048144434, + "loss": 1.227, + "theoretical_loss": 3.4305098191161076, + "tokens_seen": 2211497984 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026670411233701104, + "loss": 1.2262, + "theoretical_loss": 3.4305015785765796, + "tokens_seen": 2211563520 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026668806419257775, + "loss": 1.2759, + "theoretical_loss": 3.4304933383496143, + "tokens_seen": 2211629056 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026667201604814445, + "loss": 1.3112, + "theoretical_loss": 3.4304850984351916, + "tokens_seen": 2211694592 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026665596790371115, + "loss": 1.2538, + "theoretical_loss": 3.4304768588332895, + "tokens_seen": 2211760128 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026663991975927785, + "loss": 1.3068, + "theoretical_loss": 3.430468619543887, + "tokens_seen": 2211825664 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026662387161484455, + "loss": 1.2845, + "theoretical_loss": 3.430460380566964, + "tokens_seen": 2211891200 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026660782347041125, + "loss": 1.2817, + "theoretical_loss": 3.430452141902498, + "tokens_seen": 2211956736 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026659177532597795, + "loss": 1.2575, + "theoretical_loss": 3.430443903550469, + "tokens_seen": 2212022272 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026657572718154465, + "loss": 1.22, + "theoretical_loss": 3.4304356655108554, + "tokens_seen": 2212087808 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026655967903711135, + "loss": 1.2823, + "theoretical_loss": 3.430427427783636, + "tokens_seen": 2212153344 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026654363089267805, + "loss": 1.2623, + "theoretical_loss": 3.4304191903687897, + "tokens_seen": 2212218880 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026652758274824475, + "loss": 1.2916, + "theoretical_loss": 3.4304109532662963, + "tokens_seen": 2212284416 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026651153460381145, + "loss": 1.243, + "theoretical_loss": 3.4304027164761335, + "tokens_seen": 2212349952 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026649548645937815, + "loss": 1.3885, + "theoretical_loss": 3.4303944799982804, + "tokens_seen": 2212415488 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026647943831494485, + "loss": 1.2917, + "theoretical_loss": 3.430386243832716, + "tokens_seen": 2212481024 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026646339017051155, + "loss": 1.3126, + "theoretical_loss": 3.4303780079794204, + "tokens_seen": 2212546560 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026644734202607825, + "loss": 1.1819, + "theoretical_loss": 3.430369772438371, + "tokens_seen": 2212612096 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026643129388164496, + "loss": 1.2631, + "theoretical_loss": 3.430361537209547, + "tokens_seen": 2212677632 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026641524573721166, + "loss": 1.3772, + "theoretical_loss": 3.4303533022929273, + "tokens_seen": 2212743168 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026639919759277836, + "loss": 1.205, + "theoretical_loss": 3.4303450676884912, + "tokens_seen": 2212808704 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026638314944834506, + "loss": 1.2543, + "theoretical_loss": 3.430336833396218, + "tokens_seen": 2212874240 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026636710130391176, + "loss": 1.2203, + "theoretical_loss": 3.430328599416086, + "tokens_seen": 2212939776 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026635105315947846, + "loss": 1.2224, + "theoretical_loss": 3.4303203657480736, + "tokens_seen": 2213005312 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026633500501504516, + "loss": 1.2279, + "theoretical_loss": 3.4303121323921606, + "tokens_seen": 2213070848 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026631895687061186, + "loss": 1.2283, + "theoretical_loss": 3.430303899348326, + "tokens_seen": 2213136384 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026630290872617856, + "loss": 1.2747, + "theoretical_loss": 3.430295666616548, + "tokens_seen": 2213201920 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026628686058174526, + "loss": 1.2785, + "theoretical_loss": 3.430287434196806, + "tokens_seen": 2213267456 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002662708124373119, + "loss": 1.2318, + "theoretical_loss": 3.430279202089079, + "tokens_seen": 2213332992 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002662547642928786, + "loss": 1.271, + "theoretical_loss": 3.4302709702933454, + "tokens_seen": 2213398528 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002662387161484453, + "loss": 1.3225, + "theoretical_loss": 3.430262738809585, + "tokens_seen": 2213464064 + }, + { + "epoch": 1.05, + "learning_rate": 0.000266222668004012, + "loss": 1.2582, + "theoretical_loss": 3.430254507637776, + "tokens_seen": 2213529600 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002662066198595787, + "loss": 1.241, + "theoretical_loss": 3.430246276777898, + "tokens_seen": 2213595136 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002661905717151454, + "loss": 1.2722, + "theoretical_loss": 3.4302380462299293, + "tokens_seen": 2213660672 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002661745235707121, + "loss": 1.3708, + "theoretical_loss": 3.430229815993849, + "tokens_seen": 2213726208 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002661584754262788, + "loss": 1.2757, + "theoretical_loss": 3.4302215860696363, + "tokens_seen": 2213791744 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002661424272818455, + "loss": 1.356, + "theoretical_loss": 3.4302133564572697, + "tokens_seen": 2213857280 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002661263791374122, + "loss": 1.3148, + "theoretical_loss": 3.430205127156729, + "tokens_seen": 2213922816 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002661103309929789, + "loss": 1.284, + "theoretical_loss": 3.430196898167992, + "tokens_seen": 2213988352 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002660942828485456, + "loss": 1.3082, + "theoretical_loss": 3.430188669491039, + "tokens_seen": 2214053888 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002660782347041123, + "loss": 1.2754, + "theoretical_loss": 3.430180441125848, + "tokens_seen": 2214119424 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026606218655967907, + "loss": 1.2516, + "theoretical_loss": 3.4301722130723977, + "tokens_seen": 2214184960 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026604613841524577, + "loss": 1.3043, + "theoretical_loss": 3.430163985330668, + "tokens_seen": 2214250496 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026603009027081247, + "loss": 1.3381, + "theoretical_loss": 3.4301557579006374, + "tokens_seen": 2214316032 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026601404212637917, + "loss": 1.2035, + "theoretical_loss": 3.4301475307822846, + "tokens_seen": 2214381568 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026599799398194587, + "loss": 1.2312, + "theoretical_loss": 3.430139303975589, + "tokens_seen": 2214447104 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002659819458375126, + "loss": 1.2255, + "theoretical_loss": 3.4301310774805294, + "tokens_seen": 2214512640 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002659658976930793, + "loss": 1.2956, + "theoretical_loss": 3.430122851297085, + "tokens_seen": 2214578176 + }, + { + "epoch": 1.05, + "learning_rate": 0.000265949849548646, + "loss": 1.2434, + "theoretical_loss": 3.4301146254252344, + "tokens_seen": 2214643712 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002659338014042127, + "loss": 1.3124, + "theoretical_loss": 3.4301063998649566, + "tokens_seen": 2214709248 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002659177532597794, + "loss": 1.2511, + "theoretical_loss": 3.430098174616231, + "tokens_seen": 2214774784 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002659017051153461, + "loss": 1.2851, + "theoretical_loss": 3.4300899496790365, + "tokens_seen": 2214840320 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002658856569709128, + "loss": 1.1972, + "theoretical_loss": 3.4300817250533515, + "tokens_seen": 2214905856 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002658696088264795, + "loss": 1.1861, + "theoretical_loss": 3.430073500739155, + "tokens_seen": 2214971392 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002658535606820462, + "loss": 1.3112, + "theoretical_loss": 3.4300652767364275, + "tokens_seen": 2215036928 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 819114, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.147014856338501, + "objective/train/theoretical_loss": 3.4300570530451457, + "objective/train/tokens_used": 2235562464, + "theoretical_loss": 3.4300570530451457, + "tokens_seen": 2215102464 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002658375125376129, + "loss": 1.147, + "theoretical_loss": 3.4300570530451457, + "tokens_seen": 2215102464 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002658214643931796, + "loss": 1.306, + "theoretical_loss": 3.4300488296652905, + "tokens_seen": 2215168000 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002658054162487463, + "loss": 1.2905, + "theoretical_loss": 3.43004060659684, + "tokens_seen": 2215233536 + }, + { + "epoch": 1.05, + "learning_rate": 0.000265789368104313, + "loss": 1.3343, + "theoretical_loss": 3.430032383839773, + "tokens_seen": 2215299072 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002657733199598797, + "loss": 1.2846, + "theoretical_loss": 3.430024161394069, + "tokens_seen": 2215364608 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002657572718154464, + "loss": 1.314, + "theoretical_loss": 3.4300159392597074, + "tokens_seen": 2215430144 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002657412236710131, + "loss": 1.3675, + "theoretical_loss": 3.430007717436666, + "tokens_seen": 2215495680 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002657251755265798, + "loss": 1.2561, + "theoretical_loss": 3.4299994959249247, + "tokens_seen": 2215561216 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002657091273821465, + "loss": 1.2126, + "theoretical_loss": 3.429991274724462, + "tokens_seen": 2215626752 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002656930792377132, + "loss": 1.2702, + "theoretical_loss": 3.429983053835257, + "tokens_seen": 2215692288 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002656770310932799, + "loss": 1.222, + "theoretical_loss": 3.4299748332572895, + "tokens_seen": 2215757824 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002656609829488466, + "loss": 1.2495, + "theoretical_loss": 3.4299666129905373, + "tokens_seen": 2215823360 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026564493480441323, + "loss": 1.3392, + "theoretical_loss": 3.42995839303498, + "tokens_seen": 2215888896 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026562888665997993, + "loss": 1.2558, + "theoretical_loss": 3.4299501733905973, + "tokens_seen": 2215954432 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026561283851554663, + "loss": 1.2418, + "theoretical_loss": 3.429941954057367, + "tokens_seen": 2216019968 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026559679037111333, + "loss": 1.2784, + "theoretical_loss": 3.4299337350352683, + "tokens_seen": 2216085504 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026558074222668003, + "loss": 1.4311, + "theoretical_loss": 3.429925516324281, + "tokens_seen": 2216151040 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026556469408224674, + "loss": 1.2422, + "theoretical_loss": 3.429917297924384, + "tokens_seen": 2216216576 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026554864593781344, + "loss": 1.1655, + "theoretical_loss": 3.4299090798355554, + "tokens_seen": 2216282112 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026553259779338014, + "loss": 1.2912, + "theoretical_loss": 3.4299008620577753, + "tokens_seen": 2216347648 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026551654964894684, + "loss": 1.2277, + "theoretical_loss": 3.429892644591022, + "tokens_seen": 2216413184 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026550050150451354, + "loss": 1.2672, + "theoretical_loss": 3.4298844274352747, + "tokens_seen": 2216478720 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026548445336008024, + "loss": 1.2566, + "theoretical_loss": 3.4298762105905127, + "tokens_seen": 2216544256 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026546840521564694, + "loss": 1.2109, + "theoretical_loss": 3.429867994056715, + "tokens_seen": 2216609792 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026545235707121364, + "loss": 1.2577, + "theoretical_loss": 3.4298597778338604, + "tokens_seen": 2216675328 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026543630892678034, + "loss": 1.3, + "theoretical_loss": 3.429851561921928, + "tokens_seen": 2216740864 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026542026078234704, + "loss": 1.3641, + "theoretical_loss": 3.429843346320897, + "tokens_seen": 2216806400 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026540421263791374, + "loss": 1.1841, + "theoretical_loss": 3.429835131030746, + "tokens_seen": 2216871936 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026538816449348044, + "loss": 1.2122, + "theoretical_loss": 3.429826916051455, + "tokens_seen": 2216937472 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026537211634904714, + "loss": 1.2211, + "theoretical_loss": 3.429818701383002, + "tokens_seen": 2217003008 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026535606820461384, + "loss": 1.3645, + "theoretical_loss": 3.4298104870253665, + "tokens_seen": 2217068544 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026534002006018054, + "loss": 1.275, + "theoretical_loss": 3.429802272978528, + "tokens_seen": 2217134080 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026532397191574724, + "loss": 1.2846, + "theoretical_loss": 3.4297940592424645, + "tokens_seen": 2217199616 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026530792377131395, + "loss": 1.3519, + "theoretical_loss": 3.4297858458171557, + "tokens_seen": 2217265152 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026529187562688065, + "loss": 1.3319, + "theoretical_loss": 3.4297776327025806, + "tokens_seen": 2217330688 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026527582748244735, + "loss": 1.2126, + "theoretical_loss": 3.4297694198987188, + "tokens_seen": 2217396224 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026525977933801405, + "loss": 1.2187, + "theoretical_loss": 3.4297612074055484, + "tokens_seen": 2217461760 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026524373119358075, + "loss": 1.2209, + "theoretical_loss": 3.4297529952230486, + "tokens_seen": 2217527296 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026522768304914745, + "loss": 1.226, + "theoretical_loss": 3.4297447833511994, + "tokens_seen": 2217592832 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026521163490471415, + "loss": 1.3285, + "theoretical_loss": 3.429736571789979, + "tokens_seen": 2217658368 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026519558676028085, + "loss": 1.4251, + "theoretical_loss": 3.4297283605393662, + "tokens_seen": 2217723904 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026517953861584755, + "loss": 1.3455, + "theoretical_loss": 3.4297201495993415, + "tokens_seen": 2217789440 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026516349047141425, + "loss": 1.3599, + "theoretical_loss": 3.4297119389698825, + "tokens_seen": 2217854976 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026514744232698095, + "loss": 1.3141, + "theoretical_loss": 3.429703728650969, + "tokens_seen": 2217920512 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026513139418254765, + "loss": 1.224, + "theoretical_loss": 3.4296955186425797, + "tokens_seen": 2217986048 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026511534603811435, + "loss": 1.2622, + "theoretical_loss": 3.4296873089446938, + "tokens_seen": 2218051584 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026509929789368105, + "loss": 1.3039, + "theoretical_loss": 3.4296790995572906, + "tokens_seen": 2218117120 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026508324974924775, + "loss": 1.2731, + "theoretical_loss": 3.4296708904803492, + "tokens_seen": 2218182656 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026506720160481446, + "loss": 1.2681, + "theoretical_loss": 3.4296626817138485, + "tokens_seen": 2218248192 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026505115346038116, + "loss": 1.2551, + "theoretical_loss": 3.4296544732577674, + "tokens_seen": 2218313728 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026503510531594786, + "loss": 1.3625, + "theoretical_loss": 3.4296462651120856, + "tokens_seen": 2218379264 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026501905717151456, + "loss": 1.2674, + "theoretical_loss": 3.4296380572767813, + "tokens_seen": 2218444800 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026500300902708126, + "loss": 1.4131, + "theoretical_loss": 3.4296298497518345, + "tokens_seen": 2218510336 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026498696088264796, + "loss": 1.3762, + "theoretical_loss": 3.429621642537224, + "tokens_seen": 2218575872 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026497091273821466, + "loss": 1.3482, + "theoretical_loss": 3.4296134356329286, + "tokens_seen": 2218641408 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026495486459378136, + "loss": 1.2478, + "theoretical_loss": 3.429605229038928, + "tokens_seen": 2218706944 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026493881644934806, + "loss": 1.3071, + "theoretical_loss": 3.4295970227552, + "tokens_seen": 2218772480 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026492276830491476, + "loss": 1.1556, + "theoretical_loss": 3.4295888167817252, + "tokens_seen": 2218838016 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026490672016048146, + "loss": 1.2575, + "theoretical_loss": 3.429580611118482, + "tokens_seen": 2218903552 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026489067201604816, + "loss": 1.2218, + "theoretical_loss": 3.42957240576545, + "tokens_seen": 2218969088 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002648746238716148, + "loss": 1.2928, + "theoretical_loss": 3.4295642007226075, + "tokens_seen": 2219034624 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026485857572718156, + "loss": 1.3164, + "theoretical_loss": 3.429555995989934, + "tokens_seen": 2219100160 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026484252758274826, + "loss": 1.2853, + "theoretical_loss": 3.429547791567409, + "tokens_seen": 2219165696 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026482647943831496, + "loss": 1.3427, + "theoretical_loss": 3.429539587455011, + "tokens_seen": 2219231232 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026481043129388167, + "loss": 1.2696, + "theoretical_loss": 3.4295313836527197, + "tokens_seen": 2219296768 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026479438314944837, + "loss": 1.2026, + "theoretical_loss": 3.4295231801605137, + "tokens_seen": 2219362304 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026477833500501507, + "loss": 1.225, + "theoretical_loss": 3.4295149769783726, + "tokens_seen": 2219427840 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026476228686058177, + "loss": 1.2638, + "theoretical_loss": 3.429506774106275, + "tokens_seen": 2219493376 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026474623871614847, + "loss": 1.3652, + "theoretical_loss": 3.4294985715442, + "tokens_seen": 2219558912 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026473019057171517, + "loss": 1.1763, + "theoretical_loss": 3.429490369292128, + "tokens_seen": 2219624448 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026471414242728187, + "loss": 1.2243, + "theoretical_loss": 3.429482167350036, + "tokens_seen": 2219689984 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026469809428284857, + "loss": 1.1194, + "theoretical_loss": 3.4294739657179054, + "tokens_seen": 2219755520 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026468204613841527, + "loss": 1.3588, + "theoretical_loss": 3.429465764395714, + "tokens_seen": 2219821056 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026466599799398197, + "loss": 1.3257, + "theoretical_loss": 3.4294575633834405, + "tokens_seen": 2219886592 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026464994984954867, + "loss": 1.2439, + "theoretical_loss": 3.4294493626810656, + "tokens_seen": 2219952128 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026463390170511537, + "loss": 1.2956, + "theoretical_loss": 3.4294411622885668, + "tokens_seen": 2220017664 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002646178535606821, + "loss": 1.1939, + "theoretical_loss": 3.429432962205924, + "tokens_seen": 2220083200 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002646018054162488, + "loss": 1.2986, + "theoretical_loss": 3.4294247624331167, + "tokens_seen": 2220148736 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002645857572718155, + "loss": 1.3367, + "theoretical_loss": 3.4294165629701236, + "tokens_seen": 2220214272 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002645697091273822, + "loss": 1.3666, + "theoretical_loss": 3.429408363816924, + "tokens_seen": 2220279808 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002645536609829489, + "loss": 1.2443, + "theoretical_loss": 3.4294001649734973, + "tokens_seen": 2220345344 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002645376128385156, + "loss": 1.245, + "theoretical_loss": 3.429391966439822, + "tokens_seen": 2220410880 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002645215646940823, + "loss": 1.3305, + "theoretical_loss": 3.429383768215877, + "tokens_seen": 2220476416 + }, + { + "epoch": 1.05, + "learning_rate": 0.000264505516549649, + "loss": 1.2855, + "theoretical_loss": 3.429375570301643, + "tokens_seen": 2220541952 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002644894684052157, + "loss": 1.2587, + "theoretical_loss": 3.4293673726970977, + "tokens_seen": 2220607488 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002644734202607824, + "loss": 1.2748, + "theoretical_loss": 3.429359175402221, + "tokens_seen": 2220673024 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002644573721163491, + "loss": 1.2426, + "theoretical_loss": 3.4293509784169918, + "tokens_seen": 2220738560 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002644413239719158, + "loss": 1.2859, + "theoretical_loss": 3.429342781741389, + "tokens_seen": 2220804096 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002644252758274825, + "loss": 1.4114, + "theoretical_loss": 3.4293345853753925, + "tokens_seen": 2220869632 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002644092276830492, + "loss": 1.283, + "theoretical_loss": 3.429326389318981, + "tokens_seen": 2220935168 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002643931795386159, + "loss": 1.2272, + "theoretical_loss": 3.4293181935721337, + "tokens_seen": 2221000704 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002643771313941826, + "loss": 1.3553, + "theoretical_loss": 3.42930999813483, + "tokens_seen": 2221066240 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002643610832497493, + "loss": 1.2995, + "theoretical_loss": 3.429301803007048, + "tokens_seen": 2221131776 + }, + { + "epoch": 1.05, + "learning_rate": 0.000264345035105316, + "loss": 1.3241, + "theoretical_loss": 3.4292936081887686, + "tokens_seen": 2221197312 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002643289869608827, + "loss": 1.3159, + "theoretical_loss": 3.42928541367997, + "tokens_seen": 2221262848 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002643129388164494, + "loss": 1.2274, + "theoretical_loss": 3.4292772194806314, + "tokens_seen": 2221328384 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002642968906720161, + "loss": 1.2886, + "theoretical_loss": 3.429269025590732, + "tokens_seen": 2221393920 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002642808425275828, + "loss": 1.3562, + "theoretical_loss": 3.429260832010251, + "tokens_seen": 2221459456 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002642647943831495, + "loss": 1.22, + "theoretical_loss": 3.4292526387391677, + "tokens_seen": 2221524992 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002642487462387162, + "loss": 1.2818, + "theoretical_loss": 3.429244445777462, + "tokens_seen": 2221590528 + }, + { + "epoch": 1.05, + "objective/train/docs_used": 821676, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.295888900756836, + "objective/train/theoretical_loss": 3.4292362531251115, + "objective/train/tokens_used": 2242116064, + "theoretical_loss": 3.4292362531251115, + "tokens_seen": 2221656064 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026423269809428283, + "loss": 1.2959, + "theoretical_loss": 3.4292362531251115, + "tokens_seen": 2221656064 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026421664994984953, + "loss": 1.2438, + "theoretical_loss": 3.429228060782096, + "tokens_seen": 2221721600 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026420060180541624, + "loss": 1.2381, + "theoretical_loss": 3.4292198687483957, + "tokens_seen": 2221787136 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026418455366098294, + "loss": 1.3152, + "theoretical_loss": 3.4292116770239884, + "tokens_seen": 2221852672 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026416850551654964, + "loss": 1.3016, + "theoretical_loss": 3.4292034856088547, + "tokens_seen": 2221918208 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026415245737211634, + "loss": 1.351, + "theoretical_loss": 3.4291952945029722, + "tokens_seen": 2221983744 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026413640922768304, + "loss": 1.2179, + "theoretical_loss": 3.4291871037063215, + "tokens_seen": 2222049280 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026412036108324974, + "loss": 1.1959, + "theoretical_loss": 3.4291789132188812, + "tokens_seen": 2222114816 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026410431293881644, + "loss": 1.3085, + "theoretical_loss": 3.42917072304063, + "tokens_seen": 2222180352 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026408826479438314, + "loss": 1.3017, + "theoretical_loss": 3.4291625331715485, + "tokens_seen": 2222245888 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026407221664994984, + "loss": 1.2236, + "theoretical_loss": 3.4291543436116148, + "tokens_seen": 2222311424 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026405616850551654, + "loss": 1.2686, + "theoretical_loss": 3.429146154360808, + "tokens_seen": 2222376960 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026404012036108324, + "loss": 1.2105, + "theoretical_loss": 3.429137965419108, + "tokens_seen": 2222442496 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026402407221664994, + "loss": 1.2765, + "theoretical_loss": 3.429129776786494, + "tokens_seen": 2222508032 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026400802407221664, + "loss": 1.2037, + "theoretical_loss": 3.4291215884629445, + "tokens_seen": 2222573568 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026399197592778334, + "loss": 1.1948, + "theoretical_loss": 3.4291134004484394, + "tokens_seen": 2222639104 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026397592778335004, + "loss": 1.2726, + "theoretical_loss": 3.4291052127429578, + "tokens_seen": 2222704640 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026395987963891674, + "loss": 1.1118, + "theoretical_loss": 3.4290970253464783, + "tokens_seen": 2222770176 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026394383149448345, + "loss": 1.3286, + "theoretical_loss": 3.4290888382589815, + "tokens_seen": 2222835712 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026392778335005015, + "loss": 1.188, + "theoretical_loss": 3.429080651480445, + "tokens_seen": 2222901248 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026391173520561685, + "loss": 1.2344, + "theoretical_loss": 3.429072465010849, + "tokens_seen": 2222966784 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026389568706118355, + "loss": 1.2324, + "theoretical_loss": 3.4290642788501726, + "tokens_seen": 2223032320 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026387963891675025, + "loss": 1.269, + "theoretical_loss": 3.4290560929983953, + "tokens_seen": 2223097856 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026386359077231695, + "loss": 1.3847, + "theoretical_loss": 3.4290479074554954, + "tokens_seen": 2223163392 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026384754262788365, + "loss": 1.3449, + "theoretical_loss": 3.4290397222214533, + "tokens_seen": 2223228928 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026383149448345035, + "loss": 1.1922, + "theoretical_loss": 3.4290315372962477, + "tokens_seen": 2223294464 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026381544633901705, + "loss": 1.3287, + "theoretical_loss": 3.4290233526798573, + "tokens_seen": 2223360000 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026379939819458375, + "loss": 1.2958, + "theoretical_loss": 3.4290151683722625, + "tokens_seen": 2223425536 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026378335005015045, + "loss": 1.3135, + "theoretical_loss": 3.4290069843734416, + "tokens_seen": 2223491072 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026376730190571715, + "loss": 1.3658, + "theoretical_loss": 3.428998800683374, + "tokens_seen": 2223556608 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026375125376128385, + "loss": 1.2912, + "theoretical_loss": 3.4289906173020395, + "tokens_seen": 2223622144 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026373520561685055, + "loss": 1.3604, + "theoretical_loss": 3.428982434229417, + "tokens_seen": 2223687680 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026371915747241725, + "loss": 1.1474, + "theoretical_loss": 3.4289742514654855, + "tokens_seen": 2223753216 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026370310932798395, + "loss": 1.1652, + "theoretical_loss": 3.428966069010225, + "tokens_seen": 2223818752 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026368706118355066, + "loss": 1.311, + "theoretical_loss": 3.4289578868636137, + "tokens_seen": 2223884288 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026367101303911736, + "loss": 1.1839, + "theoretical_loss": 3.428949705025632, + "tokens_seen": 2223949824 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002636549648946841, + "loss": 1.3267, + "theoretical_loss": 3.428941523496258, + "tokens_seen": 2224015360 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002636389167502508, + "loss": 1.3617, + "theoretical_loss": 3.4289333422754718, + "tokens_seen": 2224080896 + }, + { + "epoch": 1.05, + "learning_rate": 0.0002636228686058175, + "loss": 1.2368, + "theoretical_loss": 3.4289251613632525, + "tokens_seen": 2224146432 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026360682046138416, + "loss": 1.2551, + "theoretical_loss": 3.4289169807595794, + "tokens_seen": 2224211968 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026359077231695086, + "loss": 1.2499, + "theoretical_loss": 3.4289088004644315, + "tokens_seen": 2224277504 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026357472417251756, + "loss": 1.2921, + "theoretical_loss": 3.428900620477788, + "tokens_seen": 2224343040 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026355867602808426, + "loss": 1.2191, + "theoretical_loss": 3.428892440799629, + "tokens_seen": 2224408576 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026354262788365096, + "loss": 1.3485, + "theoretical_loss": 3.428884261429933, + "tokens_seen": 2224474112 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026352657973921766, + "loss": 1.3443, + "theoretical_loss": 3.4288760823686792, + "tokens_seen": 2224539648 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026351053159478436, + "loss": 1.293, + "theoretical_loss": 3.4288679036158474, + "tokens_seen": 2224605184 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026349448345035106, + "loss": 1.2901, + "theoretical_loss": 3.428859725171417, + "tokens_seen": 2224670720 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026347843530591776, + "loss": 1.2388, + "theoretical_loss": 3.428851547035366, + "tokens_seen": 2224736256 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026346238716148446, + "loss": 1.3171, + "theoretical_loss": 3.4288433692076756, + "tokens_seen": 2224801792 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026344633901705117, + "loss": 1.2347, + "theoretical_loss": 3.4288351916883237, + "tokens_seen": 2224867328 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026343029087261787, + "loss": 1.3572, + "theoretical_loss": 3.4288270144772897, + "tokens_seen": 2224932864 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026341424272818457, + "loss": 1.2491, + "theoretical_loss": 3.428818837574554, + "tokens_seen": 2224998400 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026339819458375127, + "loss": 1.24, + "theoretical_loss": 3.428810660980094, + "tokens_seen": 2225063936 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026338214643931797, + "loss": 1.1442, + "theoretical_loss": 3.428802484693891, + "tokens_seen": 2225129472 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026336609829488467, + "loss": 1.2784, + "theoretical_loss": 3.4287943087159234, + "tokens_seen": 2225195008 + }, + { + "epoch": 1.05, + "learning_rate": 0.00026335005015045137, + "loss": 1.3336, + "theoretical_loss": 3.42878613304617, + "tokens_seen": 2225260544 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026333400200601807, + "loss": 1.1961, + "theoretical_loss": 3.428777957684611, + "tokens_seen": 2225326080 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026331795386158477, + "loss": 1.2267, + "theoretical_loss": 3.428769782631225, + "tokens_seen": 2225391616 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026330190571715147, + "loss": 1.2199, + "theoretical_loss": 3.428761607885992, + "tokens_seen": 2225457152 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026328585757271817, + "loss": 1.2607, + "theoretical_loss": 3.4287534334488905, + "tokens_seen": 2225522688 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026326980942828487, + "loss": 1.2959, + "theoretical_loss": 3.428745259319901, + "tokens_seen": 2225588224 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026325376128385157, + "loss": 1.3539, + "theoretical_loss": 3.4287370854990007, + "tokens_seen": 2225653760 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002632377131394183, + "loss": 1.4203, + "theoretical_loss": 3.4287289119861715, + "tokens_seen": 2225719296 + }, + { + "epoch": 1.06, + "learning_rate": 0.000263221664994985, + "loss": 1.2568, + "theoretical_loss": 3.428720738781391, + "tokens_seen": 2225784832 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002632056168505517, + "loss": 1.2717, + "theoretical_loss": 3.428712565884639, + "tokens_seen": 2225850368 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002631895687061184, + "loss": 1.3071, + "theoretical_loss": 3.428704393295895, + "tokens_seen": 2225915904 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002631735205616851, + "loss": 1.2387, + "theoretical_loss": 3.428696221015138, + "tokens_seen": 2225981440 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002631574724172518, + "loss": 1.2146, + "theoretical_loss": 3.4286880490423473, + "tokens_seen": 2226046976 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002631414242728185, + "loss": 1.2241, + "theoretical_loss": 3.428679877377503, + "tokens_seen": 2226112512 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002631253761283852, + "loss": 1.2701, + "theoretical_loss": 3.428671706020584, + "tokens_seen": 2226178048 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002631093279839519, + "loss": 1.3697, + "theoretical_loss": 3.4286635349715686, + "tokens_seen": 2226243584 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002630932798395186, + "loss": 1.1985, + "theoretical_loss": 3.4286553642304374, + "tokens_seen": 2226309120 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002630772316950853, + "loss": 1.3686, + "theoretical_loss": 3.428647193797169, + "tokens_seen": 2226374656 + }, + { + "epoch": 1.06, + "learning_rate": 0.000263061183550652, + "loss": 1.2085, + "theoretical_loss": 3.4286390236717437, + "tokens_seen": 2226440192 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002630451354062187, + "loss": 1.3217, + "theoretical_loss": 3.42863085385414, + "tokens_seen": 2226505728 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002630290872617854, + "loss": 1.2675, + "theoretical_loss": 3.4286226843443375, + "tokens_seen": 2226571264 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002630130391173521, + "loss": 1.3369, + "theoretical_loss": 3.428614515142315, + "tokens_seen": 2226636800 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002629969909729188, + "loss": 1.2434, + "theoretical_loss": 3.428606346248053, + "tokens_seen": 2226702336 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002629809428284855, + "loss": 1.3768, + "theoretical_loss": 3.42859817766153, + "tokens_seen": 2226767872 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002629648946840522, + "loss": 1.1612, + "theoretical_loss": 3.4285900093827255, + "tokens_seen": 2226833408 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002629488465396189, + "loss": 1.1801, + "theoretical_loss": 3.428581841411619, + "tokens_seen": 2226898944 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002629327983951856, + "loss": 1.3, + "theoretical_loss": 3.4285736737481898, + "tokens_seen": 2226964480 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002629167502507523, + "loss": 1.3125, + "theoretical_loss": 3.428565506392417, + "tokens_seen": 2227030016 + }, + { + "epoch": 1.06, + "learning_rate": 0.000262900702106319, + "loss": 1.2486, + "theoretical_loss": 3.4285573393442803, + "tokens_seen": 2227095552 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002628846539618857, + "loss": 1.2601, + "theoretical_loss": 3.428549172603759, + "tokens_seen": 2227161088 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002628686058174524, + "loss": 1.3297, + "theoretical_loss": 3.428541006170833, + "tokens_seen": 2227226624 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002628525576730191, + "loss": 1.1878, + "theoretical_loss": 3.42853284004548, + "tokens_seen": 2227292160 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026283650952858573, + "loss": 1.2508, + "theoretical_loss": 3.4285246742276807, + "tokens_seen": 2227357696 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026282046138415244, + "loss": 1.2521, + "theoretical_loss": 3.4285165087174145, + "tokens_seen": 2227423232 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026280441323971914, + "loss": 1.2266, + "theoretical_loss": 3.42850834351466, + "tokens_seen": 2227488768 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026278836509528584, + "loss": 1.2463, + "theoretical_loss": 3.4285001786193976, + "tokens_seen": 2227554304 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026277231695085254, + "loss": 1.3699, + "theoretical_loss": 3.428492014031606, + "tokens_seen": 2227619840 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026275626880641924, + "loss": 1.2871, + "theoretical_loss": 3.428483849751265, + "tokens_seen": 2227685376 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026274022066198594, + "loss": 1.3004, + "theoretical_loss": 3.428475685778353, + "tokens_seen": 2227750912 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026272417251755264, + "loss": 1.1842, + "theoretical_loss": 3.42846752211285, + "tokens_seen": 2227816448 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026270812437311934, + "loss": 1.2113, + "theoretical_loss": 3.428459358754736, + "tokens_seen": 2227881984 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026269207622868604, + "loss": 1.2699, + "theoretical_loss": 3.4284511957039894, + "tokens_seen": 2227947520 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026267602808425274, + "loss": 1.3589, + "theoretical_loss": 3.4284430329605904, + "tokens_seen": 2228013056 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026265997993981944, + "loss": 1.3512, + "theoretical_loss": 3.4284348705245176, + "tokens_seen": 2228078592 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026264393179538614, + "loss": 1.1871, + "theoretical_loss": 3.428426708395751, + "tokens_seen": 2228144128 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 823939, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2798750400543213, + "objective/train/theoretical_loss": 3.4284185465742696, + "objective/train/tokens_used": 2248669664, + "theoretical_loss": 3.4284185465742696, + "tokens_seen": 2228209664 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026262788365095284, + "loss": 1.2799, + "theoretical_loss": 3.4284185465742696, + "tokens_seen": 2228209664 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026261183550651954, + "loss": 1.3207, + "theoretical_loss": 3.428410385060053, + "tokens_seen": 2228275200 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026259578736208624, + "loss": 1.3377, + "theoretical_loss": 3.4284022238530807, + "tokens_seen": 2228340736 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026257973921765295, + "loss": 1.321, + "theoretical_loss": 3.428394062953332, + "tokens_seen": 2228406272 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026256369107321965, + "loss": 1.2606, + "theoretical_loss": 3.4283859023607857, + "tokens_seen": 2228471808 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026254764292878635, + "loss": 1.2381, + "theoretical_loss": 3.4283777420754227, + "tokens_seen": 2228537344 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026253159478435305, + "loss": 1.3087, + "theoretical_loss": 3.4283695820972206, + "tokens_seen": 2228602880 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026251554663991975, + "loss": 1.262, + "theoretical_loss": 3.42836142242616, + "tokens_seen": 2228668416 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026249949849548645, + "loss": 1.2914, + "theoretical_loss": 3.4283532630622204, + "tokens_seen": 2228733952 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026248345035105315, + "loss": 1.2255, + "theoretical_loss": 3.42834510400538, + "tokens_seen": 2228799488 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026246740220661985, + "loss": 1.2579, + "theoretical_loss": 3.4283369452556194, + "tokens_seen": 2228865024 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002624513540621866, + "loss": 1.359, + "theoretical_loss": 3.428328786812918, + "tokens_seen": 2228930560 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002624353059177533, + "loss": 1.1489, + "theoretical_loss": 3.4283206286772545, + "tokens_seen": 2228996096 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026241925777332, + "loss": 1.2925, + "theoretical_loss": 3.4283124708486086, + "tokens_seen": 2229061632 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002624032096288867, + "loss": 1.2162, + "theoretical_loss": 3.4283043133269597, + "tokens_seen": 2229127168 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002623871614844534, + "loss": 1.3455, + "theoretical_loss": 3.428296156112287, + "tokens_seen": 2229192704 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002623711133400201, + "loss": 1.178, + "theoretical_loss": 3.4282879992045707, + "tokens_seen": 2229258240 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002623550651955868, + "loss": 1.2455, + "theoretical_loss": 3.4282798426037897, + "tokens_seen": 2229323776 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002623390170511535, + "loss": 1.2484, + "theoretical_loss": 3.4282716863099236, + "tokens_seen": 2229389312 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002623229689067202, + "loss": 1.1718, + "theoretical_loss": 3.4282635303229516, + "tokens_seen": 2229454848 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002623069207622869, + "loss": 1.2369, + "theoretical_loss": 3.428255374642853, + "tokens_seen": 2229520384 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002622908726178536, + "loss": 1.3145, + "theoretical_loss": 3.4282472192696076, + "tokens_seen": 2229585920 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002622748244734203, + "loss": 1.1987, + "theoretical_loss": 3.428239064203195, + "tokens_seen": 2229651456 + }, + { + "epoch": 1.06, + "learning_rate": 0.000262258776328987, + "loss": 1.2556, + "theoretical_loss": 3.428230909443594, + "tokens_seen": 2229716992 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002622427281845537, + "loss": 1.2493, + "theoretical_loss": 3.428222754990784, + "tokens_seen": 2229782528 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002622266800401204, + "loss": 1.2503, + "theoretical_loss": 3.428214600844745, + "tokens_seen": 2229848064 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026221063189568706, + "loss": 1.2482, + "theoretical_loss": 3.428206447005457, + "tokens_seen": 2229913600 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026219458375125376, + "loss": 1.1916, + "theoretical_loss": 3.428198293472898, + "tokens_seen": 2229979136 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026217853560682046, + "loss": 1.2743, + "theoretical_loss": 3.4281901402470485, + "tokens_seen": 2230044672 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026216248746238716, + "loss": 1.3454, + "theoretical_loss": 3.4281819873278874, + "tokens_seen": 2230110208 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026214643931795386, + "loss": 1.2224, + "theoretical_loss": 3.4281738347153947, + "tokens_seen": 2230175744 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026213039117352056, + "loss": 1.359, + "theoretical_loss": 3.428165682409549, + "tokens_seen": 2230241280 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026211434302908726, + "loss": 1.1978, + "theoretical_loss": 3.4281575304103304, + "tokens_seen": 2230306816 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026209829488465396, + "loss": 1.2669, + "theoretical_loss": 3.4281493787177184, + "tokens_seen": 2230372352 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026208224674022066, + "loss": 1.3177, + "theoretical_loss": 3.4281412273316922, + "tokens_seen": 2230437888 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026206619859578737, + "loss": 1.166, + "theoretical_loss": 3.4281330762522315, + "tokens_seen": 2230503424 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026205015045135407, + "loss": 1.2969, + "theoretical_loss": 3.4281249254793154, + "tokens_seen": 2230568960 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026203410230692077, + "loss": 1.2367, + "theoretical_loss": 3.428116775012924, + "tokens_seen": 2230634496 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026201805416248747, + "loss": 1.2132, + "theoretical_loss": 3.428108624853036, + "tokens_seen": 2230700032 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026200200601805417, + "loss": 1.2904, + "theoretical_loss": 3.428100474999631, + "tokens_seen": 2230765568 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026198595787362087, + "loss": 1.3006, + "theoretical_loss": 3.4280923254526887, + "tokens_seen": 2230831104 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026196990972918757, + "loss": 1.3489, + "theoretical_loss": 3.428084176212189, + "tokens_seen": 2230896640 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026195386158475427, + "loss": 1.2356, + "theoretical_loss": 3.4280760272781103, + "tokens_seen": 2230962176 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026193781344032097, + "loss": 1.2864, + "theoretical_loss": 3.4280678786504333, + "tokens_seen": 2231027712 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026192176529588767, + "loss": 1.2395, + "theoretical_loss": 3.428059730329137, + "tokens_seen": 2231093248 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026190571715145437, + "loss": 1.2895, + "theoretical_loss": 3.4280515823142, + "tokens_seen": 2231158784 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026188966900702107, + "loss": 1.1387, + "theoretical_loss": 3.4280434346056032, + "tokens_seen": 2231224320 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002618736208625878, + "loss": 1.1421, + "theoretical_loss": 3.428035287203325, + "tokens_seen": 2231289856 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002618575727181545, + "loss": 1.1212, + "theoretical_loss": 3.4280271401073454, + "tokens_seen": 2231355392 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002618415245737212, + "loss": 1.2923, + "theoretical_loss": 3.4280189933176444, + "tokens_seen": 2231420928 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002618254764292879, + "loss": 1.2128, + "theoretical_loss": 3.4280108468342005, + "tokens_seen": 2231486464 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002618094282848546, + "loss": 1.2441, + "theoretical_loss": 3.4280027006569935, + "tokens_seen": 2231552000 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002617933801404213, + "loss": 1.2363, + "theoretical_loss": 3.427994554786003, + "tokens_seen": 2231617536 + }, + { + "epoch": 1.06, + "learning_rate": 0.000261777331995988, + "loss": 1.2607, + "theoretical_loss": 3.4279864092212087, + "tokens_seen": 2231683072 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002617612838515547, + "loss": 1.3015, + "theoretical_loss": 3.4279782639625895, + "tokens_seen": 2231748608 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002617452357071214, + "loss": 1.2705, + "theoretical_loss": 3.4279701190101255, + "tokens_seen": 2231814144 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002617291875626881, + "loss": 1.2605, + "theoretical_loss": 3.427961974363796, + "tokens_seen": 2231879680 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002617131394182548, + "loss": 1.2859, + "theoretical_loss": 3.427953830023581, + "tokens_seen": 2231945216 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002616970912738215, + "loss": 1.2997, + "theoretical_loss": 3.427945685989459, + "tokens_seen": 2232010752 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002616810431293882, + "loss": 1.2662, + "theoretical_loss": 3.42793754226141, + "tokens_seen": 2232076288 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002616649949849549, + "loss": 1.1551, + "theoretical_loss": 3.4279293988394133, + "tokens_seen": 2232141824 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002616489468405216, + "loss": 1.3132, + "theoretical_loss": 3.427921255723449, + "tokens_seen": 2232207360 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002616328986960883, + "loss": 1.3813, + "theoretical_loss": 3.427913112913496, + "tokens_seen": 2232272896 + }, + { + "epoch": 1.06, + "learning_rate": 0.000261616850551655, + "loss": 1.3041, + "theoretical_loss": 3.4279049704095343, + "tokens_seen": 2232338432 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002616008024072217, + "loss": 1.3127, + "theoretical_loss": 3.4278968282115434, + "tokens_seen": 2232403968 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002615847542627884, + "loss": 1.3223, + "theoretical_loss": 3.427888686319502, + "tokens_seen": 2232469504 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002615687061183551, + "loss": 1.356, + "theoretical_loss": 3.427880544733391, + "tokens_seen": 2232535040 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002615526579739218, + "loss": 1.3161, + "theoretical_loss": 3.4278724034531884, + "tokens_seen": 2232600576 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002615366098294885, + "loss": 1.311, + "theoretical_loss": 3.4278642624788747, + "tokens_seen": 2232666112 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002615205616850552, + "loss": 1.3526, + "theoretical_loss": 3.4278561218104295, + "tokens_seen": 2232731648 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002615045135406219, + "loss": 1.3475, + "theoretical_loss": 3.427847981447832, + "tokens_seen": 2232797184 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002614884653961886, + "loss": 1.3513, + "theoretical_loss": 3.427839841391062, + "tokens_seen": 2232862720 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002614724172517553, + "loss": 1.2398, + "theoretical_loss": 3.4278317016400983, + "tokens_seen": 2232928256 + }, + { + "epoch": 1.06, + "learning_rate": 0.000261456369107322, + "loss": 1.2884, + "theoretical_loss": 3.427823562194921, + "tokens_seen": 2232993792 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026144032096288864, + "loss": 1.2855, + "theoretical_loss": 3.4278154230555096, + "tokens_seen": 2233059328 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026142427281845534, + "loss": 1.2573, + "theoretical_loss": 3.427807284221844, + "tokens_seen": 2233124864 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026140822467402204, + "loss": 1.163, + "theoretical_loss": 3.4277991456939025, + "tokens_seen": 2233190400 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026139217652958874, + "loss": 1.2764, + "theoretical_loss": 3.4277910074716664, + "tokens_seen": 2233255936 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026137612838515544, + "loss": 1.1954, + "theoretical_loss": 3.4277828695551142, + "tokens_seen": 2233321472 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026136008024072214, + "loss": 1.18, + "theoretical_loss": 3.4277747319442256, + "tokens_seen": 2233387008 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026134403209628884, + "loss": 1.1998, + "theoretical_loss": 3.42776659463898, + "tokens_seen": 2233452544 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026132798395185554, + "loss": 1.3064, + "theoretical_loss": 3.427758457639357, + "tokens_seen": 2233518080 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026131193580742224, + "loss": 1.1901, + "theoretical_loss": 3.427750320945336, + "tokens_seen": 2233583616 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026129588766298894, + "loss": 1.2885, + "theoretical_loss": 3.4277421845568976, + "tokens_seen": 2233649152 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026127983951855564, + "loss": 1.2582, + "theoretical_loss": 3.42773404847402, + "tokens_seen": 2233714688 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026126379137412234, + "loss": 1.259, + "theoretical_loss": 3.4277259126966837, + "tokens_seen": 2233780224 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002612477432296891, + "loss": 1.1956, + "theoretical_loss": 3.427717777224868, + "tokens_seen": 2233845760 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002612316950852558, + "loss": 1.3117, + "theoretical_loss": 3.4277096420585518, + "tokens_seen": 2233911296 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002612156469408225, + "loss": 1.1732, + "theoretical_loss": 3.4277015071977157, + "tokens_seen": 2233976832 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002611995987963892, + "loss": 1.1544, + "theoretical_loss": 3.4276933726423384, + "tokens_seen": 2234042368 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002611835506519559, + "loss": 1.2355, + "theoretical_loss": 3.4276852383924004, + "tokens_seen": 2234107904 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002611675025075226, + "loss": 1.2989, + "theoretical_loss": 3.42767710444788, + "tokens_seen": 2234173440 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002611514543630893, + "loss": 1.1937, + "theoretical_loss": 3.4276689708087584, + "tokens_seen": 2234238976 + }, + { + "epoch": 1.06, + "learning_rate": 0.000261135406218656, + "loss": 1.3201, + "theoretical_loss": 3.4276608374750137, + "tokens_seen": 2234304512 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002611193580742227, + "loss": 1.1832, + "theoretical_loss": 3.427652704446626, + "tokens_seen": 2234370048 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002611033099297894, + "loss": 1.2515, + "theoretical_loss": 3.4276445717235755, + "tokens_seen": 2234435584 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002610872617853561, + "loss": 1.2706, + "theoretical_loss": 3.4276364393058407, + "tokens_seen": 2234501120 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002610712136409228, + "loss": 1.2919, + "theoretical_loss": 3.427628307193402, + "tokens_seen": 2234566656 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002610551654964895, + "loss": 1.2074, + "theoretical_loss": 3.4276201753862385, + "tokens_seen": 2234632192 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002610391173520562, + "loss": 1.2907, + "theoretical_loss": 3.4276120438843303, + "tokens_seen": 2234697728 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 826239, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2904040813446045, + "objective/train/theoretical_loss": 3.4276039126876565, + "objective/train/tokens_used": 2255223264, + "theoretical_loss": 3.4276039126876565, + "tokens_seen": 2234763264 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002610230692076229, + "loss": 1.2904, + "theoretical_loss": 3.4276039126876565, + "tokens_seen": 2234763264 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002610070210631896, + "loss": 1.2355, + "theoretical_loss": 3.427595781796197, + "tokens_seen": 2234828800 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002609909729187563, + "loss": 1.2292, + "theoretical_loss": 3.427587651209931, + "tokens_seen": 2234894336 + }, + { + "epoch": 1.06, + "learning_rate": 0.000260974924774323, + "loss": 1.1197, + "theoretical_loss": 3.4275795209288384, + "tokens_seen": 2234959872 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002609588766298897, + "loss": 1.1755, + "theoretical_loss": 3.427571390952899, + "tokens_seen": 2235025408 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002609428284854564, + "loss": 1.2787, + "theoretical_loss": 3.4275632612820917, + "tokens_seen": 2235090944 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002609267803410231, + "loss": 1.2448, + "theoretical_loss": 3.427555131916397, + "tokens_seen": 2235156480 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002609107321965898, + "loss": 1.2154, + "theoretical_loss": 3.427547002855794, + "tokens_seen": 2235222016 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002608946840521565, + "loss": 1.3056, + "theoretical_loss": 3.427538874100262, + "tokens_seen": 2235287552 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002608786359077232, + "loss": 1.4163, + "theoretical_loss": 3.4275307456497814, + "tokens_seen": 2235353088 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002608625877632899, + "loss": 1.2524, + "theoretical_loss": 3.427522617504331, + "tokens_seen": 2235418624 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002608465396188566, + "loss": 1.247, + "theoretical_loss": 3.427514489663891, + "tokens_seen": 2235484160 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002608304914744233, + "loss": 1.3515, + "theoretical_loss": 3.427506362128441, + "tokens_seen": 2235549696 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026081444332999, + "loss": 1.3376, + "theoretical_loss": 3.4274982348979606, + "tokens_seen": 2235615232 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026079839518555666, + "loss": 1.295, + "theoretical_loss": 3.4274901079724285, + "tokens_seen": 2235680768 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026078234704112336, + "loss": 1.2958, + "theoretical_loss": 3.4274819813518254, + "tokens_seen": 2235746304 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026076629889669006, + "loss": 1.133, + "theoretical_loss": 3.427473855036131, + "tokens_seen": 2235811840 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026075025075225676, + "loss": 1.1807, + "theoretical_loss": 3.427465729025324, + "tokens_seen": 2235877376 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026073420260782346, + "loss": 1.2055, + "theoretical_loss": 3.4274576033193846, + "tokens_seen": 2235942912 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026071815446339016, + "loss": 1.193, + "theoretical_loss": 3.4274494779182922, + "tokens_seen": 2236008448 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026070210631895687, + "loss": 1.3088, + "theoretical_loss": 3.4274413528220267, + "tokens_seen": 2236073984 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026068605817452357, + "loss": 1.2388, + "theoretical_loss": 3.427433228030568, + "tokens_seen": 2236139520 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026067001003009027, + "loss": 1.3118, + "theoretical_loss": 3.4274251035438947, + "tokens_seen": 2236205056 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026065396188565697, + "loss": 1.1877, + "theoretical_loss": 3.4274169793619875, + "tokens_seen": 2236270592 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026063791374122367, + "loss": 1.1773, + "theoretical_loss": 3.427408855484826, + "tokens_seen": 2236336128 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026062186559679037, + "loss": 1.3622, + "theoretical_loss": 3.427400731912389, + "tokens_seen": 2236401664 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026060581745235707, + "loss": 1.3945, + "theoretical_loss": 3.4273926086446562, + "tokens_seen": 2236467200 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026058976930792377, + "loss": 1.2658, + "theoretical_loss": 3.4273844856816083, + "tokens_seen": 2236532736 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026057372116349047, + "loss": 1.1837, + "theoretical_loss": 3.4273763630232237, + "tokens_seen": 2236598272 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026055767301905717, + "loss": 1.3369, + "theoretical_loss": 3.427368240669483, + "tokens_seen": 2236663808 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026054162487462387, + "loss": 1.2079, + "theoretical_loss": 3.4273601186203653, + "tokens_seen": 2236729344 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026052557673019057, + "loss": 1.1451, + "theoretical_loss": 3.427351996875851, + "tokens_seen": 2236794880 + }, + { + "epoch": 1.06, + "learning_rate": 0.00026050952858575727, + "loss": 1.2856, + "theoretical_loss": 3.4273438754359185, + "tokens_seen": 2236860416 + }, + { + "epoch": 1.06, + "learning_rate": 0.000260493480441324, + "loss": 1.1844, + "theoretical_loss": 3.4273357543005485, + "tokens_seen": 2236925952 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002604774322968907, + "loss": 1.2634, + "theoretical_loss": 3.4273276334697202, + "tokens_seen": 2236991488 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002604613841524574, + "loss": 1.248, + "theoretical_loss": 3.4273195129434133, + "tokens_seen": 2237057024 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002604453360080241, + "loss": 1.1963, + "theoretical_loss": 3.4273113927216077, + "tokens_seen": 2237122560 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002604292878635908, + "loss": 1.216, + "theoretical_loss": 3.4273032728042825, + "tokens_seen": 2237188096 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002604132397191575, + "loss": 1.2297, + "theoretical_loss": 3.4272951531914178, + "tokens_seen": 2237253632 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002603971915747242, + "loss": 1.2999, + "theoretical_loss": 3.4272870338829935, + "tokens_seen": 2237319168 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002603811434302909, + "loss": 1.286, + "theoretical_loss": 3.427278914878989, + "tokens_seen": 2237384704 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002603650952858576, + "loss": 1.2186, + "theoretical_loss": 3.4272707961793833, + "tokens_seen": 2237450240 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002603490471414243, + "loss": 1.227, + "theoretical_loss": 3.4272626777841575, + "tokens_seen": 2237515776 + }, + { + "epoch": 1.06, + "learning_rate": 0.000260332998996991, + "loss": 1.2199, + "theoretical_loss": 3.42725455969329, + "tokens_seen": 2237581312 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002603169508525577, + "loss": 1.3319, + "theoretical_loss": 3.427246441906761, + "tokens_seen": 2237646848 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002603009027081244, + "loss": 1.1563, + "theoretical_loss": 3.4272383244245503, + "tokens_seen": 2237712384 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002602848545636911, + "loss": 1.228, + "theoretical_loss": 3.4272302072466374, + "tokens_seen": 2237777920 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002602688064192578, + "loss": 1.3005, + "theoretical_loss": 3.427222090373002, + "tokens_seen": 2237843456 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002602527582748245, + "loss": 1.2695, + "theoretical_loss": 3.4272139738036236, + "tokens_seen": 2237908992 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002602367101303912, + "loss": 1.252, + "theoretical_loss": 3.4272058575384823, + "tokens_seen": 2237974528 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002602206619859579, + "loss": 1.1664, + "theoretical_loss": 3.4271977415775576, + "tokens_seen": 2238040064 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002602046138415246, + "loss": 1.2503, + "theoretical_loss": 3.427189625920829, + "tokens_seen": 2238105600 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002601885656970913, + "loss": 1.2164, + "theoretical_loss": 3.4271815105682766, + "tokens_seen": 2238171136 + }, + { + "epoch": 1.06, + "learning_rate": 0.000260172517552658, + "loss": 1.2556, + "theoretical_loss": 3.4271733955198793, + "tokens_seen": 2238236672 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002601564694082247, + "loss": 1.139, + "theoretical_loss": 3.427165280775618, + "tokens_seen": 2238302208 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002601404212637914, + "loss": 1.1925, + "theoretical_loss": 3.427157166335471, + "tokens_seen": 2238367744 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002601243731193581, + "loss": 1.1525, + "theoretical_loss": 3.427149052199419, + "tokens_seen": 2238433280 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002601083249749248, + "loss": 1.2456, + "theoretical_loss": 3.4271409383674416, + "tokens_seen": 2238498816 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002600922768304915, + "loss": 1.2207, + "theoretical_loss": 3.4271328248395183, + "tokens_seen": 2238564352 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002600762286860582, + "loss": 1.2227, + "theoretical_loss": 3.4271247116156287, + "tokens_seen": 2238629888 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002600601805416249, + "loss": 1.1205, + "theoretical_loss": 3.427116598695753, + "tokens_seen": 2238695424 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002600441323971916, + "loss": 1.1962, + "theoretical_loss": 3.42710848607987, + "tokens_seen": 2238760960 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002600280842527583, + "loss": 1.1378, + "theoretical_loss": 3.4271003737679604, + "tokens_seen": 2238826496 + }, + { + "epoch": 1.06, + "learning_rate": 0.000260012036108325, + "loss": 1.262, + "theoretical_loss": 3.427092261760003, + "tokens_seen": 2238892032 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002599959879638917, + "loss": 1.247, + "theoretical_loss": 3.4270841500559786, + "tokens_seen": 2238957568 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002599799398194584, + "loss": 1.3187, + "theoretical_loss": 3.4270760386558656, + "tokens_seen": 2239023104 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002599638916750251, + "loss": 1.2155, + "theoretical_loss": 3.427067927559645, + "tokens_seen": 2239088640 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002599478435305918, + "loss": 1.1355, + "theoretical_loss": 3.427059816767296, + "tokens_seen": 2239154176 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002599317953861585, + "loss": 1.2543, + "theoretical_loss": 3.4270517062787977, + "tokens_seen": 2239219712 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002599157472417252, + "loss": 1.2288, + "theoretical_loss": 3.427043596094131, + "tokens_seen": 2239285248 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002598996990972919, + "loss": 1.2512, + "theoretical_loss": 3.4270354862132746, + "tokens_seen": 2239350784 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002598836509528586, + "loss": 1.2274, + "theoretical_loss": 3.4270273766362087, + "tokens_seen": 2239416320 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002598676028084253, + "loss": 1.3332, + "theoretical_loss": 3.427019267362913, + "tokens_seen": 2239481856 + }, + { + "epoch": 1.06, + "learning_rate": 0.000259851554663992, + "loss": 1.1301, + "theoretical_loss": 3.427011158393367, + "tokens_seen": 2239547392 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002598355065195587, + "loss": 1.3134, + "theoretical_loss": 3.427003049727551, + "tokens_seen": 2239612928 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002598194583751254, + "loss": 1.2792, + "theoretical_loss": 3.4269949413654444, + "tokens_seen": 2239678464 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002598034102306921, + "loss": 1.2364, + "theoretical_loss": 3.426986833307027, + "tokens_seen": 2239744000 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002597873620862588, + "loss": 1.1809, + "theoretical_loss": 3.426978725552278, + "tokens_seen": 2239809536 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002597713139418255, + "loss": 1.2488, + "theoretical_loss": 3.426970618101178, + "tokens_seen": 2239875072 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002597552657973922, + "loss": 1.2353, + "theoretical_loss": 3.426962510953706, + "tokens_seen": 2239940608 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002597392176529589, + "loss": 1.191, + "theoretical_loss": 3.4269544041098423, + "tokens_seen": 2240006144 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002597231695085256, + "loss": 1.2809, + "theoretical_loss": 3.4269462975695664, + "tokens_seen": 2240071680 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002597071213640923, + "loss": 1.1863, + "theoretical_loss": 3.4269381913328583, + "tokens_seen": 2240137216 + }, + { + "epoch": 1.06, + "learning_rate": 0.000259691073219659, + "loss": 1.2511, + "theoretical_loss": 3.426930085399697, + "tokens_seen": 2240202752 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002596750250752257, + "loss": 1.256, + "theoretical_loss": 3.4269219797700634, + "tokens_seen": 2240268288 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002596589769307924, + "loss": 1.2478, + "theoretical_loss": 3.4269138744439362, + "tokens_seen": 2240333824 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002596429287863591, + "loss": 1.3154, + "theoretical_loss": 3.426905769421296, + "tokens_seen": 2240399360 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002596268806419258, + "loss": 1.3067, + "theoretical_loss": 3.426897664702122, + "tokens_seen": 2240464896 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002596108324974925, + "loss": 1.284, + "theoretical_loss": 3.426889560286394, + "tokens_seen": 2240530432 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002595947843530592, + "loss": 1.2464, + "theoretical_loss": 3.4268814561740917, + "tokens_seen": 2240595968 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002595787362086259, + "loss": 1.2212, + "theoretical_loss": 3.4268733523651953, + "tokens_seen": 2240661504 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002595626880641926, + "loss": 1.1799, + "theoretical_loss": 3.4268652488596847, + "tokens_seen": 2240727040 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002595466399197593, + "loss": 1.2397, + "theoretical_loss": 3.4268571456575385, + "tokens_seen": 2240792576 + }, + { + "epoch": 1.06, + "learning_rate": 0.000259530591775326, + "loss": 1.1976, + "theoretical_loss": 3.4268490427587377, + "tokens_seen": 2240858112 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002595145436308927, + "loss": 1.2734, + "theoretical_loss": 3.426840940163262, + "tokens_seen": 2240923648 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002594984954864594, + "loss": 1.3183, + "theoretical_loss": 3.4268328378710904, + "tokens_seen": 2240989184 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002594824473420261, + "loss": 1.2478, + "theoretical_loss": 3.426824735882203, + "tokens_seen": 2241054720 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002594663991975928, + "loss": 1.284, + "theoretical_loss": 3.42681663419658, + "tokens_seen": 2241120256 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002594503510531595, + "loss": 1.2358, + "theoretical_loss": 3.4268085328142006, + "tokens_seen": 2241185792 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002594343029087262, + "loss": 1.1925, + "theoretical_loss": 3.4268004317350447, + "tokens_seen": 2241251328 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 828683, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2182340621948242, + "objective/train/theoretical_loss": 3.4267923309590924, + "objective/train/tokens_used": 2261776864, + "theoretical_loss": 3.4267923309590924, + "tokens_seen": 2241316864 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002594182547642929, + "loss": 1.2182, + "theoretical_loss": 3.4267923309590924, + "tokens_seen": 2241316864 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025940220661985956, + "loss": 1.2356, + "theoretical_loss": 3.4267842304863234, + "tokens_seen": 2241382400 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025938615847542626, + "loss": 1.3206, + "theoretical_loss": 3.4267761303167172, + "tokens_seen": 2241447936 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025937011033099296, + "loss": 1.2847, + "theoretical_loss": 3.426768030450254, + "tokens_seen": 2241513472 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025935406218655966, + "loss": 1.2385, + "theoretical_loss": 3.426759930886913, + "tokens_seen": 2241579008 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025933801404212636, + "loss": 1.2529, + "theoretical_loss": 3.4267518316266745, + "tokens_seen": 2241644544 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025932196589769307, + "loss": 1.2995, + "theoretical_loss": 3.4267437326695185, + "tokens_seen": 2241710080 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025930591775325977, + "loss": 1.4001, + "theoretical_loss": 3.426735634015424, + "tokens_seen": 2241775616 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025928986960882647, + "loss": 1.1879, + "theoretical_loss": 3.4267275356643716, + "tokens_seen": 2241841152 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025927382146439317, + "loss": 1.1573, + "theoretical_loss": 3.426719437616341, + "tokens_seen": 2241906688 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025925777331995987, + "loss": 1.2365, + "theoretical_loss": 3.426711339871311, + "tokens_seen": 2241972224 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025924172517552657, + "loss": 1.2939, + "theoretical_loss": 3.4267032424292627, + "tokens_seen": 2242037760 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025922567703109327, + "loss": 1.2384, + "theoretical_loss": 3.4266951452901755, + "tokens_seen": 2242103296 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025920962888665997, + "loss": 1.1591, + "theoretical_loss": 3.4266870484540286, + "tokens_seen": 2242168832 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025919358074222667, + "loss": 1.3183, + "theoretical_loss": 3.4266789519208025, + "tokens_seen": 2242234368 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025917753259779337, + "loss": 1.2467, + "theoretical_loss": 3.4266708556904772, + "tokens_seen": 2242299904 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025916148445336007, + "loss": 1.1744, + "theoretical_loss": 3.426662759763032, + "tokens_seen": 2242365440 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025914543630892677, + "loss": 1.1917, + "theoretical_loss": 3.4266546641384465, + "tokens_seen": 2242430976 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002591293881644935, + "loss": 1.1726, + "theoretical_loss": 3.426646568816701, + "tokens_seen": 2242496512 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002591133400200602, + "loss": 1.2071, + "theoretical_loss": 3.4266384737977753, + "tokens_seen": 2242562048 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002590972918756269, + "loss": 1.2709, + "theoretical_loss": 3.426630379081649, + "tokens_seen": 2242627584 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002590812437311936, + "loss": 1.2229, + "theoretical_loss": 3.4266222846683023, + "tokens_seen": 2242693120 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002590651955867603, + "loss": 1.3021, + "theoretical_loss": 3.4266141905577143, + "tokens_seen": 2242758656 + }, + { + "epoch": 1.06, + "learning_rate": 0.000259049147442327, + "loss": 1.2339, + "theoretical_loss": 3.426606096749866, + "tokens_seen": 2242824192 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002590330992978937, + "loss": 1.2748, + "theoretical_loss": 3.426598003244736, + "tokens_seen": 2242889728 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002590170511534604, + "loss": 1.2494, + "theoretical_loss": 3.4265899100423045, + "tokens_seen": 2242955264 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002590010030090271, + "loss": 1.2038, + "theoretical_loss": 3.426581817142552, + "tokens_seen": 2243020800 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002589849548645938, + "loss": 1.2413, + "theoretical_loss": 3.4265737245454577, + "tokens_seen": 2243086336 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002589689067201605, + "loss": 1.2634, + "theoretical_loss": 3.4265656322510014, + "tokens_seen": 2243151872 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002589528585757272, + "loss": 1.3247, + "theoretical_loss": 3.426557540259163, + "tokens_seen": 2243217408 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002589368104312939, + "loss": 1.2098, + "theoretical_loss": 3.4265494485699226, + "tokens_seen": 2243282944 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002589207622868606, + "loss": 1.3415, + "theoretical_loss": 3.42654135718326, + "tokens_seen": 2243348480 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002589047141424273, + "loss": 1.1951, + "theoretical_loss": 3.426533266099155, + "tokens_seen": 2243414016 + }, + { + "epoch": 1.06, + "learning_rate": 0.000258888665997994, + "loss": 1.2947, + "theoretical_loss": 3.4265251753175874, + "tokens_seen": 2243479552 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002588726178535607, + "loss": 1.3429, + "theoretical_loss": 3.426517084838537, + "tokens_seen": 2243545088 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025885656970912744, + "loss": 1.2473, + "theoretical_loss": 3.4265089946619836, + "tokens_seen": 2243610624 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025884052156469414, + "loss": 1.2353, + "theoretical_loss": 3.426500904787907, + "tokens_seen": 2243676160 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025882447342026084, + "loss": 1.2497, + "theoretical_loss": 3.426492815216288, + "tokens_seen": 2243741696 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025880842527582754, + "loss": 1.1176, + "theoretical_loss": 3.4264847259471045, + "tokens_seen": 2243807232 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025879237713139424, + "loss": 1.2379, + "theoretical_loss": 3.4264766369803383, + "tokens_seen": 2243872768 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025877632898696094, + "loss": 1.3097, + "theoretical_loss": 3.4264685483159685, + "tokens_seen": 2243938304 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002587602808425276, + "loss": 1.2261, + "theoretical_loss": 3.426460459953975, + "tokens_seen": 2244003840 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002587442326980943, + "loss": 1.2535, + "theoretical_loss": 3.426452371894337, + "tokens_seen": 2244069376 + }, + { + "epoch": 1.06, + "learning_rate": 0.000258728184553661, + "loss": 1.2498, + "theoretical_loss": 3.4264442841370357, + "tokens_seen": 2244134912 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002587121364092277, + "loss": 1.2711, + "theoretical_loss": 3.4264361966820496, + "tokens_seen": 2244200448 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002586960882647944, + "loss": 1.2545, + "theoretical_loss": 3.4264281095293594, + "tokens_seen": 2244265984 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002586800401203611, + "loss": 1.2417, + "theoretical_loss": 3.426420022678945, + "tokens_seen": 2244331520 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002586639919759278, + "loss": 1.2503, + "theoretical_loss": 3.426411936130786, + "tokens_seen": 2244397056 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002586479438314945, + "loss": 1.2568, + "theoretical_loss": 3.4264038498848626, + "tokens_seen": 2244462592 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002586318956870612, + "loss": 1.1328, + "theoretical_loss": 3.4263957639411546, + "tokens_seen": 2244528128 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002586158475426279, + "loss": 1.2313, + "theoretical_loss": 3.4263876782996414, + "tokens_seen": 2244593664 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002585997993981946, + "loss": 1.1317, + "theoretical_loss": 3.426379592960303, + "tokens_seen": 2244659200 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002585837512537613, + "loss": 1.3269, + "theoretical_loss": 3.4263715079231196, + "tokens_seen": 2244724736 + }, + { + "epoch": 1.06, + "learning_rate": 0.000258567703109328, + "loss": 1.213, + "theoretical_loss": 3.426363423188071, + "tokens_seen": 2244790272 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002585516549648947, + "loss": 1.1629, + "theoretical_loss": 3.4263553387551373, + "tokens_seen": 2244855808 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002585356068204614, + "loss": 1.2544, + "theoretical_loss": 3.426347254624298, + "tokens_seen": 2244921344 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002585195586760281, + "loss": 1.2539, + "theoretical_loss": 3.426339170795533, + "tokens_seen": 2244986880 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002585035105315948, + "loss": 1.214, + "theoretical_loss": 3.4263310872688226, + "tokens_seen": 2245052416 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002584874623871615, + "loss": 1.2071, + "theoretical_loss": 3.4263230040441464, + "tokens_seen": 2245117952 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002584714142427282, + "loss": 1.3149, + "theoretical_loss": 3.426314921121484, + "tokens_seen": 2245183488 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002584553660982949, + "loss": 1.2158, + "theoretical_loss": 3.4263068385008157, + "tokens_seen": 2245249024 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002584393179538616, + "loss": 1.2318, + "theoretical_loss": 3.4262987561821214, + "tokens_seen": 2245314560 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002584232698094283, + "loss": 1.2132, + "theoretical_loss": 3.426290674165381, + "tokens_seen": 2245380096 + }, + { + "epoch": 1.06, + "learning_rate": 0.000258407221664995, + "loss": 1.1682, + "theoretical_loss": 3.426282592450575, + "tokens_seen": 2245445632 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002583911735205617, + "loss": 1.2259, + "theoretical_loss": 3.4262745110376818, + "tokens_seen": 2245511168 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002583751253761284, + "loss": 1.1581, + "theoretical_loss": 3.4262664299266827, + "tokens_seen": 2245576704 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002583590772316951, + "loss": 1.3068, + "theoretical_loss": 3.426258349117557, + "tokens_seen": 2245642240 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002583430290872618, + "loss": 1.2071, + "theoretical_loss": 3.4262502686102843, + "tokens_seen": 2245707776 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002583269809428285, + "loss": 1.2484, + "theoretical_loss": 3.426242188404845, + "tokens_seen": 2245773312 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002583109327983952, + "loss": 1.3954, + "theoretical_loss": 3.426234108501219, + "tokens_seen": 2245838848 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002582948846539619, + "loss": 1.1778, + "theoretical_loss": 3.4262260288993867, + "tokens_seen": 2245904384 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002582788365095286, + "loss": 1.1476, + "theoretical_loss": 3.4262179495993266, + "tokens_seen": 2245969920 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002582627883650953, + "loss": 1.2677, + "theoretical_loss": 3.42620987060102, + "tokens_seen": 2246035456 + }, + { + "epoch": 1.06, + "learning_rate": 0.000258246740220662, + "loss": 1.2628, + "theoretical_loss": 3.4262017919044463, + "tokens_seen": 2246100992 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002582306920762287, + "loss": 1.2202, + "theoretical_loss": 3.4261937135095852, + "tokens_seen": 2246166528 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002582146439317954, + "loss": 1.274, + "theoretical_loss": 3.426185635416417, + "tokens_seen": 2246232064 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002581985957873621, + "loss": 1.2381, + "theoretical_loss": 3.426177557624922, + "tokens_seen": 2246297600 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002581825476429288, + "loss": 1.206, + "theoretical_loss": 3.426169480135079, + "tokens_seen": 2246363136 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002581664994984955, + "loss": 1.2008, + "theoretical_loss": 3.4261614029468688, + "tokens_seen": 2246428672 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002581504513540622, + "loss": 1.1987, + "theoretical_loss": 3.426153326060271, + "tokens_seen": 2246494208 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002581344032096289, + "loss": 1.1982, + "theoretical_loss": 3.4261452494752658, + "tokens_seen": 2246559744 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002581183550651956, + "loss": 1.1811, + "theoretical_loss": 3.426137173191833, + "tokens_seen": 2246625280 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002581023069207623, + "loss": 1.2158, + "theoretical_loss": 3.4261290972099525, + "tokens_seen": 2246690816 + }, + { + "epoch": 1.06, + "learning_rate": 0.000258086258776329, + "loss": 1.2819, + "theoretical_loss": 3.426121021529604, + "tokens_seen": 2246756352 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002580702106318957, + "loss": 1.2426, + "theoretical_loss": 3.4261129461507682, + "tokens_seen": 2246821888 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002580541624874624, + "loss": 1.3224, + "theoretical_loss": 3.4261048710734245, + "tokens_seen": 2246887424 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002580381143430291, + "loss": 1.2021, + "theoretical_loss": 3.4260967962975526, + "tokens_seen": 2246952960 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002580220661985958, + "loss": 1.2092, + "theoretical_loss": 3.4260887218231333, + "tokens_seen": 2247018496 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002580060180541625, + "loss": 1.2199, + "theoretical_loss": 3.4260806476501457, + "tokens_seen": 2247084032 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025798996990972916, + "loss": 1.2387, + "theoretical_loss": 3.42607257377857, + "tokens_seen": 2247149568 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025797392176529586, + "loss": 1.2143, + "theoretical_loss": 3.4260645002083865, + "tokens_seen": 2247215104 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025795787362086257, + "loss": 1.2487, + "theoretical_loss": 3.426056426939575, + "tokens_seen": 2247280640 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025794182547642927, + "loss": 1.2219, + "theoretical_loss": 3.426048353972115, + "tokens_seen": 2247346176 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025792577733199597, + "loss": 1.202, + "theoretical_loss": 3.426040281305987, + "tokens_seen": 2247411712 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025790972918756267, + "loss": 1.1704, + "theoretical_loss": 3.4260322089411708, + "tokens_seen": 2247477248 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025789368104312937, + "loss": 1.3069, + "theoretical_loss": 3.4260241368776465, + "tokens_seen": 2247542784 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025787763289869607, + "loss": 1.3488, + "theoretical_loss": 3.4260160651153937, + "tokens_seen": 2247608320 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025786158475426277, + "loss": 1.1911, + "theoretical_loss": 3.426007993654393, + "tokens_seen": 2247673856 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025784553660982947, + "loss": 1.3078, + "theoretical_loss": 3.4259999224946234, + "tokens_seen": 2247739392 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025782948846539617, + "loss": 1.2696, + "theoretical_loss": 3.425991851636066, + "tokens_seen": 2247804928 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 830920, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1300736665725708, + "objective/train/theoretical_loss": 3.4259837810787, + "objective/train/tokens_used": 2268330464, + "theoretical_loss": 3.4259837810787, + "tokens_seen": 2247870464 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025781344032096287, + "loss": 1.1301, + "theoretical_loss": 3.4259837810787, + "tokens_seen": 2247870464 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025779739217652957, + "loss": 1.3303, + "theoretical_loss": 3.4259757108225055, + "tokens_seen": 2247936000 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025778134403209627, + "loss": 1.2688, + "theoretical_loss": 3.425967640867463, + "tokens_seen": 2248001536 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025776529588766297, + "loss": 1.2338, + "theoretical_loss": 3.4259595712135518, + "tokens_seen": 2248067072 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002577492477432297, + "loss": 1.2946, + "theoretical_loss": 3.425951501860752, + "tokens_seen": 2248132608 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002577331995987964, + "loss": 1.2371, + "theoretical_loss": 3.4259434328090443, + "tokens_seen": 2248198144 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002577171514543631, + "loss": 1.3098, + "theoretical_loss": 3.4259353640584074, + "tokens_seen": 2248263680 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002577011033099298, + "loss": 1.2906, + "theoretical_loss": 3.4259272956088225, + "tokens_seen": 2248329216 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002576850551654965, + "loss": 1.2179, + "theoretical_loss": 3.4259192274602692, + "tokens_seen": 2248394752 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002576690070210632, + "loss": 1.1384, + "theoretical_loss": 3.4259111596127276, + "tokens_seen": 2248460288 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025765295887662993, + "loss": 1.1968, + "theoretical_loss": 3.425903092066177, + "tokens_seen": 2248525824 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025763691073219663, + "loss": 1.1582, + "theoretical_loss": 3.425895024820598, + "tokens_seen": 2248591360 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025762086258776333, + "loss": 1.2679, + "theoretical_loss": 3.4258869578759708, + "tokens_seen": 2248656896 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025760481444333003, + "loss": 1.2826, + "theoretical_loss": 3.425878891232275, + "tokens_seen": 2248722432 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025758876629889673, + "loss": 1.2109, + "theoretical_loss": 3.4258708248894907, + "tokens_seen": 2248787968 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025757271815446343, + "loss": 1.255, + "theoretical_loss": 3.425862758847598, + "tokens_seen": 2248853504 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025755667001003014, + "loss": 1.253, + "theoretical_loss": 3.4258546931065768, + "tokens_seen": 2248919040 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025754062186559684, + "loss": 1.1953, + "theoretical_loss": 3.4258466276664072, + "tokens_seen": 2248984576 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025752457372116354, + "loss": 1.2401, + "theoretical_loss": 3.425838562527069, + "tokens_seen": 2249050112 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025750852557673024, + "loss": 1.1976, + "theoretical_loss": 3.4258304976885423, + "tokens_seen": 2249115648 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025749247743229694, + "loss": 1.1382, + "theoretical_loss": 3.425822433150808, + "tokens_seen": 2249181184 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025747642928786364, + "loss": 1.1549, + "theoretical_loss": 3.4258143689138443, + "tokens_seen": 2249246720 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025746038114343034, + "loss": 1.2201, + "theoretical_loss": 3.4258063049776326, + "tokens_seen": 2249312256 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025744433299899704, + "loss": 1.2377, + "theoretical_loss": 3.425798241342153, + "tokens_seen": 2249377792 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025742828485456374, + "loss": 1.2077, + "theoretical_loss": 3.425790178007384, + "tokens_seen": 2249443328 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025741223671013044, + "loss": 1.2101, + "theoretical_loss": 3.425782114973307, + "tokens_seen": 2249508864 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025739618856569714, + "loss": 1.2673, + "theoretical_loss": 3.4257740522399027, + "tokens_seen": 2249574400 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025738014042126384, + "loss": 1.2466, + "theoretical_loss": 3.4257659898071493, + "tokens_seen": 2249639936 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002573640922768305, + "loss": 1.2533, + "theoretical_loss": 3.425757927675028, + "tokens_seen": 2249705472 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002573480441323972, + "loss": 1.2721, + "theoretical_loss": 3.4257498658435184, + "tokens_seen": 2249771008 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002573319959879639, + "loss": 1.2168, + "theoretical_loss": 3.4257418043126004, + "tokens_seen": 2249836544 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002573159478435306, + "loss": 1.2516, + "theoretical_loss": 3.4257337430822545, + "tokens_seen": 2249902080 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002572998996990973, + "loss": 1.1923, + "theoretical_loss": 3.4257256821524606, + "tokens_seen": 2249967616 + }, + { + "epoch": 1.06, + "learning_rate": 0.000257283851554664, + "loss": 1.3072, + "theoretical_loss": 3.425717621523199, + "tokens_seen": 2250033152 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002572678034102307, + "loss": 1.2212, + "theoretical_loss": 3.4257095611944486, + "tokens_seen": 2250098688 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002572517552657974, + "loss": 1.2069, + "theoretical_loss": 3.425701501166191, + "tokens_seen": 2250164224 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002572357071213641, + "loss": 1.2657, + "theoretical_loss": 3.4256934414384053, + "tokens_seen": 2250229760 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002572196589769308, + "loss": 1.2645, + "theoretical_loss": 3.4256853820110713, + "tokens_seen": 2250295296 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002572036108324975, + "loss": 1.2429, + "theoretical_loss": 3.42567732288417, + "tokens_seen": 2250360832 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002571875626880642, + "loss": 1.315, + "theoretical_loss": 3.4256692640576807, + "tokens_seen": 2250426368 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002571715145436309, + "loss": 1.288, + "theoretical_loss": 3.425661205531584, + "tokens_seen": 2250491904 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002571554663991976, + "loss": 1.2671, + "theoretical_loss": 3.4256531473058596, + "tokens_seen": 2250557440 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002571394182547643, + "loss": 1.305, + "theoretical_loss": 3.4256450893804873, + "tokens_seen": 2250622976 + }, + { + "epoch": 1.06, + "learning_rate": 0.000257123370110331, + "loss": 1.2828, + "theoretical_loss": 3.4256370317554476, + "tokens_seen": 2250688512 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002571073219658977, + "loss": 1.2496, + "theoretical_loss": 3.4256289744307207, + "tokens_seen": 2250754048 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002570912738214644, + "loss": 1.1425, + "theoretical_loss": 3.4256209174062864, + "tokens_seen": 2250819584 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002570752256770311, + "loss": 1.1959, + "theoretical_loss": 3.4256128606821243, + "tokens_seen": 2250885120 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002570591775325978, + "loss": 1.2531, + "theoretical_loss": 3.4256048042582155, + "tokens_seen": 2250950656 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002570431293881645, + "loss": 1.2234, + "theoretical_loss": 3.425596748134539, + "tokens_seen": 2251016192 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002570270812437312, + "loss": 1.2328, + "theoretical_loss": 3.4255886923110754, + "tokens_seen": 2251081728 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002570110330992979, + "loss": 1.2517, + "theoretical_loss": 3.4255806367878052, + "tokens_seen": 2251147264 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002569949849548646, + "loss": 1.2256, + "theoretical_loss": 3.425572581564708, + "tokens_seen": 2251212800 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002569789368104313, + "loss": 1.3154, + "theoretical_loss": 3.4255645266417636, + "tokens_seen": 2251278336 + }, + { + "epoch": 1.06, + "learning_rate": 0.000256962888665998, + "loss": 1.214, + "theoretical_loss": 3.425556472018952, + "tokens_seen": 2251343872 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002569468405215647, + "loss": 1.2791, + "theoretical_loss": 3.4255484176962545, + "tokens_seen": 2251409408 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002569307923771314, + "loss": 1.3332, + "theoretical_loss": 3.4255403636736497, + "tokens_seen": 2251474944 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002569147442326981, + "loss": 1.2523, + "theoretical_loss": 3.4255323099511186, + "tokens_seen": 2251540480 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002568986960882648, + "loss": 1.1581, + "theoretical_loss": 3.425524256528641, + "tokens_seen": 2251606016 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002568826479438315, + "loss": 1.1846, + "theoretical_loss": 3.425516203406197, + "tokens_seen": 2251671552 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002568665997993982, + "loss": 1.2457, + "theoretical_loss": 3.4255081505837666, + "tokens_seen": 2251737088 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002568505516549649, + "loss": 1.1868, + "theoretical_loss": 3.42550009806133, + "tokens_seen": 2251802624 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002568345035105316, + "loss": 1.2205, + "theoretical_loss": 3.425492045838867, + "tokens_seen": 2251868160 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002568184553660983, + "loss": 1.2612, + "theoretical_loss": 3.425483993916359, + "tokens_seen": 2251933696 + }, + { + "epoch": 1.06, + "learning_rate": 0.000256802407221665, + "loss": 1.3704, + "theoretical_loss": 3.425475942293784, + "tokens_seen": 2251999232 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002567863590772317, + "loss": 1.2151, + "theoretical_loss": 3.425467890971124, + "tokens_seen": 2252064768 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002567703109327984, + "loss": 1.1973, + "theoretical_loss": 3.425459839948357, + "tokens_seen": 2252130304 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002567542627883651, + "loss": 1.223, + "theoretical_loss": 3.4254517892254657, + "tokens_seen": 2252195840 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002567382146439318, + "loss": 1.1763, + "theoretical_loss": 3.425443738802428, + "tokens_seen": 2252261376 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002567221664994985, + "loss": 1.1662, + "theoretical_loss": 3.4254356886792254, + "tokens_seen": 2252326912 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002567061183550652, + "loss": 1.2592, + "theoretical_loss": 3.4254276388558376, + "tokens_seen": 2252392448 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002566900702106319, + "loss": 1.1343, + "theoretical_loss": 3.425419589332244, + "tokens_seen": 2252457984 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002566740220661986, + "loss": 1.3206, + "theoretical_loss": 3.425411540108426, + "tokens_seen": 2252523520 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002566579739217653, + "loss": 1.2175, + "theoretical_loss": 3.4254034911843627, + "tokens_seen": 2252589056 + }, + { + "epoch": 1.06, + "learning_rate": 0.000256641925777332, + "loss": 1.3199, + "theoretical_loss": 3.4253954425600344, + "tokens_seen": 2252654592 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002566258776328987, + "loss": 1.2583, + "theoretical_loss": 3.4253873942354214, + "tokens_seen": 2252720128 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002566098294884654, + "loss": 1.2849, + "theoretical_loss": 3.425379346210504, + "tokens_seen": 2252785664 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025659378134403206, + "loss": 1.1959, + "theoretical_loss": 3.425371298485262, + "tokens_seen": 2252851200 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025657773319959877, + "loss": 1.2791, + "theoretical_loss": 3.4253632510596757, + "tokens_seen": 2252916736 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025656168505516547, + "loss": 1.2144, + "theoretical_loss": 3.425355203933725, + "tokens_seen": 2252982272 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025654563691073217, + "loss": 1.323, + "theoretical_loss": 3.42534715710739, + "tokens_seen": 2253047808 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025652958876629887, + "loss": 1.3617, + "theoretical_loss": 3.4253391105806514, + "tokens_seen": 2253113344 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025651354062186557, + "loss": 1.2705, + "theoretical_loss": 3.425331064353489, + "tokens_seen": 2253178880 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025649749247743227, + "loss": 1.2225, + "theoretical_loss": 3.4253230184258827, + "tokens_seen": 2253244416 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025648144433299897, + "loss": 1.2934, + "theoretical_loss": 3.425314972797813, + "tokens_seen": 2253309952 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025646539618856567, + "loss": 1.2413, + "theoretical_loss": 3.4253069274692596, + "tokens_seen": 2253375488 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002564493480441324, + "loss": 1.1743, + "theoretical_loss": 3.4252988824402033, + "tokens_seen": 2253441024 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002564332998996991, + "loss": 1.1801, + "theoretical_loss": 3.425290837710623, + "tokens_seen": 2253506560 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002564172517552658, + "loss": 1.1848, + "theoretical_loss": 3.4252827932805006, + "tokens_seen": 2253572096 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002564012036108325, + "loss": 1.2644, + "theoretical_loss": 3.425274749149815, + "tokens_seen": 2253637632 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025638515546639923, + "loss": 1.2111, + "theoretical_loss": 3.4252667053185464, + "tokens_seen": 2253703168 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025636910732196593, + "loss": 1.3522, + "theoretical_loss": 3.4252586617866756, + "tokens_seen": 2253768704 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025635305917753263, + "loss": 1.2535, + "theoretical_loss": 3.4252506185541822, + "tokens_seen": 2253834240 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025633701103309933, + "loss": 1.3653, + "theoretical_loss": 3.4252425756210467, + "tokens_seen": 2253899776 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025632096288866603, + "loss": 1.2559, + "theoretical_loss": 3.4252345329872487, + "tokens_seen": 2253965312 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025630491474423273, + "loss": 1.1497, + "theoretical_loss": 3.425226490652769, + "tokens_seen": 2254030848 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025628886659979943, + "loss": 1.3, + "theoretical_loss": 3.4252184486175876, + "tokens_seen": 2254096384 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025627281845536613, + "loss": 1.3143, + "theoretical_loss": 3.425210406881684, + "tokens_seen": 2254161920 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025625677031093283, + "loss": 1.2174, + "theoretical_loss": 3.4252023654450396, + "tokens_seen": 2254227456 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025624072216649953, + "loss": 1.2537, + "theoretical_loss": 3.4251943243076335, + "tokens_seen": 2254292992 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025622467402206623, + "loss": 1.2081, + "theoretical_loss": 3.425186283469446, + "tokens_seen": 2254358528 + }, + { + "epoch": 1.06, + "objective/train/docs_used": 833282, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2214281558990479, + "objective/train/theoretical_loss": 3.4251782429304583, + "objective/train/tokens_used": 2274884064, + "theoretical_loss": 3.4251782429304583, + "tokens_seen": 2254424064 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025620862587763293, + "loss": 1.2214, + "theoretical_loss": 3.4251782429304583, + "tokens_seen": 2254424064 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025619257773319964, + "loss": 1.2231, + "theoretical_loss": 3.425170202690649, + "tokens_seen": 2254489600 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025617652958876634, + "loss": 1.295, + "theoretical_loss": 3.4251621627499995, + "tokens_seen": 2254555136 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025616048144433304, + "loss": 1.2059, + "theoretical_loss": 3.4251541231084897, + "tokens_seen": 2254620672 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025614443329989974, + "loss": 1.3001, + "theoretical_loss": 3.425146083766099, + "tokens_seen": 2254686208 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025612838515546644, + "loss": 1.2325, + "theoretical_loss": 3.4251380447228086, + "tokens_seen": 2254751744 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025611233701103314, + "loss": 1.3021, + "theoretical_loss": 3.4251300059785983, + "tokens_seen": 2254817280 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025609628886659984, + "loss": 1.2254, + "theoretical_loss": 3.425121967533448, + "tokens_seen": 2254882816 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025608024072216654, + "loss": 1.276, + "theoretical_loss": 3.4251139293873383, + "tokens_seen": 2254948352 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025606419257773324, + "loss": 1.1895, + "theoretical_loss": 3.425105891540249, + "tokens_seen": 2255013888 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025604814443329994, + "loss": 1.2169, + "theoretical_loss": 3.4250978539921606, + "tokens_seen": 2255079424 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025603209628886664, + "loss": 1.2113, + "theoretical_loss": 3.4250898167430535, + "tokens_seen": 2255144960 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025601604814443334, + "loss": 1.2847, + "theoretical_loss": 3.425081779792907, + "tokens_seen": 2255210496 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025600000000000004, + "loss": 1.2877, + "theoretical_loss": 3.425073743141702, + "tokens_seen": 2255276032 + }, + { + "epoch": 1.06, + "learning_rate": 0.00025598395185556674, + "loss": 1.2014, + "theoretical_loss": 3.425065706789419, + "tokens_seen": 2255341568 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002559679037111334, + "loss": 1.2406, + "theoretical_loss": 3.425057670736037, + "tokens_seen": 2255407104 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002559518555667001, + "loss": 1.2925, + "theoretical_loss": 3.425049634981538, + "tokens_seen": 2255472640 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002559358074222668, + "loss": 1.1102, + "theoretical_loss": 3.4250415995259003, + "tokens_seen": 2255538176 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002559197592778335, + "loss": 1.2531, + "theoretical_loss": 3.425033564369105, + "tokens_seen": 2255603712 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002559037111334002, + "loss": 1.2409, + "theoretical_loss": 3.425025529511133, + "tokens_seen": 2255669248 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002558876629889669, + "loss": 1.1867, + "theoretical_loss": 3.425017494951963, + "tokens_seen": 2255734784 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002558716148445336, + "loss": 1.2058, + "theoretical_loss": 3.425009460691576, + "tokens_seen": 2255800320 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002558555667001003, + "loss": 1.2085, + "theoretical_loss": 3.4250014267299527, + "tokens_seen": 2255865856 + }, + { + "epoch": 1.06, + "learning_rate": 0.000255839518555667, + "loss": 1.2606, + "theoretical_loss": 3.4249933930670724, + "tokens_seen": 2255931392 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002558234704112337, + "loss": 1.2487, + "theoretical_loss": 3.424985359702916, + "tokens_seen": 2255996928 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002558074222668004, + "loss": 1.3594, + "theoretical_loss": 3.424977326637463, + "tokens_seen": 2256062464 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002557913741223671, + "loss": 1.241, + "theoretical_loss": 3.424969293870694, + "tokens_seen": 2256128000 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002557753259779338, + "loss": 1.2691, + "theoretical_loss": 3.42496126140259, + "tokens_seen": 2256193536 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002557592778335005, + "loss": 1.2812, + "theoretical_loss": 3.4249532292331297, + "tokens_seen": 2256259072 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002557432296890672, + "loss": 1.105, + "theoretical_loss": 3.424945197362294, + "tokens_seen": 2256324608 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002557271815446339, + "loss": 1.2292, + "theoretical_loss": 3.424937165790064, + "tokens_seen": 2256390144 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002557111334002006, + "loss": 1.317, + "theoretical_loss": 3.4249291345164186, + "tokens_seen": 2256455680 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002556950852557673, + "loss": 1.235, + "theoretical_loss": 3.4249211035413385, + "tokens_seen": 2256521216 + }, + { + "epoch": 1.06, + "learning_rate": 0.000255679037111334, + "loss": 1.2516, + "theoretical_loss": 3.4249130728648045, + "tokens_seen": 2256586752 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002556629889669007, + "loss": 1.2685, + "theoretical_loss": 3.424905042486796, + "tokens_seen": 2256652288 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002556469408224674, + "loss": 1.2201, + "theoretical_loss": 3.4248970124072935, + "tokens_seen": 2256717824 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002556308926780341, + "loss": 1.267, + "theoretical_loss": 3.424888982626278, + "tokens_seen": 2256783360 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002556148445336008, + "loss": 1.2601, + "theoretical_loss": 3.4248809531437283, + "tokens_seen": 2256848896 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002555987963891675, + "loss": 1.1852, + "theoretical_loss": 3.4248729239596254, + "tokens_seen": 2256914432 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002555827482447342, + "loss": 1.222, + "theoretical_loss": 3.42486489507395, + "tokens_seen": 2256979968 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002555667001003009, + "loss": 1.3427, + "theoretical_loss": 3.424856866486681, + "tokens_seen": 2257045504 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002555506519558676, + "loss": 1.2157, + "theoretical_loss": 3.4248488381978004, + "tokens_seen": 2257111040 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002555346038114343, + "loss": 1.2632, + "theoretical_loss": 3.4248408102072876, + "tokens_seen": 2257176576 + }, + { + "epoch": 1.06, + "learning_rate": 0.000255518555667001, + "loss": 1.3357, + "theoretical_loss": 3.4248327825151224, + "tokens_seen": 2257242112 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002555025075225677, + "loss": 1.2885, + "theoretical_loss": 3.424824755121285, + "tokens_seen": 2257307648 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002554864593781344, + "loss": 1.3242, + "theoretical_loss": 3.424816728025757, + "tokens_seen": 2257373184 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002554704112337011, + "loss": 1.3278, + "theoretical_loss": 3.4248087012285175, + "tokens_seen": 2257438720 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002554543630892678, + "loss": 1.2246, + "theoretical_loss": 3.4248006747295467, + "tokens_seen": 2257504256 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002554383149448345, + "loss": 1.32, + "theoretical_loss": 3.4247926485288254, + "tokens_seen": 2257569792 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002554222668004012, + "loss": 1.2146, + "theoretical_loss": 3.424784622626334, + "tokens_seen": 2257635328 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002554062186559679, + "loss": 1.1884, + "theoretical_loss": 3.424776597022052, + "tokens_seen": 2257700864 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002553901705115346, + "loss": 1.2271, + "theoretical_loss": 3.42476857171596, + "tokens_seen": 2257766400 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002553741223671013, + "loss": 1.3592, + "theoretical_loss": 3.4247605467080384, + "tokens_seen": 2257831936 + }, + { + "epoch": 1.06, + "learning_rate": 0.000255358074222668, + "loss": 1.2754, + "theoretical_loss": 3.4247525219982675, + "tokens_seen": 2257897472 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002553420260782347, + "loss": 1.2641, + "theoretical_loss": 3.4247444975866275, + "tokens_seen": 2257963008 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002553259779338014, + "loss": 1.2116, + "theoretical_loss": 3.4247364734730983, + "tokens_seen": 2258028544 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002553099297893681, + "loss": 1.3028, + "theoretical_loss": 3.4247284496576613, + "tokens_seen": 2258094080 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002552938816449348, + "loss": 1.2594, + "theoretical_loss": 3.4247204261402953, + "tokens_seen": 2258159616 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002552778335005015, + "loss": 1.2714, + "theoretical_loss": 3.4247124029209814, + "tokens_seen": 2258225152 + }, + { + "epoch": 1.06, + "learning_rate": 0.0002552617853560682, + "loss": 1.3189, + "theoretical_loss": 3.4247043799996995, + "tokens_seen": 2258290688 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002552457372116349, + "loss": 1.359, + "theoretical_loss": 3.4246963573764306, + "tokens_seen": 2258356224 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002552296890672016, + "loss": 1.2302, + "theoretical_loss": 3.4246883350511546, + "tokens_seen": 2258421760 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002552136409227683, + "loss": 1.2244, + "theoretical_loss": 3.4246803130238512, + "tokens_seen": 2258487296 + }, + { + "epoch": 1.07, + "learning_rate": 0.000255197592778335, + "loss": 1.3162, + "theoretical_loss": 3.4246722912945016, + "tokens_seen": 2258552832 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002551815446339017, + "loss": 1.2894, + "theoretical_loss": 3.4246642698630856, + "tokens_seen": 2258618368 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002551654964894684, + "loss": 1.2231, + "theoretical_loss": 3.4246562487295833, + "tokens_seen": 2258683904 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002551494483450351, + "loss": 1.1843, + "theoretical_loss": 3.424648227893975, + "tokens_seen": 2258749440 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002551334002006018, + "loss": 1.334, + "theoretical_loss": 3.424640207356242, + "tokens_seen": 2258814976 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002551173520561685, + "loss": 1.2489, + "theoretical_loss": 3.4246321871163636, + "tokens_seen": 2258880512 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002551013039117352, + "loss": 1.2477, + "theoretical_loss": 3.4246241671743203, + "tokens_seen": 2258946048 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002550852557673019, + "loss": 1.1877, + "theoretical_loss": 3.424616147530092, + "tokens_seen": 2259011584 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002550692076228686, + "loss": 1.1959, + "theoretical_loss": 3.42460812818366, + "tokens_seen": 2259077120 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002550531594784353, + "loss": 1.2548, + "theoretical_loss": 3.424600109135004, + "tokens_seen": 2259142656 + }, + { + "epoch": 1.07, + "learning_rate": 0.000255037111334002, + "loss": 1.2725, + "theoretical_loss": 3.4245920903841043, + "tokens_seen": 2259208192 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025502106318956873, + "loss": 1.1767, + "theoretical_loss": 3.424584071930941, + "tokens_seen": 2259273728 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025500501504513543, + "loss": 1.3534, + "theoretical_loss": 3.424576053775495, + "tokens_seen": 2259339264 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025498896690070213, + "loss": 1.3163, + "theoretical_loss": 3.424568035917746, + "tokens_seen": 2259404800 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025497291875626883, + "loss": 1.2772, + "theoretical_loss": 3.4245600183576745, + "tokens_seen": 2259470336 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025495687061183553, + "loss": 1.1946, + "theoretical_loss": 3.424552001095261, + "tokens_seen": 2259535872 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025494082246740223, + "loss": 1.2311, + "theoretical_loss": 3.424543984130486, + "tokens_seen": 2259601408 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025492477432296893, + "loss": 1.2843, + "theoretical_loss": 3.424535967463329, + "tokens_seen": 2259666944 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025490872617853563, + "loss": 1.1828, + "theoretical_loss": 3.4245279510937716, + "tokens_seen": 2259732480 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025489267803410233, + "loss": 1.2416, + "theoretical_loss": 3.424519935021793, + "tokens_seen": 2259798016 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025487662988966903, + "loss": 1.2444, + "theoretical_loss": 3.4245119192473736, + "tokens_seen": 2259863552 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025486058174523573, + "loss": 1.2078, + "theoretical_loss": 3.4245039037704945, + "tokens_seen": 2259929088 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025484453360080243, + "loss": 1.2392, + "theoretical_loss": 3.424495888591135, + "tokens_seen": 2259994624 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025482848545636913, + "loss": 1.3182, + "theoretical_loss": 3.4244878737092765, + "tokens_seen": 2260060160 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025481243731193584, + "loss": 1.2004, + "theoretical_loss": 3.424479859124899, + "tokens_seen": 2260125696 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025479638916750254, + "loss": 1.27, + "theoretical_loss": 3.4244718448379823, + "tokens_seen": 2260191232 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025478034102306924, + "loss": 1.293, + "theoretical_loss": 3.424463830848507, + "tokens_seen": 2260256768 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025476429287863594, + "loss": 1.2625, + "theoretical_loss": 3.4244558171564536, + "tokens_seen": 2260322304 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025474824473420264, + "loss": 1.2268, + "theoretical_loss": 3.4244478037618022, + "tokens_seen": 2260387840 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025473219658976934, + "loss": 1.3131, + "theoretical_loss": 3.4244397906645334, + "tokens_seen": 2260453376 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025471614844533604, + "loss": 1.2845, + "theoretical_loss": 3.424431777864628, + "tokens_seen": 2260518912 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025470010030090274, + "loss": 1.2976, + "theoretical_loss": 3.4244237653620653, + "tokens_seen": 2260584448 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025468405215646944, + "loss": 1.1556, + "theoretical_loss": 3.424415753156826, + "tokens_seen": 2260649984 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025466800401203614, + "loss": 1.2063, + "theoretical_loss": 3.4244077412488907, + "tokens_seen": 2260715520 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025465195586760284, + "loss": 1.2763, + "theoretical_loss": 3.42439972963824, + "tokens_seen": 2260781056 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025463590772316954, + "loss": 1.26, + "theoretical_loss": 3.4243917183248533, + "tokens_seen": 2260846592 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025461985957873624, + "loss": 1.2512, + "theoretical_loss": 3.4243837073087118, + "tokens_seen": 2260912128 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 836050, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2712182998657227, + "objective/train/theoretical_loss": 3.4243756965897956, + "objective/train/tokens_used": 2281437664, + "theoretical_loss": 3.4243756965897956, + "tokens_seen": 2260977664 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025460381143430294, + "loss": 1.2712, + "theoretical_loss": 3.4243756965897956, + "tokens_seen": 2260977664 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025458776328986964, + "loss": 1.132, + "theoretical_loss": 3.424367686168085, + "tokens_seen": 2261043200 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025457171514543634, + "loss": 1.2842, + "theoretical_loss": 3.4243596760435606, + "tokens_seen": 2261108736 + }, + { + "epoch": 1.07, + "learning_rate": 0.000254555667001003, + "loss": 1.2039, + "theoretical_loss": 3.4243516662162024, + "tokens_seen": 2261174272 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002545396188565697, + "loss": 1.2667, + "theoretical_loss": 3.424343656685991, + "tokens_seen": 2261239808 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002545235707121364, + "loss": 1.2058, + "theoretical_loss": 3.4243356474529065, + "tokens_seen": 2261305344 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002545075225677031, + "loss": 1.2263, + "theoretical_loss": 3.42432763851693, + "tokens_seen": 2261370880 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002544914744232698, + "loss": 1.1941, + "theoretical_loss": 3.424319629878041, + "tokens_seen": 2261436416 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002544754262788365, + "loss": 1.1617, + "theoretical_loss": 3.42431162153622, + "tokens_seen": 2261501952 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002544593781344032, + "loss": 1.1736, + "theoretical_loss": 3.424303613491448, + "tokens_seen": 2261567488 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002544433299899699, + "loss": 1.2976, + "theoretical_loss": 3.4242956057437044, + "tokens_seen": 2261633024 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002544272818455366, + "loss": 1.1631, + "theoretical_loss": 3.4242875982929704, + "tokens_seen": 2261698560 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002544112337011033, + "loss": 1.3425, + "theoretical_loss": 3.4242795911392263, + "tokens_seen": 2261764096 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025439518555667, + "loss": 1.2181, + "theoretical_loss": 3.424271584282452, + "tokens_seen": 2261829632 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002543791374122367, + "loss": 1.1942, + "theoretical_loss": 3.4242635777226287, + "tokens_seen": 2261895168 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002543630892678034, + "loss": 1.3521, + "theoretical_loss": 3.4242555714597356, + "tokens_seen": 2261960704 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002543470411233701, + "loss": 1.2515, + "theoretical_loss": 3.424247565493754, + "tokens_seen": 2262026240 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002543309929789368, + "loss": 1.2305, + "theoretical_loss": 3.424239559824664, + "tokens_seen": 2262091776 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002543149448345035, + "loss": 1.3352, + "theoretical_loss": 3.4242315544524464, + "tokens_seen": 2262157312 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002542988966900702, + "loss": 1.3211, + "theoretical_loss": 3.4242235493770807, + "tokens_seen": 2262222848 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002542828485456369, + "loss": 1.2576, + "theoretical_loss": 3.424215544598548, + "tokens_seen": 2262288384 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002542668004012036, + "loss": 1.1924, + "theoretical_loss": 3.424207540116828, + "tokens_seen": 2262353920 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002542507522567703, + "loss": 1.3566, + "theoretical_loss": 3.424199535931902, + "tokens_seen": 2262419456 + }, + { + "epoch": 1.07, + "learning_rate": 0.000254234704112337, + "loss": 1.244, + "theoretical_loss": 3.4241915320437504, + "tokens_seen": 2262484992 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002542186559679037, + "loss": 1.2334, + "theoretical_loss": 3.4241835284523527, + "tokens_seen": 2262550528 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002542026078234704, + "loss": 1.2282, + "theoretical_loss": 3.42417552515769, + "tokens_seen": 2262616064 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002541865596790371, + "loss": 1.2164, + "theoretical_loss": 3.4241675221597427, + "tokens_seen": 2262681600 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002541705115346038, + "loss": 1.2521, + "theoretical_loss": 3.424159519458491, + "tokens_seen": 2262747136 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002541544633901705, + "loss": 1.3146, + "theoretical_loss": 3.4241515170539145, + "tokens_seen": 2262812672 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002541384152457372, + "loss": 1.2473, + "theoretical_loss": 3.424143514945995, + "tokens_seen": 2262878208 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002541223671013039, + "loss": 1.318, + "theoretical_loss": 3.424135513134712, + "tokens_seen": 2262943744 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002541063189568706, + "loss": 1.3395, + "theoretical_loss": 3.424127511620047, + "tokens_seen": 2263009280 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002540902708124373, + "loss": 1.2776, + "theoretical_loss": 3.424119510401979, + "tokens_seen": 2263074816 + }, + { + "epoch": 1.07, + "learning_rate": 0.000254074222668004, + "loss": 1.2451, + "theoretical_loss": 3.4241115094804893, + "tokens_seen": 2263140352 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002540581745235707, + "loss": 1.1321, + "theoretical_loss": 3.4241035088555583, + "tokens_seen": 2263205888 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025404212637913747, + "loss": 1.1705, + "theoretical_loss": 3.4240955085271656, + "tokens_seen": 2263271424 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025402607823470417, + "loss": 1.1646, + "theoretical_loss": 3.424087508495293, + "tokens_seen": 2263336960 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025401003009027087, + "loss": 1.2581, + "theoretical_loss": 3.4240795087599194, + "tokens_seen": 2263402496 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025399398194583757, + "loss": 1.2031, + "theoretical_loss": 3.4240715093210268, + "tokens_seen": 2263468032 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025397793380140427, + "loss": 1.1045, + "theoretical_loss": 3.4240635101785943, + "tokens_seen": 2263533568 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025396188565697097, + "loss": 1.215, + "theoretical_loss": 3.424055511332603, + "tokens_seen": 2263599104 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025394583751253767, + "loss": 1.3226, + "theoretical_loss": 3.4240475127830328, + "tokens_seen": 2263664640 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002539297893681043, + "loss": 1.196, + "theoretical_loss": 3.424039514529865, + "tokens_seen": 2263730176 + }, + { + "epoch": 1.07, + "learning_rate": 0.000253913741223671, + "loss": 1.1822, + "theoretical_loss": 3.4240315165730797, + "tokens_seen": 2263795712 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002538976930792377, + "loss": 1.1913, + "theoretical_loss": 3.424023518912657, + "tokens_seen": 2263861248 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002538816449348044, + "loss": 1.2903, + "theoretical_loss": 3.424015521548577, + "tokens_seen": 2263926784 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002538655967903711, + "loss": 1.2243, + "theoretical_loss": 3.424007524480821, + "tokens_seen": 2263992320 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002538495486459378, + "loss": 1.1614, + "theoretical_loss": 3.4239995277093693, + "tokens_seen": 2264057856 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002538335005015045, + "loss": 1.2513, + "theoretical_loss": 3.4239915312342024, + "tokens_seen": 2264123392 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002538174523570712, + "loss": 1.2623, + "theoretical_loss": 3.4239835350553003, + "tokens_seen": 2264188928 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002538014042126379, + "loss": 1.2728, + "theoretical_loss": 3.4239755391726434, + "tokens_seen": 2264254464 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002537853560682046, + "loss": 1.2458, + "theoretical_loss": 3.4239675435862122, + "tokens_seen": 2264320000 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002537693079237713, + "loss": 1.2764, + "theoretical_loss": 3.4239595482959877, + "tokens_seen": 2264385536 + }, + { + "epoch": 1.07, + "learning_rate": 0.000253753259779338, + "loss": 1.2442, + "theoretical_loss": 3.4239515533019502, + "tokens_seen": 2264451072 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002537372116349047, + "loss": 1.1701, + "theoretical_loss": 3.42394355860408, + "tokens_seen": 2264516608 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002537211634904714, + "loss": 1.2119, + "theoretical_loss": 3.423935564202357, + "tokens_seen": 2264582144 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002537051153460381, + "loss": 1.2355, + "theoretical_loss": 3.4239275700967626, + "tokens_seen": 2264647680 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002536890672016048, + "loss": 1.2428, + "theoretical_loss": 3.4239195762872767, + "tokens_seen": 2264713216 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002536730190571715, + "loss": 1.2344, + "theoretical_loss": 3.4239115827738797, + "tokens_seen": 2264778752 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002536569709127382, + "loss": 1.2274, + "theoretical_loss": 3.4239035895565526, + "tokens_seen": 2264844288 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025364092276830493, + "loss": 1.2335, + "theoretical_loss": 3.4238955966352758, + "tokens_seen": 2264909824 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025362487462387163, + "loss": 1.2749, + "theoretical_loss": 3.4238876040100292, + "tokens_seen": 2264975360 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025360882647943833, + "loss": 1.1714, + "theoretical_loss": 3.4238796116807935, + "tokens_seen": 2265040896 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025359277833500503, + "loss": 1.2781, + "theoretical_loss": 3.4238716196475494, + "tokens_seen": 2265106432 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025357673019057173, + "loss": 1.2285, + "theoretical_loss": 3.423863627910277, + "tokens_seen": 2265171968 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025356068204613843, + "loss": 1.3324, + "theoretical_loss": 3.423855636468957, + "tokens_seen": 2265237504 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025354463390170513, + "loss": 1.1733, + "theoretical_loss": 3.42384764532357, + "tokens_seen": 2265303040 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025352858575727183, + "loss": 1.2432, + "theoretical_loss": 3.4238396544740963, + "tokens_seen": 2265368576 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025351253761283853, + "loss": 1.2604, + "theoretical_loss": 3.4238316639205166, + "tokens_seen": 2265434112 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025349648946840523, + "loss": 1.2301, + "theoretical_loss": 3.4238236736628114, + "tokens_seen": 2265499648 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025348044132397193, + "loss": 1.223, + "theoretical_loss": 3.423815683700961, + "tokens_seen": 2265565184 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025346439317953863, + "loss": 1.2556, + "theoretical_loss": 3.4238076940349456, + "tokens_seen": 2265630720 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025344834503510534, + "loss": 1.2855, + "theoretical_loss": 3.423799704664746, + "tokens_seen": 2265696256 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025343229689067204, + "loss": 1.3014, + "theoretical_loss": 3.423791715590343, + "tokens_seen": 2265761792 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025341624874623874, + "loss": 1.2709, + "theoretical_loss": 3.4237837268117164, + "tokens_seen": 2265827328 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025340020060180544, + "loss": 1.2134, + "theoretical_loss": 3.4237757383288474, + "tokens_seen": 2265892864 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025338415245737214, + "loss": 1.2873, + "theoretical_loss": 3.423767750141716, + "tokens_seen": 2265958400 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025336810431293884, + "loss": 1.284, + "theoretical_loss": 3.423759762250303, + "tokens_seen": 2266023936 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025335205616850554, + "loss": 1.3085, + "theoretical_loss": 3.4237517746545887, + "tokens_seen": 2266089472 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025333600802407224, + "loss": 1.1521, + "theoretical_loss": 3.4237437873545535, + "tokens_seen": 2266155008 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025331995987963894, + "loss": 1.2851, + "theoretical_loss": 3.4237358003501783, + "tokens_seen": 2266220544 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025330391173520564, + "loss": 1.2079, + "theoretical_loss": 3.423727813641443, + "tokens_seen": 2266286080 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025328786359077234, + "loss": 1.2267, + "theoretical_loss": 3.423719827228329, + "tokens_seen": 2266351616 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025327181544633904, + "loss": 1.232, + "theoretical_loss": 3.423711841110816, + "tokens_seen": 2266417152 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025325576730190574, + "loss": 1.2819, + "theoretical_loss": 3.423703855288885, + "tokens_seen": 2266482688 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025323971915747244, + "loss": 1.3319, + "theoretical_loss": 3.4236958697625166, + "tokens_seen": 2266548224 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025322367101303914, + "loss": 1.2047, + "theoretical_loss": 3.4236878845316903, + "tokens_seen": 2266613760 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025320762286860584, + "loss": 1.1853, + "theoretical_loss": 3.423679899596388, + "tokens_seen": 2266679296 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025319157472417255, + "loss": 1.2532, + "theoretical_loss": 3.4236719149565893, + "tokens_seen": 2266744832 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025317552657973925, + "loss": 1.174, + "theoretical_loss": 3.423663930612275, + "tokens_seen": 2266810368 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002531594784353059, + "loss": 1.2654, + "theoretical_loss": 3.4236559465634255, + "tokens_seen": 2266875904 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002531434302908726, + "loss": 1.3308, + "theoretical_loss": 3.423647962810022, + "tokens_seen": 2266941440 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002531273821464393, + "loss": 1.1732, + "theoretical_loss": 3.423639979352044, + "tokens_seen": 2267006976 + }, + { + "epoch": 1.07, + "learning_rate": 0.000253111334002006, + "loss": 1.2485, + "theoretical_loss": 3.4236319961894726, + "tokens_seen": 2267072512 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002530952858575727, + "loss": 1.2692, + "theoretical_loss": 3.4236240133222884, + "tokens_seen": 2267138048 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002530792377131394, + "loss": 1.2437, + "theoretical_loss": 3.4236160307504715, + "tokens_seen": 2267203584 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002530631895687061, + "loss": 1.2005, + "theoretical_loss": 3.4236080484740024, + "tokens_seen": 2267269120 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002530471414242728, + "loss": 1.2358, + "theoretical_loss": 3.4236000664928623, + "tokens_seen": 2267334656 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002530310932798395, + "loss": 1.32, + "theoretical_loss": 3.423592084807032, + "tokens_seen": 2267400192 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002530150451354062, + "loss": 1.2659, + "theoretical_loss": 3.4235841034164904, + "tokens_seen": 2267465728 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 838484, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3156183958053589, + "objective/train/theoretical_loss": 3.4235761223212196, + "objective/train/tokens_used": 2287991264, + "theoretical_loss": 3.4235761223212196, + "tokens_seen": 2267531264 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002529989969909729, + "loss": 1.3156, + "theoretical_loss": 3.4235761223212196, + "tokens_seen": 2267531264 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002529829488465396, + "loss": 1.185, + "theoretical_loss": 3.4235681415211996, + "tokens_seen": 2267596800 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002529669007021063, + "loss": 1.309, + "theoretical_loss": 3.4235601610164106, + "tokens_seen": 2267662336 + }, + { + "epoch": 1.07, + "learning_rate": 0.000252950852557673, + "loss": 1.2407, + "theoretical_loss": 3.4235521808068334, + "tokens_seen": 2267727872 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002529348044132397, + "loss": 1.2052, + "theoretical_loss": 3.423544200892449, + "tokens_seen": 2267793408 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002529187562688064, + "loss": 1.3972, + "theoretical_loss": 3.423536221273238, + "tokens_seen": 2267858944 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002529027081243731, + "loss": 1.2169, + "theoretical_loss": 3.42352824194918, + "tokens_seen": 2267924480 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002528866599799398, + "loss": 1.2621, + "theoretical_loss": 3.423520262920256, + "tokens_seen": 2267990016 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002528706118355065, + "loss": 1.3227, + "theoretical_loss": 3.4235122841864465, + "tokens_seen": 2268055552 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002528545636910732, + "loss": 1.2798, + "theoretical_loss": 3.423504305747733, + "tokens_seen": 2268121088 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025283851554663996, + "loss": 1.3343, + "theoretical_loss": 3.4234963276040946, + "tokens_seen": 2268186624 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025282246740220666, + "loss": 1.2217, + "theoretical_loss": 3.423488349755513, + "tokens_seen": 2268252160 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025280641925777336, + "loss": 1.2382, + "theoretical_loss": 3.4234803722019675, + "tokens_seen": 2268317696 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025279037111334006, + "loss": 1.2435, + "theoretical_loss": 3.42347239494344, + "tokens_seen": 2268383232 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025277432296890676, + "loss": 1.1518, + "theoretical_loss": 3.4234644179799107, + "tokens_seen": 2268448768 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025275827482447346, + "loss": 1.1831, + "theoretical_loss": 3.4234564413113597, + "tokens_seen": 2268514304 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025274222668004016, + "loss": 1.2568, + "theoretical_loss": 3.423448464937768, + "tokens_seen": 2268579840 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025272617853560686, + "loss": 1.331, + "theoretical_loss": 3.423440488859116, + "tokens_seen": 2268645376 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025271013039117356, + "loss": 1.1995, + "theoretical_loss": 3.4234325130753844, + "tokens_seen": 2268710912 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025269408224674026, + "loss": 1.2588, + "theoretical_loss": 3.4234245375865537, + "tokens_seen": 2268776448 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025267803410230697, + "loss": 1.2318, + "theoretical_loss": 3.423416562392604, + "tokens_seen": 2268841984 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025266198595787367, + "loss": 1.2621, + "theoretical_loss": 3.4234085874935167, + "tokens_seen": 2268907520 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025264593781344037, + "loss": 1.2354, + "theoretical_loss": 3.423400612889272, + "tokens_seen": 2268973056 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025262988966900707, + "loss": 1.2445, + "theoretical_loss": 3.4233926385798505, + "tokens_seen": 2269038592 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025261384152457377, + "loss": 1.2248, + "theoretical_loss": 3.423384664565233, + "tokens_seen": 2269104128 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025259779338014047, + "loss": 1.2347, + "theoretical_loss": 3.4233766908453998, + "tokens_seen": 2269169664 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025258174523570717, + "loss": 1.2207, + "theoretical_loss": 3.4233687174203316, + "tokens_seen": 2269235200 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025256569709127387, + "loss": 1.2202, + "theoretical_loss": 3.423360744290009, + "tokens_seen": 2269300736 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025254964894684057, + "loss": 1.1588, + "theoretical_loss": 3.423352771454412, + "tokens_seen": 2269366272 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025253360080240727, + "loss": 1.3803, + "theoretical_loss": 3.423344798913522, + "tokens_seen": 2269431808 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002525175526579739, + "loss": 1.2442, + "theoretical_loss": 3.42333682666732, + "tokens_seen": 2269497344 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002525015045135406, + "loss": 1.2673, + "theoretical_loss": 3.423328854715785, + "tokens_seen": 2269562880 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002524854563691073, + "loss": 1.1773, + "theoretical_loss": 3.4233208830588993, + "tokens_seen": 2269628416 + }, + { + "epoch": 1.07, + "learning_rate": 0.000252469408224674, + "loss": 1.2562, + "theoretical_loss": 3.4233129116966423, + "tokens_seen": 2269693952 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002524533600802407, + "loss": 1.1525, + "theoretical_loss": 3.4233049406289955, + "tokens_seen": 2269759488 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002524373119358074, + "loss": 1.2632, + "theoretical_loss": 3.423296969855939, + "tokens_seen": 2269825024 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002524212637913741, + "loss": 1.2389, + "theoretical_loss": 3.423288999377453, + "tokens_seen": 2269890560 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002524052156469408, + "loss": 1.2989, + "theoretical_loss": 3.423281029193519, + "tokens_seen": 2269956096 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002523891675025075, + "loss": 1.257, + "theoretical_loss": 3.4232730593041167, + "tokens_seen": 2270021632 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002523731193580742, + "loss": 1.2942, + "theoretical_loss": 3.4232650897092274, + "tokens_seen": 2270087168 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002523570712136409, + "loss": 1.2479, + "theoretical_loss": 3.423257120408832, + "tokens_seen": 2270152704 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002523410230692076, + "loss": 1.2016, + "theoretical_loss": 3.42324915140291, + "tokens_seen": 2270218240 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002523249749247743, + "loss": 1.2344, + "theoretical_loss": 3.423241182691443, + "tokens_seen": 2270283776 + }, + { + "epoch": 1.07, + "learning_rate": 0.000252308926780341, + "loss": 1.2821, + "theoretical_loss": 3.423233214274411, + "tokens_seen": 2270349312 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002522928786359077, + "loss": 1.2017, + "theoretical_loss": 3.423225246151795, + "tokens_seen": 2270414848 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025227683049147443, + "loss": 1.1078, + "theoretical_loss": 3.423217278323576, + "tokens_seen": 2270480384 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025226078234704113, + "loss": 1.1842, + "theoretical_loss": 3.4232093107897335, + "tokens_seen": 2270545920 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025224473420260783, + "loss": 1.2296, + "theoretical_loss": 3.423201343550249, + "tokens_seen": 2270611456 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025222868605817453, + "loss": 1.2232, + "theoretical_loss": 3.423193376605103, + "tokens_seen": 2270676992 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025221263791374123, + "loss": 1.2168, + "theoretical_loss": 3.4231854099542756, + "tokens_seen": 2270742528 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025219658976930793, + "loss": 1.2236, + "theoretical_loss": 3.4231774435977482, + "tokens_seen": 2270808064 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025218054162487463, + "loss": 1.2815, + "theoretical_loss": 3.423169477535501, + "tokens_seen": 2270873600 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025216449348044133, + "loss": 1.3124, + "theoretical_loss": 3.423161511767515, + "tokens_seen": 2270939136 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025214844533600803, + "loss": 1.2913, + "theoretical_loss": 3.4231535462937703, + "tokens_seen": 2271004672 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025213239719157473, + "loss": 1.1776, + "theoretical_loss": 3.423145581114248, + "tokens_seen": 2271070208 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025211634904714143, + "loss": 1.2561, + "theoretical_loss": 3.4231376162289284, + "tokens_seen": 2271135744 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025210030090270813, + "loss": 1.2547, + "theoretical_loss": 3.4231296516377925, + "tokens_seen": 2271201280 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025208425275827483, + "loss": 1.1748, + "theoretical_loss": 3.423121687340821, + "tokens_seen": 2271266816 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025206820461384154, + "loss": 1.3548, + "theoretical_loss": 3.423113723337994, + "tokens_seen": 2271332352 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025205215646940824, + "loss": 1.1899, + "theoretical_loss": 3.423105759629292, + "tokens_seen": 2271397888 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025203610832497494, + "loss": 1.1497, + "theoretical_loss": 3.4230977962146962, + "tokens_seen": 2271463424 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025202006018054164, + "loss": 1.2281, + "theoretical_loss": 3.4230898330941875, + "tokens_seen": 2271528960 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025200401203610834, + "loss": 1.2112, + "theoretical_loss": 3.423081870267746, + "tokens_seen": 2271594496 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025198796389167504, + "loss": 1.1759, + "theoretical_loss": 3.423073907735353, + "tokens_seen": 2271660032 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025197191574724174, + "loss": 1.2194, + "theoretical_loss": 3.4230659454969885, + "tokens_seen": 2271725568 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025195586760280844, + "loss": 1.1692, + "theoretical_loss": 3.423057983552633, + "tokens_seen": 2271791104 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025193981945837514, + "loss": 1.2657, + "theoretical_loss": 3.423050021902268, + "tokens_seen": 2271856640 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025192377131394184, + "loss": 1.1762, + "theoretical_loss": 3.423042060545874, + "tokens_seen": 2271922176 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025190772316950854, + "loss": 1.3007, + "theoretical_loss": 3.4230340994834303, + "tokens_seen": 2271987712 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025189167502507524, + "loss": 1.3855, + "theoretical_loss": 3.4230261387149197, + "tokens_seen": 2272053248 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025187562688064194, + "loss": 1.2867, + "theoretical_loss": 3.423018178240321, + "tokens_seen": 2272118784 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025185957873620864, + "loss": 1.2717, + "theoretical_loss": 3.4230102180596162, + "tokens_seen": 2272184320 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025184353059177534, + "loss": 1.218, + "theoretical_loss": 3.4230022581727853, + "tokens_seen": 2272249856 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025182748244734204, + "loss": 1.2608, + "theoretical_loss": 3.4229942985798094, + "tokens_seen": 2272315392 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025181143430290875, + "loss": 1.1622, + "theoretical_loss": 3.4229863392806683, + "tokens_seen": 2272380928 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025179538615847545, + "loss": 1.1575, + "theoretical_loss": 3.422978380275344, + "tokens_seen": 2272446464 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025177933801404215, + "loss": 1.251, + "theoretical_loss": 3.422970421563816, + "tokens_seen": 2272512000 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025176328986960885, + "loss": 1.2532, + "theoretical_loss": 3.4229624631460656, + "tokens_seen": 2272577536 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002517472417251755, + "loss": 1.2713, + "theoretical_loss": 3.4229545050220733, + "tokens_seen": 2272643072 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002517311935807422, + "loss": 1.2564, + "theoretical_loss": 3.42294654719182, + "tokens_seen": 2272708608 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002517151454363089, + "loss": 1.2283, + "theoretical_loss": 3.422938589655286, + "tokens_seen": 2272774144 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002516990972918756, + "loss": 1.3439, + "theoretical_loss": 3.422930632412452, + "tokens_seen": 2272839680 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002516830491474423, + "loss": 1.1948, + "theoretical_loss": 3.4229226754632993, + "tokens_seen": 2272905216 + }, + { + "epoch": 1.07, + "learning_rate": 0.000251667001003009, + "loss": 1.2843, + "theoretical_loss": 3.4229147188078084, + "tokens_seen": 2272970752 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025165095285857575, + "loss": 1.357, + "theoretical_loss": 3.422906762445959, + "tokens_seen": 2273036288 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025163490471414245, + "loss": 1.1805, + "theoretical_loss": 3.4228988063777335, + "tokens_seen": 2273101824 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025161885656970915, + "loss": 1.2444, + "theoretical_loss": 3.422890850603111, + "tokens_seen": 2273167360 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025160280842527585, + "loss": 1.2026, + "theoretical_loss": 3.422882895122073, + "tokens_seen": 2273232896 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025158676028084255, + "loss": 1.3298, + "theoretical_loss": 3.4228749399346, + "tokens_seen": 2273298432 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025157071213640926, + "loss": 1.3419, + "theoretical_loss": 3.422866985040673, + "tokens_seen": 2273363968 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025155466399197596, + "loss": 1.2766, + "theoretical_loss": 3.4228590304402724, + "tokens_seen": 2273429504 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025153861584754266, + "loss": 1.326, + "theoretical_loss": 3.4228510761333792, + "tokens_seen": 2273495040 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025152256770310936, + "loss": 1.208, + "theoretical_loss": 3.422843122119974, + "tokens_seen": 2273560576 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025150651955867606, + "loss": 1.1515, + "theoretical_loss": 3.422835168400037, + "tokens_seen": 2273626112 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025149047141424276, + "loss": 1.2421, + "theoretical_loss": 3.42282721497355, + "tokens_seen": 2273691648 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025147442326980946, + "loss": 1.1588, + "theoretical_loss": 3.422819261840492, + "tokens_seen": 2273757184 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025145837512537616, + "loss": 1.3579, + "theoretical_loss": 3.4228113090008456, + "tokens_seen": 2273822720 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025144232698094286, + "loss": 1.2588, + "theoretical_loss": 3.4228033564545903, + "tokens_seen": 2273888256 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025142627883650956, + "loss": 1.2625, + "theoretical_loss": 3.4227954042017075, + "tokens_seen": 2273953792 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025141023069207626, + "loss": 1.2461, + "theoretical_loss": 3.4227874522421775, + "tokens_seen": 2274019328 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 840714, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2033460140228271, + "objective/train/theoretical_loss": 3.422779500575981, + "objective/train/tokens_used": 2294544864, + "theoretical_loss": 3.422779500575981, + "tokens_seen": 2274084864 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025139418254764296, + "loss": 1.2033, + "theoretical_loss": 3.422779500575981, + "tokens_seen": 2274084864 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025137813440320966, + "loss": 1.2478, + "theoretical_loss": 3.422771549203099, + "tokens_seen": 2274150400 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025136208625877636, + "loss": 1.369, + "theoretical_loss": 3.4227635981235123, + "tokens_seen": 2274215936 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025134603811434306, + "loss": 1.2525, + "theoretical_loss": 3.422755647337201, + "tokens_seen": 2274281472 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025132998996990976, + "loss": 1.3818, + "theoretical_loss": 3.422747696844147, + "tokens_seen": 2274347008 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025131394182547647, + "loss": 1.3049, + "theoretical_loss": 3.42273974664433, + "tokens_seen": 2274412544 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025129789368104317, + "loss": 1.2774, + "theoretical_loss": 3.4227317967377306, + "tokens_seen": 2274478080 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025128184553660987, + "loss": 1.2629, + "theoretical_loss": 3.42272384712433, + "tokens_seen": 2274543616 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025126579739217657, + "loss": 1.2321, + "theoretical_loss": 3.4227158978041095, + "tokens_seen": 2274609152 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025124974924774327, + "loss": 1.2835, + "theoretical_loss": 3.422707948777049, + "tokens_seen": 2274674688 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025123370110330997, + "loss": 1.3972, + "theoretical_loss": 3.42270000004313, + "tokens_seen": 2274740224 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025121765295887667, + "loss": 1.2002, + "theoretical_loss": 3.422692051602332, + "tokens_seen": 2274805760 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025120160481444337, + "loss": 1.2062, + "theoretical_loss": 3.4226841034546367, + "tokens_seen": 2274871296 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025118555667001007, + "loss": 1.329, + "theoretical_loss": 3.4226761556000245, + "tokens_seen": 2274936832 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025116950852557677, + "loss": 1.3034, + "theoretical_loss": 3.4226682080384765, + "tokens_seen": 2275002368 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025115346038114347, + "loss": 1.1909, + "theoretical_loss": 3.422660260769973, + "tokens_seen": 2275067904 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025113741223671017, + "loss": 1.2511, + "theoretical_loss": 3.4226523137944955, + "tokens_seen": 2275133440 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002511213640922768, + "loss": 1.2903, + "theoretical_loss": 3.422644367112024, + "tokens_seen": 2275198976 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002511053159478435, + "loss": 1.1731, + "theoretical_loss": 3.4226364207225393, + "tokens_seen": 2275264512 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002510892678034102, + "loss": 1.2162, + "theoretical_loss": 3.422628474626023, + "tokens_seen": 2275330048 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002510732196589769, + "loss": 1.2511, + "theoretical_loss": 3.4226205288224545, + "tokens_seen": 2275395584 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002510571715145436, + "loss": 1.2493, + "theoretical_loss": 3.4226125833118157, + "tokens_seen": 2275461120 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002510411233701103, + "loss": 1.2712, + "theoretical_loss": 3.4226046380940867, + "tokens_seen": 2275526656 + }, + { + "epoch": 1.07, + "learning_rate": 0.000251025075225677, + "loss": 1.2703, + "theoretical_loss": 3.4225966931692486, + "tokens_seen": 2275592192 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002510090270812437, + "loss": 1.3155, + "theoretical_loss": 3.422588748537282, + "tokens_seen": 2275657728 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002509929789368104, + "loss": 1.2574, + "theoretical_loss": 3.4225808041981685, + "tokens_seen": 2275723264 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002509769307923771, + "loss": 1.1613, + "theoretical_loss": 3.422572860151887, + "tokens_seen": 2275788800 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002509608826479438, + "loss": 1.2426, + "theoretical_loss": 3.4225649163984206, + "tokens_seen": 2275854336 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002509448345035105, + "loss": 1.2116, + "theoretical_loss": 3.4225569729377483, + "tokens_seen": 2275919872 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002509287863590772, + "loss": 1.2339, + "theoretical_loss": 3.4225490297698515, + "tokens_seen": 2275985408 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002509127382146439, + "loss": 1.2105, + "theoretical_loss": 3.422541086894711, + "tokens_seen": 2276050944 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025089669007021063, + "loss": 1.1373, + "theoretical_loss": 3.422533144312307, + "tokens_seen": 2276116480 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025088064192577733, + "loss": 1.2335, + "theoretical_loss": 3.422525202022621, + "tokens_seen": 2276182016 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025086459378134403, + "loss": 1.3264, + "theoretical_loss": 3.4225172600256344, + "tokens_seen": 2276247552 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025084854563691073, + "loss": 1.4059, + "theoretical_loss": 3.4225093183213264, + "tokens_seen": 2276313088 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025083249749247743, + "loss": 1.1731, + "theoretical_loss": 3.4225013769096786, + "tokens_seen": 2276378624 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025081644934804413, + "loss": 1.2574, + "theoretical_loss": 3.4224934357906722, + "tokens_seen": 2276444160 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025080040120361083, + "loss": 1.2915, + "theoretical_loss": 3.4224854949642873, + "tokens_seen": 2276509696 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025078435305917753, + "loss": 1.2202, + "theoretical_loss": 3.422477554430505, + "tokens_seen": 2276575232 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025076830491474423, + "loss": 1.2727, + "theoretical_loss": 3.422469614189306, + "tokens_seen": 2276640768 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025075225677031093, + "loss": 1.2709, + "theoretical_loss": 3.4224616742406706, + "tokens_seen": 2276706304 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025073620862587763, + "loss": 1.3001, + "theoretical_loss": 3.4224537345845807, + "tokens_seen": 2276771840 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025072016048144433, + "loss": 1.3036, + "theoretical_loss": 3.4224457952210168, + "tokens_seen": 2276837376 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025070411233701104, + "loss": 1.3019, + "theoretical_loss": 3.422437856149959, + "tokens_seen": 2276902912 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025068806419257774, + "loss": 1.2876, + "theoretical_loss": 3.422429917371389, + "tokens_seen": 2276968448 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025067201604814444, + "loss": 1.1895, + "theoretical_loss": 3.4224219788852865, + "tokens_seen": 2277033984 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025065596790371114, + "loss": 1.2887, + "theoretical_loss": 3.422414040691633, + "tokens_seen": 2277099520 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025063991975927784, + "loss": 1.2158, + "theoretical_loss": 3.4224061027904096, + "tokens_seen": 2277165056 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025062387161484454, + "loss": 1.1569, + "theoretical_loss": 3.4223981651815967, + "tokens_seen": 2277230592 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025060782347041124, + "loss": 1.3236, + "theoretical_loss": 3.422390227865175, + "tokens_seen": 2277296128 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025059177532597794, + "loss": 1.2136, + "theoretical_loss": 3.422382290841126, + "tokens_seen": 2277361664 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025057572718154464, + "loss": 1.1744, + "theoretical_loss": 3.4223743541094294, + "tokens_seen": 2277427200 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025055967903711134, + "loss": 1.2102, + "theoretical_loss": 3.422366417670067, + "tokens_seen": 2277492736 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025054363089267804, + "loss": 1.2475, + "theoretical_loss": 3.4223584815230192, + "tokens_seen": 2277558272 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025052758274824474, + "loss": 1.2808, + "theoretical_loss": 3.4223505456682672, + "tokens_seen": 2277623808 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025051153460381144, + "loss": 1.1934, + "theoretical_loss": 3.422342610105791, + "tokens_seen": 2277689344 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025049548645937814, + "loss": 1.316, + "theoretical_loss": 3.4223346748355725, + "tokens_seen": 2277754880 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025047943831494484, + "loss": 1.3046, + "theoretical_loss": 3.422326739857591, + "tokens_seen": 2277820416 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025046339017051154, + "loss": 1.3041, + "theoretical_loss": 3.4223188051718294, + "tokens_seen": 2277885952 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025044734202607825, + "loss": 1.3261, + "theoretical_loss": 3.422310870778267, + "tokens_seen": 2277951488 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025043129388164495, + "loss": 1.2671, + "theoretical_loss": 3.422302936676885, + "tokens_seen": 2278017024 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025041524573721165, + "loss": 1.2244, + "theoretical_loss": 3.4222950028676644, + "tokens_seen": 2278082560 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025039919759277835, + "loss": 1.1867, + "theoretical_loss": 3.4222870693505856, + "tokens_seen": 2278148096 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025038314944834505, + "loss": 1.1653, + "theoretical_loss": 3.42227913612563, + "tokens_seen": 2278213632 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025036710130391175, + "loss": 1.223, + "theoretical_loss": 3.4222712031927784, + "tokens_seen": 2278279168 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025035105315947845, + "loss": 1.2007, + "theoretical_loss": 3.422263270552011, + "tokens_seen": 2278344704 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025033500501504515, + "loss": 1.2628, + "theoretical_loss": 3.4222553382033096, + "tokens_seen": 2278410240 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025031895687061185, + "loss": 1.3235, + "theoretical_loss": 3.4222474061466546, + "tokens_seen": 2278475776 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025030290872617855, + "loss": 1.2714, + "theoretical_loss": 3.4222394743820264, + "tokens_seen": 2278541312 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025028686058174525, + "loss": 1.2408, + "theoretical_loss": 3.4222315429094063, + "tokens_seen": 2278606848 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025027081243731195, + "loss": 1.269, + "theoretical_loss": 3.422223611728775, + "tokens_seen": 2278672384 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025025476429287865, + "loss": 1.2694, + "theoretical_loss": 3.4222156808401136, + "tokens_seen": 2278737920 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025023871614844535, + "loss": 1.3899, + "theoretical_loss": 3.422207750243403, + "tokens_seen": 2278803456 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025022266800401205, + "loss": 1.3022, + "theoretical_loss": 3.4221998199386237, + "tokens_seen": 2278868992 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025020661985957875, + "loss": 1.1638, + "theoretical_loss": 3.422191889925757, + "tokens_seen": 2278934528 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025019057171514546, + "loss": 1.1909, + "theoretical_loss": 3.4221839602047828, + "tokens_seen": 2279000064 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025017452357071216, + "loss": 1.3029, + "theoretical_loss": 3.422176030775683, + "tokens_seen": 2279065600 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025015847542627886, + "loss": 1.1994, + "theoretical_loss": 3.4221681016384387, + "tokens_seen": 2279131136 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025014242728184556, + "loss": 1.2164, + "theoretical_loss": 3.4221601727930295, + "tokens_seen": 2279196672 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025012637913741226, + "loss": 1.3702, + "theoretical_loss": 3.422152244239437, + "tokens_seen": 2279262208 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025011033099297896, + "loss": 1.2566, + "theoretical_loss": 3.422144315977642, + "tokens_seen": 2279327744 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025009428284854566, + "loss": 1.3015, + "theoretical_loss": 3.422136388007625, + "tokens_seen": 2279393280 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025007823470411236, + "loss": 1.2438, + "theoretical_loss": 3.4221284603293682, + "tokens_seen": 2279458816 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025006218655967906, + "loss": 1.211, + "theoretical_loss": 3.4221205329428512, + "tokens_seen": 2279524352 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025004613841524576, + "loss": 1.2461, + "theoretical_loss": 3.4221126058480547, + "tokens_seen": 2279589888 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025003009027081246, + "loss": 1.28, + "theoretical_loss": 3.4221046790449607, + "tokens_seen": 2279655424 + }, + { + "epoch": 1.07, + "learning_rate": 0.00025001404212637916, + "loss": 1.2271, + "theoretical_loss": 3.422096752533549, + "tokens_seen": 2279720960 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024999799398194586, + "loss": 1.3045, + "theoretical_loss": 3.4220888263138014, + "tokens_seen": 2279786496 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024998194583751256, + "loss": 1.2157, + "theoretical_loss": 3.4220809003856982, + "tokens_seen": 2279852032 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024996589769307926, + "loss": 1.2267, + "theoretical_loss": 3.42207297474922, + "tokens_seen": 2279917568 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024994984954864597, + "loss": 1.271, + "theoretical_loss": 3.422065049404349, + "tokens_seen": 2279983104 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024993380140421267, + "loss": 1.0971, + "theoretical_loss": 3.4220571243510642, + "tokens_seen": 2280048640 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024991775325977937, + "loss": 1.153, + "theoretical_loss": 3.422049199589348, + "tokens_seen": 2280114176 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024990170511534607, + "loss": 1.2107, + "theoretical_loss": 3.422041275119181, + "tokens_seen": 2280179712 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024988565697091277, + "loss": 1.1825, + "theoretical_loss": 3.422033350940543, + "tokens_seen": 2280245248 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024986960882647947, + "loss": 1.3556, + "theoretical_loss": 3.4220254270534167, + "tokens_seen": 2280310784 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024985356068204617, + "loss": 1.1851, + "theoretical_loss": 3.4220175034577816, + "tokens_seen": 2280376320 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024983751253761287, + "loss": 1.2388, + "theoretical_loss": 3.4220095801536194, + "tokens_seen": 2280441856 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024982146439317957, + "loss": 1.2616, + "theoretical_loss": 3.4220016571409104, + "tokens_seen": 2280507392 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024980541624874627, + "loss": 1.1933, + "theoretical_loss": 3.421993734419636, + "tokens_seen": 2280572928 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 843116, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2210228443145752, + "objective/train/theoretical_loss": 3.4219858119897766, + "objective/train/tokens_used": 2301098464, + "theoretical_loss": 3.4219858119897766, + "tokens_seen": 2280638464 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024978936810431297, + "loss": 1.221, + "theoretical_loss": 3.4219858119897766, + "tokens_seen": 2280638464 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024977331995987967, + "loss": 1.2083, + "theoretical_loss": 3.4219778898513136, + "tokens_seen": 2280704000 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024975727181544637, + "loss": 1.3206, + "theoretical_loss": 3.4219699680042277, + "tokens_seen": 2280769536 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002497412236710131, + "loss": 1.2978, + "theoretical_loss": 3.4219620464485, + "tokens_seen": 2280835072 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002497251755265797, + "loss": 1.104, + "theoretical_loss": 3.4219541251841106, + "tokens_seen": 2280900608 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002497091273821464, + "loss": 1.3051, + "theoretical_loss": 3.4219462042110416, + "tokens_seen": 2280966144 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002496930792377131, + "loss": 1.1863, + "theoretical_loss": 3.421938283529273, + "tokens_seen": 2281031680 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002496770310932798, + "loss": 1.2737, + "theoretical_loss": 3.4219303631387863, + "tokens_seen": 2281097216 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002496609829488465, + "loss": 1.2444, + "theoretical_loss": 3.421922443039562, + "tokens_seen": 2281162752 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002496449348044132, + "loss": 1.2668, + "theoretical_loss": 3.4219145232315813, + "tokens_seen": 2281228288 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002496288866599799, + "loss": 1.1663, + "theoretical_loss": 3.421906603714825, + "tokens_seen": 2281293824 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002496128385155466, + "loss": 1.187, + "theoretical_loss": 3.4218986844892743, + "tokens_seen": 2281359360 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002495967903711133, + "loss": 1.2822, + "theoretical_loss": 3.42189076555491, + "tokens_seen": 2281424896 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024958074222668, + "loss": 1.3182, + "theoretical_loss": 3.4218828469117124, + "tokens_seen": 2281490432 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002495646940822467, + "loss": 1.322, + "theoretical_loss": 3.4218749285596632, + "tokens_seen": 2281555968 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002495486459378134, + "loss": 1.1373, + "theoretical_loss": 3.421867010498743, + "tokens_seen": 2281621504 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024953259779338013, + "loss": 1.1789, + "theoretical_loss": 3.4218590927289334, + "tokens_seen": 2281687040 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024951654964894683, + "loss": 1.2954, + "theoretical_loss": 3.421851175250214, + "tokens_seen": 2281752576 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024950050150451353, + "loss": 1.2478, + "theoretical_loss": 3.421843258062567, + "tokens_seen": 2281818112 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024948445336008023, + "loss": 1.1966, + "theoretical_loss": 3.4218353411659725, + "tokens_seen": 2281883648 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024946840521564693, + "loss": 1.2652, + "theoretical_loss": 3.4218274245604117, + "tokens_seen": 2281949184 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024945235707121363, + "loss": 1.2277, + "theoretical_loss": 3.421819508245866, + "tokens_seen": 2282014720 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024943630892678033, + "loss": 1.193, + "theoretical_loss": 3.4218115922223156, + "tokens_seen": 2282080256 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024942026078234703, + "loss": 1.3294, + "theoretical_loss": 3.421803676489742, + "tokens_seen": 2282145792 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024940421263791373, + "loss": 1.1829, + "theoretical_loss": 3.421795761048126, + "tokens_seen": 2282211328 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024938816449348043, + "loss": 1.3136, + "theoretical_loss": 3.4217878458974487, + "tokens_seen": 2282276864 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024937211634904713, + "loss": 1.2624, + "theoretical_loss": 3.4217799310376904, + "tokens_seen": 2282342400 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024935606820461383, + "loss": 1.2047, + "theoretical_loss": 3.4217720164688328, + "tokens_seen": 2282407936 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024934002006018053, + "loss": 1.302, + "theoretical_loss": 3.4217641021908562, + "tokens_seen": 2282473472 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024932397191574724, + "loss": 1.1964, + "theoretical_loss": 3.4217561882037426, + "tokens_seen": 2282539008 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024930792377131394, + "loss": 1.2608, + "theoretical_loss": 3.421748274507472, + "tokens_seen": 2282604544 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024929187562688064, + "loss": 1.161, + "theoretical_loss": 3.4217403611020254, + "tokens_seen": 2282670080 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024927582748244734, + "loss": 1.271, + "theoretical_loss": 3.421732447987384, + "tokens_seen": 2282735616 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024925977933801404, + "loss": 1.375, + "theoretical_loss": 3.4217245351635293, + "tokens_seen": 2282801152 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002492437311935808, + "loss": 1.2479, + "theoretical_loss": 3.4217166226304414, + "tokens_seen": 2282866688 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002492276830491475, + "loss": 1.3976, + "theoretical_loss": 3.421708710388102, + "tokens_seen": 2282932224 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002492116349047142, + "loss": 1.1935, + "theoretical_loss": 3.421700798436491, + "tokens_seen": 2282997760 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002491955867602809, + "loss": 1.3179, + "theoretical_loss": 3.4216928867755905, + "tokens_seen": 2283063296 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002491795386158476, + "loss": 1.2121, + "theoretical_loss": 3.421684975405381, + "tokens_seen": 2283128832 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002491634904714143, + "loss": 1.2215, + "theoretical_loss": 3.421677064325843, + "tokens_seen": 2283194368 + }, + { + "epoch": 1.07, + "learning_rate": 0.000249147442326981, + "loss": 1.2646, + "theoretical_loss": 3.4216691535369588, + "tokens_seen": 2283259904 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002491313941825477, + "loss": 1.2225, + "theoretical_loss": 3.421661243038708, + "tokens_seen": 2283325440 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002491153460381144, + "loss": 1.2781, + "theoretical_loss": 3.421653332831072, + "tokens_seen": 2283390976 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002490992978936811, + "loss": 1.1954, + "theoretical_loss": 3.4216454229140325, + "tokens_seen": 2283456512 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024908324974924775, + "loss": 1.2322, + "theoretical_loss": 3.42163751328757, + "tokens_seen": 2283522048 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024906720160481445, + "loss": 1.136, + "theoretical_loss": 3.421629603951665, + "tokens_seen": 2283587584 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024905115346038115, + "loss": 1.2482, + "theoretical_loss": 3.4216216949062987, + "tokens_seen": 2283653120 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024903510531594785, + "loss": 1.2633, + "theoretical_loss": 3.4216137861514526, + "tokens_seen": 2283718656 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024901905717151455, + "loss": 1.225, + "theoretical_loss": 3.4216058776871074, + "tokens_seen": 2283784192 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024900300902708125, + "loss": 1.1563, + "theoretical_loss": 3.421597969513244, + "tokens_seen": 2283849728 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024898696088264795, + "loss": 1.2652, + "theoretical_loss": 3.4215900616298436, + "tokens_seen": 2283915264 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024897091273821465, + "loss": 1.2171, + "theoretical_loss": 3.4215821540368867, + "tokens_seen": 2283980800 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024895486459378135, + "loss": 1.3077, + "theoretical_loss": 3.421574246734355, + "tokens_seen": 2284046336 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024893881644934805, + "loss": 1.2676, + "theoretical_loss": 3.421566339722229, + "tokens_seen": 2284111872 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024892276830491475, + "loss": 1.1451, + "theoretical_loss": 3.42155843300049, + "tokens_seen": 2284177408 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024890672016048145, + "loss": 1.196, + "theoretical_loss": 3.4215505265691184, + "tokens_seen": 2284242944 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024889067201604815, + "loss": 1.2421, + "theoretical_loss": 3.421542620428096, + "tokens_seen": 2284308480 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024887462387161485, + "loss": 1.1842, + "theoretical_loss": 3.4215347145774038, + "tokens_seen": 2284374016 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024885857572718155, + "loss": 1.2493, + "theoretical_loss": 3.421526809017022, + "tokens_seen": 2284439552 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024884252758274825, + "loss": 1.2589, + "theoretical_loss": 3.4215189037469322, + "tokens_seen": 2284505088 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024882647943831496, + "loss": 1.1651, + "theoretical_loss": 3.4215109987671153, + "tokens_seen": 2284570624 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024881043129388166, + "loss": 1.2331, + "theoretical_loss": 3.4215030940775524, + "tokens_seen": 2284636160 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024879438314944836, + "loss": 1.2644, + "theoretical_loss": 3.4214951896782244, + "tokens_seen": 2284701696 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024877833500501506, + "loss": 1.3013, + "theoretical_loss": 3.4214872855691123, + "tokens_seen": 2284767232 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024876228686058176, + "loss": 1.2282, + "theoretical_loss": 3.4214793817501974, + "tokens_seen": 2284832768 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024874623871614846, + "loss": 1.2186, + "theoretical_loss": 3.4214714782214606, + "tokens_seen": 2284898304 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024873019057171516, + "loss": 1.2093, + "theoretical_loss": 3.4214635749828823, + "tokens_seen": 2284963840 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024871414242728186, + "loss": 1.2441, + "theoretical_loss": 3.4214556720344445, + "tokens_seen": 2285029376 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024869809428284856, + "loss": 1.2004, + "theoretical_loss": 3.421447769376128, + "tokens_seen": 2285094912 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024868204613841526, + "loss": 1.2267, + "theoretical_loss": 3.421439867007913, + "tokens_seen": 2285160448 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024866599799398196, + "loss": 1.2173, + "theoretical_loss": 3.4214319649297815, + "tokens_seen": 2285225984 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024864994984954866, + "loss": 1.2186, + "theoretical_loss": 3.421424063141714, + "tokens_seen": 2285291520 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024863390170511536, + "loss": 1.1173, + "theoretical_loss": 3.421416161643692, + "tokens_seen": 2285357056 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024861785356068206, + "loss": 1.2524, + "theoretical_loss": 3.4214082604356957, + "tokens_seen": 2285422592 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024860180541624876, + "loss": 1.1767, + "theoretical_loss": 3.4214003595177074, + "tokens_seen": 2285488128 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024858575727181546, + "loss": 1.1912, + "theoretical_loss": 3.4213924588897067, + "tokens_seen": 2285553664 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024856970912738217, + "loss": 1.1473, + "theoretical_loss": 3.421384558551676, + "tokens_seen": 2285619200 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024855366098294887, + "loss": 1.2747, + "theoretical_loss": 3.421376658503595, + "tokens_seen": 2285684736 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024853761283851557, + "loss": 1.2499, + "theoretical_loss": 3.421368758745446, + "tokens_seen": 2285750272 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024852156469408227, + "loss": 1.1975, + "theoretical_loss": 3.42136085927721, + "tokens_seen": 2285815808 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024850551654964897, + "loss": 1.1717, + "theoretical_loss": 3.421352960098867, + "tokens_seen": 2285881344 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024848946840521567, + "loss": 1.3448, + "theoretical_loss": 3.421345061210398, + "tokens_seen": 2285946880 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024847342026078237, + "loss": 1.3092, + "theoretical_loss": 3.421337162611785, + "tokens_seen": 2286012416 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024845737211634907, + "loss": 1.3023, + "theoretical_loss": 3.4213292643030093, + "tokens_seen": 2286077952 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024844132397191577, + "loss": 1.2572, + "theoretical_loss": 3.421321366284051, + "tokens_seen": 2286143488 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024842527582748247, + "loss": 1.2554, + "theoretical_loss": 3.4213134685548914, + "tokens_seen": 2286209024 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024840922768304917, + "loss": 1.1615, + "theoretical_loss": 3.421305571115512, + "tokens_seen": 2286274560 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024839317953861587, + "loss": 1.124, + "theoretical_loss": 3.4212976739658933, + "tokens_seen": 2286340096 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002483771313941826, + "loss": 1.2072, + "theoretical_loss": 3.4212897771060167, + "tokens_seen": 2286405632 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002483610832497493, + "loss": 1.3665, + "theoretical_loss": 3.421281880535863, + "tokens_seen": 2286471168 + }, + { + "epoch": 1.07, + "learning_rate": 0.000248345035105316, + "loss": 1.3074, + "theoretical_loss": 3.4212739842554134, + "tokens_seen": 2286536704 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002483289869608827, + "loss": 1.2307, + "theoretical_loss": 3.4212660882646495, + "tokens_seen": 2286602240 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002483129388164493, + "loss": 1.1714, + "theoretical_loss": 3.4212581925635512, + "tokens_seen": 2286667776 + }, + { + "epoch": 1.07, + "learning_rate": 0.000248296890672016, + "loss": 1.2695, + "theoretical_loss": 3.421250297152101, + "tokens_seen": 2286733312 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002482808425275827, + "loss": 1.2311, + "theoretical_loss": 3.4212424020302787, + "tokens_seen": 2286798848 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002482647943831494, + "loss": 1.1947, + "theoretical_loss": 3.421234507198066, + "tokens_seen": 2286864384 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002482487462387161, + "loss": 1.2164, + "theoretical_loss": 3.421226612655444, + "tokens_seen": 2286929920 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002482326980942828, + "loss": 1.2774, + "theoretical_loss": 3.4212187184023937, + "tokens_seen": 2286995456 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002482166499498495, + "loss": 1.2662, + "theoretical_loss": 3.4212108244388957, + "tokens_seen": 2287060992 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002482006018054162, + "loss": 1.3366, + "theoretical_loss": 3.421202930764932, + "tokens_seen": 2287126528 + }, + { + "epoch": 1.07, + "objective/train/docs_used": 845618, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2732019424438477, + "objective/train/theoretical_loss": 3.4211950373804827, + "objective/train/tokens_used": 2307652064, + "theoretical_loss": 3.4211950373804827, + "tokens_seen": 2287192064 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002481845536609829, + "loss": 1.2732, + "theoretical_loss": 3.4211950373804827, + "tokens_seen": 2287192064 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002481685055165496, + "loss": 1.2514, + "theoretical_loss": 3.42118714428553, + "tokens_seen": 2287257600 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024815245737211633, + "loss": 1.2792, + "theoretical_loss": 3.421179251480054, + "tokens_seen": 2287323136 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024813640922768303, + "loss": 1.1569, + "theoretical_loss": 3.4211713589640365, + "tokens_seen": 2287388672 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024812036108324973, + "loss": 1.2657, + "theoretical_loss": 3.421163466737458, + "tokens_seen": 2287454208 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024810431293881643, + "loss": 1.2451, + "theoretical_loss": 3.4211555748003, + "tokens_seen": 2287519744 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024808826479438313, + "loss": 1.2986, + "theoretical_loss": 3.4211476831525434, + "tokens_seen": 2287585280 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024807221664994983, + "loss": 1.1664, + "theoretical_loss": 3.421139791794169, + "tokens_seen": 2287650816 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024805616850551653, + "loss": 1.1942, + "theoretical_loss": 3.4211319007251584, + "tokens_seen": 2287716352 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002480401203610833, + "loss": 1.2177, + "theoretical_loss": 3.421124009945493, + "tokens_seen": 2287781888 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024802407221665, + "loss": 1.317, + "theoretical_loss": 3.421116119455153, + "tokens_seen": 2287847424 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002480080240722167, + "loss": 1.3124, + "theoretical_loss": 3.42110822925412, + "tokens_seen": 2287912960 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002479919759277834, + "loss": 1.2439, + "theoretical_loss": 3.421100339342375, + "tokens_seen": 2287978496 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002479759277833501, + "loss": 1.2637, + "theoretical_loss": 3.4210924497198993, + "tokens_seen": 2288044032 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002479598796389168, + "loss": 1.2094, + "theoretical_loss": 3.4210845603866744, + "tokens_seen": 2288109568 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002479438314944835, + "loss": 1.2545, + "theoretical_loss": 3.4210766713426803, + "tokens_seen": 2288175104 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002479277833500502, + "loss": 1.3086, + "theoretical_loss": 3.4210687825878985, + "tokens_seen": 2288240640 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002479117352056169, + "loss": 1.2786, + "theoretical_loss": 3.421060894122311, + "tokens_seen": 2288306176 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002478956870611836, + "loss": 1.2096, + "theoretical_loss": 3.4210530059458977, + "tokens_seen": 2288371712 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002478796389167503, + "loss": 1.2951, + "theoretical_loss": 3.4210451180586405, + "tokens_seen": 2288437248 + }, + { + "epoch": 1.07, + "learning_rate": 0.000247863590772317, + "loss": 1.248, + "theoretical_loss": 3.42103723046052, + "tokens_seen": 2288502784 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002478475426278837, + "loss": 1.2432, + "theoretical_loss": 3.4210293431515177, + "tokens_seen": 2288568320 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002478314944834504, + "loss": 1.2399, + "theoretical_loss": 3.421021456131615, + "tokens_seen": 2288633856 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002478154463390171, + "loss": 1.2056, + "theoretical_loss": 3.421013569400792, + "tokens_seen": 2288699392 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002477993981945838, + "loss": 1.1418, + "theoretical_loss": 3.4210056829590307, + "tokens_seen": 2288764928 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002477833500501505, + "loss": 1.177, + "theoretical_loss": 3.4209977968063123, + "tokens_seen": 2288830464 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002477673019057172, + "loss": 1.2197, + "theoretical_loss": 3.4209899109426174, + "tokens_seen": 2288896000 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002477512537612839, + "loss": 1.1761, + "theoretical_loss": 3.4209820253679273, + "tokens_seen": 2288961536 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002477352056168506, + "loss": 1.265, + "theoretical_loss": 3.4209741400822233, + "tokens_seen": 2289027072 + }, + { + "epoch": 1.07, + "learning_rate": 0.0002477191574724173, + "loss": 1.3439, + "theoretical_loss": 3.4209662550854865, + "tokens_seen": 2289092608 + }, + { + "epoch": 1.07, + "learning_rate": 0.000247703109327984, + "loss": 1.1761, + "theoretical_loss": 3.420958370377698, + "tokens_seen": 2289158144 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024768706118355065, + "loss": 1.3325, + "theoretical_loss": 3.4209504859588384, + "tokens_seen": 2289223680 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024767101303911735, + "loss": 1.3111, + "theoretical_loss": 3.42094260182889, + "tokens_seen": 2289289216 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024765496489468405, + "loss": 1.1373, + "theoretical_loss": 3.4209347179878327, + "tokens_seen": 2289354752 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024763891675025075, + "loss": 1.2353, + "theoretical_loss": 3.4209268344356487, + "tokens_seen": 2289420288 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024762286860581745, + "loss": 1.3303, + "theoretical_loss": 3.4209189511723186, + "tokens_seen": 2289485824 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024760682046138415, + "loss": 1.2933, + "theoretical_loss": 3.4209110681978236, + "tokens_seen": 2289551360 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024759077231695085, + "loss": 1.2674, + "theoretical_loss": 3.420903185512145, + "tokens_seen": 2289616896 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024757472417251755, + "loss": 1.261, + "theoretical_loss": 3.4208953031152634, + "tokens_seen": 2289682432 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024755867602808425, + "loss": 1.2854, + "theoretical_loss": 3.4208874210071607, + "tokens_seen": 2289747968 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024754262788365095, + "loss": 1.2811, + "theoretical_loss": 3.4208795391878177, + "tokens_seen": 2289813504 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024752657973921765, + "loss": 1.3009, + "theoretical_loss": 3.420871657657216, + "tokens_seen": 2289879040 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024751053159478435, + "loss": 1.295, + "theoretical_loss": 3.420863776415336, + "tokens_seen": 2289944576 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024749448345035105, + "loss": 1.1988, + "theoretical_loss": 3.420855895462159, + "tokens_seen": 2290010112 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024747843530591775, + "loss": 1.2096, + "theoretical_loss": 3.4208480147976665, + "tokens_seen": 2290075648 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024746238716148445, + "loss": 1.3062, + "theoretical_loss": 3.42084013442184, + "tokens_seen": 2290141184 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024744633901705116, + "loss": 1.2604, + "theoretical_loss": 3.42083225433466, + "tokens_seen": 2290206720 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024743029087261786, + "loss": 1.2041, + "theoretical_loss": 3.4208243745361075, + "tokens_seen": 2290272256 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024741424272818456, + "loss": 1.1935, + "theoretical_loss": 3.4208164950261644, + "tokens_seen": 2290337792 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024739819458375126, + "loss": 1.2483, + "theoretical_loss": 3.420808615804811, + "tokens_seen": 2290403328 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024738214643931796, + "loss": 1.3618, + "theoretical_loss": 3.42080073687203, + "tokens_seen": 2290468864 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024736609829488466, + "loss": 1.1503, + "theoretical_loss": 3.4207928582278004, + "tokens_seen": 2290534400 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024735005015045136, + "loss": 1.2621, + "theoretical_loss": 3.4207849798721055, + "tokens_seen": 2290599936 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024733400200601806, + "loss": 1.2723, + "theoretical_loss": 3.420777101804925, + "tokens_seen": 2290665472 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024731795386158476, + "loss": 1.1908, + "theoretical_loss": 3.420769224026241, + "tokens_seen": 2290731008 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024730190571715146, + "loss": 1.2184, + "theoretical_loss": 3.420761346536034, + "tokens_seen": 2290796544 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024728585757271816, + "loss": 1.2991, + "theoretical_loss": 3.4207534693342856, + "tokens_seen": 2290862080 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024726980942828486, + "loss": 1.3014, + "theoretical_loss": 3.4207455924209764, + "tokens_seen": 2290927616 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024725376128385156, + "loss": 1.3515, + "theoretical_loss": 3.420737715796089, + "tokens_seen": 2290993152 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024723771313941826, + "loss": 1.2531, + "theoretical_loss": 3.420729839459603, + "tokens_seen": 2291058688 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024722166499498496, + "loss": 1.2273, + "theoretical_loss": 3.4207219634115003, + "tokens_seen": 2291124224 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024720561685055167, + "loss": 1.249, + "theoretical_loss": 3.4207140876517617, + "tokens_seen": 2291189760 + }, + { + "epoch": 1.07, + "learning_rate": 0.00024718956870611837, + "loss": 1.2642, + "theoretical_loss": 3.4207062121803693, + "tokens_seen": 2291255296 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024717352056168507, + "loss": 1.2716, + "theoretical_loss": 3.4206983369973036, + "tokens_seen": 2291320832 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024715747241725177, + "loss": 1.2736, + "theoretical_loss": 3.4206904621025456, + "tokens_seen": 2291386368 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024714142427281847, + "loss": 1.2884, + "theoretical_loss": 3.420682587496077, + "tokens_seen": 2291451904 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024712537612838517, + "loss": 1.2406, + "theoretical_loss": 3.4206747131778785, + "tokens_seen": 2291517440 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024710932798395187, + "loss": 1.2017, + "theoretical_loss": 3.4206668391479322, + "tokens_seen": 2291582976 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024709327983951857, + "loss": 1.3286, + "theoretical_loss": 3.4206589654062185, + "tokens_seen": 2291648512 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024707723169508527, + "loss": 1.2899, + "theoretical_loss": 3.4206510919527187, + "tokens_seen": 2291714048 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024706118355065197, + "loss": 1.2098, + "theoretical_loss": 3.420643218787414, + "tokens_seen": 2291779584 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024704513540621867, + "loss": 1.2782, + "theoretical_loss": 3.420635345910286, + "tokens_seen": 2291845120 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024702908726178537, + "loss": 1.2797, + "theoretical_loss": 3.4206274733213156, + "tokens_seen": 2291910656 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024701303911735207, + "loss": 1.2284, + "theoretical_loss": 3.420619601020484, + "tokens_seen": 2291976192 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002469969909729188, + "loss": 1.2482, + "theoretical_loss": 3.4206117290077724, + "tokens_seen": 2292041728 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002469809428284855, + "loss": 1.2647, + "theoretical_loss": 3.4206038572831625, + "tokens_seen": 2292107264 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002469648946840522, + "loss": 1.2284, + "theoretical_loss": 3.4205959858466346, + "tokens_seen": 2292172800 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002469488465396189, + "loss": 1.176, + "theoretical_loss": 3.4205881146981705, + "tokens_seen": 2292238336 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002469327983951856, + "loss": 1.2478, + "theoretical_loss": 3.420580243837752, + "tokens_seen": 2292303872 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002469167502507522, + "loss": 1.2113, + "theoretical_loss": 3.420572373265359, + "tokens_seen": 2292369408 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002469007021063189, + "loss": 1.247, + "theoretical_loss": 3.420564502980974, + "tokens_seen": 2292434944 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002468846539618856, + "loss": 1.246, + "theoretical_loss": 3.420556632984577, + "tokens_seen": 2292500480 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002468686058174523, + "loss": 1.34, + "theoretical_loss": 3.42054876327615, + "tokens_seen": 2292566016 + }, + { + "epoch": 1.08, + "learning_rate": 0.000246852557673019, + "loss": 1.2426, + "theoretical_loss": 3.4205408938556743, + "tokens_seen": 2292631552 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002468365095285858, + "loss": 1.2926, + "theoretical_loss": 3.420533024723131, + "tokens_seen": 2292697088 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002468204613841525, + "loss": 1.312, + "theoretical_loss": 3.4205251558785013, + "tokens_seen": 2292762624 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002468044132397192, + "loss": 1.1752, + "theoretical_loss": 3.420517287321766, + "tokens_seen": 2292828160 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002467883650952859, + "loss": 1.3608, + "theoretical_loss": 3.420509419052907, + "tokens_seen": 2292893696 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002467723169508526, + "loss": 1.3957, + "theoretical_loss": 3.4205015510719052, + "tokens_seen": 2292959232 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002467562688064193, + "loss": 1.2299, + "theoretical_loss": 3.420493683378742, + "tokens_seen": 2293024768 + }, + { + "epoch": 1.08, + "learning_rate": 0.000246740220661986, + "loss": 1.2376, + "theoretical_loss": 3.420485815973399, + "tokens_seen": 2293090304 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002467241725175527, + "loss": 1.2725, + "theoretical_loss": 3.4204779488558565, + "tokens_seen": 2293155840 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002467081243731194, + "loss": 1.2098, + "theoretical_loss": 3.4204700820260965, + "tokens_seen": 2293221376 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002466920762286861, + "loss": 1.2499, + "theoretical_loss": 3.4204622154840996, + "tokens_seen": 2293286912 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002466760280842528, + "loss": 1.2076, + "theoretical_loss": 3.420454349229848, + "tokens_seen": 2293352448 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002466599799398195, + "loss": 1.238, + "theoretical_loss": 3.420446483263322, + "tokens_seen": 2293417984 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002466439317953862, + "loss": 1.2034, + "theoretical_loss": 3.4204386175845034, + "tokens_seen": 2293483520 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002466278836509529, + "loss": 1.2845, + "theoretical_loss": 3.4204307521933734, + "tokens_seen": 2293549056 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002466118355065196, + "loss": 1.2664, + "theoretical_loss": 3.4204228870899134, + "tokens_seen": 2293614592 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002465957873620863, + "loss": 1.2503, + "theoretical_loss": 3.420415022274104, + "tokens_seen": 2293680128 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 848346, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2497282028198242, + "objective/train/theoretical_loss": 3.4204071577459274, + "objective/train/tokens_used": 2314205664, + "theoretical_loss": 3.4204071577459274, + "tokens_seen": 2293745664 + }, + { + "epoch": 1.08, + "learning_rate": 0.000246579739217653, + "loss": 1.2497, + "theoretical_loss": 3.4204071577459274, + "tokens_seen": 2293745664 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002465636910732197, + "loss": 1.1608, + "theoretical_loss": 3.4203992935053638, + "tokens_seen": 2293811200 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002465476429287864, + "loss": 1.2336, + "theoretical_loss": 3.420391429552396, + "tokens_seen": 2293876736 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002465315947843531, + "loss": 1.2046, + "theoretical_loss": 3.420383565887003, + "tokens_seen": 2293942272 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002465155466399198, + "loss": 1.3328, + "theoretical_loss": 3.420375702509168, + "tokens_seen": 2294007808 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002464994984954865, + "loss": 1.3478, + "theoretical_loss": 3.420367839418872, + "tokens_seen": 2294073344 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002464834503510532, + "loss": 1.1689, + "theoretical_loss": 3.420359976616095, + "tokens_seen": 2294138880 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002464674022066199, + "loss": 1.2187, + "theoretical_loss": 3.42035211410082, + "tokens_seen": 2294204416 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002464513540621866, + "loss": 1.2731, + "theoretical_loss": 3.4203442518730274, + "tokens_seen": 2294269952 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002464353059177533, + "loss": 1.2928, + "theoretical_loss": 3.4203363899326984, + "tokens_seen": 2294335488 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024641925777332, + "loss": 1.2312, + "theoretical_loss": 3.420328528279814, + "tokens_seen": 2294401024 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002464032096288867, + "loss": 1.2179, + "theoretical_loss": 3.4203206669143564, + "tokens_seen": 2294466560 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002463871614844534, + "loss": 1.1738, + "theoretical_loss": 3.420312805836306, + "tokens_seen": 2294532096 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002463711133400201, + "loss": 1.2785, + "theoretical_loss": 3.420304945045645, + "tokens_seen": 2294597632 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002463550651955868, + "loss": 1.2358, + "theoretical_loss": 3.4202970845423537, + "tokens_seen": 2294663168 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002463390170511535, + "loss": 1.171, + "theoretical_loss": 3.420289224326414, + "tokens_seen": 2294728704 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002463229689067202, + "loss": 1.2248, + "theoretical_loss": 3.420281364397807, + "tokens_seen": 2294794240 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002463069207622869, + "loss": 1.3073, + "theoretical_loss": 3.420273504756514, + "tokens_seen": 2294859776 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024629087261785355, + "loss": 1.2414, + "theoretical_loss": 3.420265645402516, + "tokens_seen": 2294925312 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024627482447342025, + "loss": 1.2627, + "theoretical_loss": 3.420257786335795, + "tokens_seen": 2294990848 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024625877632898695, + "loss": 1.1977, + "theoretical_loss": 3.4202499275563323, + "tokens_seen": 2295056384 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024624272818455365, + "loss": 1.2868, + "theoretical_loss": 3.420242069064108, + "tokens_seen": 2295121920 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024622668004012035, + "loss": 1.3311, + "theoretical_loss": 3.420234210859104, + "tokens_seen": 2295187456 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024621063189568705, + "loss": 1.2189, + "theoretical_loss": 3.420226352941303, + "tokens_seen": 2295252992 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024619458375125375, + "loss": 1.3158, + "theoretical_loss": 3.4202184953106842, + "tokens_seen": 2295318528 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024617853560682045, + "loss": 1.2501, + "theoretical_loss": 3.42021063796723, + "tokens_seen": 2295384064 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024616248746238715, + "loss": 1.2123, + "theoretical_loss": 3.4202027809109214, + "tokens_seen": 2295449600 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024614643931795385, + "loss": 1.3815, + "theoretical_loss": 3.42019492414174, + "tokens_seen": 2295515136 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024613039117352055, + "loss": 1.2133, + "theoretical_loss": 3.420187067659667, + "tokens_seen": 2295580672 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024611434302908725, + "loss": 1.328, + "theoretical_loss": 3.4201792114646836, + "tokens_seen": 2295646208 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024609829488465395, + "loss": 1.3195, + "theoretical_loss": 3.420171355556771, + "tokens_seen": 2295711744 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024608224674022066, + "loss": 1.2892, + "theoretical_loss": 3.420163499935911, + "tokens_seen": 2295777280 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024606619859578736, + "loss": 1.3356, + "theoretical_loss": 3.420155644602084, + "tokens_seen": 2295842816 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024605015045135406, + "loss": 1.2364, + "theoretical_loss": 3.4201477895552723, + "tokens_seen": 2295908352 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024603410230692076, + "loss": 1.373, + "theoretical_loss": 3.420139934795457, + "tokens_seen": 2295973888 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024601805416248746, + "loss": 1.2589, + "theoretical_loss": 3.420132080322619, + "tokens_seen": 2296039424 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024600200601805416, + "loss": 1.1964, + "theoretical_loss": 3.42012422613674, + "tokens_seen": 2296104960 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024598595787362086, + "loss": 1.2632, + "theoretical_loss": 3.420116372237801, + "tokens_seen": 2296170496 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024596990972918756, + "loss": 1.2148, + "theoretical_loss": 3.420108518625784, + "tokens_seen": 2296236032 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024595386158475426, + "loss": 1.2288, + "theoretical_loss": 3.4201006653006694, + "tokens_seen": 2296301568 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024593781344032096, + "loss": 1.3107, + "theoretical_loss": 3.420092812262439, + "tokens_seen": 2296367104 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024592176529588766, + "loss": 1.2038, + "theoretical_loss": 3.4200849595110743, + "tokens_seen": 2296432640 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024590571715145436, + "loss": 1.2007, + "theoretical_loss": 3.4200771070465565, + "tokens_seen": 2296498176 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024588966900702106, + "loss": 1.0978, + "theoretical_loss": 3.4200692548688667, + "tokens_seen": 2296563712 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024587362086258776, + "loss": 1.243, + "theoretical_loss": 3.4200614029779866, + "tokens_seen": 2296629248 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024585757271815446, + "loss": 1.2841, + "theoretical_loss": 3.4200535513738974, + "tokens_seen": 2296694784 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024584152457372116, + "loss": 1.2204, + "theoretical_loss": 3.4200457000565803, + "tokens_seen": 2296760320 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024582547642928787, + "loss": 1.1684, + "theoretical_loss": 3.420037849026017, + "tokens_seen": 2296825856 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024580942828485457, + "loss": 1.1613, + "theoretical_loss": 3.4200299982821885, + "tokens_seen": 2296891392 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024579338014042127, + "loss": 1.2285, + "theoretical_loss": 3.420022147825076, + "tokens_seen": 2296956928 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024577733199598797, + "loss": 1.3849, + "theoretical_loss": 3.420014297654661, + "tokens_seen": 2297022464 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024576128385155467, + "loss": 1.2469, + "theoretical_loss": 3.4200064477709256, + "tokens_seen": 2297088000 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024574523570712137, + "loss": 1.2701, + "theoretical_loss": 3.41999859817385, + "tokens_seen": 2297153536 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024572918756268807, + "loss": 1.3115, + "theoretical_loss": 3.4199907488634165, + "tokens_seen": 2297219072 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024571313941825477, + "loss": 1.217, + "theoretical_loss": 3.4199828998396056, + "tokens_seen": 2297284608 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024569709127382147, + "loss": 1.2919, + "theoretical_loss": 3.419975051102399, + "tokens_seen": 2297350144 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024568104312938817, + "loss": 1.2568, + "theoretical_loss": 3.4199672026517787, + "tokens_seen": 2297415680 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024566499498495487, + "loss": 1.2742, + "theoretical_loss": 3.419959354487725, + "tokens_seen": 2297481216 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024564894684052157, + "loss": 1.1113, + "theoretical_loss": 3.4199515066102197, + "tokens_seen": 2297546752 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002456328986960883, + "loss": 1.2546, + "theoretical_loss": 3.4199436590192445, + "tokens_seen": 2297612288 + }, + { + "epoch": 1.08, + "learning_rate": 0.000245616850551655, + "loss": 1.2529, + "theoretical_loss": 3.4199358117147804, + "tokens_seen": 2297677824 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002456008024072217, + "loss": 1.1847, + "theoretical_loss": 3.419927964696809, + "tokens_seen": 2297743360 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002455847542627884, + "loss": 1.3158, + "theoretical_loss": 3.4199201179653116, + "tokens_seen": 2297808896 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002455687061183551, + "loss": 1.3052, + "theoretical_loss": 3.419912271520269, + "tokens_seen": 2297874432 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002455526579739218, + "loss": 1.1641, + "theoretical_loss": 3.4199044253616635, + "tokens_seen": 2297939968 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002455366098294885, + "loss": 1.2394, + "theoretical_loss": 3.4198965794894756, + "tokens_seen": 2298005504 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002455205616850552, + "loss": 1.2079, + "theoretical_loss": 3.4198887339036874, + "tokens_seen": 2298071040 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002455045135406219, + "loss": 1.326, + "theoretical_loss": 3.4198808886042804, + "tokens_seen": 2298136576 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002454884653961886, + "loss": 1.291, + "theoretical_loss": 3.4198730435912355, + "tokens_seen": 2298202112 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002454724172517553, + "loss": 1.1921, + "theoretical_loss": 3.4198651988645334, + "tokens_seen": 2298267648 + }, + { + "epoch": 1.08, + "learning_rate": 0.000245456369107322, + "loss": 1.1564, + "theoretical_loss": 3.4198573544241566, + "tokens_seen": 2298333184 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002454403209628887, + "loss": 1.2613, + "theoretical_loss": 3.4198495102700863, + "tokens_seen": 2298398720 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002454242728184554, + "loss": 1.2271, + "theoretical_loss": 3.419841666402304, + "tokens_seen": 2298464256 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002454082246740221, + "loss": 1.3944, + "theoretical_loss": 3.4198338228207903, + "tokens_seen": 2298529792 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002453921765295888, + "loss": 1.2709, + "theoretical_loss": 3.4198259795255272, + "tokens_seen": 2298595328 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002453761283851555, + "loss": 1.179, + "theoretical_loss": 3.419818136516496, + "tokens_seen": 2298660864 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002453600802407222, + "loss": 1.2068, + "theoretical_loss": 3.4198102937936783, + "tokens_seen": 2298726400 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002453440320962889, + "loss": 1.2683, + "theoretical_loss": 3.4198024513570555, + "tokens_seen": 2298791936 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002453279839518556, + "loss": 1.2447, + "theoretical_loss": 3.4197946092066083, + "tokens_seen": 2298857472 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002453119358074223, + "loss": 1.273, + "theoretical_loss": 3.419786767342319, + "tokens_seen": 2298923008 + }, + { + "epoch": 1.08, + "learning_rate": 0.000245295887662989, + "loss": 1.3466, + "theoretical_loss": 3.4197789257641675, + "tokens_seen": 2298988544 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002452798395185557, + "loss": 1.1888, + "theoretical_loss": 3.4197710844721376, + "tokens_seen": 2299054080 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002452637913741224, + "loss": 1.2664, + "theoretical_loss": 3.4197632434662086, + "tokens_seen": 2299119616 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002452477432296891, + "loss": 1.2937, + "theoretical_loss": 3.419755402746363, + "tokens_seen": 2299185152 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002452316950852558, + "loss": 1.3073, + "theoretical_loss": 3.4197475623125815, + "tokens_seen": 2299250688 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002452156469408225, + "loss": 1.21, + "theoretical_loss": 3.4197397221648465, + "tokens_seen": 2299316224 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002451995987963892, + "loss": 1.204, + "theoretical_loss": 3.4197318823031386, + "tokens_seen": 2299381760 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002451835506519559, + "loss": 1.2097, + "theoretical_loss": 3.4197240427274394, + "tokens_seen": 2299447296 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002451675025075226, + "loss": 1.1725, + "theoretical_loss": 3.4197162034377304, + "tokens_seen": 2299512832 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002451514543630893, + "loss": 1.2729, + "theoretical_loss": 3.4197083644339927, + "tokens_seen": 2299578368 + }, + { + "epoch": 1.08, + "learning_rate": 0.000245135406218656, + "loss": 1.1758, + "theoretical_loss": 3.4197005257162085, + "tokens_seen": 2299643904 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002451193580742227, + "loss": 1.2353, + "theoretical_loss": 3.4196926872843583, + "tokens_seen": 2299709440 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002451033099297894, + "loss": 1.2581, + "theoretical_loss": 3.419684849138424, + "tokens_seen": 2299774976 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002450872617853561, + "loss": 1.26, + "theoretical_loss": 3.419677011278387, + "tokens_seen": 2299840512 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002450712136409228, + "loss": 1.165, + "theoretical_loss": 3.4196691737042286, + "tokens_seen": 2299906048 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002450551654964895, + "loss": 1.31, + "theoretical_loss": 3.4196613364159303, + "tokens_seen": 2299971584 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002450391173520562, + "loss": 1.2361, + "theoretical_loss": 3.419653499413474, + "tokens_seen": 2300037120 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002450230692076229, + "loss": 1.2669, + "theoretical_loss": 3.41964566269684, + "tokens_seen": 2300102656 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002450070210631896, + "loss": 1.3298, + "theoretical_loss": 3.419637826266011, + "tokens_seen": 2300168192 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002449909729187563, + "loss": 1.1692, + "theoretical_loss": 3.4196299901209675, + "tokens_seen": 2300233728 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 850642, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.221297025680542, + "objective/train/theoretical_loss": 3.419622154261691, + "objective/train/tokens_used": 2320759264, + "theoretical_loss": 3.419622154261691, + "tokens_seen": 2300299264 + }, + { + "epoch": 1.08, + "learning_rate": 0.000244974924774323, + "loss": 1.2213, + "theoretical_loss": 3.419622154261691, + "tokens_seen": 2300299264 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002449588766298897, + "loss": 1.2549, + "theoretical_loss": 3.4196143186881636, + "tokens_seen": 2300364800 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002449428284854564, + "loss": 1.2837, + "theoretical_loss": 3.419606483400366, + "tokens_seen": 2300430336 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002449267803410231, + "loss": 1.2482, + "theoretical_loss": 3.4195986483982805, + "tokens_seen": 2300495872 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002449107321965898, + "loss": 1.1857, + "theoretical_loss": 3.4195908136818876, + "tokens_seen": 2300561408 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002448946840521565, + "loss": 1.3093, + "theoretical_loss": 3.419582979251169, + "tokens_seen": 2300626944 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024487863590772315, + "loss": 1.424, + "theoretical_loss": 3.419575145106107, + "tokens_seen": 2300692480 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024486258776328985, + "loss": 1.2882, + "theoretical_loss": 3.419567311246682, + "tokens_seen": 2300758016 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024484653961885655, + "loss": 1.2814, + "theoretical_loss": 3.419559477672876, + "tokens_seen": 2300823552 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024483049147442325, + "loss": 1.2016, + "theoretical_loss": 3.41955164438467, + "tokens_seen": 2300889088 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024481444332998995, + "loss": 1.3115, + "theoretical_loss": 3.4195438113820456, + "tokens_seen": 2300954624 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024479839518555665, + "loss": 1.2603, + "theoretical_loss": 3.4195359786649844, + "tokens_seen": 2301020160 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024478234704112335, + "loss": 1.1704, + "theoretical_loss": 3.419528146233468, + "tokens_seen": 2301085696 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024476629889669005, + "loss": 1.3196, + "theoretical_loss": 3.4195203140874777, + "tokens_seen": 2301151232 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024475025075225675, + "loss": 1.2546, + "theoretical_loss": 3.419512482226995, + "tokens_seen": 2301216768 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024473420260782345, + "loss": 1.2005, + "theoretical_loss": 3.4195046506520015, + "tokens_seen": 2301282304 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024471815446339016, + "loss": 1.1833, + "theoretical_loss": 3.419496819362478, + "tokens_seen": 2301347840 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024470210631895686, + "loss": 1.2012, + "theoretical_loss": 3.4194889883584065, + "tokens_seen": 2301413376 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024468605817452356, + "loss": 1.2237, + "theoretical_loss": 3.419481157639769, + "tokens_seen": 2301478912 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024467001003009026, + "loss": 1.2534, + "theoretical_loss": 3.4194733272065454, + "tokens_seen": 2301544448 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024465396188565696, + "loss": 1.322, + "theoretical_loss": 3.4194654970587193, + "tokens_seen": 2301609984 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024463791374122366, + "loss": 1.2111, + "theoretical_loss": 3.41945766719627, + "tokens_seen": 2301675520 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024462186559679036, + "loss": 1.2585, + "theoretical_loss": 3.4194498376191804, + "tokens_seen": 2301741056 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024460581745235706, + "loss": 1.2361, + "theoretical_loss": 3.4194420083274317, + "tokens_seen": 2301806592 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024458976930792376, + "loss": 1.2047, + "theoretical_loss": 3.419434179321005, + "tokens_seen": 2301872128 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024457372116349046, + "loss": 1.222, + "theoretical_loss": 3.419426350599882, + "tokens_seen": 2301937664 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024455767301905716, + "loss": 1.1761, + "theoretical_loss": 3.4194185221640447, + "tokens_seen": 2302003200 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024454162487462386, + "loss": 1.1487, + "theoretical_loss": 3.4194106940134734, + "tokens_seen": 2302068736 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024452557673019056, + "loss": 1.2404, + "theoretical_loss": 3.419402866148151, + "tokens_seen": 2302134272 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024450952858575726, + "loss": 1.3096, + "theoretical_loss": 3.419395038568058, + "tokens_seen": 2302199808 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024449348044132396, + "loss": 1.2706, + "theoretical_loss": 3.419387211273176, + "tokens_seen": 2302265344 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024447743229689066, + "loss": 1.2544, + "theoretical_loss": 3.4193793842634865, + "tokens_seen": 2302330880 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024446138415245737, + "loss": 1.1897, + "theoretical_loss": 3.4193715575389714, + "tokens_seen": 2302396416 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024444533600802407, + "loss": 1.2447, + "theoretical_loss": 3.4193637310996117, + "tokens_seen": 2302461952 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002444292878635908, + "loss": 1.229, + "theoretical_loss": 3.419355904945389, + "tokens_seen": 2302527488 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002444132397191575, + "loss": 1.2977, + "theoretical_loss": 3.4193480790762854, + "tokens_seen": 2302593024 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002443971915747242, + "loss": 1.1778, + "theoretical_loss": 3.4193402534922814, + "tokens_seen": 2302658560 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002443811434302909, + "loss": 1.234, + "theoretical_loss": 3.4193324281933593, + "tokens_seen": 2302724096 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002443650952858576, + "loss": 1.2052, + "theoretical_loss": 3.4193246031795, + "tokens_seen": 2302789632 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002443490471414243, + "loss": 1.2275, + "theoretical_loss": 3.419316778450686, + "tokens_seen": 2302855168 + }, + { + "epoch": 1.08, + "learning_rate": 0.000244332998996991, + "loss": 1.3265, + "theoretical_loss": 3.4193089540068975, + "tokens_seen": 2302920704 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002443169508525577, + "loss": 1.2179, + "theoretical_loss": 3.419301129848117, + "tokens_seen": 2302986240 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002443009027081244, + "loss": 1.1992, + "theoretical_loss": 3.419293305974325, + "tokens_seen": 2303051776 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002442848545636911, + "loss": 1.2281, + "theoretical_loss": 3.4192854823855043, + "tokens_seen": 2303117312 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002442688064192578, + "loss": 1.2155, + "theoretical_loss": 3.4192776590816356, + "tokens_seen": 2303182848 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002442527582748245, + "loss": 1.2925, + "theoretical_loss": 3.4192698360627007, + "tokens_seen": 2303248384 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002442367101303912, + "loss": 1.2308, + "theoretical_loss": 3.4192620133286806, + "tokens_seen": 2303313920 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002442206619859579, + "loss": 1.2565, + "theoretical_loss": 3.4192541908795575, + "tokens_seen": 2303379456 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002442046138415246, + "loss": 1.2013, + "theoretical_loss": 3.4192463687153127, + "tokens_seen": 2303444992 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002441885656970913, + "loss": 1.2348, + "theoretical_loss": 3.419238546835927, + "tokens_seen": 2303510528 + }, + { + "epoch": 1.08, + "learning_rate": 0.000244172517552658, + "loss": 1.2632, + "theoretical_loss": 3.419230725241383, + "tokens_seen": 2303576064 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002441564694082247, + "loss": 1.2155, + "theoretical_loss": 3.4192229039316624, + "tokens_seen": 2303641600 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002441404212637914, + "loss": 1.2792, + "theoretical_loss": 3.4192150829067454, + "tokens_seen": 2303707136 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002441243731193581, + "loss": 1.3642, + "theoretical_loss": 3.4192072621666143, + "tokens_seen": 2303772672 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002441083249749248, + "loss": 1.1529, + "theoretical_loss": 3.4191994417112506, + "tokens_seen": 2303838208 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002440922768304915, + "loss": 1.1825, + "theoretical_loss": 3.419191621540636, + "tokens_seen": 2303903744 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002440762286860582, + "loss": 1.2844, + "theoretical_loss": 3.4191838016547518, + "tokens_seen": 2303969280 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024406018054162488, + "loss": 1.2022, + "theoretical_loss": 3.4191759820535794, + "tokens_seen": 2304034816 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024404413239719158, + "loss": 1.1802, + "theoretical_loss": 3.4191681627371007, + "tokens_seen": 2304100352 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024402808425275828, + "loss": 1.2251, + "theoretical_loss": 3.419160343705297, + "tokens_seen": 2304165888 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024401203610832498, + "loss": 1.3466, + "theoretical_loss": 3.4191525249581494, + "tokens_seen": 2304231424 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024399598796389168, + "loss": 1.2794, + "theoretical_loss": 3.4191447064956404, + "tokens_seen": 2304296960 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024397993981945838, + "loss": 1.3043, + "theoretical_loss": 3.419136888317751, + "tokens_seen": 2304362496 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024396389167502508, + "loss": 1.364, + "theoretical_loss": 3.419129070424463, + "tokens_seen": 2304428032 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024394784353059179, + "loss": 1.2634, + "theoretical_loss": 3.4191212528157573, + "tokens_seen": 2304493568 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024393179538615849, + "loss": 1.2292, + "theoretical_loss": 3.4191134354916164, + "tokens_seen": 2304559104 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002439157472417252, + "loss": 1.1571, + "theoretical_loss": 3.419105618452021, + "tokens_seen": 2304624640 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002438996990972919, + "loss": 1.2469, + "theoretical_loss": 3.419097801696953, + "tokens_seen": 2304690176 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002438836509528586, + "loss": 1.3112, + "theoretical_loss": 3.4190899852263943, + "tokens_seen": 2304755712 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002438676028084253, + "loss": 1.2611, + "theoretical_loss": 3.4190821690403257, + "tokens_seen": 2304821248 + }, + { + "epoch": 1.08, + "learning_rate": 0.000243851554663992, + "loss": 1.2407, + "theoretical_loss": 3.4190743531387295, + "tokens_seen": 2304886784 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002438355065195587, + "loss": 1.2982, + "theoretical_loss": 3.4190665375215867, + "tokens_seen": 2304952320 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002438194583751254, + "loss": 1.3712, + "theoretical_loss": 3.4190587221888795, + "tokens_seen": 2305017856 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002438034102306921, + "loss": 1.2426, + "theoretical_loss": 3.419050907140589, + "tokens_seen": 2305083392 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002437873620862588, + "loss": 1.2671, + "theoretical_loss": 3.4190430923766963, + "tokens_seen": 2305148928 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002437713139418255, + "loss": 1.3025, + "theoretical_loss": 3.4190352778971835, + "tokens_seen": 2305214464 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002437552657973922, + "loss": 1.2779, + "theoretical_loss": 3.4190274637020326, + "tokens_seen": 2305280000 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002437392176529589, + "loss": 1.3622, + "theoretical_loss": 3.4190196497912244, + "tokens_seen": 2305345536 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002437231695085256, + "loss": 1.2006, + "theoretical_loss": 3.4190118361647412, + "tokens_seen": 2305411072 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002437071213640923, + "loss": 1.2484, + "theoretical_loss": 3.4190040228225635, + "tokens_seen": 2305476608 + }, + { + "epoch": 1.08, + "learning_rate": 0.000243691073219659, + "loss": 1.2167, + "theoretical_loss": 3.418996209764674, + "tokens_seen": 2305542144 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024367502507522567, + "loss": 1.2545, + "theoretical_loss": 3.418988396991054, + "tokens_seen": 2305607680 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024365897693079237, + "loss": 1.2122, + "theoretical_loss": 3.4189805845016847, + "tokens_seen": 2305673216 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024364292878635907, + "loss": 1.23, + "theoretical_loss": 3.4189727722965477, + "tokens_seen": 2305738752 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024362688064192577, + "loss": 1.1518, + "theoretical_loss": 3.4189649603756247, + "tokens_seen": 2305804288 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024361083249749247, + "loss": 1.2326, + "theoretical_loss": 3.4189571487388974, + "tokens_seen": 2305869824 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024359478435305917, + "loss": 1.2265, + "theoretical_loss": 3.4189493373863478, + "tokens_seen": 2305935360 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024357873620862587, + "loss": 1.2299, + "theoretical_loss": 3.418941526317956, + "tokens_seen": 2306000896 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024356268806419257, + "loss": 1.2694, + "theoretical_loss": 3.418933715533705, + "tokens_seen": 2306066432 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024354663991975927, + "loss": 1.1828, + "theoretical_loss": 3.4189259050335767, + "tokens_seen": 2306131968 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024353059177532597, + "loss": 1.3424, + "theoretical_loss": 3.4189180948175513, + "tokens_seen": 2306197504 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024351454363089268, + "loss": 1.3321, + "theoretical_loss": 3.418910284885611, + "tokens_seen": 2306263040 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024349849548645938, + "loss": 1.2147, + "theoretical_loss": 3.4189024752377377, + "tokens_seen": 2306328576 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024348244734202608, + "loss": 1.2059, + "theoretical_loss": 3.4188946658739128, + "tokens_seen": 2306394112 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024346639919759278, + "loss": 1.2278, + "theoretical_loss": 3.4188868567941175, + "tokens_seen": 2306459648 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024345035105315948, + "loss": 1.1782, + "theoretical_loss": 3.418879047998334, + "tokens_seen": 2306525184 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024343430290872618, + "loss": 1.2175, + "theoretical_loss": 3.4188712394865437, + "tokens_seen": 2306590720 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024341825476429288, + "loss": 1.3042, + "theoretical_loss": 3.4188634312587283, + "tokens_seen": 2306656256 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024340220661985958, + "loss": 1.0767, + "theoretical_loss": 3.418855623314869, + "tokens_seen": 2306721792 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024338615847542628, + "loss": 1.2876, + "theoretical_loss": 3.4188478156549476, + "tokens_seen": 2306787328 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 852977, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2708866596221924, + "objective/train/theoretical_loss": 3.418840008278946, + "objective/train/tokens_used": 2327312864, + "theoretical_loss": 3.418840008278946, + "tokens_seen": 2306852864 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024337011033099298, + "loss": 1.2709, + "theoretical_loss": 3.418840008278946, + "tokens_seen": 2306852864 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024335406218655968, + "loss": 1.1387, + "theoretical_loss": 3.4188322011868455, + "tokens_seen": 2306918400 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024333801404212638, + "loss": 1.317, + "theoretical_loss": 3.4188243943786274, + "tokens_seen": 2306983936 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024332196589769308, + "loss": 1.2326, + "theoretical_loss": 3.4188165878542742, + "tokens_seen": 2307049472 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024330591775325978, + "loss": 1.2382, + "theoretical_loss": 3.418808781613767, + "tokens_seen": 2307115008 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024328986960882646, + "loss": 1.2444, + "theoretical_loss": 3.4188009756570876, + "tokens_seen": 2307180544 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024327382146439316, + "loss": 1.2116, + "theoretical_loss": 3.418793169984217, + "tokens_seen": 2307246080 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024325777331995986, + "loss": 1.3413, + "theoretical_loss": 3.4187853645951374, + "tokens_seen": 2307311616 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002432417251755266, + "loss": 1.2022, + "theoretical_loss": 3.4187775594898304, + "tokens_seen": 2307377152 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024322567703109331, + "loss": 1.2528, + "theoretical_loss": 3.4187697546682774, + "tokens_seen": 2307442688 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024320962888666001, + "loss": 1.245, + "theoretical_loss": 3.4187619501304605, + "tokens_seen": 2307508224 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024319358074222672, + "loss": 1.2244, + "theoretical_loss": 3.4187541458763606, + "tokens_seen": 2307573760 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024317753259779342, + "loss": 1.3074, + "theoretical_loss": 3.41874634190596, + "tokens_seen": 2307639296 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024316148445336012, + "loss": 1.2077, + "theoretical_loss": 3.4187385382192397, + "tokens_seen": 2307704832 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024314543630892682, + "loss": 1.2603, + "theoretical_loss": 3.418730734816182, + "tokens_seen": 2307770368 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024312938816449352, + "loss": 1.2769, + "theoretical_loss": 3.418722931696768, + "tokens_seen": 2307835904 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024311334002006022, + "loss": 1.1836, + "theoretical_loss": 3.4187151288609794, + "tokens_seen": 2307901440 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024309729187562692, + "loss": 1.2668, + "theoretical_loss": 3.4187073263087986, + "tokens_seen": 2307966976 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024308124373119362, + "loss": 1.1676, + "theoretical_loss": 3.418699524040206, + "tokens_seen": 2308032512 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024306519558676032, + "loss": 1.2703, + "theoretical_loss": 3.418691722055184, + "tokens_seen": 2308098048 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024304914744232702, + "loss": 1.1876, + "theoretical_loss": 3.4186839203537143, + "tokens_seen": 2308163584 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002430330992978937, + "loss": 1.194, + "theoretical_loss": 3.418676118935778, + "tokens_seen": 2308229120 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002430170511534604, + "loss": 1.2841, + "theoretical_loss": 3.4186683178013575, + "tokens_seen": 2308294656 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002430010030090271, + "loss": 1.2862, + "theoretical_loss": 3.4186605169504336, + "tokens_seen": 2308360192 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002429849548645938, + "loss": 1.1694, + "theoretical_loss": 3.4186527163829887, + "tokens_seen": 2308425728 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002429689067201605, + "loss": 1.1706, + "theoretical_loss": 3.4186449160990042, + "tokens_seen": 2308491264 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002429528585757272, + "loss": 1.2481, + "theoretical_loss": 3.4186371160984614, + "tokens_seen": 2308556800 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002429368104312939, + "loss": 1.1679, + "theoretical_loss": 3.4186293163813426, + "tokens_seen": 2308622336 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002429207622868606, + "loss": 1.1628, + "theoretical_loss": 3.4186215169476286, + "tokens_seen": 2308687872 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002429047141424273, + "loss": 1.3082, + "theoretical_loss": 3.418613717797302, + "tokens_seen": 2308753408 + }, + { + "epoch": 1.08, + "learning_rate": 0.000242888665997994, + "loss": 1.3036, + "theoretical_loss": 3.4186059189303437, + "tokens_seen": 2308818944 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002428726178535607, + "loss": 1.1768, + "theoretical_loss": 3.418598120346736, + "tokens_seen": 2308884480 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002428565697091274, + "loss": 1.2641, + "theoretical_loss": 3.4185903220464597, + "tokens_seen": 2308950016 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002428405215646941, + "loss": 1.2763, + "theoretical_loss": 3.418582524029498, + "tokens_seen": 2309015552 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002428244734202608, + "loss": 1.2554, + "theoretical_loss": 3.4185747262958306, + "tokens_seen": 2309081088 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002428084252758275, + "loss": 1.2015, + "theoretical_loss": 3.4185669288454403, + "tokens_seen": 2309146624 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002427923771313942, + "loss": 1.2341, + "theoretical_loss": 3.4185591316783093, + "tokens_seen": 2309212160 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002427763289869609, + "loss": 1.2679, + "theoretical_loss": 3.418551334794418, + "tokens_seen": 2309277696 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002427602808425276, + "loss": 1.3266, + "theoretical_loss": 3.4185435381937483, + "tokens_seen": 2309343232 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002427442326980943, + "loss": 1.256, + "theoretical_loss": 3.418535741876283, + "tokens_seen": 2309408768 + }, + { + "epoch": 1.08, + "learning_rate": 0.000242728184553661, + "loss": 1.2043, + "theoretical_loss": 3.4185279458420026, + "tokens_seen": 2309474304 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002427121364092277, + "loss": 1.2342, + "theoretical_loss": 3.418520150090889, + "tokens_seen": 2309539840 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002426960882647944, + "loss": 1.2986, + "theoretical_loss": 3.4185123546229246, + "tokens_seen": 2309605376 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002426800401203611, + "loss": 1.3055, + "theoretical_loss": 3.41850455943809, + "tokens_seen": 2309670912 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002426639919759278, + "loss": 1.1932, + "theoretical_loss": 3.418496764536368, + "tokens_seen": 2309736448 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024264794383149448, + "loss": 1.2128, + "theoretical_loss": 3.4184889699177394, + "tokens_seen": 2309801984 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024263189568706118, + "loss": 1.3186, + "theoretical_loss": 3.418481175582186, + "tokens_seen": 2309867520 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024261584754262788, + "loss": 1.1818, + "theoretical_loss": 3.41847338152969, + "tokens_seen": 2309933056 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024259979939819458, + "loss": 1.18, + "theoretical_loss": 3.418465587760233, + "tokens_seen": 2309998592 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024258375125376129, + "loss": 1.2823, + "theoretical_loss": 3.418457794273796, + "tokens_seen": 2310064128 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024256770310932799, + "loss": 1.1901, + "theoretical_loss": 3.4184500010703616, + "tokens_seen": 2310129664 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002425516549648947, + "loss": 1.3153, + "theoretical_loss": 3.418442208149911, + "tokens_seen": 2310195200 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002425356068204614, + "loss": 1.2086, + "theoretical_loss": 3.4184344155124258, + "tokens_seen": 2310260736 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002425195586760281, + "loss": 1.2178, + "theoretical_loss": 3.4184266231578877, + "tokens_seen": 2310326272 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002425035105315948, + "loss": 1.2427, + "theoretical_loss": 3.4184188310862784, + "tokens_seen": 2310391808 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002424874623871615, + "loss": 1.2275, + "theoretical_loss": 3.41841103929758, + "tokens_seen": 2310457344 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002424714142427282, + "loss": 1.2612, + "theoretical_loss": 3.4184032477917743, + "tokens_seen": 2310522880 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002424553660982949, + "loss": 1.2239, + "theoretical_loss": 3.418395456568842, + "tokens_seen": 2310588416 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002424393179538616, + "loss": 1.2115, + "theoretical_loss": 3.4183876656287664, + "tokens_seen": 2310653952 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002424232698094283, + "loss": 1.174, + "theoretical_loss": 3.4183798749715275, + "tokens_seen": 2310719488 + }, + { + "epoch": 1.08, + "learning_rate": 0.000242407221664995, + "loss": 1.2666, + "theoretical_loss": 3.418372084597108, + "tokens_seen": 2310785024 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002423911735205617, + "loss": 1.3065, + "theoretical_loss": 3.4183642945054897, + "tokens_seen": 2310850560 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002423751253761284, + "loss": 1.2102, + "theoretical_loss": 3.4183565046966535, + "tokens_seen": 2310916096 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002423590772316951, + "loss": 1.2778, + "theoretical_loss": 3.4183487151705823, + "tokens_seen": 2310981632 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002423430290872618, + "loss": 1.2538, + "theoretical_loss": 3.418340925927257, + "tokens_seen": 2311047168 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002423269809428285, + "loss": 1.2363, + "theoretical_loss": 3.418333136966659, + "tokens_seen": 2311112704 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002423109327983952, + "loss": 1.3439, + "theoretical_loss": 3.4183253482887705, + "tokens_seen": 2311178240 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002422948846539619, + "loss": 1.2115, + "theoretical_loss": 3.4183175598935738, + "tokens_seen": 2311243776 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002422788365095286, + "loss": 1.21, + "theoretical_loss": 3.4183097717810496, + "tokens_seen": 2311309312 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024226278836509527, + "loss": 1.1462, + "theoretical_loss": 3.4183019839511806, + "tokens_seen": 2311374848 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024224674022066197, + "loss": 1.2794, + "theoretical_loss": 3.4182941964039473, + "tokens_seen": 2311440384 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024223069207622867, + "loss": 1.3129, + "theoretical_loss": 3.4182864091393323, + "tokens_seen": 2311505920 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024221464393179537, + "loss": 1.286, + "theoretical_loss": 3.4182786221573176, + "tokens_seen": 2311571456 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024219859578736207, + "loss": 1.2527, + "theoretical_loss": 3.418270835457884, + "tokens_seen": 2311636992 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024218254764292877, + "loss": 1.2606, + "theoretical_loss": 3.418263049041014, + "tokens_seen": 2311702528 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024216649949849547, + "loss": 1.1706, + "theoretical_loss": 3.418255262906689, + "tokens_seen": 2311768064 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024215045135406218, + "loss": 1.2618, + "theoretical_loss": 3.4182474770548907, + "tokens_seen": 2311833600 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024213440320962888, + "loss": 1.2746, + "theoretical_loss": 3.418239691485601, + "tokens_seen": 2311899136 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024211835506519558, + "loss": 1.1744, + "theoretical_loss": 3.418231906198802, + "tokens_seen": 2311964672 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024210230692076228, + "loss": 1.3376, + "theoretical_loss": 3.4182241211944744, + "tokens_seen": 2312030208 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024208625877632898, + "loss": 1.2724, + "theoretical_loss": 3.418216336472601, + "tokens_seen": 2312095744 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024207021063189568, + "loss": 1.192, + "theoretical_loss": 3.4182085520331627, + "tokens_seen": 2312161280 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024205416248746238, + "loss": 1.2211, + "theoretical_loss": 3.418200767876142, + "tokens_seen": 2312226816 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024203811434302913, + "loss": 1.2426, + "theoretical_loss": 3.41819298400152, + "tokens_seen": 2312292352 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002420220661985958, + "loss": 1.2022, + "theoretical_loss": 3.418185200409279, + "tokens_seen": 2312357888 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002420060180541625, + "loss": 1.2093, + "theoretical_loss": 3.418177417099401, + "tokens_seen": 2312423424 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002419899699097292, + "loss": 1.2923, + "theoretical_loss": 3.4181696340718664, + "tokens_seen": 2312488960 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002419739217652959, + "loss": 1.2575, + "theoretical_loss": 3.418161851326658, + "tokens_seen": 2312554496 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002419578736208626, + "loss": 1.1748, + "theoretical_loss": 3.4181540688637577, + "tokens_seen": 2312620032 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002419418254764293, + "loss": 1.3577, + "theoretical_loss": 3.4181462866831467, + "tokens_seen": 2312685568 + }, + { + "epoch": 1.08, + "learning_rate": 0.000241925777331996, + "loss": 1.2433, + "theoretical_loss": 3.418138504784807, + "tokens_seen": 2312751104 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002419097291875627, + "loss": 1.2615, + "theoretical_loss": 3.418130723168721, + "tokens_seen": 2312816640 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002418936810431294, + "loss": 1.2153, + "theoretical_loss": 3.418122941834869, + "tokens_seen": 2312882176 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002418776328986961, + "loss": 1.2436, + "theoretical_loss": 3.418115160783234, + "tokens_seen": 2312947712 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024186158475426281, + "loss": 1.1825, + "theoretical_loss": 3.4181073800137973, + "tokens_seen": 2313013248 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024184553660982951, + "loss": 1.2574, + "theoretical_loss": 3.418099599526541, + "tokens_seen": 2313078784 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024182948846539622, + "loss": 1.1164, + "theoretical_loss": 3.4180918193214467, + "tokens_seen": 2313144320 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024181344032096292, + "loss": 1.2374, + "theoretical_loss": 3.4180840393984955, + "tokens_seen": 2313209856 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024179739217652962, + "loss": 1.1497, + "theoretical_loss": 3.41807625975767, + "tokens_seen": 2313275392 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024178134403209632, + "loss": 1.3219, + "theoretical_loss": 3.418068480398952, + "tokens_seen": 2313340928 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 855197, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.205795168876648, + "objective/train/theoretical_loss": 3.4180607013223225, + "objective/train/tokens_used": 2333866464, + "theoretical_loss": 3.4180607013223225, + "tokens_seen": 2313406464 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024176529588766302, + "loss": 1.2058, + "theoretical_loss": 3.4180607013223225, + "tokens_seen": 2313406464 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024174924774322972, + "loss": 1.235, + "theoretical_loss": 3.4180529225277643, + "tokens_seen": 2313472000 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024173319959879642, + "loss": 1.2433, + "theoretical_loss": 3.4180451440152586, + "tokens_seen": 2313537536 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024171715145436312, + "loss": 1.1968, + "theoretical_loss": 3.418037365784787, + "tokens_seen": 2313603072 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024170110330992982, + "loss": 1.1826, + "theoretical_loss": 3.418029587836332, + "tokens_seen": 2313668608 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024168505516549652, + "loss": 1.2247, + "theoretical_loss": 3.4180218101698743, + "tokens_seen": 2313734144 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024166900702106322, + "loss": 1.174, + "theoretical_loss": 3.418014032785397, + "tokens_seen": 2313799680 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024165295887662992, + "loss": 1.2398, + "theoretical_loss": 3.4180062556828807, + "tokens_seen": 2313865216 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002416369107321966, + "loss": 1.2358, + "theoretical_loss": 3.417998478862308, + "tokens_seen": 2313930752 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002416208625877633, + "loss": 1.1567, + "theoretical_loss": 3.41799070232366, + "tokens_seen": 2313996288 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024160481444333, + "loss": 1.2056, + "theoretical_loss": 3.4179829260669194, + "tokens_seen": 2314061824 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002415887662988967, + "loss": 1.3779, + "theoretical_loss": 3.4179751500920674, + "tokens_seen": 2314127360 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002415727181544634, + "loss": 1.2971, + "theoretical_loss": 3.417967374399086, + "tokens_seen": 2314192896 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002415566700100301, + "loss": 1.1871, + "theoretical_loss": 3.4179595989879563, + "tokens_seen": 2314258432 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002415406218655968, + "loss": 1.1732, + "theoretical_loss": 3.4179518238586613, + "tokens_seen": 2314323968 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002415245737211635, + "loss": 1.3117, + "theoretical_loss": 3.417944049011182, + "tokens_seen": 2314389504 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002415085255767302, + "loss": 1.2274, + "theoretical_loss": 3.4179362744455, + "tokens_seen": 2314455040 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002414924774322969, + "loss": 1.2539, + "theoretical_loss": 3.417928500161598, + "tokens_seen": 2314520576 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002414764292878636, + "loss": 1.2588, + "theoretical_loss": 3.4179207261594575, + "tokens_seen": 2314586112 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002414603811434303, + "loss": 1.2637, + "theoretical_loss": 3.41791295243906, + "tokens_seen": 2314651648 + }, + { + "epoch": 1.08, + "learning_rate": 0.000241444332998997, + "loss": 1.1281, + "theoretical_loss": 3.417905179000387, + "tokens_seen": 2314717184 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002414282848545637, + "loss": 1.2086, + "theoretical_loss": 3.4178974058434215, + "tokens_seen": 2314782720 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002414122367101304, + "loss": 1.3033, + "theoretical_loss": 3.4178896329681443, + "tokens_seen": 2314848256 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002413961885656971, + "loss": 1.2868, + "theoretical_loss": 3.4178818603745373, + "tokens_seen": 2314913792 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002413801404212638, + "loss": 1.3284, + "theoretical_loss": 3.4178740880625824, + "tokens_seen": 2314979328 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002413640922768305, + "loss": 1.1553, + "theoretical_loss": 3.4178663160322618, + "tokens_seen": 2315044864 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002413480441323972, + "loss": 1.1691, + "theoretical_loss": 3.4178585442835567, + "tokens_seen": 2315110400 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002413319959879639, + "loss": 1.2047, + "theoretical_loss": 3.41785077281645, + "tokens_seen": 2315175936 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002413159478435306, + "loss": 1.1935, + "theoretical_loss": 3.417843001630922, + "tokens_seen": 2315241472 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002412998996990973, + "loss": 1.3196, + "theoretical_loss": 3.417835230726956, + "tokens_seen": 2315307008 + }, + { + "epoch": 1.08, + "learning_rate": 0.000241283851554664, + "loss": 1.2962, + "theoretical_loss": 3.4178274601045326, + "tokens_seen": 2315372544 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002412678034102307, + "loss": 1.2853, + "theoretical_loss": 3.4178196897636344, + "tokens_seen": 2315438080 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024125175526579738, + "loss": 1.221, + "theoretical_loss": 3.4178119197042434, + "tokens_seen": 2315503616 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024123570712136408, + "loss": 1.2211, + "theoretical_loss": 3.417804149926341, + "tokens_seen": 2315569152 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024121965897693078, + "loss": 1.2394, + "theoretical_loss": 3.4177963804299085, + "tokens_seen": 2315634688 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024120361083249749, + "loss": 1.215, + "theoretical_loss": 3.4177886112149287, + "tokens_seen": 2315700224 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024118756268806419, + "loss": 1.175, + "theoretical_loss": 3.417780842281383, + "tokens_seen": 2315765760 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002411715145436309, + "loss": 1.2568, + "theoretical_loss": 3.4177730736292533, + "tokens_seen": 2315831296 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002411554663991976, + "loss": 1.3906, + "theoretical_loss": 3.4177653052585217, + "tokens_seen": 2315896832 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002411394182547643, + "loss": 1.3175, + "theoretical_loss": 3.4177575371691695, + "tokens_seen": 2315962368 + }, + { + "epoch": 1.08, + "learning_rate": 0.000241123370110331, + "loss": 1.3228, + "theoretical_loss": 3.4177497693611794, + "tokens_seen": 2316027904 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002411073219658977, + "loss": 1.2757, + "theoretical_loss": 3.417742001834532, + "tokens_seen": 2316093440 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002410912738214644, + "loss": 1.2025, + "theoretical_loss": 3.4177342345892106, + "tokens_seen": 2316158976 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002410752256770311, + "loss": 1.2395, + "theoretical_loss": 3.4177264676251955, + "tokens_seen": 2316224512 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002410591775325978, + "loss": 1.2758, + "theoretical_loss": 3.4177187009424697, + "tokens_seen": 2316290048 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002410431293881645, + "loss": 1.2568, + "theoretical_loss": 3.417710934541015, + "tokens_seen": 2316355584 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002410270812437312, + "loss": 1.2785, + "theoretical_loss": 3.4177031684208123, + "tokens_seen": 2316421120 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002410110330992979, + "loss": 1.2673, + "theoretical_loss": 3.4176954025818445, + "tokens_seen": 2316486656 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002409949849548646, + "loss": 1.2988, + "theoretical_loss": 3.417687637024093, + "tokens_seen": 2316552192 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002409789368104313, + "loss": 1.2787, + "theoretical_loss": 3.41767987174754, + "tokens_seen": 2316617728 + }, + { + "epoch": 1.08, + "learning_rate": 0.000240962888665998, + "loss": 1.1833, + "theoretical_loss": 3.4176721067521667, + "tokens_seen": 2316683264 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002409468405215647, + "loss": 1.2419, + "theoretical_loss": 3.4176643420379555, + "tokens_seen": 2316748800 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002409307923771314, + "loss": 1.081, + "theoretical_loss": 3.4176565776048884, + "tokens_seen": 2316814336 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002409147442326981, + "loss": 1.0703, + "theoretical_loss": 3.417648813452947, + "tokens_seen": 2316879872 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002408986960882648, + "loss": 1.1524, + "theoretical_loss": 3.4176410495821132, + "tokens_seen": 2316945408 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002408826479438315, + "loss": 1.2427, + "theoretical_loss": 3.4176332859923684, + "tokens_seen": 2317010944 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024086659979939817, + "loss": 1.1894, + "theoretical_loss": 3.417625522683695, + "tokens_seen": 2317076480 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024085055165496487, + "loss": 1.2363, + "theoretical_loss": 3.4176177596560753, + "tokens_seen": 2317142016 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024083450351053163, + "loss": 1.2356, + "theoretical_loss": 3.41760999690949, + "tokens_seen": 2317207552 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024081845536609833, + "loss": 1.2139, + "theoretical_loss": 3.417602234443922, + "tokens_seen": 2317273088 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024080240722166503, + "loss": 1.2232, + "theoretical_loss": 3.417594472259353, + "tokens_seen": 2317338624 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024078635907723173, + "loss": 1.2662, + "theoretical_loss": 3.4175867103557644, + "tokens_seen": 2317404160 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024077031093279843, + "loss": 1.2974, + "theoretical_loss": 3.4175789487331385, + "tokens_seen": 2317469696 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024075426278836513, + "loss": 1.2279, + "theoretical_loss": 3.417571187391457, + "tokens_seen": 2317535232 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024073821464393183, + "loss": 1.1965, + "theoretical_loss": 3.417563426330702, + "tokens_seen": 2317600768 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024072216649949853, + "loss": 1.2931, + "theoretical_loss": 3.4175556655508554, + "tokens_seen": 2317666304 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024070611835506523, + "loss": 1.2203, + "theoretical_loss": 3.4175479050518986, + "tokens_seen": 2317731840 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024069007021063193, + "loss": 1.2465, + "theoretical_loss": 3.417540144833814, + "tokens_seen": 2317797376 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024067402206619863, + "loss": 1.2382, + "theoretical_loss": 3.4175323848965835, + "tokens_seen": 2317862912 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024065797392176533, + "loss": 1.1961, + "theoretical_loss": 3.4175246252401887, + "tokens_seen": 2317928448 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024064192577733203, + "loss": 1.2031, + "theoretical_loss": 3.4175168658646116, + "tokens_seen": 2317993984 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002406258776328987, + "loss": 1.2964, + "theoretical_loss": 3.4175091067698338, + "tokens_seen": 2318059520 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002406098294884654, + "loss": 1.148, + "theoretical_loss": 3.417501347955838, + "tokens_seen": 2318125056 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002405937813440321, + "loss": 1.132, + "theoretical_loss": 3.4174935894226053, + "tokens_seen": 2318190592 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002405777331995988, + "loss": 1.3729, + "theoretical_loss": 3.417485831170118, + "tokens_seen": 2318256128 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002405616850551655, + "loss": 1.1869, + "theoretical_loss": 3.417478073198358, + "tokens_seen": 2318321664 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002405456369107322, + "loss": 1.2427, + "theoretical_loss": 3.417470315507307, + "tokens_seen": 2318387200 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002405295887662989, + "loss": 1.2249, + "theoretical_loss": 3.4174625580969473, + "tokens_seen": 2318452736 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002405135406218656, + "loss": 1.1711, + "theoretical_loss": 3.4174548009672603, + "tokens_seen": 2318518272 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002404974924774323, + "loss": 1.2608, + "theoretical_loss": 3.4174470441182283, + "tokens_seen": 2318583808 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024048144433299901, + "loss": 1.2407, + "theoretical_loss": 3.4174392875498327, + "tokens_seen": 2318649344 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024046539618856571, + "loss": 1.2685, + "theoretical_loss": 3.4174315312620562, + "tokens_seen": 2318714880 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024044934804413242, + "loss": 1.1943, + "theoretical_loss": 3.41742377525488, + "tokens_seen": 2318780416 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024043329989969912, + "loss": 1.199, + "theoretical_loss": 3.4174160195282868, + "tokens_seen": 2318845952 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024041725175526582, + "loss": 1.1771, + "theoretical_loss": 3.417408264082258, + "tokens_seen": 2318911488 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024040120361083252, + "loss": 1.2208, + "theoretical_loss": 3.417400508916775, + "tokens_seen": 2318977024 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024038515546639922, + "loss": 1.2115, + "theoretical_loss": 3.417392754031821, + "tokens_seen": 2319042560 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024036910732196592, + "loss": 1.1557, + "theoretical_loss": 3.4173849994273766, + "tokens_seen": 2319108096 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024035305917753262, + "loss": 1.3061, + "theoretical_loss": 3.4173772451034248, + "tokens_seen": 2319173632 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024033701103309932, + "loss": 1.1965, + "theoretical_loss": 3.4173694910599473, + "tokens_seen": 2319239168 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024032096288866602, + "loss": 1.179, + "theoretical_loss": 3.417361737296925, + "tokens_seen": 2319304704 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024030491474423272, + "loss": 1.2009, + "theoretical_loss": 3.417353983814341, + "tokens_seen": 2319370240 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024028886659979942, + "loss": 1.1884, + "theoretical_loss": 3.417346230612177, + "tokens_seen": 2319435776 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024027281845536612, + "loss": 1.21, + "theoretical_loss": 3.4173384776904148, + "tokens_seen": 2319501312 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024025677031093282, + "loss": 1.2517, + "theoretical_loss": 3.417330725049036, + "tokens_seen": 2319566848 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002402407221664995, + "loss": 1.3204, + "theoretical_loss": 3.4173229726880234, + "tokens_seen": 2319632384 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002402246740220662, + "loss": 1.1845, + "theoretical_loss": 3.417315220607358, + "tokens_seen": 2319697920 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002402086258776329, + "loss": 1.2348, + "theoretical_loss": 3.4173074688070226, + "tokens_seen": 2319763456 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002401925777331996, + "loss": 1.3011, + "theoretical_loss": 3.4172997172869986, + "tokens_seen": 2319828992 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002401765295887663, + "loss": 1.3899, + "theoretical_loss": 3.4172919660472676, + "tokens_seen": 2319894528 + }, + { + "epoch": 1.08, + "objective/train/docs_used": 857406, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2064659595489502, + "objective/train/theoretical_loss": 3.417284215087812, + "objective/train/tokens_used": 2340420064, + "theoretical_loss": 3.417284215087812, + "tokens_seen": 2319960064 + }, + { + "epoch": 1.08, + "learning_rate": 0.000240160481444333, + "loss": 1.2065, + "theoretical_loss": 3.417284215087812, + "tokens_seen": 2319960064 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002401444332998997, + "loss": 1.2695, + "theoretical_loss": 3.417276464408614, + "tokens_seen": 2320025600 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002401283851554664, + "loss": 1.2611, + "theoretical_loss": 3.4172687140096554, + "tokens_seen": 2320091136 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002401123370110331, + "loss": 1.2423, + "theoretical_loss": 3.417260963890918, + "tokens_seen": 2320156672 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002400962888665998, + "loss": 1.2424, + "theoretical_loss": 3.4172532140523835, + "tokens_seen": 2320222208 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002400802407221665, + "loss": 1.1639, + "theoretical_loss": 3.4172454644940347, + "tokens_seen": 2320287744 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002400641925777332, + "loss": 1.1861, + "theoretical_loss": 3.417237715215853, + "tokens_seen": 2320353280 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002400481444332999, + "loss": 1.2059, + "theoretical_loss": 3.41722996621782, + "tokens_seen": 2320418816 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002400320962888666, + "loss": 1.1618, + "theoretical_loss": 3.4172222174999183, + "tokens_seen": 2320484352 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002400160481444333, + "loss": 1.1921, + "theoretical_loss": 3.417214469062129, + "tokens_seen": 2320549888 + }, + { + "epoch": 1.08, + "learning_rate": 0.00024, + "loss": 1.1972, + "theoretical_loss": 3.417206720904435, + "tokens_seen": 2320615424 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002399839518555667, + "loss": 1.1471, + "theoretical_loss": 3.417198973026818, + "tokens_seen": 2320680960 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002399679037111334, + "loss": 1.2865, + "theoretical_loss": 3.41719122542926, + "tokens_seen": 2320746496 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002399518555667001, + "loss": 1.1566, + "theoretical_loss": 3.4171834781117427, + "tokens_seen": 2320812032 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002399358074222668, + "loss": 1.2493, + "theoretical_loss": 3.4171757310742485, + "tokens_seen": 2320877568 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002399197592778335, + "loss": 1.0807, + "theoretical_loss": 3.417167984316759, + "tokens_seen": 2320943104 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002399037111334002, + "loss": 1.2124, + "theoretical_loss": 3.417160237839256, + "tokens_seen": 2321008640 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002398876629889669, + "loss": 1.263, + "theoretical_loss": 3.4171524916417217, + "tokens_seen": 2321074176 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002398716148445336, + "loss": 1.2812, + "theoretical_loss": 3.417144745724139, + "tokens_seen": 2321139712 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023985556670010028, + "loss": 1.1741, + "theoretical_loss": 3.417137000086488, + "tokens_seen": 2321205248 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023983951855566699, + "loss": 1.236, + "theoretical_loss": 3.417129254728752, + "tokens_seen": 2321270784 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023982347041123369, + "loss": 1.2707, + "theoretical_loss": 3.4171215096509124, + "tokens_seen": 2321336320 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002398074222668004, + "loss": 1.2523, + "theoretical_loss": 3.417113764852952, + "tokens_seen": 2321401856 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002397913741223671, + "loss": 1.179, + "theoretical_loss": 3.417106020334852, + "tokens_seen": 2321467392 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002397753259779338, + "loss": 1.1888, + "theoretical_loss": 3.4170982760965947, + "tokens_seen": 2321532928 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002397592778335005, + "loss": 1.2612, + "theoretical_loss": 3.417090532138162, + "tokens_seen": 2321598464 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002397432296890672, + "loss": 1.2414, + "theoretical_loss": 3.4170827884595356, + "tokens_seen": 2321664000 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002397271815446339, + "loss": 1.2201, + "theoretical_loss": 3.417075045060698, + "tokens_seen": 2321729536 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002397111334002006, + "loss": 1.2092, + "theoretical_loss": 3.417067301941631, + "tokens_seen": 2321795072 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002396950852557673, + "loss": 1.2383, + "theoretical_loss": 3.4170595591023165, + "tokens_seen": 2321860608 + }, + { + "epoch": 1.08, + "learning_rate": 0.000239679037111334, + "loss": 1.1426, + "theoretical_loss": 3.417051816542737, + "tokens_seen": 2321926144 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002396629889669007, + "loss": 1.2928, + "theoretical_loss": 3.417044074262874, + "tokens_seen": 2321991680 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002396469408224674, + "loss": 1.2063, + "theoretical_loss": 3.4170363322627093, + "tokens_seen": 2322057216 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023963089267803415, + "loss": 1.1807, + "theoretical_loss": 3.417028590542225, + "tokens_seen": 2322122752 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023961484453360085, + "loss": 1.2176, + "theoretical_loss": 3.4170208491014034, + "tokens_seen": 2322188288 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023959879638916752, + "loss": 1.2594, + "theoretical_loss": 3.4170131079402264, + "tokens_seen": 2322253824 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023958274824473422, + "loss": 1.2848, + "theoretical_loss": 3.4170053670586764, + "tokens_seen": 2322319360 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023956670010030092, + "loss": 1.2415, + "theoretical_loss": 3.4169976264567348, + "tokens_seen": 2322384896 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023955065195586762, + "loss": 1.1625, + "theoretical_loss": 3.4169898861343837, + "tokens_seen": 2322450432 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023953460381143432, + "loss": 1.1965, + "theoretical_loss": 3.416982146091605, + "tokens_seen": 2322515968 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023951855566700103, + "loss": 1.2929, + "theoretical_loss": 3.4169744063283813, + "tokens_seen": 2322581504 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023950250752256773, + "loss": 1.2557, + "theoretical_loss": 3.416966666844694, + "tokens_seen": 2322647040 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023948645937813443, + "loss": 1.1579, + "theoretical_loss": 3.416958927640526, + "tokens_seen": 2322712576 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023947041123370113, + "loss": 1.2661, + "theoretical_loss": 3.416951188715858, + "tokens_seen": 2322778112 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023945436308926783, + "loss": 1.2716, + "theoretical_loss": 3.416943450070673, + "tokens_seen": 2322843648 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023943831494483453, + "loss": 1.2695, + "theoretical_loss": 3.4169357117049524, + "tokens_seen": 2322909184 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023942226680040123, + "loss": 1.2334, + "theoretical_loss": 3.4169279736186793, + "tokens_seen": 2322974720 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023940621865596793, + "loss": 1.2987, + "theoretical_loss": 3.4169202358118342, + "tokens_seen": 2323040256 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023939017051153463, + "loss": 1.181, + "theoretical_loss": 3.4169124982844004, + "tokens_seen": 2323105792 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023937412236710133, + "loss": 1.2747, + "theoretical_loss": 3.416904761036359, + "tokens_seen": 2323171328 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023935807422266803, + "loss": 1.2355, + "theoretical_loss": 3.416897024067693, + "tokens_seen": 2323236864 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023934202607823473, + "loss": 1.1991, + "theoretical_loss": 3.4168892873783836, + "tokens_seen": 2323302400 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023932597793380143, + "loss": 1.228, + "theoretical_loss": 3.416881550968413, + "tokens_seen": 2323367936 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023930992978936813, + "loss": 1.164, + "theoretical_loss": 3.4168738148377638, + "tokens_seen": 2323433472 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023929388164493483, + "loss": 1.2093, + "theoretical_loss": 3.416866078986417, + "tokens_seen": 2323499008 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023927783350050153, + "loss": 1.2527, + "theoretical_loss": 3.416858343414356, + "tokens_seen": 2323564544 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023926178535606824, + "loss": 1.2649, + "theoretical_loss": 3.4168506081215617, + "tokens_seen": 2323630080 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023924573721163494, + "loss": 1.1313, + "theoretical_loss": 3.4168428731080165, + "tokens_seen": 2323695616 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023922968906720164, + "loss": 1.1803, + "theoretical_loss": 3.4168351383737026, + "tokens_seen": 2323761152 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002392136409227683, + "loss": 1.2387, + "theoretical_loss": 3.4168274039186017, + "tokens_seen": 2323826688 + }, + { + "epoch": 1.08, + "learning_rate": 0.000239197592778335, + "loss": 1.1654, + "theoretical_loss": 3.4168196697426962, + "tokens_seen": 2323892224 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002391815446339017, + "loss": 1.172, + "theoretical_loss": 3.416811935845968, + "tokens_seen": 2323957760 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002391654964894684, + "loss": 1.2166, + "theoretical_loss": 3.4168042022283993, + "tokens_seen": 2324023296 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002391494483450351, + "loss": 1.2338, + "theoretical_loss": 3.416796468889972, + "tokens_seen": 2324088832 + }, + { + "epoch": 1.08, + "learning_rate": 0.0002391334002006018, + "loss": 1.1321, + "theoretical_loss": 3.416788735830668, + "tokens_seen": 2324154368 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023911735205616851, + "loss": 1.1033, + "theoretical_loss": 3.4167810030504695, + "tokens_seen": 2324219904 + }, + { + "epoch": 1.08, + "learning_rate": 0.00023910130391173521, + "loss": 1.2405, + "theoretical_loss": 3.4167732705493585, + "tokens_seen": 2324285440 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023908525576730192, + "loss": 1.2662, + "theoretical_loss": 3.4167655383273177, + "tokens_seen": 2324350976 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023906920762286862, + "loss": 1.1182, + "theoretical_loss": 3.416757806384328, + "tokens_seen": 2324416512 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023905315947843532, + "loss": 1.2426, + "theoretical_loss": 3.416750074720372, + "tokens_seen": 2324482048 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023903711133400202, + "loss": 1.3235, + "theoretical_loss": 3.4167423433354323, + "tokens_seen": 2324547584 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023902106318956872, + "loss": 1.244, + "theoretical_loss": 3.41673461222949, + "tokens_seen": 2324613120 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023900501504513542, + "loss": 1.2517, + "theoretical_loss": 3.4167268814025284, + "tokens_seen": 2324678656 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023898896690070212, + "loss": 1.178, + "theoretical_loss": 3.416719150854528, + "tokens_seen": 2324744192 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023897291875626882, + "loss": 1.1669, + "theoretical_loss": 3.4167114205854725, + "tokens_seen": 2324809728 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023895687061183552, + "loss": 1.2336, + "theoretical_loss": 3.4167036905953423, + "tokens_seen": 2324875264 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023894082246740222, + "loss": 1.1818, + "theoretical_loss": 3.416695960884121, + "tokens_seen": 2324940800 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023892477432296892, + "loss": 1.235, + "theoretical_loss": 3.41668823145179, + "tokens_seen": 2325006336 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023890872617853562, + "loss": 1.2203, + "theoretical_loss": 3.416680502298331, + "tokens_seen": 2325071872 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023889267803410232, + "loss": 1.2084, + "theoretical_loss": 3.4166727734237266, + "tokens_seen": 2325137408 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023887662988966902, + "loss": 1.257, + "theoretical_loss": 3.4166650448279587, + "tokens_seen": 2325202944 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023886058174523572, + "loss": 1.2802, + "theoretical_loss": 3.4166573165110097, + "tokens_seen": 2325268480 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023884453360080242, + "loss": 1.2167, + "theoretical_loss": 3.416649588472861, + "tokens_seen": 2325334016 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002388284854563691, + "loss": 1.1921, + "theoretical_loss": 3.4166418607134954, + "tokens_seen": 2325399552 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002388124373119358, + "loss": 1.3216, + "theoretical_loss": 3.4166341332328947, + "tokens_seen": 2325465088 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002387963891675025, + "loss": 1.2282, + "theoretical_loss": 3.416626406031041, + "tokens_seen": 2325530624 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002387803410230692, + "loss": 1.2119, + "theoretical_loss": 3.4166186791079163, + "tokens_seen": 2325596160 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002387642928786359, + "loss": 1.2784, + "theoretical_loss": 3.4166109524635027, + "tokens_seen": 2325661696 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002387482447342026, + "loss": 1.1352, + "theoretical_loss": 3.4166032260977826, + "tokens_seen": 2325727232 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002387321965897693, + "loss": 1.3177, + "theoretical_loss": 3.4165955000107378, + "tokens_seen": 2325792768 + }, + { + "epoch": 1.09, + "learning_rate": 0.000238716148445336, + "loss": 1.2454, + "theoretical_loss": 3.41658777420235, + "tokens_seen": 2325858304 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002387001003009027, + "loss": 1.2021, + "theoretical_loss": 3.416580048672602, + "tokens_seen": 2325923840 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002386840521564694, + "loss": 1.1919, + "theoretical_loss": 3.4165723234214758, + "tokens_seen": 2325989376 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002386680040120361, + "loss": 1.2612, + "theoretical_loss": 3.416564598448953, + "tokens_seen": 2326054912 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002386519558676028, + "loss": 1.213, + "theoretical_loss": 3.4165568737550167, + "tokens_seen": 2326120448 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002386359077231695, + "loss": 1.244, + "theoretical_loss": 3.416549149339648, + "tokens_seen": 2326185984 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002386198595787362, + "loss": 1.2564, + "theoretical_loss": 3.416541425202829, + "tokens_seen": 2326251520 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002386038114343029, + "loss": 1.2566, + "theoretical_loss": 3.4165337013445427, + "tokens_seen": 2326317056 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002385877632898696, + "loss": 1.1659, + "theoretical_loss": 3.41652597776477, + "tokens_seen": 2326382592 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002385717151454363, + "loss": 1.163, + "theoretical_loss": 3.416518254463494, + "tokens_seen": 2326448128 + }, + { + "epoch": 1.09, + "objective/train/docs_used": 859752, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1562694311141968, + "objective/train/theoretical_loss": 3.416510531440697, + "objective/train/tokens_used": 2346973664, + "theoretical_loss": 3.416510531440697, + "tokens_seen": 2326513664 + }, + { + "epoch": 1.09, + "learning_rate": 0.000238555667001003, + "loss": 1.1563, + "theoretical_loss": 3.416510531440697, + "tokens_seen": 2326513664 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002385396188565697, + "loss": 1.266, + "theoretical_loss": 3.41650280869636, + "tokens_seen": 2326579200 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002385235707121364, + "loss": 1.2262, + "theoretical_loss": 3.416495086230466, + "tokens_seen": 2326644736 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002385075225677031, + "loss": 1.2493, + "theoretical_loss": 3.416487364042997, + "tokens_seen": 2326710272 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002384914744232698, + "loss": 1.2181, + "theoretical_loss": 3.4164796421339347, + "tokens_seen": 2326775808 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002384754262788365, + "loss": 1.2995, + "theoretical_loss": 3.4164719205032617, + "tokens_seen": 2326841344 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002384593781344032, + "loss": 1.2687, + "theoretical_loss": 3.4164641991509597, + "tokens_seen": 2326906880 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023844332998996989, + "loss": 1.1158, + "theoretical_loss": 3.416456478077011, + "tokens_seen": 2326972416 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023842728184553664, + "loss": 1.2211, + "theoretical_loss": 3.4164487572813984, + "tokens_seen": 2327037952 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023841123370110334, + "loss": 1.2697, + "theoretical_loss": 3.4164410367641027, + "tokens_seen": 2327103488 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023839518555667004, + "loss": 1.2232, + "theoretical_loss": 3.416433316525107, + "tokens_seen": 2327169024 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023837913741223674, + "loss": 1.2707, + "theoretical_loss": 3.416425596564393, + "tokens_seen": 2327234560 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023836308926780344, + "loss": 1.2688, + "theoretical_loss": 3.416417876881943, + "tokens_seen": 2327300096 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023834704112337014, + "loss": 1.3084, + "theoretical_loss": 3.4164101574777392, + "tokens_seen": 2327365632 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023833099297893684, + "loss": 1.1715, + "theoretical_loss": 3.4164024383517635, + "tokens_seen": 2327431168 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023831494483450355, + "loss": 1.2196, + "theoretical_loss": 3.4163947195039985, + "tokens_seen": 2327496704 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023829889669007025, + "loss": 1.2841, + "theoretical_loss": 3.4163870009344257, + "tokens_seen": 2327562240 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023828284854563695, + "loss": 1.2076, + "theoretical_loss": 3.416379282643028, + "tokens_seen": 2327627776 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023826680040120365, + "loss": 1.3132, + "theoretical_loss": 3.4163715646297868, + "tokens_seen": 2327693312 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023825075225677035, + "loss": 1.1875, + "theoretical_loss": 3.4163638468946846, + "tokens_seen": 2327758848 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023823470411233705, + "loss": 1.2361, + "theoretical_loss": 3.4163561294377036, + "tokens_seen": 2327824384 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023821865596790375, + "loss": 1.367, + "theoretical_loss": 3.416348412258826, + "tokens_seen": 2327889920 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023820260782347042, + "loss": 1.1423, + "theoretical_loss": 3.4163406953580333, + "tokens_seen": 2327955456 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023818655967903712, + "loss": 1.2539, + "theoretical_loss": 3.4163329787353085, + "tokens_seen": 2328020992 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023817051153460382, + "loss": 1.1577, + "theoretical_loss": 3.4163252623906337, + "tokens_seen": 2328086528 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023815446339017052, + "loss": 1.1507, + "theoretical_loss": 3.4163175463239903, + "tokens_seen": 2328152064 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023813841524573723, + "loss": 1.2139, + "theoretical_loss": 3.416309830535361, + "tokens_seen": 2328217600 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023812236710130393, + "loss": 1.1918, + "theoretical_loss": 3.4163021150247284, + "tokens_seen": 2328283136 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023810631895687063, + "loss": 1.25, + "theoretical_loss": 3.4162943997920734, + "tokens_seen": 2328348672 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023809027081243733, + "loss": 1.1541, + "theoretical_loss": 3.416286684837379, + "tokens_seen": 2328414208 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023807422266800403, + "loss": 1.0849, + "theoretical_loss": 3.4162789701606275, + "tokens_seen": 2328479744 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023805817452357073, + "loss": 1.1802, + "theoretical_loss": 3.416271255761801, + "tokens_seen": 2328545280 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023804212637913743, + "loss": 1.2173, + "theoretical_loss": 3.4162635416408813, + "tokens_seen": 2328610816 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023802607823470413, + "loss": 1.2397, + "theoretical_loss": 3.4162558277978508, + "tokens_seen": 2328676352 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023801003009027083, + "loss": 1.1347, + "theoretical_loss": 3.4162481142326917, + "tokens_seen": 2328741888 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023799398194583753, + "loss": 1.2116, + "theoretical_loss": 3.416240400945386, + "tokens_seen": 2328807424 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023797793380140423, + "loss": 1.1471, + "theoretical_loss": 3.416232687935916, + "tokens_seen": 2328872960 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023796188565697093, + "loss": 1.3043, + "theoretical_loss": 3.4162249752042637, + "tokens_seen": 2328938496 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023794583751253763, + "loss": 1.2761, + "theoretical_loss": 3.4162172627504117, + "tokens_seen": 2329004032 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023792978936810433, + "loss": 1.28, + "theoretical_loss": 3.416209550574342, + "tokens_seen": 2329069568 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023791374122367103, + "loss": 1.21, + "theoretical_loss": 3.416201838676036, + "tokens_seen": 2329135104 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023789769307923773, + "loss": 1.2481, + "theoretical_loss": 3.416194127055477, + "tokens_seen": 2329200640 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023788164493480444, + "loss": 1.1894, + "theoretical_loss": 3.416186415712647, + "tokens_seen": 2329266176 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023786559679037114, + "loss": 1.3171, + "theoretical_loss": 3.4161787046475274, + "tokens_seen": 2329331712 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023784954864593784, + "loss": 1.1424, + "theoretical_loss": 3.4161709938601015, + "tokens_seen": 2329397248 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023783350050150454, + "loss": 1.2392, + "theoretical_loss": 3.4161632833503504, + "tokens_seen": 2329462784 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002378174523570712, + "loss": 1.139, + "theoretical_loss": 3.416155573118257, + "tokens_seen": 2329528320 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002378014042126379, + "loss": 1.1965, + "theoretical_loss": 3.416147863163803, + "tokens_seen": 2329593856 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002377853560682046, + "loss": 1.2299, + "theoretical_loss": 3.4161401534869715, + "tokens_seen": 2329659392 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002377693079237713, + "loss": 1.1522, + "theoretical_loss": 3.4161324440877436, + "tokens_seen": 2329724928 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023775325977933801, + "loss": 1.1721, + "theoretical_loss": 3.416124734966102, + "tokens_seen": 2329790464 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023773721163490471, + "loss": 1.2395, + "theoretical_loss": 3.4161170261220293, + "tokens_seen": 2329856000 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023772116349047141, + "loss": 1.2344, + "theoretical_loss": 3.416109317555507, + "tokens_seen": 2329921536 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023770511534603812, + "loss": 1.1598, + "theoretical_loss": 3.416101609266517, + "tokens_seen": 2329987072 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023768906720160482, + "loss": 1.3099, + "theoretical_loss": 3.4160939012550426, + "tokens_seen": 2330052608 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023767301905717152, + "loss": 1.2272, + "theoretical_loss": 3.4160861935210654, + "tokens_seen": 2330118144 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023765697091273822, + "loss": 1.2163, + "theoretical_loss": 3.416078486064568, + "tokens_seen": 2330183680 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023764092276830492, + "loss": 1.1709, + "theoretical_loss": 3.4160707788855316, + "tokens_seen": 2330249216 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023762487462387162, + "loss": 1.3022, + "theoretical_loss": 3.4160630719839395, + "tokens_seen": 2330314752 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023760882647943832, + "loss": 1.232, + "theoretical_loss": 3.4160553653597736, + "tokens_seen": 2330380288 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023759277833500502, + "loss": 1.1676, + "theoretical_loss": 3.4160476590130155, + "tokens_seen": 2330445824 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023757673019057172, + "loss": 1.1692, + "theoretical_loss": 3.416039952943648, + "tokens_seen": 2330511360 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023756068204613842, + "loss": 1.2558, + "theoretical_loss": 3.4160322471516538, + "tokens_seen": 2330576896 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023754463390170512, + "loss": 1.2577, + "theoretical_loss": 3.416024541637014, + "tokens_seen": 2330642432 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023752858575727182, + "loss": 1.2766, + "theoretical_loss": 3.416016836399712, + "tokens_seen": 2330707968 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023751253761283852, + "loss": 1.3134, + "theoretical_loss": 3.4160091314397287, + "tokens_seen": 2330773504 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023749648946840522, + "loss": 1.2974, + "theoretical_loss": 3.4160014267570475, + "tokens_seen": 2330839040 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023748044132397192, + "loss": 1.2354, + "theoretical_loss": 3.41599372235165, + "tokens_seen": 2330904576 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023746439317953863, + "loss": 1.2071, + "theoretical_loss": 3.4159860182235184, + "tokens_seen": 2330970112 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023744834503510533, + "loss": 1.2919, + "theoretical_loss": 3.4159783143726354, + "tokens_seen": 2331035648 + }, + { + "epoch": 1.09, + "learning_rate": 0.000237432296890672, + "loss": 1.2036, + "theoretical_loss": 3.4159706107989827, + "tokens_seen": 2331101184 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002374162487462387, + "loss": 1.1152, + "theoretical_loss": 3.415962907502543, + "tokens_seen": 2331166720 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002374002006018054, + "loss": 1.2393, + "theoretical_loss": 3.4159552044832977, + "tokens_seen": 2331232256 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002373841524573721, + "loss": 1.2138, + "theoretical_loss": 3.41594750174123, + "tokens_seen": 2331297792 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002373681043129388, + "loss": 1.2321, + "theoretical_loss": 3.415939799276322, + "tokens_seen": 2331363328 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002373520561685055, + "loss": 1.1988, + "theoretical_loss": 3.415932097088555, + "tokens_seen": 2331428864 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002373360080240722, + "loss": 1.2368, + "theoretical_loss": 3.4159243951779126, + "tokens_seen": 2331494400 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002373199598796389, + "loss": 1.1737, + "theoretical_loss": 3.415916693544376, + "tokens_seen": 2331559936 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002373039117352056, + "loss": 1.1922, + "theoretical_loss": 3.415908992187928, + "tokens_seen": 2331625472 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002372878635907723, + "loss": 1.2354, + "theoretical_loss": 3.4159012911085505, + "tokens_seen": 2331691008 + }, + { + "epoch": 1.09, + "learning_rate": 0.000237271815446339, + "loss": 1.177, + "theoretical_loss": 3.4158935903062257, + "tokens_seen": 2331756544 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002372557673019057, + "loss": 1.1816, + "theoretical_loss": 3.4158858897809363, + "tokens_seen": 2331822080 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002372397191574724, + "loss": 1.2934, + "theoretical_loss": 3.4158781895326644, + "tokens_seen": 2331887616 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023722367101303916, + "loss": 1.2379, + "theoretical_loss": 3.4158704895613923, + "tokens_seen": 2331953152 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023720762286860586, + "loss": 1.2274, + "theoretical_loss": 3.4158627898671017, + "tokens_seen": 2332018688 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023719157472417256, + "loss": 1.1652, + "theoretical_loss": 3.4158550904497753, + "tokens_seen": 2332084224 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023717552657973924, + "loss": 1.2012, + "theoretical_loss": 3.4158473913093954, + "tokens_seen": 2332149760 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023715947843530594, + "loss": 1.1821, + "theoretical_loss": 3.415839692445944, + "tokens_seen": 2332215296 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023714343029087264, + "loss": 1.1712, + "theoretical_loss": 3.4158319938594035, + "tokens_seen": 2332280832 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023712738214643934, + "loss": 1.1408, + "theoretical_loss": 3.4158242955497564, + "tokens_seen": 2332346368 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023711133400200604, + "loss": 1.1962, + "theoretical_loss": 3.4158165975169847, + "tokens_seen": 2332411904 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023709528585757274, + "loss": 1.0874, + "theoretical_loss": 3.4158088997610703, + "tokens_seen": 2332477440 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023707923771313944, + "loss": 1.3624, + "theoretical_loss": 3.4158012022819966, + "tokens_seen": 2332542976 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023706318956870614, + "loss": 1.2557, + "theoretical_loss": 3.4157935050797446, + "tokens_seen": 2332608512 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023704714142427284, + "loss": 1.1252, + "theoretical_loss": 3.415785808154297, + "tokens_seen": 2332674048 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023703109327983954, + "loss": 1.2124, + "theoretical_loss": 3.4157781115056363, + "tokens_seen": 2332739584 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023701504513540624, + "loss": 1.2203, + "theoretical_loss": 3.415770415133745, + "tokens_seen": 2332805120 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023699899699097294, + "loss": 1.2734, + "theoretical_loss": 3.4157627190386046, + "tokens_seen": 2332870656 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023698294884653964, + "loss": 1.1326, + "theoretical_loss": 3.4157550232201976, + "tokens_seen": 2332936192 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023696690070210634, + "loss": 1.1931, + "theoretical_loss": 3.415747327678507, + "tokens_seen": 2333001728 + }, + { + "epoch": 1.09, + "objective/train/docs_used": 862463, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.176609754562378, + "objective/train/theoretical_loss": 3.415739632413514, + "objective/train/tokens_used": 2353527264, + "theoretical_loss": 3.415739632413514, + "tokens_seen": 2333067264 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023695085255767305, + "loss": 1.1766, + "theoretical_loss": 3.415739632413514, + "tokens_seen": 2333067264 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023693480441323975, + "loss": 1.1398, + "theoretical_loss": 3.4157319374252015, + "tokens_seen": 2333132800 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023691875626880645, + "loss": 1.0731, + "theoretical_loss": 3.4157242427135523, + "tokens_seen": 2333198336 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023690270812437315, + "loss": 1.2309, + "theoretical_loss": 3.415716548278547, + "tokens_seen": 2333263872 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023688665997993985, + "loss": 1.1079, + "theoretical_loss": 3.41570885412017, + "tokens_seen": 2333329408 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023687061183550655, + "loss": 1.1517, + "theoretical_loss": 3.4157011602384024, + "tokens_seen": 2333394944 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023685456369107325, + "loss": 1.1515, + "theoretical_loss": 3.415693466633226, + "tokens_seen": 2333460480 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023683851554663995, + "loss": 1.2269, + "theoretical_loss": 3.415685773304624, + "tokens_seen": 2333526016 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023682246740220665, + "loss": 1.241, + "theoretical_loss": 3.4156780802525786, + "tokens_seen": 2333591552 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023680641925777335, + "loss": 1.163, + "theoretical_loss": 3.4156703874770717, + "tokens_seen": 2333657088 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023679037111334002, + "loss": 1.1956, + "theoretical_loss": 3.415662694978086, + "tokens_seen": 2333722624 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023677432296890673, + "loss": 1.1382, + "theoretical_loss": 3.4156550027556034, + "tokens_seen": 2333788160 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023675827482447343, + "loss": 1.2552, + "theoretical_loss": 3.4156473108096064, + "tokens_seen": 2333853696 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023674222668004013, + "loss": 1.1704, + "theoretical_loss": 3.4156396191400775, + "tokens_seen": 2333919232 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023672617853560683, + "loss": 1.2342, + "theoretical_loss": 3.4156319277469986, + "tokens_seen": 2333984768 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023671013039117353, + "loss": 1.183, + "theoretical_loss": 3.415624236630352, + "tokens_seen": 2334050304 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023669408224674023, + "loss": 1.349, + "theoretical_loss": 3.4156165457901206, + "tokens_seen": 2334115840 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023667803410230693, + "loss": 1.2792, + "theoretical_loss": 3.415608855226286, + "tokens_seen": 2334181376 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023666198595787363, + "loss": 1.1309, + "theoretical_loss": 3.415601164938831, + "tokens_seen": 2334246912 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023664593781344033, + "loss": 1.1899, + "theoretical_loss": 3.4155934749277375, + "tokens_seen": 2334312448 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023662988966900703, + "loss": 1.2154, + "theoretical_loss": 3.4155857851929885, + "tokens_seen": 2334377984 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023661384152457373, + "loss": 1.1344, + "theoretical_loss": 3.4155780957345656, + "tokens_seen": 2334443520 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023659779338014043, + "loss": 1.2738, + "theoretical_loss": 3.415570406552451, + "tokens_seen": 2334509056 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023658174523570713, + "loss": 1.1877, + "theoretical_loss": 3.415562717646628, + "tokens_seen": 2334574592 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023656569709127383, + "loss": 1.1396, + "theoretical_loss": 3.4155550290170775, + "tokens_seen": 2334640128 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023654964894684053, + "loss": 1.2339, + "theoretical_loss": 3.415547340663783, + "tokens_seen": 2334705664 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023653360080240723, + "loss": 1.2012, + "theoretical_loss": 3.4155396525867263, + "tokens_seen": 2334771200 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023651755265797394, + "loss": 1.183, + "theoretical_loss": 3.41553196478589, + "tokens_seen": 2334836736 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023650150451354064, + "loss": 1.158, + "theoretical_loss": 3.415524277261256, + "tokens_seen": 2334902272 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023648545636910734, + "loss": 1.2596, + "theoretical_loss": 3.4155165900128077, + "tokens_seen": 2334967808 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023648545636910734, + "loss": 1.2912, + "theoretical_loss": 3.415508903040526, + "tokens_seen": 2335033344 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023646940822467404, + "loss": 1.1683, + "theoretical_loss": 3.415501216344394, + "tokens_seen": 2335098880 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023645336008024074, + "loss": 1.2301, + "theoretical_loss": 3.4154935299243934, + "tokens_seen": 2335164416 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023643731193580744, + "loss": 1.1919, + "theoretical_loss": 3.415485843780507, + "tokens_seen": 2335229952 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023642126379137414, + "loss": 1.2168, + "theoretical_loss": 3.415478157912718, + "tokens_seen": 2335295488 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002364052156469408, + "loss": 1.2224, + "theoretical_loss": 3.415470472321007, + "tokens_seen": 2335361024 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002363891675025075, + "loss": 1.1856, + "theoretical_loss": 3.4154627870053575, + "tokens_seen": 2335426560 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023637311935807421, + "loss": 1.2249, + "theoretical_loss": 3.415455101965752, + "tokens_seen": 2335492096 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023635707121364091, + "loss": 1.2829, + "theoretical_loss": 3.415447417202172, + "tokens_seen": 2335557632 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023634102306920762, + "loss": 1.2266, + "theoretical_loss": 3.4154397327146, + "tokens_seen": 2335623168 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023632497492477432, + "loss": 1.2233, + "theoretical_loss": 3.4154320485030185, + "tokens_seen": 2335688704 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023630892678034102, + "loss": 1.2497, + "theoretical_loss": 3.4154243645674103, + "tokens_seen": 2335754240 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023629287863590772, + "loss": 1.2133, + "theoretical_loss": 3.4154166809077573, + "tokens_seen": 2335819776 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023627683049147442, + "loss": 1.2685, + "theoretical_loss": 3.415408997524042, + "tokens_seen": 2335885312 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023626078234704112, + "loss": 1.2165, + "theoretical_loss": 3.4154013144162465, + "tokens_seen": 2335950848 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023624473420260782, + "loss": 1.2156, + "theoretical_loss": 3.415393631584353, + "tokens_seen": 2336016384 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023622868605817452, + "loss": 1.2445, + "theoretical_loss": 3.4153859490283445, + "tokens_seen": 2336081920 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023621263791374122, + "loss": 1.2056, + "theoretical_loss": 3.415378266748203, + "tokens_seen": 2336147456 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023619658976930792, + "loss": 1.2219, + "theoretical_loss": 3.415370584743911, + "tokens_seen": 2336212992 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023618054162487462, + "loss": 1.1791, + "theoretical_loss": 3.4153629030154504, + "tokens_seen": 2336278528 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023616449348044132, + "loss": 1.2348, + "theoretical_loss": 3.415355221562804, + "tokens_seen": 2336344064 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023614844533600802, + "loss": 1.2743, + "theoretical_loss": 3.4153475403859543, + "tokens_seen": 2336409600 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023613239719157472, + "loss": 1.072, + "theoretical_loss": 3.415339859484883, + "tokens_seen": 2336475136 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023611634904714142, + "loss": 1.1815, + "theoretical_loss": 3.415332178859573, + "tokens_seen": 2336540672 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023610030090270812, + "loss": 1.2202, + "theoretical_loss": 3.4153244985100066, + "tokens_seen": 2336606208 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023608425275827483, + "loss": 1.3098, + "theoretical_loss": 3.415316818436166, + "tokens_seen": 2336671744 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023606820461384153, + "loss": 1.2294, + "theoretical_loss": 3.415309138638034, + "tokens_seen": 2336737280 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023605215646940823, + "loss": 1.1988, + "theoretical_loss": 3.4153014591155926, + "tokens_seen": 2336802816 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023603610832497493, + "loss": 1.2256, + "theoretical_loss": 3.415293779868824, + "tokens_seen": 2336868352 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023602006018054166, + "loss": 1.2328, + "theoretical_loss": 3.4152861008977107, + "tokens_seen": 2336933888 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023600401203610836, + "loss": 1.1775, + "theoretical_loss": 3.415278422202235, + "tokens_seen": 2336999424 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023598796389167506, + "loss": 1.233, + "theoretical_loss": 3.41527074378238, + "tokens_seen": 2337064960 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023597191574724176, + "loss": 1.2302, + "theoretical_loss": 3.4152630656381273, + "tokens_seen": 2337130496 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023595586760280846, + "loss": 1.2369, + "theoretical_loss": 3.4152553877694594, + "tokens_seen": 2337196032 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023593981945837516, + "loss": 1.126, + "theoretical_loss": 3.4152477101763585, + "tokens_seen": 2337261568 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023592377131394186, + "loss": 1.2024, + "theoretical_loss": 3.415240032858808, + "tokens_seen": 2337327104 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023590772316950856, + "loss": 1.1569, + "theoretical_loss": 3.415232355816789, + "tokens_seen": 2337392640 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023589167502507526, + "loss": 1.284, + "theoretical_loss": 3.4152246790502843, + "tokens_seen": 2337458176 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023587562688064196, + "loss": 1.1806, + "theoretical_loss": 3.4152170025592765, + "tokens_seen": 2337523712 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023585957873620866, + "loss": 1.1975, + "theoretical_loss": 3.4152093263437484, + "tokens_seen": 2337589248 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023584353059177536, + "loss": 1.251, + "theoretical_loss": 3.4152016504036817, + "tokens_seen": 2337654784 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023582748244734206, + "loss": 1.2838, + "theoretical_loss": 3.4151939747390587, + "tokens_seen": 2337720320 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023581143430290876, + "loss": 1.2069, + "theoretical_loss": 3.415186299349862, + "tokens_seen": 2337785856 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023579538615847546, + "loss": 1.2825, + "theoretical_loss": 3.4151786242360744, + "tokens_seen": 2337851392 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023577933801404214, + "loss": 1.181, + "theoretical_loss": 3.415170949397678, + "tokens_seen": 2337916928 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023576328986960884, + "loss": 1.208, + "theoretical_loss": 3.415163274834655, + "tokens_seen": 2337982464 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023574724172517554, + "loss": 1.204, + "theoretical_loss": 3.415155600546988, + "tokens_seen": 2338048000 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023573119358074224, + "loss": 1.1211, + "theoretical_loss": 3.41514792653466, + "tokens_seen": 2338113536 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023571514543630894, + "loss": 1.209, + "theoretical_loss": 3.4151402527976518, + "tokens_seen": 2338179072 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023569909729187564, + "loss": 1.2311, + "theoretical_loss": 3.4151325793359475, + "tokens_seen": 2338244608 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023568304914744234, + "loss": 1.2839, + "theoretical_loss": 3.4151249061495283, + "tokens_seen": 2338310144 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023566700100300904, + "loss": 1.217, + "theoretical_loss": 3.4151172332383775, + "tokens_seen": 2338375680 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023565095285857574, + "loss": 1.2951, + "theoretical_loss": 3.415109560602477, + "tokens_seen": 2338441216 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023563490471414244, + "loss": 1.2356, + "theoretical_loss": 3.4151018882418094, + "tokens_seen": 2338506752 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023561885656970914, + "loss": 1.3007, + "theoretical_loss": 3.4150942161563567, + "tokens_seen": 2338572288 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023560280842527584, + "loss": 1.2486, + "theoretical_loss": 3.415086544346102, + "tokens_seen": 2338637824 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023558676028084255, + "loss": 1.2742, + "theoretical_loss": 3.4150788728110273, + "tokens_seen": 2338703360 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023557071213640925, + "loss": 1.2982, + "theoretical_loss": 3.415071201551115, + "tokens_seen": 2338768896 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023555466399197595, + "loss": 1.2495, + "theoretical_loss": 3.415063530566348, + "tokens_seen": 2338834432 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023553861584754265, + "loss": 1.1718, + "theoretical_loss": 3.4150558598567082, + "tokens_seen": 2338899968 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023552256770310935, + "loss": 1.25, + "theoretical_loss": 3.415048189422178, + "tokens_seen": 2338965504 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023550651955867605, + "loss": 1.1524, + "theoretical_loss": 3.4150405192627398, + "tokens_seen": 2339031040 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023549047141424275, + "loss": 1.1775, + "theoretical_loss": 3.4150328493783766, + "tokens_seen": 2339096576 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023547442326980945, + "loss": 1.2307, + "theoretical_loss": 3.4150251797690703, + "tokens_seen": 2339162112 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023545837512537615, + "loss": 1.1952, + "theoretical_loss": 3.4150175104348035, + "tokens_seen": 2339227648 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023544232698094285, + "loss": 1.2292, + "theoretical_loss": 3.4150098413755585, + "tokens_seen": 2339293184 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023542627883650955, + "loss": 1.316, + "theoretical_loss": 3.415002172591318, + "tokens_seen": 2339358720 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023541023069207625, + "loss": 1.2618, + "theoretical_loss": 3.414994504082064, + "tokens_seen": 2339424256 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023539418254764293, + "loss": 1.221, + "theoretical_loss": 3.4149868358477793, + "tokens_seen": 2339489792 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023537813440320963, + "loss": 1.3253, + "theoretical_loss": 3.4149791678884465, + "tokens_seen": 2339555328 + }, + { + "epoch": 1.09, + "objective/train/docs_used": 864994, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.314795970916748, + "objective/train/theoretical_loss": 3.4149715002040475, + "objective/train/tokens_used": 2360080864, + "theoretical_loss": 3.4149715002040475, + "tokens_seen": 2339620864 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023536208625877633, + "loss": 1.3148, + "theoretical_loss": 3.4149715002040475, + "tokens_seen": 2339620864 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023534603811434303, + "loss": 1.3441, + "theoretical_loss": 3.4149638327945655, + "tokens_seen": 2339686400 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023532998996990973, + "loss": 1.1593, + "theoretical_loss": 3.414956165659982, + "tokens_seen": 2339751936 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023531394182547643, + "loss": 1.2568, + "theoretical_loss": 3.41494849880028, + "tokens_seen": 2339817472 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023529789368104313, + "loss": 1.2244, + "theoretical_loss": 3.414940832215442, + "tokens_seen": 2339883008 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023528184553660983, + "loss": 1.2361, + "theoretical_loss": 3.4149331659054507, + "tokens_seen": 2339948544 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023526579739217653, + "loss": 1.2221, + "theoretical_loss": 3.4149254998702876, + "tokens_seen": 2340014080 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023524974924774323, + "loss": 1.2584, + "theoretical_loss": 3.4149178341099358, + "tokens_seen": 2340079616 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023523370110330993, + "loss": 1.2869, + "theoretical_loss": 3.4149101686243775, + "tokens_seen": 2340145152 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023521765295887663, + "loss": 1.259, + "theoretical_loss": 3.414902503413596, + "tokens_seen": 2340210688 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023520160481444333, + "loss": 1.308, + "theoretical_loss": 3.414894838477572, + "tokens_seen": 2340276224 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023518555667001003, + "loss": 1.3149, + "theoretical_loss": 3.41488717381629, + "tokens_seen": 2340341760 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023516950852557673, + "loss": 1.1808, + "theoretical_loss": 3.4148795094297313, + "tokens_seen": 2340407296 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023515346038114344, + "loss": 1.306, + "theoretical_loss": 3.414871845317878, + "tokens_seen": 2340472832 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023513741223671014, + "loss": 1.2493, + "theoretical_loss": 3.4148641814807137, + "tokens_seen": 2340538368 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023512136409227684, + "loss": 1.1555, + "theoretical_loss": 3.41485651791822, + "tokens_seen": 2340603904 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023510531594784354, + "loss": 1.2026, + "theoretical_loss": 3.41484885463038, + "tokens_seen": 2340669440 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023508926780341024, + "loss": 1.2185, + "theoretical_loss": 3.4148411916171755, + "tokens_seen": 2340734976 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023507321965897694, + "loss": 1.2275, + "theoretical_loss": 3.414833528878589, + "tokens_seen": 2340800512 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023505717151454364, + "loss": 1.25, + "theoretical_loss": 3.414825866414604, + "tokens_seen": 2340866048 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023504112337011034, + "loss": 1.2354, + "theoretical_loss": 3.4148182042252015, + "tokens_seen": 2340931584 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023502507522567704, + "loss": 1.1919, + "theoretical_loss": 3.414810542310365, + "tokens_seen": 2340997120 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023500902708124371, + "loss": 1.2358, + "theoretical_loss": 3.414802880670077, + "tokens_seen": 2341062656 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023499297893681041, + "loss": 1.1806, + "theoretical_loss": 3.4147952193043194, + "tokens_seen": 2341128192 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023497693079237711, + "loss": 1.2932, + "theoretical_loss": 3.4147875582130744, + "tokens_seen": 2341193728 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023496088264794382, + "loss": 1.2381, + "theoretical_loss": 3.414779897396326, + "tokens_seen": 2341259264 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023494483450351052, + "loss": 1.1718, + "theoretical_loss": 3.4147722368540547, + "tokens_seen": 2341324800 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023492878635907722, + "loss": 1.321, + "theoretical_loss": 3.414764576586245, + "tokens_seen": 2341390336 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023491273821464392, + "loss": 1.2621, + "theoretical_loss": 3.4147569165928777, + "tokens_seen": 2341455872 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023489669007021062, + "loss": 1.11, + "theoretical_loss": 3.414749256873936, + "tokens_seen": 2341521408 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023488064192577732, + "loss": 1.1642, + "theoretical_loss": 3.414741597429402, + "tokens_seen": 2341586944 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023486459378134402, + "loss": 1.1989, + "theoretical_loss": 3.414733938259259, + "tokens_seen": 2341652480 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023484854563691072, + "loss": 1.1776, + "theoretical_loss": 3.414726279363489, + "tokens_seen": 2341718016 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023483249749247747, + "loss": 1.1917, + "theoretical_loss": 3.4147186207420748, + "tokens_seen": 2341783552 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023481644934804418, + "loss": 1.2215, + "theoretical_loss": 3.4147109623949983, + "tokens_seen": 2341849088 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023480040120361088, + "loss": 1.2346, + "theoretical_loss": 3.414703304322242, + "tokens_seen": 2341914624 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023478435305917758, + "loss": 1.1974, + "theoretical_loss": 3.414695646523789, + "tokens_seen": 2341980160 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023476830491474425, + "loss": 1.3289, + "theoretical_loss": 3.4146879889996216, + "tokens_seen": 2342045696 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023475225677031095, + "loss": 1.2761, + "theoretical_loss": 3.414680331749722, + "tokens_seen": 2342111232 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023473620862587765, + "loss": 1.188, + "theoretical_loss": 3.4146726747740725, + "tokens_seen": 2342176768 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023472016048144435, + "loss": 1.2453, + "theoretical_loss": 3.4146650180726565, + "tokens_seen": 2342242304 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023470411233701105, + "loss": 1.2623, + "theoretical_loss": 3.414657361645456, + "tokens_seen": 2342307840 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023468806419257775, + "loss": 1.2606, + "theoretical_loss": 3.4146497054924536, + "tokens_seen": 2342373376 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023467201604814445, + "loss": 1.1179, + "theoretical_loss": 3.4146420496136316, + "tokens_seen": 2342438912 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023465596790371115, + "loss": 1.2768, + "theoretical_loss": 3.4146343940089725, + "tokens_seen": 2342504448 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023463991975927786, + "loss": 1.1546, + "theoretical_loss": 3.414626738678459, + "tokens_seen": 2342569984 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023462387161484456, + "loss": 1.2717, + "theoretical_loss": 3.414619083622074, + "tokens_seen": 2342635520 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023460782347041126, + "loss": 1.2473, + "theoretical_loss": 3.414611428839799, + "tokens_seen": 2342701056 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023459177532597796, + "loss": 1.2747, + "theoretical_loss": 3.4146037743316175, + "tokens_seen": 2342766592 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023457572718154466, + "loss": 1.2278, + "theoretical_loss": 3.414596120097511, + "tokens_seen": 2342832128 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023455967903711136, + "loss": 1.1445, + "theoretical_loss": 3.4145884661374635, + "tokens_seen": 2342897664 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023454363089267806, + "loss": 1.2011, + "theoretical_loss": 3.4145808124514563, + "tokens_seen": 2342963200 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023452758274824476, + "loss": 1.1735, + "theoretical_loss": 3.414573159039472, + "tokens_seen": 2343028736 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023451153460381146, + "loss": 1.2684, + "theoretical_loss": 3.4145655059014937, + "tokens_seen": 2343094272 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023449548645937816, + "loss": 1.3009, + "theoretical_loss": 3.414557853037503, + "tokens_seen": 2343159808 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023447943831494486, + "loss": 1.2338, + "theoretical_loss": 3.4145502004474837, + "tokens_seen": 2343225344 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023446339017051156, + "loss": 1.1561, + "theoretical_loss": 3.414542548131418, + "tokens_seen": 2343290880 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023444734202607826, + "loss": 1.1912, + "theoretical_loss": 3.4145348960892874, + "tokens_seen": 2343356416 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023443129388164496, + "loss": 1.32, + "theoretical_loss": 3.4145272443210755, + "tokens_seen": 2343421952 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023441524573721166, + "loss": 1.2336, + "theoretical_loss": 3.4145195928267644, + "tokens_seen": 2343487488 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023439919759277836, + "loss": 1.2589, + "theoretical_loss": 3.414511941606337, + "tokens_seen": 2343553024 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023438314944834504, + "loss": 1.2292, + "theoretical_loss": 3.4145042906597753, + "tokens_seen": 2343618560 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023436710130391174, + "loss": 1.2915, + "theoretical_loss": 3.414496639987062, + "tokens_seen": 2343684096 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023435105315947844, + "loss": 1.3361, + "theoretical_loss": 3.41448898958818, + "tokens_seen": 2343749632 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023433500501504514, + "loss": 1.2214, + "theoretical_loss": 3.4144813394631113, + "tokens_seen": 2343815168 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023431895687061184, + "loss": 1.2195, + "theoretical_loss": 3.414473689611839, + "tokens_seen": 2343880704 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023430290872617854, + "loss": 1.2639, + "theoretical_loss": 3.414466040034345, + "tokens_seen": 2343946240 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023428686058174524, + "loss": 1.1233, + "theoretical_loss": 3.4144583907306125, + "tokens_seen": 2344011776 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023427081243731194, + "loss": 1.2627, + "theoretical_loss": 3.414450741700624, + "tokens_seen": 2344077312 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023425476429287864, + "loss": 1.3121, + "theoretical_loss": 3.4144430929443614, + "tokens_seen": 2344142848 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023423871614844534, + "loss": 1.2741, + "theoretical_loss": 3.414435444461808, + "tokens_seen": 2344208384 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023422266800401204, + "loss": 1.1966, + "theoretical_loss": 3.4144277962529457, + "tokens_seen": 2344273920 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023420661985957875, + "loss": 1.2517, + "theoretical_loss": 3.4144201483177574, + "tokens_seen": 2344339456 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023419057171514545, + "loss": 1.1794, + "theoretical_loss": 3.414412500656226, + "tokens_seen": 2344404992 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023417452357071215, + "loss": 1.1745, + "theoretical_loss": 3.4144048532683335, + "tokens_seen": 2344470528 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023415847542627885, + "loss": 1.147, + "theoretical_loss": 3.4143972061540624, + "tokens_seen": 2344536064 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023414242728184555, + "loss": 1.2731, + "theoretical_loss": 3.414389559313396, + "tokens_seen": 2344601600 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023412637913741225, + "loss": 1.2033, + "theoretical_loss": 3.414381912746316, + "tokens_seen": 2344667136 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023411033099297895, + "loss": 1.1656, + "theoretical_loss": 3.414374266452805, + "tokens_seen": 2344732672 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023409428284854565, + "loss": 1.1144, + "theoretical_loss": 3.4143666204328467, + "tokens_seen": 2344798208 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023407823470411235, + "loss": 1.2306, + "theoretical_loss": 3.4143589746864227, + "tokens_seen": 2344863744 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023406218655967905, + "loss": 1.2949, + "theoretical_loss": 3.414351329213516, + "tokens_seen": 2344929280 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023404613841524575, + "loss": 1.1535, + "theoretical_loss": 3.414343684014108, + "tokens_seen": 2344994816 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023403009027081245, + "loss": 1.2522, + "theoretical_loss": 3.414336039088183, + "tokens_seen": 2345060352 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023401404212637915, + "loss": 1.3233, + "theoretical_loss": 3.4143283944357226, + "tokens_seen": 2345125888 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023399799398194583, + "loss": 1.2056, + "theoretical_loss": 3.4143207500567097, + "tokens_seen": 2345191424 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023398194583751253, + "loss": 1.2583, + "theoretical_loss": 3.4143131059511265, + "tokens_seen": 2345256960 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023396589769307923, + "loss": 1.1013, + "theoretical_loss": 3.4143054621189557, + "tokens_seen": 2345322496 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023394984954864593, + "loss": 1.2448, + "theoretical_loss": 3.41429781856018, + "tokens_seen": 2345388032 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023393380140421263, + "loss": 1.1968, + "theoretical_loss": 3.4142901752747825, + "tokens_seen": 2345453568 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023391775325977933, + "loss": 1.2232, + "theoretical_loss": 3.4142825322627446, + "tokens_seen": 2345519104 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023390170511534603, + "loss": 1.2642, + "theoretical_loss": 3.41427488952405, + "tokens_seen": 2345584640 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023388565697091273, + "loss": 1.2332, + "theoretical_loss": 3.4142672470586803, + "tokens_seen": 2345650176 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023386960882647943, + "loss": 1.2178, + "theoretical_loss": 3.414259604866619, + "tokens_seen": 2345715712 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023385356068204613, + "loss": 1.2394, + "theoretical_loss": 3.4142519629478483, + "tokens_seen": 2345781248 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023383751253761283, + "loss": 1.2643, + "theoretical_loss": 3.4142443213023506, + "tokens_seen": 2345846784 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023382146439317953, + "loss": 1.2611, + "theoretical_loss": 3.414236679930109, + "tokens_seen": 2345912320 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023380541624874623, + "loss": 1.1879, + "theoretical_loss": 3.4142290388311056, + "tokens_seen": 2345977856 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023378936810431293, + "loss": 1.2497, + "theoretical_loss": 3.4142213980053233, + "tokens_seen": 2346043392 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023377331995987964, + "loss": 1.2479, + "theoretical_loss": 3.4142137574527442, + "tokens_seen": 2346108928 + }, + { + "epoch": 1.09, + "objective/train/docs_used": 867338, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1086442470550537, + "objective/train/theoretical_loss": 3.414206117173352, + "objective/train/tokens_used": 2366634464, + "theoretical_loss": 3.414206117173352, + "tokens_seen": 2346174464 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023375727181544634, + "loss": 1.1086, + "theoretical_loss": 3.414206117173352, + "tokens_seen": 2346174464 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023374122367101304, + "loss": 1.2957, + "theoretical_loss": 3.414198477167128, + "tokens_seen": 2346240000 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023372517552657974, + "loss": 1.3573, + "theoretical_loss": 3.4141908374340555, + "tokens_seen": 2346305536 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023370912738214644, + "loss": 1.3292, + "theoretical_loss": 3.414183197974117, + "tokens_seen": 2346371072 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023369307923771314, + "loss": 1.1498, + "theoretical_loss": 3.4141755587872953, + "tokens_seen": 2346436608 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023367703109327984, + "loss": 1.1894, + "theoretical_loss": 3.414167919873573, + "tokens_seen": 2346502144 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023366098294884654, + "loss": 1.1738, + "theoretical_loss": 3.414160281232932, + "tokens_seen": 2346567680 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023364493480441324, + "loss": 1.1584, + "theoretical_loss": 3.4141526428653552, + "tokens_seen": 2346633216 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023362888665997997, + "loss": 1.1941, + "theoretical_loss": 3.414145004770826, + "tokens_seen": 2346698752 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023361283851554667, + "loss": 1.2695, + "theoretical_loss": 3.4141373669493262, + "tokens_seen": 2346764288 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023359679037111337, + "loss": 1.2414, + "theoretical_loss": 3.414129729400839, + "tokens_seen": 2346829824 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023358074222668007, + "loss": 1.1695, + "theoretical_loss": 3.4141220921253463, + "tokens_seen": 2346895360 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023356469408224677, + "loss": 1.2908, + "theoretical_loss": 3.4141144551228315, + "tokens_seen": 2346960896 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023354864593781347, + "loss": 1.3766, + "theoretical_loss": 3.414106818393276, + "tokens_seen": 2347026432 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023353259779338017, + "loss": 1.2169, + "theoretical_loss": 3.414099181936664, + "tokens_seen": 2347091968 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023351654964894687, + "loss": 1.2666, + "theoretical_loss": 3.4140915457529775, + "tokens_seen": 2347157504 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023350050150451357, + "loss": 1.2009, + "theoretical_loss": 3.4140839098421987, + "tokens_seen": 2347223040 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023348445336008027, + "loss": 1.2816, + "theoretical_loss": 3.4140762742043105, + "tokens_seen": 2347288576 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023346840521564697, + "loss": 1.295, + "theoretical_loss": 3.4140686388392956, + "tokens_seen": 2347354112 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023345235707121368, + "loss": 1.2149, + "theoretical_loss": 3.414061003747137, + "tokens_seen": 2347419648 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023343630892678038, + "loss": 1.2612, + "theoretical_loss": 3.4140533689278163, + "tokens_seen": 2347485184 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023342026078234708, + "loss": 1.2798, + "theoretical_loss": 3.4140457343813173, + "tokens_seen": 2347550720 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023340421263791378, + "loss": 1.1932, + "theoretical_loss": 3.4140381001076214, + "tokens_seen": 2347616256 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023338816449348048, + "loss": 1.1968, + "theoretical_loss": 3.4140304661067127, + "tokens_seen": 2347681792 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023337211634904718, + "loss": 1.2568, + "theoretical_loss": 3.4140228323785724, + "tokens_seen": 2347747328 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023335606820461385, + "loss": 1.2341, + "theoretical_loss": 3.414015198923184, + "tokens_seen": 2347812864 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023334002006018055, + "loss": 1.1873, + "theoretical_loss": 3.41400756574053, + "tokens_seen": 2347878400 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023332397191574725, + "loss": 1.227, + "theoretical_loss": 3.413999932830593, + "tokens_seen": 2347943936 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023330792377131395, + "loss": 1.2586, + "theoretical_loss": 3.413992300193356, + "tokens_seen": 2348009472 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023329187562688065, + "loss": 1.227, + "theoretical_loss": 3.4139846678288013, + "tokens_seen": 2348075008 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023327582748244736, + "loss": 1.2382, + "theoretical_loss": 3.413977035736911, + "tokens_seen": 2348140544 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023325977933801406, + "loss": 1.1509, + "theoretical_loss": 3.4139694039176685, + "tokens_seen": 2348206080 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023324373119358076, + "loss": 1.2482, + "theoretical_loss": 3.413961772371056, + "tokens_seen": 2348271616 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023322768304914746, + "loss": 1.1818, + "theoretical_loss": 3.4139541410970566, + "tokens_seen": 2348337152 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023321163490471416, + "loss": 1.2214, + "theoretical_loss": 3.413946510095653, + "tokens_seen": 2348402688 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023319558676028086, + "loss": 1.3012, + "theoretical_loss": 3.4139388793668273, + "tokens_seen": 2348468224 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023317953861584756, + "loss": 1.3084, + "theoretical_loss": 3.413931248910562, + "tokens_seen": 2348533760 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023316349047141426, + "loss": 1.1684, + "theoretical_loss": 3.413923618726841, + "tokens_seen": 2348599296 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023314744232698096, + "loss": 1.2222, + "theoretical_loss": 3.4139159888156456, + "tokens_seen": 2348664832 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023313139418254766, + "loss": 1.1807, + "theoretical_loss": 3.4139083591769595, + "tokens_seen": 2348730368 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023311534603811436, + "loss": 1.2517, + "theoretical_loss": 3.4139007298107646, + "tokens_seen": 2348795904 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023309929789368106, + "loss": 1.1784, + "theoretical_loss": 3.413893100717044, + "tokens_seen": 2348861440 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023308324974924776, + "loss": 1.3007, + "theoretical_loss": 3.4138854718957803, + "tokens_seen": 2348926976 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023306720160481446, + "loss": 1.2169, + "theoretical_loss": 3.4138778433469557, + "tokens_seen": 2348992512 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023305115346038116, + "loss": 1.166, + "theoretical_loss": 3.4138702150705535, + "tokens_seen": 2349058048 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023303510531594786, + "loss": 1.2596, + "theoretical_loss": 3.413862587066556, + "tokens_seen": 2349123584 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023301905717151457, + "loss": 1.1682, + "theoretical_loss": 3.4138549593349463, + "tokens_seen": 2349189120 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023300300902708127, + "loss": 1.2911, + "theoretical_loss": 3.413847331875707, + "tokens_seen": 2349254656 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023298696088264797, + "loss": 1.2449, + "theoretical_loss": 3.41383970468882, + "tokens_seen": 2349320192 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023297091273821464, + "loss": 1.2655, + "theoretical_loss": 3.413832077774269, + "tokens_seen": 2349385728 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023295486459378134, + "loss": 1.279, + "theoretical_loss": 3.4138244511320357, + "tokens_seen": 2349451264 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023293881644934804, + "loss": 1.1808, + "theoretical_loss": 3.4138168247621037, + "tokens_seen": 2349516800 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023292276830491474, + "loss": 1.2719, + "theoretical_loss": 3.413809198664455, + "tokens_seen": 2349582336 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023290672016048144, + "loss": 1.2752, + "theoretical_loss": 3.413801572839073, + "tokens_seen": 2349647872 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023289067201604814, + "loss": 1.2557, + "theoretical_loss": 3.4137939472859395, + "tokens_seen": 2349713408 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023287462387161484, + "loss": 1.1976, + "theoretical_loss": 3.413786322005038, + "tokens_seen": 2349778944 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023285857572718154, + "loss": 1.3603, + "theoretical_loss": 3.4137786969963506, + "tokens_seen": 2349844480 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023284252758274825, + "loss": 1.232, + "theoretical_loss": 3.4137710722598604, + "tokens_seen": 2349910016 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023282647943831495, + "loss": 1.3227, + "theoretical_loss": 3.41376344779555, + "tokens_seen": 2349975552 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023281043129388165, + "loss": 1.2092, + "theoretical_loss": 3.4137558236034016, + "tokens_seen": 2350041088 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023279438314944835, + "loss": 1.2835, + "theoretical_loss": 3.4137481996833987, + "tokens_seen": 2350106624 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023277833500501505, + "loss": 1.1749, + "theoretical_loss": 3.4137405760355235, + "tokens_seen": 2350172160 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023276228686058175, + "loss": 1.2185, + "theoretical_loss": 3.4137329526597586, + "tokens_seen": 2350237696 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023274623871614845, + "loss": 1.2391, + "theoretical_loss": 3.413725329556087, + "tokens_seen": 2350303232 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023273019057171515, + "loss": 1.2246, + "theoretical_loss": 3.4137177067244915, + "tokens_seen": 2350368768 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023271414242728185, + "loss": 1.1338, + "theoretical_loss": 3.413710084164954, + "tokens_seen": 2350434304 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023269809428284855, + "loss": 1.2247, + "theoretical_loss": 3.4137024618774587, + "tokens_seen": 2350499840 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023268204613841525, + "loss": 1.2816, + "theoretical_loss": 3.4136948398619866, + "tokens_seen": 2350565376 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023266599799398195, + "loss": 1.2922, + "theoretical_loss": 3.413687218118522, + "tokens_seen": 2350630912 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023264994984954865, + "loss": 1.2156, + "theoretical_loss": 3.413679596647046, + "tokens_seen": 2350696448 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023263390170511535, + "loss": 1.272, + "theoretical_loss": 3.413671975447543, + "tokens_seen": 2350761984 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023261785356068205, + "loss": 1.1796, + "theoretical_loss": 3.4136643545199945, + "tokens_seen": 2350827520 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023260180541624875, + "loss": 1.2974, + "theoretical_loss": 3.4136567338643835, + "tokens_seen": 2350893056 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023258575727181543, + "loss": 1.3459, + "theoretical_loss": 3.4136491134806928, + "tokens_seen": 2350958592 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023256970912738213, + "loss": 1.2722, + "theoretical_loss": 3.413641493368905, + "tokens_seen": 2351024128 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023255366098294883, + "loss": 1.2854, + "theoretical_loss": 3.413633873529003, + "tokens_seen": 2351089664 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023253761283851553, + "loss": 1.2749, + "theoretical_loss": 3.4136262539609694, + "tokens_seen": 2351155200 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023252156469408223, + "loss": 1.2566, + "theoretical_loss": 3.413618634664787, + "tokens_seen": 2351220736 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023250551654964893, + "loss": 1.1871, + "theoretical_loss": 3.4136110156404387, + "tokens_seen": 2351286272 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023248946840521563, + "loss": 1.2257, + "theoretical_loss": 3.4136033968879067, + "tokens_seen": 2351351808 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023247342026078233, + "loss": 1.1869, + "theoretical_loss": 3.413595778407174, + "tokens_seen": 2351417344 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023245737211634903, + "loss": 1.2386, + "theoretical_loss": 3.413588160198224, + "tokens_seen": 2351482880 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023244132397191573, + "loss": 1.2189, + "theoretical_loss": 3.4135805422610384, + "tokens_seen": 2351548416 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002324252758274825, + "loss": 1.1043, + "theoretical_loss": 3.4135729245956004, + "tokens_seen": 2351613952 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002324092276830492, + "loss": 1.2136, + "theoretical_loss": 3.4135653072018926, + "tokens_seen": 2351679488 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002323931795386159, + "loss": 1.1696, + "theoretical_loss": 3.4135576900798976, + "tokens_seen": 2351745024 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002323771313941826, + "loss": 1.3192, + "theoretical_loss": 3.4135500732295987, + "tokens_seen": 2351810560 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002323610832497493, + "loss": 1.193, + "theoretical_loss": 3.413542456650978, + "tokens_seen": 2351876096 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023234503510531596, + "loss": 1.2152, + "theoretical_loss": 3.4135348403440187, + "tokens_seen": 2351941632 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023232898696088267, + "loss": 1.2379, + "theoretical_loss": 3.4135272243087034, + "tokens_seen": 2352007168 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023231293881644937, + "loss": 1.283, + "theoretical_loss": 3.4135196085450143, + "tokens_seen": 2352072704 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023229689067201607, + "loss": 1.2444, + "theoretical_loss": 3.413511993052935, + "tokens_seen": 2352138240 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023228084252758277, + "loss": 1.2626, + "theoretical_loss": 3.4135043778324485, + "tokens_seen": 2352203776 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023226479438314947, + "loss": 1.2041, + "theoretical_loss": 3.413496762883536, + "tokens_seen": 2352269312 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023224874623871617, + "loss": 1.2847, + "theoretical_loss": 3.4134891482061818, + "tokens_seen": 2352334848 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023223269809428287, + "loss": 1.2525, + "theoretical_loss": 3.413481533800368, + "tokens_seen": 2352400384 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023221664994984957, + "loss": 1.1319, + "theoretical_loss": 3.4134739196660773, + "tokens_seen": 2352465920 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023220060180541627, + "loss": 1.2197, + "theoretical_loss": 3.413466305803292, + "tokens_seen": 2352531456 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023218455366098297, + "loss": 1.172, + "theoretical_loss": 3.413458692211996, + "tokens_seen": 2352596992 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023216850551654967, + "loss": 1.29, + "theoretical_loss": 3.4134510788921713, + "tokens_seen": 2352662528 + }, + { + "epoch": 1.09, + "objective/train/docs_used": 869751, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2676352262496948, + "objective/train/theoretical_loss": 3.413443465843801, + "objective/train/tokens_used": 2373188064, + "theoretical_loss": 3.413443465843801, + "tokens_seen": 2352728064 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023215245737211637, + "loss": 1.2676, + "theoretical_loss": 3.413443465843801, + "tokens_seen": 2352728064 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023213640922768307, + "loss": 1.2044, + "theoretical_loss": 3.4134358530668676, + "tokens_seen": 2352793600 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023212036108324977, + "loss": 1.2007, + "theoretical_loss": 3.4134282405613536, + "tokens_seen": 2352859136 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023210431293881647, + "loss": 1.3495, + "theoretical_loss": 3.4134206283272426, + "tokens_seen": 2352924672 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023208826479438317, + "loss": 1.2533, + "theoretical_loss": 3.4134130163645167, + "tokens_seen": 2352990208 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023207221664994988, + "loss": 1.3194, + "theoretical_loss": 3.413405404673159, + "tokens_seen": 2353055744 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023205616850551658, + "loss": 1.2277, + "theoretical_loss": 3.413397793253152, + "tokens_seen": 2353121280 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023204012036108328, + "loss": 1.1229, + "theoretical_loss": 3.413390182104478, + "tokens_seen": 2353186816 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023202407221664998, + "loss": 1.2481, + "theoretical_loss": 3.413382571227121, + "tokens_seen": 2353252352 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023200802407221668, + "loss": 1.1785, + "theoretical_loss": 3.413374960621063, + "tokens_seen": 2353317888 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023199197592778338, + "loss": 1.2322, + "theoretical_loss": 3.4133673502862867, + "tokens_seen": 2353383424 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023197592778335008, + "loss": 1.1254, + "theoretical_loss": 3.4133597402227753, + "tokens_seen": 2353448960 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023195987963891675, + "loss": 1.388, + "theoretical_loss": 3.413352130430511, + "tokens_seen": 2353514496 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023194383149448345, + "loss": 1.1322, + "theoretical_loss": 3.4133445209094773, + "tokens_seen": 2353580032 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023192778335005015, + "loss": 1.2677, + "theoretical_loss": 3.4133369116596564, + "tokens_seen": 2353645568 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023191173520561685, + "loss": 1.2218, + "theoretical_loss": 3.4133293026810314, + "tokens_seen": 2353711104 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023189568706118356, + "loss": 1.1617, + "theoretical_loss": 3.413321693973585, + "tokens_seen": 2353776640 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023187963891675026, + "loss": 1.1989, + "theoretical_loss": 3.4133140855372996, + "tokens_seen": 2353842176 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023186359077231696, + "loss": 1.2109, + "theoretical_loss": 3.413306477372159, + "tokens_seen": 2353907712 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023184754262788366, + "loss": 1.2175, + "theoretical_loss": 3.4132988694781448, + "tokens_seen": 2353973248 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023183149448345036, + "loss": 1.2969, + "theoretical_loss": 3.4132912618552407, + "tokens_seen": 2354038784 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023181544633901706, + "loss": 1.2117, + "theoretical_loss": 3.4132836545034286, + "tokens_seen": 2354104320 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023179939819458376, + "loss": 1.1593, + "theoretical_loss": 3.413276047422692, + "tokens_seen": 2354169856 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023178335005015046, + "loss": 1.2635, + "theoretical_loss": 3.413268440613014, + "tokens_seen": 2354235392 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023176730190571716, + "loss": 1.2748, + "theoretical_loss": 3.4132608340743764, + "tokens_seen": 2354300928 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023175125376128386, + "loss": 1.1344, + "theoretical_loss": 3.4132532278067624, + "tokens_seen": 2354366464 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023173520561685056, + "loss": 1.1905, + "theoretical_loss": 3.4132456218101552, + "tokens_seen": 2354432000 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023171915747241726, + "loss": 1.2438, + "theoretical_loss": 3.413238016084537, + "tokens_seen": 2354497536 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023170310932798396, + "loss": 1.1539, + "theoretical_loss": 3.4132304106298914, + "tokens_seen": 2354563072 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023168706118355066, + "loss": 1.3124, + "theoretical_loss": 3.4132228054462006, + "tokens_seen": 2354628608 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023167101303911736, + "loss": 1.3294, + "theoretical_loss": 3.4132152005334473, + "tokens_seen": 2354694144 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023165496489468407, + "loss": 1.3386, + "theoretical_loss": 3.413207595891614, + "tokens_seen": 2354759680 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023163891675025077, + "loss": 1.0754, + "theoretical_loss": 3.4131999915206848, + "tokens_seen": 2354825216 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023162286860581747, + "loss": 1.338, + "theoretical_loss": 3.413192387420642, + "tokens_seen": 2354890752 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023160682046138417, + "loss": 1.174, + "theoretical_loss": 3.4131847835914675, + "tokens_seen": 2354956288 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023159077231695087, + "loss": 1.2142, + "theoretical_loss": 3.413177180033145, + "tokens_seen": 2355021824 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023157472417251754, + "loss": 1.18, + "theoretical_loss": 3.4131695767456565, + "tokens_seen": 2355087360 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023155867602808424, + "loss": 1.1991, + "theoretical_loss": 3.413161973728986, + "tokens_seen": 2355152896 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023154262788365094, + "loss": 1.2155, + "theoretical_loss": 3.413154370983116, + "tokens_seen": 2355218432 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023152657973921764, + "loss": 1.2727, + "theoretical_loss": 3.413146768508028, + "tokens_seen": 2355283968 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023151053159478434, + "loss": 1.22, + "theoretical_loss": 3.413139166303707, + "tokens_seen": 2355349504 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023149448345035104, + "loss": 1.2936, + "theoretical_loss": 3.4131315643701337, + "tokens_seen": 2355415040 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023147843530591774, + "loss": 1.2625, + "theoretical_loss": 3.4131239627072927, + "tokens_seen": 2355480576 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023146238716148445, + "loss": 1.2909, + "theoretical_loss": 3.413116361315166, + "tokens_seen": 2355546112 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023144633901705115, + "loss": 1.3058, + "theoretical_loss": 3.4131087601937358, + "tokens_seen": 2355611648 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023143029087261785, + "loss": 1.3211, + "theoretical_loss": 3.413101159342986, + "tokens_seen": 2355677184 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023141424272818455, + "loss": 1.1483, + "theoretical_loss": 3.413093558762899, + "tokens_seen": 2355742720 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023139819458375125, + "loss": 1.2672, + "theoretical_loss": 3.4130859584534576, + "tokens_seen": 2355808256 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023138214643931795, + "loss": 1.2286, + "theoretical_loss": 3.4130783584146442, + "tokens_seen": 2355873792 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023136609829488465, + "loss": 1.264, + "theoretical_loss": 3.413070758646443, + "tokens_seen": 2355939328 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023135005015045135, + "loss": 1.2308, + "theoretical_loss": 3.4130631591488356, + "tokens_seen": 2356004864 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023133400200601805, + "loss": 1.2586, + "theoretical_loss": 3.413055559921805, + "tokens_seen": 2356070400 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023131795386158475, + "loss": 1.1485, + "theoretical_loss": 3.4130479609653346, + "tokens_seen": 2356135936 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023130190571715145, + "loss": 1.2814, + "theoretical_loss": 3.4130403622794065, + "tokens_seen": 2356201472 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023128585757271815, + "loss": 1.2976, + "theoretical_loss": 3.413032763864004, + "tokens_seen": 2356267008 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023126980942828485, + "loss": 1.2238, + "theoretical_loss": 3.41302516571911, + "tokens_seen": 2356332544 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023125376128385155, + "loss": 1.1156, + "theoretical_loss": 3.4130175678447072, + "tokens_seen": 2356398080 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023123771313941825, + "loss": 1.2602, + "theoretical_loss": 3.4130099702407786, + "tokens_seen": 2356463616 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023122166499498498, + "loss": 1.2242, + "theoretical_loss": 3.4130023729073065, + "tokens_seen": 2356529152 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023120561685055168, + "loss": 1.202, + "theoretical_loss": 3.4129947758442745, + "tokens_seen": 2356594688 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023118956870611838, + "loss": 1.2729, + "theoretical_loss": 3.4129871790516653, + "tokens_seen": 2356660224 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023117352056168508, + "loss": 1.28, + "theoretical_loss": 3.412979582529461, + "tokens_seen": 2356725760 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023115747241725178, + "loss": 1.2448, + "theoretical_loss": 3.412971986277645, + "tokens_seen": 2356791296 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023114142427281849, + "loss": 1.2374, + "theoretical_loss": 3.412964390296201, + "tokens_seen": 2356856832 + }, + { + "epoch": 1.09, + "learning_rate": 0.00023112537612838519, + "loss": 1.2839, + "theoretical_loss": 3.41295679458511, + "tokens_seen": 2356922368 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002311093279839519, + "loss": 1.3438, + "theoretical_loss": 3.412949199144357, + "tokens_seen": 2356987904 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002310932798395186, + "loss": 1.2104, + "theoretical_loss": 3.412941603973923, + "tokens_seen": 2357053440 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002310772316950853, + "loss": 1.2312, + "theoretical_loss": 3.412934009073792, + "tokens_seen": 2357118976 + }, + { + "epoch": 1.09, + "learning_rate": 0.000231061183550652, + "loss": 1.2093, + "theoretical_loss": 3.412926414443946, + "tokens_seen": 2357184512 + }, + { + "epoch": 1.09, + "learning_rate": 0.0002310451354062187, + "loss": 1.2201, + "theoretical_loss": 3.4129188200843688, + "tokens_seen": 2357250048 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002310290872617854, + "loss": 1.2676, + "theoretical_loss": 3.4129112259950425, + "tokens_seen": 2357315584 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002310130391173521, + "loss": 1.2286, + "theoretical_loss": 3.4129036321759507, + "tokens_seen": 2357381120 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002309969909729188, + "loss": 1.256, + "theoretical_loss": 3.4128960386270757, + "tokens_seen": 2357446656 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002309809428284855, + "loss": 1.184, + "theoretical_loss": 3.4128884453484005, + "tokens_seen": 2357512192 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002309648946840522, + "loss": 1.3333, + "theoretical_loss": 3.412880852339908, + "tokens_seen": 2357577728 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002309488465396189, + "loss": 1.2516, + "theoretical_loss": 3.412873259601581, + "tokens_seen": 2357643264 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023093279839518557, + "loss": 1.2638, + "theoretical_loss": 3.412865667133403, + "tokens_seen": 2357708800 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023091675025075227, + "loss": 1.1747, + "theoretical_loss": 3.4128580749353556, + "tokens_seen": 2357774336 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023090070210631897, + "loss": 1.2193, + "theoretical_loss": 3.4128504830074227, + "tokens_seen": 2357839872 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023088465396188567, + "loss": 1.1914, + "theoretical_loss": 3.412842891349587, + "tokens_seen": 2357905408 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023086860581745237, + "loss": 1.2848, + "theoretical_loss": 3.412835299961831, + "tokens_seen": 2357970944 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023085255767301907, + "loss": 1.2933, + "theoretical_loss": 3.412827708844138, + "tokens_seen": 2358036480 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023083650952858577, + "loss": 1.2498, + "theoretical_loss": 3.412820117996491, + "tokens_seen": 2358102016 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023082046138415247, + "loss": 1.2366, + "theoretical_loss": 3.4128125274188728, + "tokens_seen": 2358167552 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023080441323971917, + "loss": 1.2764, + "theoretical_loss": 3.4128049371112663, + "tokens_seen": 2358233088 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023078836509528587, + "loss": 1.2798, + "theoretical_loss": 3.4127973470736537, + "tokens_seen": 2358298624 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023077231695085257, + "loss": 1.249, + "theoretical_loss": 3.4127897573060184, + "tokens_seen": 2358364160 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023075626880641927, + "loss": 1.216, + "theoretical_loss": 3.4127821678083436, + "tokens_seen": 2358429696 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023074022066198597, + "loss": 1.0743, + "theoretical_loss": 3.412774578580612, + "tokens_seen": 2358495232 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023072417251755267, + "loss": 1.2982, + "theoretical_loss": 3.4127669896228063, + "tokens_seen": 2358560768 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023070812437311938, + "loss": 1.1748, + "theoretical_loss": 3.4127594009349096, + "tokens_seen": 2358626304 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023069207622868608, + "loss": 1.2371, + "theoretical_loss": 3.4127518125169045, + "tokens_seen": 2358691840 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023067602808425278, + "loss": 1.2509, + "theoretical_loss": 3.4127442243687742, + "tokens_seen": 2358757376 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023065997993981948, + "loss": 1.1919, + "theoretical_loss": 3.412736636490502, + "tokens_seen": 2358822912 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023064393179538618, + "loss": 1.2227, + "theoretical_loss": 3.4127290488820696, + "tokens_seen": 2358888448 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023062788365095288, + "loss": 1.2162, + "theoretical_loss": 3.412721461543461, + "tokens_seen": 2358953984 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023061183550651958, + "loss": 1.2413, + "theoretical_loss": 3.412713874474659, + "tokens_seen": 2359019520 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023059578736208628, + "loss": 1.211, + "theoretical_loss": 3.412706287675646, + "tokens_seen": 2359085056 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023057973921765298, + "loss": 1.2478, + "theoretical_loss": 3.412698701146405, + "tokens_seen": 2359150592 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023056369107321968, + "loss": 1.2806, + "theoretical_loss": 3.4126911148869192, + "tokens_seen": 2359216128 + }, + { + "epoch": 1.1, + "objective/train/docs_used": 871975, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3010919094085693, + "objective/train/theoretical_loss": 3.4126835288971717, + "objective/train/tokens_used": 2379741664, + "theoretical_loss": 3.4126835288971717, + "tokens_seen": 2359281664 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023054764292878635, + "loss": 1.3011, + "theoretical_loss": 3.4126835288971717, + "tokens_seen": 2359281664 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023053159478435306, + "loss": 1.1642, + "theoretical_loss": 3.412675943177145, + "tokens_seen": 2359347200 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023051554663991976, + "loss": 1.1928, + "theoretical_loss": 3.412668357726822, + "tokens_seen": 2359412736 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023049949849548646, + "loss": 1.2663, + "theoretical_loss": 3.412660772546186, + "tokens_seen": 2359478272 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023048345035105316, + "loss": 1.2375, + "theoretical_loss": 3.4126531876352195, + "tokens_seen": 2359543808 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023046740220661986, + "loss": 1.1829, + "theoretical_loss": 3.412645602993906, + "tokens_seen": 2359609344 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023045135406218656, + "loss": 1.2035, + "theoretical_loss": 3.412638018622227, + "tokens_seen": 2359674880 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023043530591775326, + "loss": 1.188, + "theoretical_loss": 3.4126304345201675, + "tokens_seen": 2359740416 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023041925777331996, + "loss": 1.2544, + "theoretical_loss": 3.4126228506877085, + "tokens_seen": 2359805952 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023040320962888666, + "loss": 1.2714, + "theoretical_loss": 3.4126152671248344, + "tokens_seen": 2359871488 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023038716148445336, + "loss": 1.2842, + "theoretical_loss": 3.412607683831528, + "tokens_seen": 2359937024 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023037111334002006, + "loss": 1.1952, + "theoretical_loss": 3.412600100807771, + "tokens_seen": 2360002560 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023035506519558676, + "loss": 1.2605, + "theoretical_loss": 3.4125925180535472, + "tokens_seen": 2360068096 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023033901705115346, + "loss": 1.1191, + "theoretical_loss": 3.4125849355688396, + "tokens_seen": 2360133632 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023032296890672016, + "loss": 1.2585, + "theoretical_loss": 3.412577353353631, + "tokens_seen": 2360199168 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023030692076228686, + "loss": 1.1996, + "theoretical_loss": 3.412569771407904, + "tokens_seen": 2360264704 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023029087261785356, + "loss": 1.2396, + "theoretical_loss": 3.4125621897316423, + "tokens_seen": 2360330240 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023027482447342027, + "loss": 1.1848, + "theoretical_loss": 3.4125546083248284, + "tokens_seen": 2360395776 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023025877632898697, + "loss": 1.1842, + "theoretical_loss": 3.412547027187445, + "tokens_seen": 2360461312 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023024272818455367, + "loss": 1.1707, + "theoretical_loss": 3.4125394463194754, + "tokens_seen": 2360526848 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023022668004012037, + "loss": 1.1695, + "theoretical_loss": 3.4125318657209025, + "tokens_seen": 2360592384 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023021063189568707, + "loss": 1.2522, + "theoretical_loss": 3.4125242853917093, + "tokens_seen": 2360657920 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023019458375125377, + "loss": 1.2488, + "theoretical_loss": 3.4125167053318783, + "tokens_seen": 2360723456 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023017853560682047, + "loss": 1.1423, + "theoretical_loss": 3.412509125541393, + "tokens_seen": 2360788992 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023016248746238714, + "loss": 1.2371, + "theoretical_loss": 3.412501546020236, + "tokens_seen": 2360854528 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023014643931795384, + "loss": 1.2117, + "theoretical_loss": 3.4124939667683907, + "tokens_seen": 2360920064 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023013039117352054, + "loss": 1.3085, + "theoretical_loss": 3.4124863877858393, + "tokens_seen": 2360985600 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023011434302908724, + "loss": 1.2947, + "theoretical_loss": 3.4124788090725655, + "tokens_seen": 2361051136 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023009829488465395, + "loss": 1.1805, + "theoretical_loss": 3.412471230628552, + "tokens_seen": 2361116672 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023008224674022065, + "loss": 1.1759, + "theoretical_loss": 3.4124636524537815, + "tokens_seen": 2361182208 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023006619859578735, + "loss": 1.2073, + "theoretical_loss": 3.4124560745482375, + "tokens_seen": 2361247744 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023005015045135405, + "loss": 1.1404, + "theoretical_loss": 3.412448496911902, + "tokens_seen": 2361313280 + }, + { + "epoch": 1.1, + "learning_rate": 0.00023003410230692075, + "loss": 1.2397, + "theoretical_loss": 3.4124409195447596, + "tokens_seen": 2361378816 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002300180541624875, + "loss": 1.1379, + "theoretical_loss": 3.4124333424467914, + "tokens_seen": 2361444352 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002300020060180542, + "loss": 1.2368, + "theoretical_loss": 3.4124257656179817, + "tokens_seen": 2361509888 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002299859578736209, + "loss": 1.2062, + "theoretical_loss": 3.412418189058313, + "tokens_seen": 2361575424 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002299699097291876, + "loss": 1.0752, + "theoretical_loss": 3.4124106127677685, + "tokens_seen": 2361640960 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002299538615847543, + "loss": 1.2706, + "theoretical_loss": 3.4124030367463303, + "tokens_seen": 2361706496 + }, + { + "epoch": 1.1, + "learning_rate": 0.000229937813440321, + "loss": 1.1464, + "theoretical_loss": 3.4123954609939826, + "tokens_seen": 2361772032 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022992176529588768, + "loss": 1.2153, + "theoretical_loss": 3.4123878855107073, + "tokens_seen": 2361837568 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022990571715145438, + "loss": 1.0867, + "theoretical_loss": 3.4123803102964887, + "tokens_seen": 2361903104 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022988966900702108, + "loss": 1.1906, + "theoretical_loss": 3.4123727353513083, + "tokens_seen": 2361968640 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022987362086258778, + "loss": 1.1948, + "theoretical_loss": 3.41236516067515, + "tokens_seen": 2362034176 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022985757271815448, + "loss": 1.1682, + "theoretical_loss": 3.4123575862679965, + "tokens_seen": 2362099712 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022984152457372118, + "loss": 1.1889, + "theoretical_loss": 3.412350012129831, + "tokens_seen": 2362165248 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022982547642928788, + "loss": 1.2641, + "theoretical_loss": 3.4123424382606355, + "tokens_seen": 2362230784 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022980942828485458, + "loss": 1.1699, + "theoretical_loss": 3.412334864660395, + "tokens_seen": 2362296320 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022979338014042128, + "loss": 1.1947, + "theoretical_loss": 3.4123272913290905, + "tokens_seen": 2362361856 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022977733199598799, + "loss": 1.1777, + "theoretical_loss": 3.4123197182667058, + "tokens_seen": 2362427392 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022976128385155469, + "loss": 1.2572, + "theoretical_loss": 3.4123121454732237, + "tokens_seen": 2362492928 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022974523570712139, + "loss": 1.3054, + "theoretical_loss": 3.412304572948628, + "tokens_seen": 2362558464 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002297291875626881, + "loss": 1.1402, + "theoretical_loss": 3.4122970006929005, + "tokens_seen": 2362624000 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002297131394182548, + "loss": 1.1861, + "theoretical_loss": 3.412289428706025, + "tokens_seen": 2362689536 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002296970912738215, + "loss": 1.2252, + "theoretical_loss": 3.4122818569879843, + "tokens_seen": 2362755072 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002296810431293882, + "loss": 1.2145, + "theoretical_loss": 3.412274285538761, + "tokens_seen": 2362820608 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002296649949849549, + "loss": 1.1505, + "theoretical_loss": 3.4122667143583385, + "tokens_seen": 2362886144 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002296489468405216, + "loss": 1.3924, + "theoretical_loss": 3.4122591434467, + "tokens_seen": 2362951680 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002296328986960883, + "loss": 1.3346, + "theoretical_loss": 3.4122515728038283, + "tokens_seen": 2363017216 + }, + { + "epoch": 1.1, + "learning_rate": 0.000229616850551655, + "loss": 1.2415, + "theoretical_loss": 3.412244002429706, + "tokens_seen": 2363082752 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002296008024072217, + "loss": 1.1918, + "theoretical_loss": 3.412236432324317, + "tokens_seen": 2363148288 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002295847542627884, + "loss": 1.19, + "theoretical_loss": 3.412228862487643, + "tokens_seen": 2363213824 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002295687061183551, + "loss": 1.2018, + "theoretical_loss": 3.4122212929196682, + "tokens_seen": 2363279360 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002295526579739218, + "loss": 1.2173, + "theoretical_loss": 3.412213723620375, + "tokens_seen": 2363344896 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022953660982948847, + "loss": 1.1903, + "theoretical_loss": 3.4122061545897466, + "tokens_seen": 2363410432 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022952056168505517, + "loss": 1.1197, + "theoretical_loss": 3.4121985858277664, + "tokens_seen": 2363475968 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022950451354062187, + "loss": 1.1836, + "theoretical_loss": 3.4121910173344165, + "tokens_seen": 2363541504 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022948846539618857, + "loss": 1.2614, + "theoretical_loss": 3.4121834491096807, + "tokens_seen": 2363607040 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022947241725175527, + "loss": 1.217, + "theoretical_loss": 3.412175881153542, + "tokens_seen": 2363672576 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022945636910732197, + "loss": 1.3338, + "theoretical_loss": 3.4121683134659833, + "tokens_seen": 2363738112 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022944032096288867, + "loss": 1.2536, + "theoretical_loss": 3.412160746046987, + "tokens_seen": 2363803648 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022942427281845537, + "loss": 1.1872, + "theoretical_loss": 3.4121531788965367, + "tokens_seen": 2363869184 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022940822467402207, + "loss": 1.268, + "theoretical_loss": 3.4121456120146156, + "tokens_seen": 2363934720 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022939217652958877, + "loss": 1.2008, + "theoretical_loss": 3.412138045401206, + "tokens_seen": 2364000256 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022937612838515547, + "loss": 1.1845, + "theoretical_loss": 3.412130479056292, + "tokens_seen": 2364065792 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022936008024072217, + "loss": 1.2036, + "theoretical_loss": 3.412122912979856, + "tokens_seen": 2364131328 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022934403209628888, + "loss": 1.2548, + "theoretical_loss": 3.4121153471718806, + "tokens_seen": 2364196864 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022932798395185558, + "loss": 1.186, + "theoretical_loss": 3.4121077816323497, + "tokens_seen": 2364262400 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022931193580742228, + "loss": 1.2557, + "theoretical_loss": 3.4121002163612455, + "tokens_seen": 2364327936 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022929588766298898, + "loss": 1.1742, + "theoretical_loss": 3.412092651358552, + "tokens_seen": 2364393472 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022927983951855568, + "loss": 1.2268, + "theoretical_loss": 3.4120850866242516, + "tokens_seen": 2364459008 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022926379137412238, + "loss": 1.3556, + "theoretical_loss": 3.4120775221583273, + "tokens_seen": 2364524544 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022924774322968908, + "loss": 1.2143, + "theoretical_loss": 3.412069957960762, + "tokens_seen": 2364590080 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022923169508525578, + "loss": 1.2388, + "theoretical_loss": 3.4120623940315395, + "tokens_seen": 2364655616 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022921564694082248, + "loss": 1.1904, + "theoretical_loss": 3.4120548303706424, + "tokens_seen": 2364721152 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022919959879638918, + "loss": 1.3605, + "theoretical_loss": 3.4120472669780537, + "tokens_seen": 2364786688 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022918355065195588, + "loss": 1.1076, + "theoretical_loss": 3.4120397038537567, + "tokens_seen": 2364852224 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022916750250752258, + "loss": 1.3415, + "theoretical_loss": 3.4120321409977334, + "tokens_seen": 2364917760 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022915145436308926, + "loss": 1.2624, + "theoretical_loss": 3.4120245784099685, + "tokens_seen": 2364983296 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022913540621865596, + "loss": 1.209, + "theoretical_loss": 3.4120170160904437, + "tokens_seen": 2365048832 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022911935807422266, + "loss": 1.2142, + "theoretical_loss": 3.4120094540391426, + "tokens_seen": 2365114368 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022910330992978936, + "loss": 1.2056, + "theoretical_loss": 3.4120018922560487, + "tokens_seen": 2365179904 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022908726178535606, + "loss": 1.1464, + "theoretical_loss": 3.411994330741144, + "tokens_seen": 2365245440 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022907121364092276, + "loss": 1.2055, + "theoretical_loss": 3.4119867694944124, + "tokens_seen": 2365310976 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022905516549648946, + "loss": 1.22, + "theoretical_loss": 3.411979208515837, + "tokens_seen": 2365376512 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022903911735205616, + "loss": 1.2599, + "theoretical_loss": 3.4119716478054003, + "tokens_seen": 2365442048 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022902306920762286, + "loss": 1.2406, + "theoretical_loss": 3.4119640873630854, + "tokens_seen": 2365507584 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022900702106318956, + "loss": 1.2432, + "theoretical_loss": 3.4119565271888757, + "tokens_seen": 2365573120 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022899097291875626, + "loss": 1.2491, + "theoretical_loss": 3.4119489672827545, + "tokens_seen": 2365638656 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022897492477432296, + "loss": 1.2579, + "theoretical_loss": 3.4119414076447043, + "tokens_seen": 2365704192 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022895887662988966, + "loss": 1.3425, + "theoretical_loss": 3.411933848274708, + "tokens_seen": 2365769728 + }, + { + "epoch": 1.1, + "objective/train/docs_used": 874603, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2623955011367798, + "objective/train/theoretical_loss": 3.4119262891727495, + "objective/train/tokens_used": 2386295264, + "theoretical_loss": 3.4119262891727495, + "tokens_seen": 2365835264 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022894282848545636, + "loss": 1.2624, + "theoretical_loss": 3.4119262891727495, + "tokens_seen": 2365835264 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022892678034102306, + "loss": 1.2066, + "theoretical_loss": 3.4119187303388117, + "tokens_seen": 2365900800 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022891073219658977, + "loss": 1.2143, + "theoretical_loss": 3.4119111717728767, + "tokens_seen": 2365966336 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022889468405215647, + "loss": 1.3012, + "theoretical_loss": 3.411903613474929, + "tokens_seen": 2366031872 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022887863590772317, + "loss": 1.267, + "theoretical_loss": 3.4118960554449504, + "tokens_seen": 2366097408 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022886258776328987, + "loss": 1.2448, + "theoretical_loss": 3.411888497682925, + "tokens_seen": 2366162944 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022884653961885657, + "loss": 1.2301, + "theoretical_loss": 3.4118809401888353, + "tokens_seen": 2366228480 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022883049147442327, + "loss": 1.245, + "theoretical_loss": 3.411873382962664, + "tokens_seen": 2366294016 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022881444332999, + "loss": 1.2645, + "theoretical_loss": 3.4118658260043953, + "tokens_seen": 2366359552 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002287983951855567, + "loss": 1.2829, + "theoretical_loss": 3.411858269314011, + "tokens_seen": 2366425088 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002287823470411234, + "loss": 1.268, + "theoretical_loss": 3.4118507128914954, + "tokens_seen": 2366490624 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002287662988966901, + "loss": 1.2236, + "theoretical_loss": 3.411843156736831, + "tokens_seen": 2366556160 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002287502507522568, + "loss": 1.239, + "theoretical_loss": 3.411835600850001, + "tokens_seen": 2366621696 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002287342026078235, + "loss": 1.2099, + "theoretical_loss": 3.4118280452309877, + "tokens_seen": 2366687232 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002287181544633902, + "loss": 1.2396, + "theoretical_loss": 3.4118204898797755, + "tokens_seen": 2366752768 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002287021063189569, + "loss": 1.2706, + "theoretical_loss": 3.411812934796347, + "tokens_seen": 2366818304 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002286860581745236, + "loss": 1.259, + "theoretical_loss": 3.411805379980685, + "tokens_seen": 2366883840 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002286700100300903, + "loss": 1.2313, + "theoretical_loss": 3.4117978254327728, + "tokens_seen": 2366949376 + }, + { + "epoch": 1.1, + "learning_rate": 0.000228653961885657, + "loss": 1.1986, + "theoretical_loss": 3.411790271152593, + "tokens_seen": 2367014912 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002286379137412237, + "loss": 1.187, + "theoretical_loss": 3.41178271714013, + "tokens_seen": 2367080448 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002286218655967904, + "loss": 1.2764, + "theoretical_loss": 3.4117751633953657, + "tokens_seen": 2367145984 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002286058174523571, + "loss": 1.2995, + "theoretical_loss": 3.4117676099182837, + "tokens_seen": 2367211520 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002285897693079238, + "loss": 1.1893, + "theoretical_loss": 3.4117600567088666, + "tokens_seen": 2367277056 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002285737211634905, + "loss": 1.2343, + "theoretical_loss": 3.411752503767098, + "tokens_seen": 2367342592 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002285576730190572, + "loss": 1.2728, + "theoretical_loss": 3.4117449510929614, + "tokens_seen": 2367408128 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002285416248746239, + "loss": 1.2889, + "theoretical_loss": 3.411737398686439, + "tokens_seen": 2367473664 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022852557673019058, + "loss": 1.2744, + "theoretical_loss": 3.4117298465475145, + "tokens_seen": 2367539200 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022850952858575728, + "loss": 1.2149, + "theoretical_loss": 3.4117222946761707, + "tokens_seen": 2367604736 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022849348044132398, + "loss": 1.294, + "theoretical_loss": 3.411714743072391, + "tokens_seen": 2367670272 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022847743229689068, + "loss": 1.1821, + "theoretical_loss": 3.4117071917361583, + "tokens_seen": 2367735808 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022846138415245738, + "loss": 1.2318, + "theoretical_loss": 3.4116996406674556, + "tokens_seen": 2367801344 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022844533600802408, + "loss": 1.2138, + "theoretical_loss": 3.4116920898662664, + "tokens_seen": 2367866880 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022842928786359078, + "loss": 1.3142, + "theoretical_loss": 3.4116845393325734, + "tokens_seen": 2367932416 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022841323971915748, + "loss": 1.2637, + "theoretical_loss": 3.4116769890663603, + "tokens_seen": 2367997952 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022839719157472419, + "loss": 1.2551, + "theoretical_loss": 3.411669439067609, + "tokens_seen": 2368063488 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022838114343029089, + "loss": 1.2023, + "theoretical_loss": 3.4116618893363047, + "tokens_seen": 2368129024 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002283650952858576, + "loss": 1.2195, + "theoretical_loss": 3.4116543398724284, + "tokens_seen": 2368194560 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002283490471414243, + "loss": 1.1397, + "theoretical_loss": 3.4116467906759644, + "tokens_seen": 2368260096 + }, + { + "epoch": 1.1, + "learning_rate": 0.000228332998996991, + "loss": 1.1901, + "theoretical_loss": 3.4116392417468955, + "tokens_seen": 2368325632 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002283169508525577, + "loss": 1.2638, + "theoretical_loss": 3.411631693085205, + "tokens_seen": 2368391168 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002283009027081244, + "loss": 1.1643, + "theoretical_loss": 3.411624144690876, + "tokens_seen": 2368456704 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002282848545636911, + "loss": 1.2356, + "theoretical_loss": 3.411616596563891, + "tokens_seen": 2368522240 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002282688064192578, + "loss": 1.2204, + "theoretical_loss": 3.4116090487042348, + "tokens_seen": 2368587776 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002282527582748245, + "loss": 1.2259, + "theoretical_loss": 3.4116015011118885, + "tokens_seen": 2368653312 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002282367101303912, + "loss": 1.2219, + "theoretical_loss": 3.4115939537868365, + "tokens_seen": 2368718848 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002282206619859579, + "loss": 1.2113, + "theoretical_loss": 3.4115864067290613, + "tokens_seen": 2368784384 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002282046138415246, + "loss": 1.2625, + "theoretical_loss": 3.4115788599385466, + "tokens_seen": 2368849920 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002281885656970913, + "loss": 1.2696, + "theoretical_loss": 3.4115713134152754, + "tokens_seen": 2368915456 + }, + { + "epoch": 1.1, + "learning_rate": 0.000228172517552658, + "loss": 1.2949, + "theoretical_loss": 3.4115637671592305, + "tokens_seen": 2368980992 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002281564694082247, + "loss": 1.1227, + "theoretical_loss": 3.4115562211703954, + "tokens_seen": 2369046528 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022814042126379137, + "loss": 1.2683, + "theoretical_loss": 3.4115486754487527, + "tokens_seen": 2369112064 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022812437311935807, + "loss": 1.1176, + "theoretical_loss": 3.4115411299942866, + "tokens_seen": 2369177600 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022810832497492477, + "loss": 1.2415, + "theoretical_loss": 3.411533584806979, + "tokens_seen": 2369243136 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022809227683049147, + "loss": 1.3305, + "theoretical_loss": 3.411526039886814, + "tokens_seen": 2369308672 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022807622868605817, + "loss": 1.2976, + "theoretical_loss": 3.411518495233775, + "tokens_seen": 2369374208 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022806018054162487, + "loss": 1.2268, + "theoretical_loss": 3.411510950847844, + "tokens_seen": 2369439744 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022804413239719157, + "loss": 1.2507, + "theoretical_loss": 3.4115034067290044, + "tokens_seen": 2369505280 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022802808425275827, + "loss": 1.2419, + "theoretical_loss": 3.41149586287724, + "tokens_seen": 2369570816 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022801203610832497, + "loss": 1.1918, + "theoretical_loss": 3.4114883192925336, + "tokens_seen": 2369636352 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022799598796389167, + "loss": 1.1739, + "theoretical_loss": 3.4114807759748684, + "tokens_seen": 2369701888 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022797993981945837, + "loss": 1.2441, + "theoretical_loss": 3.4114732329242274, + "tokens_seen": 2369767424 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022796389167502508, + "loss": 1.2061, + "theoretical_loss": 3.411465690140594, + "tokens_seen": 2369832960 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022794784353059178, + "loss": 1.2246, + "theoretical_loss": 3.411458147623952, + "tokens_seen": 2369898496 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022793179538615848, + "loss": 1.1156, + "theoretical_loss": 3.4114506053742835, + "tokens_seen": 2369964032 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022791574724172518, + "loss": 1.1799, + "theoretical_loss": 3.4114430633915713, + "tokens_seen": 2370029568 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022789969909729188, + "loss": 1.1842, + "theoretical_loss": 3.4114355216758, + "tokens_seen": 2370095104 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022788365095285858, + "loss": 1.1916, + "theoretical_loss": 3.411427980226952, + "tokens_seen": 2370160640 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022786760280842528, + "loss": 1.2661, + "theoretical_loss": 3.4114204390450107, + "tokens_seen": 2370226176 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022785155466399198, + "loss": 1.1065, + "theoretical_loss": 3.4114128981299587, + "tokens_seen": 2370291712 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022783550651955868, + "loss": 1.2456, + "theoretical_loss": 3.4114053574817795, + "tokens_seen": 2370357248 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022781945837512538, + "loss": 1.2517, + "theoretical_loss": 3.4113978171004566, + "tokens_seen": 2370422784 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022780341023069208, + "loss": 1.2183, + "theoretical_loss": 3.411390276985973, + "tokens_seen": 2370488320 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022778736208625878, + "loss": 1.2134, + "theoretical_loss": 3.411382737138312, + "tokens_seen": 2370553856 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022777131394182548, + "loss": 1.2345, + "theoretical_loss": 3.4113751975574567, + "tokens_seen": 2370619392 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022775526579739216, + "loss": 1.2775, + "theoretical_loss": 3.41136765824339, + "tokens_seen": 2370684928 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022773921765295886, + "loss": 1.1814, + "theoretical_loss": 3.411360119196095, + "tokens_seen": 2370750464 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022772316950852556, + "loss": 1.2084, + "theoretical_loss": 3.4113525804155556, + "tokens_seen": 2370816000 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022770712136409226, + "loss": 1.2263, + "theoretical_loss": 3.411345041901754, + "tokens_seen": 2370881536 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022769107321965896, + "loss": 1.2273, + "theoretical_loss": 3.4113375036546745, + "tokens_seen": 2370947072 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022767502507522566, + "loss": 1.235, + "theoretical_loss": 3.4113299656742995, + "tokens_seen": 2371012608 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022765897693079236, + "loss": 1.2152, + "theoretical_loss": 3.4113224279606125, + "tokens_seen": 2371078144 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022764292878635906, + "loss": 1.2769, + "theoretical_loss": 3.4113148905135966, + "tokens_seen": 2371143680 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022762688064192582, + "loss": 1.2269, + "theoretical_loss": 3.4113073533332345, + "tokens_seen": 2371209216 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022761083249749252, + "loss": 1.2417, + "theoretical_loss": 3.4112998164195107, + "tokens_seen": 2371274752 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022759478435305922, + "loss": 1.2107, + "theoretical_loss": 3.4112922797724075, + "tokens_seen": 2371340288 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022757873620862592, + "loss": 1.241, + "theoretical_loss": 3.411284743391908, + "tokens_seen": 2371405824 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022756268806419262, + "loss": 1.149, + "theoretical_loss": 3.4112772072779953, + "tokens_seen": 2371471360 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022754663991975932, + "loss": 1.269, + "theoretical_loss": 3.4112696714306536, + "tokens_seen": 2371536896 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022753059177532602, + "loss": 1.2532, + "theoretical_loss": 3.411262135849865, + "tokens_seen": 2371602432 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022751454363089272, + "loss": 1.3542, + "theoretical_loss": 3.4112546005356132, + "tokens_seen": 2371667968 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002274984954864594, + "loss": 1.2456, + "theoretical_loss": 3.4112470654878813, + "tokens_seen": 2371733504 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002274824473420261, + "loss": 1.292, + "theoretical_loss": 3.4112395307066525, + "tokens_seen": 2371799040 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002274663991975928, + "loss": 1.2383, + "theoretical_loss": 3.41123199619191, + "tokens_seen": 2371864576 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002274503510531595, + "loss": 1.2755, + "theoretical_loss": 3.4112244619436374, + "tokens_seen": 2371930112 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002274343029087262, + "loss": 1.2934, + "theoretical_loss": 3.411216927961817, + "tokens_seen": 2371995648 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002274182547642929, + "loss": 1.2338, + "theoretical_loss": 3.411209394246433, + "tokens_seen": 2372061184 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002274022066198596, + "loss": 1.2214, + "theoretical_loss": 3.4112018607974686, + "tokens_seen": 2372126720 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002273861584754263, + "loss": 1.3103, + "theoretical_loss": 3.4111943276149064, + "tokens_seen": 2372192256 + }, + { + "epoch": 1.1, + "learning_rate": 0.000227370110330993, + "loss": 1.1711, + "theoretical_loss": 3.4111867946987298, + "tokens_seen": 2372257792 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002273540621865597, + "loss": 1.1769, + "theoretical_loss": 3.411179262048922, + "tokens_seen": 2372323328 + }, + { + "epoch": 1.1, + "objective/train/docs_used": 876954, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2211642265319824, + "objective/train/theoretical_loss": 3.4111717296654662, + "objective/train/tokens_used": 2392848864, + "theoretical_loss": 3.4111717296654662, + "tokens_seen": 2372388864 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002273380140421264, + "loss": 1.2212, + "theoretical_loss": 3.4111717296654662, + "tokens_seen": 2372388864 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002273219658976931, + "loss": 1.1793, + "theoretical_loss": 3.4111641975483455, + "tokens_seen": 2372454400 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002273059177532598, + "loss": 1.2404, + "theoretical_loss": 3.411156665697544, + "tokens_seen": 2372519936 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002272898696088265, + "loss": 1.2523, + "theoretical_loss": 3.4111491341130438, + "tokens_seen": 2372585472 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002272738214643932, + "loss": 1.1593, + "theoretical_loss": 3.4111416027948285, + "tokens_seen": 2372651008 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002272577733199599, + "loss": 1.1563, + "theoretical_loss": 3.4111340717428815, + "tokens_seen": 2372716544 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002272417251755266, + "loss": 1.1871, + "theoretical_loss": 3.411126540957186, + "tokens_seen": 2372782080 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002272256770310933, + "loss": 1.265, + "theoretical_loss": 3.4111190104377256, + "tokens_seen": 2372847616 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022720962888666, + "loss": 1.3305, + "theoretical_loss": 3.411111480184483, + "tokens_seen": 2372913152 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002271935807422267, + "loss": 1.2846, + "theoretical_loss": 3.4111039501974414, + "tokens_seen": 2372978688 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002271775325977934, + "loss": 1.2581, + "theoretical_loss": 3.4110964204765843, + "tokens_seen": 2373044224 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002271614844533601, + "loss": 1.3475, + "theoretical_loss": 3.411088891021895, + "tokens_seen": 2373109760 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002271454363089268, + "loss": 1.1846, + "theoretical_loss": 3.411081361833356, + "tokens_seen": 2373175296 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002271293881644935, + "loss": 1.1976, + "theoretical_loss": 3.4110738329109513, + "tokens_seen": 2373240832 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022711334002006018, + "loss": 1.294, + "theoretical_loss": 3.4110663042546645, + "tokens_seen": 2373306368 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022709729187562688, + "loss": 1.2097, + "theoretical_loss": 3.4110587758644777, + "tokens_seen": 2373371904 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022708124373119358, + "loss": 1.2888, + "theoretical_loss": 3.4110512477403754, + "tokens_seen": 2373437440 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022706519558676028, + "loss": 1.1169, + "theoretical_loss": 3.41104371988234, + "tokens_seen": 2373502976 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022704914744232698, + "loss": 1.2885, + "theoretical_loss": 3.4110361922903545, + "tokens_seen": 2373568512 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022703309929789369, + "loss": 1.2646, + "theoretical_loss": 3.411028664964403, + "tokens_seen": 2373634048 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022701705115346039, + "loss": 1.2271, + "theoretical_loss": 3.411021137904468, + "tokens_seen": 2373699584 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022700100300902709, + "loss": 1.3464, + "theoretical_loss": 3.411013611110534, + "tokens_seen": 2373765120 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002269849548645938, + "loss": 1.1969, + "theoretical_loss": 3.4110060845825827, + "tokens_seen": 2373830656 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002269689067201605, + "loss": 1.2377, + "theoretical_loss": 3.4109985583205984, + "tokens_seen": 2373896192 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002269528585757272, + "loss": 1.2447, + "theoretical_loss": 3.4109910323245636, + "tokens_seen": 2373961728 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002269368104312939, + "loss": 1.2335, + "theoretical_loss": 3.4109835065944623, + "tokens_seen": 2374027264 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002269207622868606, + "loss": 1.3469, + "theoretical_loss": 3.410975981130277, + "tokens_seen": 2374092800 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002269047141424273, + "loss": 1.2509, + "theoretical_loss": 3.410968455931992, + "tokens_seen": 2374158336 + }, + { + "epoch": 1.1, + "learning_rate": 0.000226888665997994, + "loss": 1.2637, + "theoretical_loss": 3.4109609309995896, + "tokens_seen": 2374223872 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002268726178535607, + "loss": 1.101, + "theoretical_loss": 3.4109534063330536, + "tokens_seen": 2374289408 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002268565697091274, + "loss": 1.1758, + "theoretical_loss": 3.410945881932367, + "tokens_seen": 2374354944 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002268405215646941, + "loss": 1.2772, + "theoretical_loss": 3.410938357797513, + "tokens_seen": 2374420480 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002268244734202608, + "loss": 1.1604, + "theoretical_loss": 3.410930833928475, + "tokens_seen": 2374486016 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002268084252758275, + "loss": 1.1774, + "theoretical_loss": 3.4109233103252365, + "tokens_seen": 2374551552 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002267923771313942, + "loss": 1.2281, + "theoretical_loss": 3.410915786987781, + "tokens_seen": 2374617088 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002267763289869609, + "loss": 1.2946, + "theoretical_loss": 3.4109082639160904, + "tokens_seen": 2374682624 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002267602808425276, + "loss": 1.2216, + "theoretical_loss": 3.4109007411101495, + "tokens_seen": 2374748160 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002267442326980943, + "loss": 1.2345, + "theoretical_loss": 3.410893218569941, + "tokens_seen": 2374813696 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022672818455366097, + "loss": 1.2441, + "theoretical_loss": 3.410885696295448, + "tokens_seen": 2374879232 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022671213640922767, + "loss": 1.3459, + "theoretical_loss": 3.4108781742866543, + "tokens_seen": 2374944768 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022669608826479437, + "loss": 1.241, + "theoretical_loss": 3.4108706525435424, + "tokens_seen": 2375010304 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022668004012036107, + "loss": 1.2285, + "theoretical_loss": 3.4108631310660966, + "tokens_seen": 2375075840 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022666399197592777, + "loss": 1.2303, + "theoretical_loss": 3.4108556098542993, + "tokens_seen": 2375141376 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022664794383149447, + "loss": 1.2593, + "theoretical_loss": 3.410848088908134, + "tokens_seen": 2375206912 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022663189568706117, + "loss": 1.23, + "theoretical_loss": 3.4108405682275844, + "tokens_seen": 2375272448 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022661584754262787, + "loss": 1.187, + "theoretical_loss": 3.410833047812633, + "tokens_seen": 2375337984 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022659979939819458, + "loss": 1.2065, + "theoretical_loss": 3.410825527663264, + "tokens_seen": 2375403520 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022658375125376128, + "loss": 1.2487, + "theoretical_loss": 3.4108180077794605, + "tokens_seen": 2375469056 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022656770310932798, + "loss": 1.224, + "theoretical_loss": 3.4108104881612054, + "tokens_seen": 2375534592 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022655165496489468, + "loss": 1.3341, + "theoretical_loss": 3.4108029688084818, + "tokens_seen": 2375600128 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022653560682046138, + "loss": 1.2973, + "theoretical_loss": 3.410795449721274, + "tokens_seen": 2375665664 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022651955867602808, + "loss": 1.1307, + "theoretical_loss": 3.410787930899564, + "tokens_seen": 2375731200 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022650351053159478, + "loss": 1.2441, + "theoretical_loss": 3.4107804123433363, + "tokens_seen": 2375796736 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022648746238716148, + "loss": 1.2036, + "theoretical_loss": 3.4107728940525734, + "tokens_seen": 2375862272 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022647141424272818, + "loss": 1.2155, + "theoretical_loss": 3.4107653760272587, + "tokens_seen": 2375927808 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022645536609829488, + "loss": 1.1645, + "theoretical_loss": 3.410757858267376, + "tokens_seen": 2375993344 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022643931795386158, + "loss": 1.3171, + "theoretical_loss": 3.4107503407729087, + "tokens_seen": 2376058880 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002264232698094283, + "loss": 1.1949, + "theoretical_loss": 3.4107428235438393, + "tokens_seen": 2376124416 + }, + { + "epoch": 1.1, + "learning_rate": 0.000226407221664995, + "loss": 1.1745, + "theoretical_loss": 3.4107353065801513, + "tokens_seen": 2376189952 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002263911735205617, + "loss": 1.2438, + "theoretical_loss": 3.4107277898818285, + "tokens_seen": 2376255488 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002263751253761284, + "loss": 1.2772, + "theoretical_loss": 3.4107202734488538, + "tokens_seen": 2376321024 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002263590772316951, + "loss": 1.2225, + "theoretical_loss": 3.410712757281211, + "tokens_seen": 2376386560 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002263430290872618, + "loss": 1.2952, + "theoretical_loss": 3.4107052413788828, + "tokens_seen": 2376452096 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002263269809428285, + "loss": 1.225, + "theoretical_loss": 3.4106977257418527, + "tokens_seen": 2376517632 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022631093279839521, + "loss": 1.207, + "theoretical_loss": 3.4106902103701042, + "tokens_seen": 2376583168 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022629488465396191, + "loss": 1.1864, + "theoretical_loss": 3.4106826952636204, + "tokens_seen": 2376648704 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022627883650952861, + "loss": 1.2236, + "theoretical_loss": 3.410675180422385, + "tokens_seen": 2376714240 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022626278836509532, + "loss": 1.2308, + "theoretical_loss": 3.410667665846381, + "tokens_seen": 2376779776 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022624674022066202, + "loss": 1.3029, + "theoretical_loss": 3.410660151535592, + "tokens_seen": 2376845312 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022623069207622872, + "loss": 1.2461, + "theoretical_loss": 3.410652637490001, + "tokens_seen": 2376910848 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022621464393179542, + "loss": 1.2261, + "theoretical_loss": 3.4106451237095916, + "tokens_seen": 2376976384 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022619859578736212, + "loss": 1.1825, + "theoretical_loss": 3.4106376101943465, + "tokens_seen": 2377041920 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022618254764292882, + "loss": 1.1924, + "theoretical_loss": 3.4106300969442502, + "tokens_seen": 2377107456 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022616649949849552, + "loss": 1.2596, + "theoretical_loss": 3.410622583959285, + "tokens_seen": 2377172992 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022615045135406222, + "loss": 1.1432, + "theoretical_loss": 3.4106150712394343, + "tokens_seen": 2377238528 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022613440320962892, + "loss": 1.1968, + "theoretical_loss": 3.4106075587846822, + "tokens_seen": 2377304064 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022611835506519562, + "loss": 1.2356, + "theoretical_loss": 3.4106000465950115, + "tokens_seen": 2377369600 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002261023069207623, + "loss": 1.3027, + "theoretical_loss": 3.4105925346704056, + "tokens_seen": 2377435136 + }, + { + "epoch": 1.1, + "learning_rate": 0.000226086258776329, + "loss": 1.2457, + "theoretical_loss": 3.4105850230108476, + "tokens_seen": 2377500672 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002260702106318957, + "loss": 1.2341, + "theoretical_loss": 3.4105775116163217, + "tokens_seen": 2377566208 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002260541624874624, + "loss": 1.2011, + "theoretical_loss": 3.41057000048681, + "tokens_seen": 2377631744 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002260381143430291, + "loss": 1.2623, + "theoretical_loss": 3.4105624896222966, + "tokens_seen": 2377697280 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002260220661985958, + "loss": 1.1872, + "theoretical_loss": 3.410554979022765, + "tokens_seen": 2377762816 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002260060180541625, + "loss": 1.1879, + "theoretical_loss": 3.4105474686881982, + "tokens_seen": 2377828352 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002259899699097292, + "loss": 1.1913, + "theoretical_loss": 3.410539958618579, + "tokens_seen": 2377893888 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002259739217652959, + "loss": 1.2828, + "theoretical_loss": 3.4105324488138926, + "tokens_seen": 2377959424 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002259578736208626, + "loss": 1.2719, + "theoretical_loss": 3.4105249392741204, + "tokens_seen": 2378024960 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002259418254764293, + "loss": 1.1702, + "theoretical_loss": 3.4105174299992465, + "tokens_seen": 2378090496 + }, + { + "epoch": 1.1, + "learning_rate": 0.000225925777331996, + "loss": 1.2622, + "theoretical_loss": 3.4105099209892544, + "tokens_seen": 2378156032 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002259097291875627, + "loss": 1.2244, + "theoretical_loss": 3.410502412244127, + "tokens_seen": 2378221568 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002258936810431294, + "loss": 1.3446, + "theoretical_loss": 3.410494903763848, + "tokens_seen": 2378287104 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002258776328986961, + "loss": 1.1652, + "theoretical_loss": 3.410487395548401, + "tokens_seen": 2378352640 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002258615847542628, + "loss": 1.2555, + "theoretical_loss": 3.410479887597769, + "tokens_seen": 2378418176 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002258455366098295, + "loss": 1.1472, + "theoretical_loss": 3.4104723799119356, + "tokens_seen": 2378483712 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002258294884653962, + "loss": 1.2272, + "theoretical_loss": 3.410464872490884, + "tokens_seen": 2378549248 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002258134403209629, + "loss": 1.2683, + "theoretical_loss": 3.4104573653345973, + "tokens_seen": 2378614784 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002257973921765296, + "loss": 1.263, + "theoretical_loss": 3.4104498584430596, + "tokens_seen": 2378680320 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002257813440320963, + "loss": 1.2283, + "theoretical_loss": 3.410442351816253, + "tokens_seen": 2378745856 + }, + { + "epoch": 1.1, + "learning_rate": 0.000225765295887663, + "loss": 1.1593, + "theoretical_loss": 3.410434845454162, + "tokens_seen": 2378811392 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002257492477432297, + "loss": 1.16, + "theoretical_loss": 3.4104273393567706, + "tokens_seen": 2378876928 + }, + { + "epoch": 1.1, + "objective/train/docs_used": 879258, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.229572057723999, + "objective/train/theoretical_loss": 3.41041983352406, + "objective/train/tokens_used": 2399402464, + "theoretical_loss": 3.41041983352406, + "tokens_seen": 2378942464 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002257331995987964, + "loss": 1.2296, + "theoretical_loss": 3.41041983352406, + "tokens_seen": 2378942464 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022571715145436308, + "loss": 1.2686, + "theoretical_loss": 3.4104123279560152, + "tokens_seen": 2379008000 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022570110330992978, + "loss": 1.1564, + "theoretical_loss": 3.4104048226526196, + "tokens_seen": 2379073536 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022568505516549648, + "loss": 1.1783, + "theoretical_loss": 3.410397317613856, + "tokens_seen": 2379139072 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022566900702106318, + "loss": 1.2322, + "theoretical_loss": 3.4103898128397074, + "tokens_seen": 2379204608 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022565295887662989, + "loss": 1.2985, + "theoretical_loss": 3.410382308330158, + "tokens_seen": 2379270144 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022563691073219659, + "loss": 1.1779, + "theoretical_loss": 3.4103748040851913, + "tokens_seen": 2379335680 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002256208625877633, + "loss": 1.1966, + "theoretical_loss": 3.41036730010479, + "tokens_seen": 2379401216 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022560481444333, + "loss": 1.1594, + "theoretical_loss": 3.4103597963889376, + "tokens_seen": 2379466752 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002255887662988967, + "loss": 1.2151, + "theoretical_loss": 3.4103522929376178, + "tokens_seen": 2379532288 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002255727181544634, + "loss": 1.2402, + "theoretical_loss": 3.4103447897508143, + "tokens_seen": 2379597824 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002255566700100301, + "loss": 1.1868, + "theoretical_loss": 3.4103372868285096, + "tokens_seen": 2379663360 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002255406218655968, + "loss": 1.2706, + "theoretical_loss": 3.4103297841706874, + "tokens_seen": 2379728896 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002255245737211635, + "loss": 1.2133, + "theoretical_loss": 3.4103222817773315, + "tokens_seen": 2379794432 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002255085255767302, + "loss": 1.2039, + "theoretical_loss": 3.410314779648425, + "tokens_seen": 2379859968 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002254924774322969, + "loss": 1.2873, + "theoretical_loss": 3.4103072777839514, + "tokens_seen": 2379925504 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002254764292878636, + "loss": 1.3457, + "theoretical_loss": 3.410299776183894, + "tokens_seen": 2379991040 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002254603811434303, + "loss": 1.2557, + "theoretical_loss": 3.410292274848236, + "tokens_seen": 2380056576 + }, + { + "epoch": 1.1, + "learning_rate": 0.000225444332998997, + "loss": 1.1576, + "theoretical_loss": 3.410284773776961, + "tokens_seen": 2380122112 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002254282848545637, + "loss": 1.2903, + "theoretical_loss": 3.4102772729700526, + "tokens_seen": 2380187648 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002254122367101304, + "loss": 1.221, + "theoretical_loss": 3.410269772427494, + "tokens_seen": 2380253184 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002253961885656971, + "loss": 1.2309, + "theoretical_loss": 3.410262272149269, + "tokens_seen": 2380318720 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002253801404212638, + "loss": 1.1627, + "theoretical_loss": 3.41025477213536, + "tokens_seen": 2380384256 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002253640922768305, + "loss": 1.1742, + "theoretical_loss": 3.4102472723857518, + "tokens_seen": 2380449792 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002253480441323972, + "loss": 1.3052, + "theoretical_loss": 3.410239772900426, + "tokens_seen": 2380515328 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022533199598796387, + "loss": 1.2163, + "theoretical_loss": 3.4102322736793678, + "tokens_seen": 2380580864 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022531594784353057, + "loss": 1.2206, + "theoretical_loss": 3.4102247747225602, + "tokens_seen": 2380646400 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022529989969909727, + "loss": 1.1755, + "theoretical_loss": 3.410217276029986, + "tokens_seen": 2380711936 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022528385155466397, + "loss": 1.2048, + "theoretical_loss": 3.4102097776016285, + "tokens_seen": 2380777472 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022526780341023067, + "loss": 1.2918, + "theoretical_loss": 3.410202279437472, + "tokens_seen": 2380843008 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022525175526579737, + "loss": 1.1942, + "theoretical_loss": 3.4101947815374993, + "tokens_seen": 2380908544 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022523570712136407, + "loss": 1.2343, + "theoretical_loss": 3.410187283901694, + "tokens_seen": 2380974080 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022521965897693083, + "loss": 1.3151, + "theoretical_loss": 3.410179786530039, + "tokens_seen": 2381039616 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022520361083249753, + "loss": 1.2508, + "theoretical_loss": 3.410172289422519, + "tokens_seen": 2381105152 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022518756268806423, + "loss": 1.1985, + "theoretical_loss": 3.410164792579116, + "tokens_seen": 2381170688 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022517151454363093, + "loss": 1.0965, + "theoretical_loss": 3.4101572959998148, + "tokens_seen": 2381236224 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022515546639919763, + "loss": 1.2314, + "theoretical_loss": 3.4101497996845973, + "tokens_seen": 2381301760 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022513941825476433, + "loss": 1.26, + "theoretical_loss": 3.410142303633448, + "tokens_seen": 2381367296 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022512337011033103, + "loss": 1.1862, + "theoretical_loss": 3.4101348078463505, + "tokens_seen": 2381432832 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022510732196589773, + "loss": 1.2074, + "theoretical_loss": 3.4101273123232874, + "tokens_seen": 2381498368 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002250912738214644, + "loss": 1.238, + "theoretical_loss": 3.410119817064243, + "tokens_seen": 2381563904 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002250752256770311, + "loss": 1.1761, + "theoretical_loss": 3.4101123220691996, + "tokens_seen": 2381629440 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002250591775325978, + "loss": 1.202, + "theoretical_loss": 3.4101048273381416, + "tokens_seen": 2381694976 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002250431293881645, + "loss": 1.2132, + "theoretical_loss": 3.410097332871052, + "tokens_seen": 2381760512 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002250270812437312, + "loss": 1.1564, + "theoretical_loss": 3.4100898386679144, + "tokens_seen": 2381826048 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002250110330992979, + "loss": 1.2521, + "theoretical_loss": 3.4100823447287123, + "tokens_seen": 2381891584 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002249949849548646, + "loss": 1.2933, + "theoretical_loss": 3.410074851053429, + "tokens_seen": 2381957120 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002249789368104313, + "loss": 1.2446, + "theoretical_loss": 3.410067357642048, + "tokens_seen": 2382022656 + }, + { + "epoch": 1.1, + "learning_rate": 0.000224962888665998, + "loss": 1.2548, + "theoretical_loss": 3.410059864494553, + "tokens_seen": 2382088192 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002249468405215647, + "loss": 1.2153, + "theoretical_loss": 3.410052371610927, + "tokens_seen": 2382153728 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022493079237713141, + "loss": 1.224, + "theoretical_loss": 3.4100448789911533, + "tokens_seen": 2382219264 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022491474423269811, + "loss": 1.1929, + "theoretical_loss": 3.4100373866352163, + "tokens_seen": 2382284800 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022489869608826482, + "loss": 1.2082, + "theoretical_loss": 3.4100298945430985, + "tokens_seen": 2382350336 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022488264794383152, + "loss": 1.227, + "theoretical_loss": 3.410022402714784, + "tokens_seen": 2382415872 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022486659979939822, + "loss": 1.2669, + "theoretical_loss": 3.4100149111502556, + "tokens_seen": 2382481408 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022485055165496492, + "loss": 1.2987, + "theoretical_loss": 3.410007419849497, + "tokens_seen": 2382546944 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022483450351053162, + "loss": 1.2237, + "theoretical_loss": 3.4099999288124923, + "tokens_seen": 2382612480 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022481845536609832, + "loss": 1.3367, + "theoretical_loss": 3.409992438039224, + "tokens_seen": 2382678016 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022480240722166502, + "loss": 1.2285, + "theoretical_loss": 3.409984947529676, + "tokens_seen": 2382743552 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022478635907723172, + "loss": 1.1494, + "theoretical_loss": 3.409977457283832, + "tokens_seen": 2382809088 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022477031093279842, + "loss": 1.282, + "theoretical_loss": 3.4099699673016755, + "tokens_seen": 2382874624 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022475426278836512, + "loss": 1.2833, + "theoretical_loss": 3.409962477583189, + "tokens_seen": 2382940160 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022473821464393182, + "loss": 1.2364, + "theoretical_loss": 3.409954988128357, + "tokens_seen": 2383005696 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022472216649949852, + "loss": 1.1876, + "theoretical_loss": 3.4099474989371625, + "tokens_seen": 2383071232 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002247061183550652, + "loss": 1.1763, + "theoretical_loss": 3.4099400100095894, + "tokens_seen": 2383136768 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002246900702106319, + "loss": 1.325, + "theoretical_loss": 3.4099325213456204, + "tokens_seen": 2383202304 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002246740220661986, + "loss": 1.166, + "theoretical_loss": 3.4099250329452397, + "tokens_seen": 2383267840 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002246579739217653, + "loss": 1.1924, + "theoretical_loss": 3.4099175448084305, + "tokens_seen": 2383333376 + }, + { + "epoch": 1.1, + "learning_rate": 0.000224641925777332, + "loss": 1.325, + "theoretical_loss": 3.4099100569351766, + "tokens_seen": 2383398912 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002246258776328987, + "loss": 1.1496, + "theoretical_loss": 3.4099025693254608, + "tokens_seen": 2383464448 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002246098294884654, + "loss": 1.2255, + "theoretical_loss": 3.4098950819792666, + "tokens_seen": 2383529984 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002245937813440321, + "loss": 1.2889, + "theoretical_loss": 3.4098875948965786, + "tokens_seen": 2383595520 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002245777331995988, + "loss": 1.1893, + "theoretical_loss": 3.409880108077379, + "tokens_seen": 2383661056 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002245616850551655, + "loss": 1.2117, + "theoretical_loss": 3.409872621521652, + "tokens_seen": 2383726592 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002245456369107322, + "loss": 1.2624, + "theoretical_loss": 3.4098651352293805, + "tokens_seen": 2383792128 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002245295887662989, + "loss": 1.1794, + "theoretical_loss": 3.409857649200549, + "tokens_seen": 2383857664 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002245135406218656, + "loss": 1.2483, + "theoretical_loss": 3.4098501634351397, + "tokens_seen": 2383923200 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002244974924774323, + "loss": 1.2645, + "theoretical_loss": 3.409842677933137, + "tokens_seen": 2383988736 + }, + { + "epoch": 1.1, + "learning_rate": 0.000224481444332999, + "loss": 1.207, + "theoretical_loss": 3.409835192694524, + "tokens_seen": 2384054272 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002244653961885657, + "loss": 1.1851, + "theoretical_loss": 3.4098277077192845, + "tokens_seen": 2384119808 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002244493480441324, + "loss": 1.2962, + "theoretical_loss": 3.4098202230074017, + "tokens_seen": 2384185344 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002244332998996991, + "loss": 1.1634, + "theoretical_loss": 3.409812738558859, + "tokens_seen": 2384250880 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002244172517552658, + "loss": 1.236, + "theoretical_loss": 3.409805254373641, + "tokens_seen": 2384316416 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002244012036108325, + "loss": 1.2455, + "theoretical_loss": 3.4097977704517293, + "tokens_seen": 2384381952 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002243851554663992, + "loss": 1.2283, + "theoretical_loss": 3.4097902867931085, + "tokens_seen": 2384447488 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002243691073219659, + "loss": 1.2183, + "theoretical_loss": 3.4097828033977624, + "tokens_seen": 2384513024 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002243530591775326, + "loss": 1.2553, + "theoretical_loss": 3.4097753202656738, + "tokens_seen": 2384578560 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002243370110330993, + "loss": 1.3177, + "theoretical_loss": 3.4097678373968265, + "tokens_seen": 2384644096 + }, + { + "epoch": 1.1, + "learning_rate": 0.000224320962888666, + "loss": 1.2046, + "theoretical_loss": 3.4097603547912043, + "tokens_seen": 2384709632 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022430491474423268, + "loss": 1.1977, + "theoretical_loss": 3.40975287244879, + "tokens_seen": 2384775168 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022428886659979939, + "loss": 1.0952, + "theoretical_loss": 3.4097453903695683, + "tokens_seen": 2384840704 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022427281845536609, + "loss": 1.3038, + "theoretical_loss": 3.409737908553521, + "tokens_seen": 2384906240 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002242567703109328, + "loss": 1.2146, + "theoretical_loss": 3.4097304270006332, + "tokens_seen": 2384971776 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002242407221664995, + "loss": 1.2321, + "theoretical_loss": 3.4097229457108877, + "tokens_seen": 2385037312 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002242246740220662, + "loss": 1.2341, + "theoretical_loss": 3.409715464684268, + "tokens_seen": 2385102848 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002242086258776329, + "loss": 1.2596, + "theoretical_loss": 3.4097079839207574, + "tokens_seen": 2385168384 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002241925777331996, + "loss": 1.3094, + "theoretical_loss": 3.40970050342034, + "tokens_seen": 2385233920 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002241765295887663, + "loss": 1.2539, + "theoretical_loss": 3.409693023182999, + "tokens_seen": 2385299456 + }, + { + "epoch": 1.1, + "learning_rate": 0.000224160481444333, + "loss": 1.3205, + "theoretical_loss": 3.4096855432087176, + "tokens_seen": 2385364992 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002241444332998997, + "loss": 1.2117, + "theoretical_loss": 3.40967806349748, + "tokens_seen": 2385430528 + }, + { + "epoch": 1.1, + "objective/train/docs_used": 881672, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.115523099899292, + "objective/train/theoretical_loss": 3.4096705840492696, + "objective/train/tokens_used": 2405956064, + "theoretical_loss": 3.4096705840492696, + "tokens_seen": 2385496064 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002241283851554664, + "loss": 1.1155, + "theoretical_loss": 3.4096705840492696, + "tokens_seen": 2385496064 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002241123370110331, + "loss": 1.2127, + "theoretical_loss": 3.4096631048640695, + "tokens_seen": 2385561600 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002240962888665998, + "loss": 1.2809, + "theoretical_loss": 3.4096556259418636, + "tokens_seen": 2385627136 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002240802407221665, + "loss": 1.1759, + "theoretical_loss": 3.4096481472826348, + "tokens_seen": 2385692672 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002240641925777332, + "loss": 1.1556, + "theoretical_loss": 3.4096406688863676, + "tokens_seen": 2385758208 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002240481444332999, + "loss": 1.1932, + "theoretical_loss": 3.409633190753045, + "tokens_seen": 2385823744 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002240320962888666, + "loss": 1.1773, + "theoretical_loss": 3.4096257128826504, + "tokens_seen": 2385889280 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022401604814443332, + "loss": 1.2624, + "theoretical_loss": 3.4096182352751674, + "tokens_seen": 2385954816 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022400000000000002, + "loss": 1.1715, + "theoretical_loss": 3.40961075793058, + "tokens_seen": 2386020352 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022398395185556672, + "loss": 1.2595, + "theoretical_loss": 3.409603280848871, + "tokens_seen": 2386085888 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022396790371113343, + "loss": 1.278, + "theoretical_loss": 3.4095958040300243, + "tokens_seen": 2386151424 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022395185556670013, + "loss": 1.2936, + "theoretical_loss": 3.4095883274740237, + "tokens_seen": 2386216960 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022393580742226683, + "loss": 1.1878, + "theoretical_loss": 3.409580851180852, + "tokens_seen": 2386282496 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022391975927783353, + "loss": 1.1896, + "theoretical_loss": 3.4095733751504937, + "tokens_seen": 2386348032 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022390371113340023, + "loss": 1.2365, + "theoretical_loss": 3.409565899382932, + "tokens_seen": 2386413568 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022388766298896693, + "loss": 1.2663, + "theoretical_loss": 3.40955842387815, + "tokens_seen": 2386479104 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022387161484453363, + "loss": 1.1767, + "theoretical_loss": 3.4095509486361317, + "tokens_seen": 2386544640 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022385556670010033, + "loss": 1.1997, + "theoretical_loss": 3.4095434736568606, + "tokens_seen": 2386610176 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022383951855566703, + "loss": 1.2164, + "theoretical_loss": 3.4095359989403202, + "tokens_seen": 2386675712 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022382347041123373, + "loss": 1.278, + "theoretical_loss": 3.4095285244864937, + "tokens_seen": 2386741248 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022380742226680043, + "loss": 1.2948, + "theoretical_loss": 3.409521050295365, + "tokens_seen": 2386806784 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022379137412236713, + "loss": 1.2044, + "theoretical_loss": 3.409513576366918, + "tokens_seen": 2386872320 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022377532597793383, + "loss": 1.2741, + "theoretical_loss": 3.409506102701136, + "tokens_seen": 2386937856 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022375927783350053, + "loss": 1.1753, + "theoretical_loss": 3.409498629298002, + "tokens_seen": 2387003392 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022374322968906723, + "loss": 1.2625, + "theoretical_loss": 3.4094911561575, + "tokens_seen": 2387068928 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022372718154463393, + "loss": 1.1659, + "theoretical_loss": 3.4094836832796136, + "tokens_seen": 2387134464 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022371113340020064, + "loss": 1.1745, + "theoretical_loss": 3.4094762106643266, + "tokens_seen": 2387200000 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022369508525576734, + "loss": 1.2589, + "theoretical_loss": 3.409468738311622, + "tokens_seen": 2387265536 + }, + { + "epoch": 1.1, + "learning_rate": 0.000223679037111334, + "loss": 1.2411, + "theoretical_loss": 3.409461266221484, + "tokens_seen": 2387331072 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002236629889669007, + "loss": 1.2563, + "theoretical_loss": 3.409453794393895, + "tokens_seen": 2387396608 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002236469408224674, + "loss": 1.1725, + "theoretical_loss": 3.40944632282884, + "tokens_seen": 2387462144 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002236308926780341, + "loss": 1.3004, + "theoretical_loss": 3.409438851526302, + "tokens_seen": 2387527680 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002236148445336008, + "loss": 1.1991, + "theoretical_loss": 3.4094313804862644, + "tokens_seen": 2387593216 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002235987963891675, + "loss": 1.0981, + "theoretical_loss": 3.4094239097087113, + "tokens_seen": 2387658752 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002235827482447342, + "loss": 1.2349, + "theoretical_loss": 3.4094164391936252, + "tokens_seen": 2387724288 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022356670010030091, + "loss": 1.1818, + "theoretical_loss": 3.4094089689409905, + "tokens_seen": 2387789824 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022355065195586761, + "loss": 1.2593, + "theoretical_loss": 3.409401498950791, + "tokens_seen": 2387855360 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022353460381143432, + "loss": 1.2973, + "theoretical_loss": 3.4093940292230096, + "tokens_seen": 2387920896 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022351855566700102, + "loss": 1.2861, + "theoretical_loss": 3.4093865597576305, + "tokens_seen": 2387986432 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022350250752256772, + "loss": 1.2027, + "theoretical_loss": 3.4093790905546366, + "tokens_seen": 2388051968 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022348645937813442, + "loss": 1.2945, + "theoretical_loss": 3.409371621614012, + "tokens_seen": 2388117504 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022347041123370112, + "loss": 1.2954, + "theoretical_loss": 3.40936415293574, + "tokens_seen": 2388183040 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022345436308926782, + "loss": 1.2355, + "theoretical_loss": 3.4093566845198047, + "tokens_seen": 2388248576 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022343831494483452, + "loss": 1.277, + "theoretical_loss": 3.4093492163661887, + "tokens_seen": 2388314112 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022342226680040122, + "loss": 1.2705, + "theoretical_loss": 3.4093417484748767, + "tokens_seen": 2388379648 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022340621865596792, + "loss": 1.2177, + "theoretical_loss": 3.4093342808458518, + "tokens_seen": 2388445184 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022339017051153462, + "loss": 1.3131, + "theoretical_loss": 3.409326813479097, + "tokens_seen": 2388510720 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022337412236710132, + "loss": 1.2571, + "theoretical_loss": 3.409319346374597, + "tokens_seen": 2388576256 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022335807422266802, + "loss": 1.1856, + "theoretical_loss": 3.4093118795323347, + "tokens_seen": 2388641792 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022334202607823472, + "loss": 1.2968, + "theoretical_loss": 3.409304412952294, + "tokens_seen": 2388707328 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022332597793380142, + "loss": 1.2411, + "theoretical_loss": 3.4092969466344583, + "tokens_seen": 2388772864 + }, + { + "epoch": 1.1, + "learning_rate": 0.00022330992978936812, + "loss": 1.283, + "theoretical_loss": 3.4092894805788108, + "tokens_seen": 2388838400 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002232938816449348, + "loss": 1.3235, + "theoretical_loss": 3.4092820147853358, + "tokens_seen": 2388903936 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002232778335005015, + "loss": 1.2206, + "theoretical_loss": 3.409274549254017, + "tokens_seen": 2388969472 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002232617853560682, + "loss": 1.1836, + "theoretical_loss": 3.4092670839848376, + "tokens_seen": 2389035008 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002232457372116349, + "loss": 1.1782, + "theoretical_loss": 3.4092596189777806, + "tokens_seen": 2389100544 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002232296890672016, + "loss": 1.1156, + "theoretical_loss": 3.409252154232831, + "tokens_seen": 2389166080 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002232136409227683, + "loss": 1.218, + "theoretical_loss": 3.4092446897499715, + "tokens_seen": 2389231616 + }, + { + "epoch": 1.1, + "learning_rate": 0.000223197592778335, + "loss": 1.2084, + "theoretical_loss": 3.4092372255291856, + "tokens_seen": 2389297152 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002231815446339017, + "loss": 1.1912, + "theoretical_loss": 3.4092297615704577, + "tokens_seen": 2389362688 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002231654964894684, + "loss": 1.2176, + "theoretical_loss": 3.4092222978737707, + "tokens_seen": 2389428224 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002231494483450351, + "loss": 1.2653, + "theoretical_loss": 3.4092148344391084, + "tokens_seen": 2389493760 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002231334002006018, + "loss": 1.2628, + "theoretical_loss": 3.4092073712664543, + "tokens_seen": 2389559296 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002231173520561685, + "loss": 1.2013, + "theoretical_loss": 3.409199908355792, + "tokens_seen": 2389624832 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002231013039117352, + "loss": 1.2338, + "theoretical_loss": 3.4091924457071054, + "tokens_seen": 2389690368 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002230852557673019, + "loss": 1.2505, + "theoretical_loss": 3.4091849833203782, + "tokens_seen": 2389755904 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002230692076228686, + "loss": 1.2267, + "theoretical_loss": 3.409177521195594, + "tokens_seen": 2389821440 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002230531594784353, + "loss": 1.2534, + "theoretical_loss": 3.4091700593327356, + "tokens_seen": 2389886976 + }, + { + "epoch": 1.1, + "learning_rate": 0.000223037111334002, + "loss": 1.284, + "theoretical_loss": 3.409162597731788, + "tokens_seen": 2389952512 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002230210631895687, + "loss": 1.1854, + "theoretical_loss": 3.4091551363927337, + "tokens_seen": 2390018048 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002230050150451354, + "loss": 1.1839, + "theoretical_loss": 3.4091476753155563, + "tokens_seen": 2390083584 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002229889669007021, + "loss": 1.1988, + "theoretical_loss": 3.4091402145002405, + "tokens_seen": 2390149120 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002229729187562688, + "loss": 1.2332, + "theoretical_loss": 3.4091327539467686, + "tokens_seen": 2390214656 + }, + { + "epoch": 1.1, + "learning_rate": 0.0002229568706118355, + "loss": 1.2654, + "theoretical_loss": 3.4091252936551255, + "tokens_seen": 2390280192 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002229408224674022, + "loss": 1.1143, + "theoretical_loss": 3.4091178336252943, + "tokens_seen": 2390345728 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002229247743229689, + "loss": 1.2201, + "theoretical_loss": 3.409110373857258, + "tokens_seen": 2390411264 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022290872617853559, + "loss": 1.2705, + "theoretical_loss": 3.409102914351001, + "tokens_seen": 2390476800 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022289267803410229, + "loss": 1.2158, + "theoretical_loss": 3.409095455106507, + "tokens_seen": 2390542336 + }, + { + "epoch": 1.11, + "learning_rate": 0.000222876629889669, + "loss": 1.2619, + "theoretical_loss": 3.4090879961237595, + "tokens_seen": 2390607872 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002228605817452357, + "loss": 1.1877, + "theoretical_loss": 3.4090805374027418, + "tokens_seen": 2390673408 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002228445336008024, + "loss": 1.137, + "theoretical_loss": 3.409073078943438, + "tokens_seen": 2390738944 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002228284854563691, + "loss": 1.2154, + "theoretical_loss": 3.4090656207458307, + "tokens_seen": 2390804480 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022281243731193584, + "loss": 1.2589, + "theoretical_loss": 3.409058162809905, + "tokens_seen": 2390870016 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022279638916750254, + "loss": 1.1432, + "theoretical_loss": 3.409050705135644, + "tokens_seen": 2390935552 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022278034102306924, + "loss": 1.2787, + "theoretical_loss": 3.409043247723031, + "tokens_seen": 2391001088 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022276429287863595, + "loss": 1.1814, + "theoretical_loss": 3.40903579057205, + "tokens_seen": 2391066624 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022274824473420265, + "loss": 1.1841, + "theoretical_loss": 3.4090283336826843, + "tokens_seen": 2391132160 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022273219658976935, + "loss": 1.1969, + "theoretical_loss": 3.4090208770549175, + "tokens_seen": 2391197696 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022271614844533605, + "loss": 1.1694, + "theoretical_loss": 3.4090134206887344, + "tokens_seen": 2391263232 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022270010030090275, + "loss": 1.2199, + "theoretical_loss": 3.409005964584117, + "tokens_seen": 2391328768 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022268405215646945, + "loss": 1.2503, + "theoretical_loss": 3.4089985087410506, + "tokens_seen": 2391394304 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022266800401203612, + "loss": 1.17, + "theoretical_loss": 3.408991053159517, + "tokens_seen": 2391459840 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022265195586760282, + "loss": 1.3101, + "theoretical_loss": 3.4089835978395016, + "tokens_seen": 2391525376 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022263590772316952, + "loss": 1.2763, + "theoretical_loss": 3.4089761427809875, + "tokens_seen": 2391590912 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022261985957873622, + "loss": 1.175, + "theoretical_loss": 3.4089686879839576, + "tokens_seen": 2391656448 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022260381143430292, + "loss": 1.1836, + "theoretical_loss": 3.4089612334483967, + "tokens_seen": 2391721984 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022258776328986963, + "loss": 1.2935, + "theoretical_loss": 3.4089537791742877, + "tokens_seen": 2391787520 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022257171514543633, + "loss": 1.2035, + "theoretical_loss": 3.4089463251616143, + "tokens_seen": 2391853056 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022255566700100303, + "loss": 1.2342, + "theoretical_loss": 3.4089388714103603, + "tokens_seen": 2391918592 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022253961885656973, + "loss": 1.2171, + "theoretical_loss": 3.4089314179205097, + "tokens_seen": 2391984128 + }, + { + "epoch": 1.11, + "objective/train/docs_used": 883997, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1598790884017944, + "objective/train/theoretical_loss": 3.408923964692046, + "objective/train/tokens_used": 2412509664, + "theoretical_loss": 3.408923964692046, + "tokens_seen": 2392049664 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022252357071213643, + "loss": 1.1599, + "theoretical_loss": 3.408923964692046, + "tokens_seen": 2392049664 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022250752256770313, + "loss": 1.2526, + "theoretical_loss": 3.4089165117249527, + "tokens_seen": 2392115200 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022249147442326983, + "loss": 1.1929, + "theoretical_loss": 3.4089090590192135, + "tokens_seen": 2392180736 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022247542627883653, + "loss": 1.1838, + "theoretical_loss": 3.408901606574812, + "tokens_seen": 2392246272 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022245937813440323, + "loss": 1.2085, + "theoretical_loss": 3.4088941543917324, + "tokens_seen": 2392311808 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022244332998996993, + "loss": 1.2437, + "theoretical_loss": 3.4088867024699576, + "tokens_seen": 2392377344 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022242728184553663, + "loss": 1.1463, + "theoretical_loss": 3.4088792508094716, + "tokens_seen": 2392442880 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022241123370110333, + "loss": 1.1648, + "theoretical_loss": 3.4088717994102584, + "tokens_seen": 2392508416 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022239518555667003, + "loss": 1.2038, + "theoretical_loss": 3.408864348272301, + "tokens_seen": 2392573952 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022237913741223673, + "loss": 1.2723, + "theoretical_loss": 3.4088568973955837, + "tokens_seen": 2392639488 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022236308926780343, + "loss": 1.2038, + "theoretical_loss": 3.4088494467800903, + "tokens_seen": 2392705024 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022234704112337013, + "loss": 1.2466, + "theoretical_loss": 3.408841996425804, + "tokens_seen": 2392770560 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022233099297893684, + "loss": 1.2167, + "theoretical_loss": 3.4088345463327085, + "tokens_seen": 2392836096 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022231494483450354, + "loss": 1.2313, + "theoretical_loss": 3.408827096500788, + "tokens_seen": 2392901632 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022229889669007024, + "loss": 1.1594, + "theoretical_loss": 3.4088196469300254, + "tokens_seen": 2392967168 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002222828485456369, + "loss": 1.2032, + "theoretical_loss": 3.4088121976204055, + "tokens_seen": 2393032704 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002222668004012036, + "loss": 1.1888, + "theoretical_loss": 3.4088047485719106, + "tokens_seen": 2393098240 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002222507522567703, + "loss": 1.2021, + "theoretical_loss": 3.408797299784526, + "tokens_seen": 2393163776 + }, + { + "epoch": 1.11, + "learning_rate": 0.000222234704112337, + "loss": 1.1477, + "theoretical_loss": 3.4087898512582337, + "tokens_seen": 2393229312 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002222186559679037, + "loss": 1.2002, + "theoretical_loss": 3.4087824029930185, + "tokens_seen": 2393294848 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002222026078234704, + "loss": 1.1933, + "theoretical_loss": 3.4087749549888637, + "tokens_seen": 2393360384 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022218655967903711, + "loss": 1.1673, + "theoretical_loss": 3.4087675072457535, + "tokens_seen": 2393425920 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022217051153460381, + "loss": 1.2009, + "theoretical_loss": 3.408760059763671, + "tokens_seen": 2393491456 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022215446339017052, + "loss": 1.2914, + "theoretical_loss": 3.4087526125426004, + "tokens_seen": 2393556992 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022213841524573722, + "loss": 1.2768, + "theoretical_loss": 3.4087451655825247, + "tokens_seen": 2393622528 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022212236710130392, + "loss": 1.344, + "theoretical_loss": 3.4087377188834282, + "tokens_seen": 2393688064 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022210631895687062, + "loss": 1.246, + "theoretical_loss": 3.4087302724452946, + "tokens_seen": 2393753600 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022209027081243732, + "loss": 1.2412, + "theoretical_loss": 3.4087228262681073, + "tokens_seen": 2393819136 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022207422266800402, + "loss": 1.2736, + "theoretical_loss": 3.4087153803518504, + "tokens_seen": 2393884672 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022205817452357072, + "loss": 1.2316, + "theoretical_loss": 3.4087079346965075, + "tokens_seen": 2393950208 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022204212637913742, + "loss": 1.1903, + "theoretical_loss": 3.4087004893020616, + "tokens_seen": 2394015744 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022202607823470412, + "loss": 1.1536, + "theoretical_loss": 3.4086930441684977, + "tokens_seen": 2394081280 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022201003009027082, + "loss": 1.3118, + "theoretical_loss": 3.4086855992957985, + "tokens_seen": 2394146816 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022199398194583752, + "loss": 1.2525, + "theoretical_loss": 3.4086781546839484, + "tokens_seen": 2394212352 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022197793380140422, + "loss": 1.1942, + "theoretical_loss": 3.4086707103329306, + "tokens_seen": 2394277888 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022196188565697092, + "loss": 1.103, + "theoretical_loss": 3.408663266242729, + "tokens_seen": 2394343424 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022194583751253762, + "loss": 1.2507, + "theoretical_loss": 3.408655822413327, + "tokens_seen": 2394408960 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022192978936810432, + "loss": 1.2238, + "theoretical_loss": 3.408648378844709, + "tokens_seen": 2394474496 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022191374122367102, + "loss": 1.2496, + "theoretical_loss": 3.4086409355368583, + "tokens_seen": 2394540032 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002218976930792377, + "loss": 1.1548, + "theoretical_loss": 3.4086334924897588, + "tokens_seen": 2394605568 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002218816449348044, + "loss": 1.1683, + "theoretical_loss": 3.4086260497033942, + "tokens_seen": 2394671104 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002218655967903711, + "loss": 1.2492, + "theoretical_loss": 3.408618607177748, + "tokens_seen": 2394736640 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002218495486459378, + "loss": 1.3303, + "theoretical_loss": 3.4086111649128044, + "tokens_seen": 2394802176 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002218335005015045, + "loss": 1.272, + "theoretical_loss": 3.4086037229085466, + "tokens_seen": 2394867712 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002218174523570712, + "loss": 1.2297, + "theoretical_loss": 3.408596281164958, + "tokens_seen": 2394933248 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002218014042126379, + "loss": 1.2789, + "theoretical_loss": 3.408588839682024, + "tokens_seen": 2394998784 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002217853560682046, + "loss": 1.1894, + "theoretical_loss": 3.4085813984597264, + "tokens_seen": 2395064320 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002217693079237713, + "loss": 1.1833, + "theoretical_loss": 3.4085739574980503, + "tokens_seen": 2395129856 + }, + { + "epoch": 1.11, + "learning_rate": 0.000221753259779338, + "loss": 1.2011, + "theoretical_loss": 3.408566516796979, + "tokens_seen": 2395195392 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002217372116349047, + "loss": 1.255, + "theoretical_loss": 3.4085590763564957, + "tokens_seen": 2395260928 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002217211634904714, + "loss": 1.2195, + "theoretical_loss": 3.4085516361765853, + "tokens_seen": 2395326464 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002217051153460381, + "loss": 1.1857, + "theoretical_loss": 3.40854419625723, + "tokens_seen": 2395392000 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002216890672016048, + "loss": 1.2394, + "theoretical_loss": 3.408536756598415, + "tokens_seen": 2395457536 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002216730190571715, + "loss": 1.154, + "theoretical_loss": 3.408529317200123, + "tokens_seen": 2395523072 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002216569709127382, + "loss": 1.1515, + "theoretical_loss": 3.4085218780623388, + "tokens_seen": 2395588608 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002216409227683049, + "loss": 1.2136, + "theoretical_loss": 3.4085144391850455, + "tokens_seen": 2395654144 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002216248746238716, + "loss": 1.2769, + "theoretical_loss": 3.4085070005682265, + "tokens_seen": 2395719680 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022160882647943834, + "loss": 1.1785, + "theoretical_loss": 3.4084995622118663, + "tokens_seen": 2395785216 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022159277833500504, + "loss": 1.27, + "theoretical_loss": 3.408492124115948, + "tokens_seen": 2395850752 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022157673019057174, + "loss": 1.1503, + "theoretical_loss": 3.408484686280456, + "tokens_seen": 2395916288 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022156068204613844, + "loss": 1.2147, + "theoretical_loss": 3.4084772487053736, + "tokens_seen": 2395981824 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022154463390170514, + "loss": 1.1646, + "theoretical_loss": 3.4084698113906846, + "tokens_seen": 2396047360 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022152858575727184, + "loss": 1.1581, + "theoretical_loss": 3.408462374336373, + "tokens_seen": 2396112896 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022151253761283854, + "loss": 1.3325, + "theoretical_loss": 3.4084549375424227, + "tokens_seen": 2396178432 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022149648946840524, + "loss": 1.2818, + "theoretical_loss": 3.408447501008817, + "tokens_seen": 2396243968 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022148044132397194, + "loss": 1.2425, + "theoretical_loss": 3.4084400647355393, + "tokens_seen": 2396309504 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022146439317953864, + "loss": 1.2871, + "theoretical_loss": 3.408432628722575, + "tokens_seen": 2396375040 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022144834503510534, + "loss": 1.24, + "theoretical_loss": 3.4084251929699056, + "tokens_seen": 2396440576 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022143229689067204, + "loss": 1.2107, + "theoretical_loss": 3.4084177574775167, + "tokens_seen": 2396506112 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022141624874623874, + "loss": 1.1879, + "theoretical_loss": 3.4084103222453916, + "tokens_seen": 2396571648 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022140020060180545, + "loss": 1.2024, + "theoretical_loss": 3.4084028872735135, + "tokens_seen": 2396637184 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022138415245737215, + "loss": 1.1644, + "theoretical_loss": 3.4083954525618667, + "tokens_seen": 2396702720 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022136810431293885, + "loss": 1.2668, + "theoretical_loss": 3.4083880181104353, + "tokens_seen": 2396768256 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022135205616850555, + "loss": 1.1923, + "theoretical_loss": 3.408380583919202, + "tokens_seen": 2396833792 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022133600802407225, + "loss": 1.2271, + "theoretical_loss": 3.4083731499881513, + "tokens_seen": 2396899328 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022131995987963895, + "loss": 1.1549, + "theoretical_loss": 3.408365716317267, + "tokens_seen": 2396964864 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022130391173520565, + "loss": 1.227, + "theoretical_loss": 3.408358282906533, + "tokens_seen": 2397030400 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022128786359077235, + "loss": 1.2086, + "theoretical_loss": 3.4083508497559327, + "tokens_seen": 2397095936 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022127181544633905, + "loss": 1.1722, + "theoretical_loss": 3.40834341686545, + "tokens_seen": 2397161472 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022125576730190572, + "loss": 1.2205, + "theoretical_loss": 3.408335984235069, + "tokens_seen": 2397227008 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022123971915747242, + "loss": 1.2846, + "theoretical_loss": 3.408328551864773, + "tokens_seen": 2397292544 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022122367101303913, + "loss": 1.1943, + "theoretical_loss": 3.408321119754546, + "tokens_seen": 2397358080 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022120762286860583, + "loss": 1.1736, + "theoretical_loss": 3.4083136879043723, + "tokens_seen": 2397423616 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022119157472417253, + "loss": 1.1982, + "theoretical_loss": 3.4083062563142343, + "tokens_seen": 2397489152 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022117552657973923, + "loss": 1.2486, + "theoretical_loss": 3.4082988249841173, + "tokens_seen": 2397554688 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022115947843530593, + "loss": 1.1634, + "theoretical_loss": 3.4082913939140043, + "tokens_seen": 2397620224 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022114343029087263, + "loss": 1.3056, + "theoretical_loss": 3.4082839631038793, + "tokens_seen": 2397685760 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022112738214643933, + "loss": 1.1633, + "theoretical_loss": 3.408276532553726, + "tokens_seen": 2397751296 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022111133400200603, + "loss": 1.3306, + "theoretical_loss": 3.408269102263528, + "tokens_seen": 2397816832 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022109528585757273, + "loss": 1.172, + "theoretical_loss": 3.40826167223327, + "tokens_seen": 2397882368 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022107923771313943, + "loss": 1.2366, + "theoretical_loss": 3.408254242462935, + "tokens_seen": 2397947904 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022106318956870613, + "loss": 1.1817, + "theoretical_loss": 3.4082468129525068, + "tokens_seen": 2398013440 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022104714142427283, + "loss": 1.2336, + "theoretical_loss": 3.4082393837019693, + "tokens_seen": 2398078976 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022103109327983953, + "loss": 1.2668, + "theoretical_loss": 3.4082319547113067, + "tokens_seen": 2398144512 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022101504513540623, + "loss": 1.1848, + "theoretical_loss": 3.4082245259805024, + "tokens_seen": 2398210048 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022099899699097293, + "loss": 1.2209, + "theoretical_loss": 3.4082170975095405, + "tokens_seen": 2398275584 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022098294884653963, + "loss": 1.177, + "theoretical_loss": 3.408209669298404, + "tokens_seen": 2398341120 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022096690070210634, + "loss": 1.1911, + "theoretical_loss": 3.408202241347078, + "tokens_seen": 2398406656 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022095085255767304, + "loss": 1.2091, + "theoretical_loss": 3.4081948136555456, + "tokens_seen": 2398472192 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022093480441323974, + "loss": 1.2999, + "theoretical_loss": 3.40818738622379, + "tokens_seen": 2398537728 + }, + { + "epoch": 1.11, + "objective/train/docs_used": 886405, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1905288696289062, + "objective/train/theoretical_loss": 3.4081799590517963, + "objective/train/tokens_used": 2419063264, + "theoretical_loss": 3.4081799590517963, + "tokens_seen": 2398603264 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022091875626880644, + "loss": 1.1905, + "theoretical_loss": 3.4081799590517963, + "tokens_seen": 2398603264 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022090270812437314, + "loss": 1.2538, + "theoretical_loss": 3.4081725321395475, + "tokens_seen": 2398668800 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022088665997993984, + "loss": 1.2308, + "theoretical_loss": 3.4081651054870274, + "tokens_seen": 2398734336 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002208706118355065, + "loss": 1.2447, + "theoretical_loss": 3.408157679094221, + "tokens_seen": 2398799872 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002208545636910732, + "loss": 1.2228, + "theoretical_loss": 3.4081502529611103, + "tokens_seen": 2398865408 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002208385155466399, + "loss": 1.2419, + "theoretical_loss": 3.40814282708768, + "tokens_seen": 2398930944 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022082246740220661, + "loss": 1.1993, + "theoretical_loss": 3.4081354014739142, + "tokens_seen": 2398996480 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022080641925777331, + "loss": 1.2821, + "theoretical_loss": 3.408127976119796, + "tokens_seen": 2399062016 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022079037111334002, + "loss": 1.1597, + "theoretical_loss": 3.40812055102531, + "tokens_seen": 2399127552 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022077432296890672, + "loss": 1.2229, + "theoretical_loss": 3.40811312619044, + "tokens_seen": 2399193088 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022075827482447342, + "loss": 1.1883, + "theoretical_loss": 3.408105701615169, + "tokens_seen": 2399258624 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022074222668004012, + "loss": 1.1623, + "theoretical_loss": 3.408098277299482, + "tokens_seen": 2399324160 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022072617853560682, + "loss": 1.1674, + "theoretical_loss": 3.408090853243362, + "tokens_seen": 2399389696 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022071013039117352, + "loss": 1.2007, + "theoretical_loss": 3.4080834294467923, + "tokens_seen": 2399455232 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022069408224674022, + "loss": 1.1179, + "theoretical_loss": 3.4080760059097583, + "tokens_seen": 2399520768 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022067803410230692, + "loss": 1.1664, + "theoretical_loss": 3.4080685826322425, + "tokens_seen": 2399586304 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022066198595787362, + "loss": 1.2699, + "theoretical_loss": 3.4080611596142294, + "tokens_seen": 2399651840 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022064593781344032, + "loss": 1.2551, + "theoretical_loss": 3.408053736855703, + "tokens_seen": 2399717376 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022062988966900702, + "loss": 1.2372, + "theoretical_loss": 3.408046314356646, + "tokens_seen": 2399782912 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022061384152457372, + "loss": 1.2061, + "theoretical_loss": 3.408038892117044, + "tokens_seen": 2399848448 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022059779338014042, + "loss": 1.2479, + "theoretical_loss": 3.40803147013688, + "tokens_seen": 2399913984 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022058174523570712, + "loss": 1.2655, + "theoretical_loss": 3.408024048416137, + "tokens_seen": 2399979520 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022056569709127382, + "loss": 1.1893, + "theoretical_loss": 3.4080166269548, + "tokens_seen": 2400045056 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022054964894684052, + "loss": 1.2251, + "theoretical_loss": 3.4080092057528524, + "tokens_seen": 2400110592 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022053360080240723, + "loss": 1.1553, + "theoretical_loss": 3.4080017848102786, + "tokens_seen": 2400176128 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022051755265797393, + "loss": 1.2037, + "theoretical_loss": 3.4079943641270614, + "tokens_seen": 2400241664 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022050150451354063, + "loss": 1.2355, + "theoretical_loss": 3.407986943703185, + "tokens_seen": 2400307200 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002204854563691073, + "loss": 1.3029, + "theoretical_loss": 3.4079795235386343, + "tokens_seen": 2400372736 + }, + { + "epoch": 1.11, + "learning_rate": 0.000220469408224674, + "loss": 1.0468, + "theoretical_loss": 3.4079721036333916, + "tokens_seen": 2400438272 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002204533600802407, + "loss": 1.386, + "theoretical_loss": 3.407964683987442, + "tokens_seen": 2400503808 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002204373119358074, + "loss": 1.2218, + "theoretical_loss": 3.4079572646007685, + "tokens_seen": 2400569344 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002204212637913741, + "loss": 1.2193, + "theoretical_loss": 3.4079498454733557, + "tokens_seen": 2400634880 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022040521564694086, + "loss": 1.2267, + "theoretical_loss": 3.407942426605187, + "tokens_seen": 2400700416 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022038916750250756, + "loss": 1.1979, + "theoretical_loss": 3.4079350079962465, + "tokens_seen": 2400765952 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022037311935807426, + "loss": 1.2352, + "theoretical_loss": 3.4079275896465173, + "tokens_seen": 2400831488 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022035707121364096, + "loss": 1.2246, + "theoretical_loss": 3.4079201715559844, + "tokens_seen": 2400897024 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022034102306920766, + "loss": 1.2327, + "theoretical_loss": 3.407912753724631, + "tokens_seen": 2400962560 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022032497492477436, + "loss": 1.2668, + "theoretical_loss": 3.4079053361524414, + "tokens_seen": 2401028096 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022030892678034106, + "loss": 1.2349, + "theoretical_loss": 3.4078979188393985, + "tokens_seen": 2401093632 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022029287863590776, + "loss": 1.3788, + "theoretical_loss": 3.4078905017854875, + "tokens_seen": 2401159168 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022027683049147446, + "loss": 1.2392, + "theoretical_loss": 3.407883084990692, + "tokens_seen": 2401224704 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022026078234704116, + "loss": 1.2339, + "theoretical_loss": 3.4078756684549947, + "tokens_seen": 2401290240 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022024473420260784, + "loss": 1.2978, + "theoretical_loss": 3.4078682521783805, + "tokens_seen": 2401355776 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022022868605817454, + "loss": 1.3029, + "theoretical_loss": 3.4078608361608334, + "tokens_seen": 2401421312 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022021263791374124, + "loss": 1.2109, + "theoretical_loss": 3.4078534204023363, + "tokens_seen": 2401486848 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022019658976930794, + "loss": 1.1863, + "theoretical_loss": 3.4078460049028743, + "tokens_seen": 2401552384 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022018054162487464, + "loss": 1.2254, + "theoretical_loss": 3.407838589662431, + "tokens_seen": 2401617920 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022016449348044134, + "loss": 1.2022, + "theoretical_loss": 3.4078311746809895, + "tokens_seen": 2401683456 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022014844533600804, + "loss": 1.1954, + "theoretical_loss": 3.407823759958534, + "tokens_seen": 2401748992 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022013239719157474, + "loss": 1.2264, + "theoretical_loss": 3.407816345495049, + "tokens_seen": 2401814528 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022011634904714144, + "loss": 1.195, + "theoretical_loss": 3.4078089312905178, + "tokens_seen": 2401880064 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022010030090270814, + "loss": 1.2592, + "theoretical_loss": 3.4078015173449243, + "tokens_seen": 2401945600 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022008425275827484, + "loss": 1.2189, + "theoretical_loss": 3.407794103658253, + "tokens_seen": 2402011136 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022006820461384154, + "loss": 1.2373, + "theoretical_loss": 3.407786690230487, + "tokens_seen": 2402076672 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022005215646940824, + "loss": 1.3611, + "theoretical_loss": 3.4077792770616107, + "tokens_seen": 2402142208 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022003610832497494, + "loss": 1.1972, + "theoretical_loss": 3.407771864151608, + "tokens_seen": 2402207744 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022002006018054165, + "loss": 1.2068, + "theoretical_loss": 3.4077644515004626, + "tokens_seen": 2402273280 + }, + { + "epoch": 1.11, + "learning_rate": 0.00022000401203610835, + "loss": 1.2181, + "theoretical_loss": 3.407757039108158, + "tokens_seen": 2402338816 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021998796389167505, + "loss": 1.2199, + "theoretical_loss": 3.407749626974679, + "tokens_seen": 2402404352 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021997191574724175, + "loss": 1.2307, + "theoretical_loss": 3.4077422151000087, + "tokens_seen": 2402469888 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021995586760280845, + "loss": 1.2605, + "theoretical_loss": 3.4077348034841317, + "tokens_seen": 2402535424 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021993981945837515, + "loss": 1.2637, + "theoretical_loss": 3.4077273921270312, + "tokens_seen": 2402600960 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021992377131394185, + "loss": 1.2793, + "theoretical_loss": 3.407719981028692, + "tokens_seen": 2402666496 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021990772316950855, + "loss": 1.2054, + "theoretical_loss": 3.407712570189097, + "tokens_seen": 2402732032 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021989167502507525, + "loss": 1.2938, + "theoretical_loss": 3.4077051596082306, + "tokens_seen": 2402797568 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021987562688064195, + "loss": 1.2532, + "theoretical_loss": 3.4076977492860765, + "tokens_seen": 2402863104 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021985957873620862, + "loss": 1.1423, + "theoretical_loss": 3.4076903392226194, + "tokens_seen": 2402928640 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021984353059177533, + "loss": 1.2771, + "theoretical_loss": 3.407682929417842, + "tokens_seen": 2402994176 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021982748244734203, + "loss": 1.2216, + "theoretical_loss": 3.407675519871729, + "tokens_seen": 2403059712 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021981143430290873, + "loss": 1.3827, + "theoretical_loss": 3.4076681105842646, + "tokens_seen": 2403125248 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021979538615847543, + "loss": 1.2126, + "theoretical_loss": 3.407660701555432, + "tokens_seen": 2403190784 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021977933801404213, + "loss": 1.1768, + "theoretical_loss": 3.407653292785215, + "tokens_seen": 2403256320 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021976328986960883, + "loss": 1.235, + "theoretical_loss": 3.407645884273598, + "tokens_seen": 2403321856 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021974724172517553, + "loss": 1.2169, + "theoretical_loss": 3.407638476020565, + "tokens_seen": 2403387392 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021973119358074223, + "loss": 1.2353, + "theoretical_loss": 3.4076310680260997, + "tokens_seen": 2403452928 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021971514543630893, + "loss": 1.1311, + "theoretical_loss": 3.407623660290186, + "tokens_seen": 2403518464 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021969909729187563, + "loss": 1.3329, + "theoretical_loss": 3.4076162528128084, + "tokens_seen": 2403584000 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021968304914744233, + "loss": 1.1778, + "theoretical_loss": 3.4076088455939497, + "tokens_seen": 2403649536 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021966700100300903, + "loss": 1.165, + "theoretical_loss": 3.407601438633595, + "tokens_seen": 2403715072 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021965095285857573, + "loss": 1.21, + "theoretical_loss": 3.407594031931727, + "tokens_seen": 2403780608 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021963490471414243, + "loss": 1.2732, + "theoretical_loss": 3.4075866254883307, + "tokens_seen": 2403846144 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021961885656970913, + "loss": 1.2355, + "theoretical_loss": 3.40757921930339, + "tokens_seen": 2403911680 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021960280842527583, + "loss": 1.1668, + "theoretical_loss": 3.407571813376888, + "tokens_seen": 2403977216 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021958676028084254, + "loss": 1.2433, + "theoretical_loss": 3.407564407708809, + "tokens_seen": 2404042752 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021957071213640924, + "loss": 1.2857, + "theoretical_loss": 3.4075570022991375, + "tokens_seen": 2404108288 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021955466399197594, + "loss": 1.2295, + "theoretical_loss": 3.407549597147857, + "tokens_seen": 2404173824 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021953861584754264, + "loss": 1.337, + "theoretical_loss": 3.407542192254951, + "tokens_seen": 2404239360 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021952256770310934, + "loss": 1.2627, + "theoretical_loss": 3.4075347876204045, + "tokens_seen": 2404304896 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021950651955867604, + "loss": 1.2343, + "theoretical_loss": 3.4075273832442003, + "tokens_seen": 2404370432 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021949047141424274, + "loss": 1.2017, + "theoretical_loss": 3.407519979126323, + "tokens_seen": 2404435968 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002194744232698094, + "loss": 1.2909, + "theoretical_loss": 3.4075125752667565, + "tokens_seen": 2404501504 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021945837512537611, + "loss": 1.1975, + "theoretical_loss": 3.407505171665485, + "tokens_seen": 2404567040 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021944232698094281, + "loss": 1.2654, + "theoretical_loss": 3.4074977683224916, + "tokens_seen": 2404632576 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021942627883650951, + "loss": 1.1685, + "theoretical_loss": 3.407490365237761, + "tokens_seen": 2404698112 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021941023069207622, + "loss": 1.2605, + "theoretical_loss": 3.407482962411277, + "tokens_seen": 2404763648 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021939418254764292, + "loss": 1.2564, + "theoretical_loss": 3.4074755598430233, + "tokens_seen": 2404829184 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021937813440320962, + "loss": 1.2995, + "theoretical_loss": 3.4074681575329837, + "tokens_seen": 2404894720 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021936208625877632, + "loss": 1.183, + "theoretical_loss": 3.407460755481143, + "tokens_seen": 2404960256 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021934603811434302, + "loss": 1.2106, + "theoretical_loss": 3.4074533536874845, + "tokens_seen": 2405025792 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021932998996990972, + "loss": 1.2177, + "theoretical_loss": 3.4074459521519924, + "tokens_seen": 2405091328 + }, + { + "epoch": 1.11, + "objective/train/docs_used": 889105, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1824032068252563, + "objective/train/theoretical_loss": 3.4074385508746503, + "objective/train/tokens_used": 2425616864, + "theoretical_loss": 3.4074385508746503, + "tokens_seen": 2405156864 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021931394182547642, + "loss": 1.1824, + "theoretical_loss": 3.4074385508746503, + "tokens_seen": 2405156864 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021929789368104312, + "loss": 1.1731, + "theoretical_loss": 3.4074311498554426, + "tokens_seen": 2405222400 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021928184553660982, + "loss": 1.1413, + "theoretical_loss": 3.4074237490943533, + "tokens_seen": 2405287936 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021926579739217652, + "loss": 1.2089, + "theoretical_loss": 3.407416348591366, + "tokens_seen": 2405353472 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021924974924774322, + "loss": 1.3284, + "theoretical_loss": 3.4074089483464647, + "tokens_seen": 2405419008 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021923370110330992, + "loss": 1.2431, + "theoretical_loss": 3.4074015483596334, + "tokens_seen": 2405484544 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021921765295887665, + "loss": 1.194, + "theoretical_loss": 3.4073941486308565, + "tokens_seen": 2405550080 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021920160481444335, + "loss": 1.273, + "theoretical_loss": 3.4073867491601177, + "tokens_seen": 2405615616 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021918555667001005, + "loss": 1.1949, + "theoretical_loss": 3.4073793499474005, + "tokens_seen": 2405681152 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021916950852557675, + "loss": 1.3096, + "theoretical_loss": 3.4073719509926894, + "tokens_seen": 2405746688 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021915346038114345, + "loss": 1.1913, + "theoretical_loss": 3.4073645522959684, + "tokens_seen": 2405812224 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021913741223671015, + "loss": 1.289, + "theoretical_loss": 3.4073571538572214, + "tokens_seen": 2405877760 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021912136409227685, + "loss": 1.1833, + "theoretical_loss": 3.407349755676432, + "tokens_seen": 2405943296 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021910531594784355, + "loss": 1.2131, + "theoretical_loss": 3.407342357753585, + "tokens_seen": 2406008832 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021908926780341026, + "loss": 1.1365, + "theoretical_loss": 3.4073349600886633, + "tokens_seen": 2406074368 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021907321965897696, + "loss": 1.1777, + "theoretical_loss": 3.407327562681652, + "tokens_seen": 2406139904 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021905717151454366, + "loss": 1.3043, + "theoretical_loss": 3.407320165532534, + "tokens_seen": 2406205440 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021904112337011036, + "loss": 1.2505, + "theoretical_loss": 3.407312768641294, + "tokens_seen": 2406270976 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021902507522567706, + "loss": 1.2023, + "theoretical_loss": 3.407305372007916, + "tokens_seen": 2406336512 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021900902708124376, + "loss": 1.2212, + "theoretical_loss": 3.4072979756323836, + "tokens_seen": 2406402048 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021899297893681046, + "loss": 1.2024, + "theoretical_loss": 3.4072905795146813, + "tokens_seen": 2406467584 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021897693079237716, + "loss": 1.1872, + "theoretical_loss": 3.4072831836547923, + "tokens_seen": 2406533120 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021896088264794386, + "loss": 1.1953, + "theoretical_loss": 3.4072757880527016, + "tokens_seen": 2406598656 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021894483450351056, + "loss": 1.1822, + "theoretical_loss": 3.407268392708392, + "tokens_seen": 2406664192 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021892878635907726, + "loss": 1.2637, + "theoretical_loss": 3.4072609976218486, + "tokens_seen": 2406729728 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021891273821464396, + "loss": 1.1732, + "theoretical_loss": 3.4072536027930553, + "tokens_seen": 2406795264 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021889669007021066, + "loss": 1.1797, + "theoretical_loss": 3.407246208221995, + "tokens_seen": 2406860800 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021888064192577736, + "loss": 1.2229, + "theoretical_loss": 3.407238813908653, + "tokens_seen": 2406926336 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021886459378134406, + "loss": 1.2399, + "theoretical_loss": 3.4072314198530127, + "tokens_seen": 2406991872 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021884854563691074, + "loss": 1.1969, + "theoretical_loss": 3.407224026055058, + "tokens_seen": 2407057408 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021883249749247744, + "loss": 1.2581, + "theoretical_loss": 3.407216632514773, + "tokens_seen": 2407122944 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021881644934804414, + "loss": 1.2015, + "theoretical_loss": 3.407209239232142, + "tokens_seen": 2407188480 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021880040120361084, + "loss": 1.3204, + "theoretical_loss": 3.4072018462071485, + "tokens_seen": 2407254016 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021878435305917754, + "loss": 1.1527, + "theoretical_loss": 3.407194453439777, + "tokens_seen": 2407319552 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021876830491474424, + "loss": 1.1549, + "theoretical_loss": 3.4071870609300117, + "tokens_seen": 2407385088 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021875225677031094, + "loss": 1.3148, + "theoretical_loss": 3.4071796686778355, + "tokens_seen": 2407450624 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021873620862587764, + "loss": 1.1533, + "theoretical_loss": 3.4071722766832333, + "tokens_seen": 2407516160 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021872016048144434, + "loss": 1.249, + "theoretical_loss": 3.407164884946189, + "tokens_seen": 2407581696 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021870411233701104, + "loss": 1.2267, + "theoretical_loss": 3.407157493466687, + "tokens_seen": 2407647232 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021868806419257774, + "loss": 1.1006, + "theoretical_loss": 3.40715010224471, + "tokens_seen": 2407712768 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021867201604814444, + "loss": 1.2548, + "theoretical_loss": 3.4071427112802435, + "tokens_seen": 2407778304 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021865596790371115, + "loss": 1.2865, + "theoretical_loss": 3.407135320573271, + "tokens_seen": 2407843840 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021863991975927785, + "loss": 1.2978, + "theoretical_loss": 3.407127930123776, + "tokens_seen": 2407909376 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021862387161484455, + "loss": 1.2625, + "theoretical_loss": 3.407120539931743, + "tokens_seen": 2407974912 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021860782347041125, + "loss": 1.154, + "theoretical_loss": 3.407113149997156, + "tokens_seen": 2408040448 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021859177532597795, + "loss": 1.172, + "theoretical_loss": 3.407105760319999, + "tokens_seen": 2408105984 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021857572718154465, + "loss": 1.2487, + "theoretical_loss": 3.4070983709002562, + "tokens_seen": 2408171520 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021855967903711135, + "loss": 1.2648, + "theoretical_loss": 3.407090981737911, + "tokens_seen": 2408237056 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021854363089267805, + "loss": 1.2296, + "theoretical_loss": 3.4070835928329486, + "tokens_seen": 2408302592 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021852758274824475, + "loss": 1.3194, + "theoretical_loss": 3.407076204185352, + "tokens_seen": 2408368128 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021851153460381145, + "loss": 1.177, + "theoretical_loss": 3.4070688157951055, + "tokens_seen": 2408433664 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021849548645937815, + "loss": 1.2, + "theoretical_loss": 3.407061427662193, + "tokens_seen": 2408499200 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021847943831494485, + "loss": 1.3252, + "theoretical_loss": 3.407054039786599, + "tokens_seen": 2408564736 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021846339017051153, + "loss": 1.2703, + "theoretical_loss": 3.407046652168307, + "tokens_seen": 2408630272 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021844734202607823, + "loss": 1.2798, + "theoretical_loss": 3.4070392648073016, + "tokens_seen": 2408695808 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021843129388164493, + "loss": 1.3023, + "theoretical_loss": 3.407031877703566, + "tokens_seen": 2408761344 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021841524573721163, + "loss": 1.3336, + "theoretical_loss": 3.4070244908570855, + "tokens_seen": 2408826880 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021839919759277833, + "loss": 1.3086, + "theoretical_loss": 3.407017104267843, + "tokens_seen": 2408892416 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021838314944834503, + "loss": 1.1487, + "theoretical_loss": 3.407009717935823, + "tokens_seen": 2408957952 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021836710130391173, + "loss": 1.1956, + "theoretical_loss": 3.407002331861009, + "tokens_seen": 2409023488 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021835105315947843, + "loss": 1.3021, + "theoretical_loss": 3.406994946043387, + "tokens_seen": 2409089024 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021833500501504513, + "loss": 1.2136, + "theoretical_loss": 3.4069875604829383, + "tokens_seen": 2409154560 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021831895687061183, + "loss": 1.1742, + "theoretical_loss": 3.4069801751796485, + "tokens_seen": 2409220096 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021830290872617853, + "loss": 1.1332, + "theoretical_loss": 3.4069727901335014, + "tokens_seen": 2409285632 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021828686058174523, + "loss": 1.2353, + "theoretical_loss": 3.406965405344481, + "tokens_seen": 2409351168 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021827081243731193, + "loss": 1.1948, + "theoretical_loss": 3.4069580208125716, + "tokens_seen": 2409416704 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021825476429287863, + "loss": 1.2675, + "theoretical_loss": 3.4069506365377573, + "tokens_seen": 2409482240 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021823871614844533, + "loss": 1.174, + "theoretical_loss": 3.4069432525200214, + "tokens_seen": 2409547776 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021822266800401204, + "loss": 1.2444, + "theoretical_loss": 3.4069358687593487, + "tokens_seen": 2409613312 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021820661985957874, + "loss": 1.2796, + "theoretical_loss": 3.4069284852557233, + "tokens_seen": 2409678848 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021819057171514544, + "loss": 1.3045, + "theoretical_loss": 3.406921102009129, + "tokens_seen": 2409744384 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021817452357071214, + "loss": 1.2589, + "theoretical_loss": 3.406913719019549, + "tokens_seen": 2409809920 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021815847542627884, + "loss": 1.1762, + "theoretical_loss": 3.406906336286969, + "tokens_seen": 2409875456 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021814242728184554, + "loss": 1.2921, + "theoretical_loss": 3.406898953811372, + "tokens_seen": 2409940992 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021812637913741224, + "loss": 1.2004, + "theoretical_loss": 3.406891571592743, + "tokens_seen": 2410006528 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021811033099297894, + "loss": 1.2181, + "theoretical_loss": 3.4068841896310644, + "tokens_seen": 2410072064 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021809428284854564, + "loss": 1.2811, + "theoretical_loss": 3.406876807926322, + "tokens_seen": 2410137600 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021807823470411231, + "loss": 1.0998, + "theoretical_loss": 3.4068694264784987, + "tokens_seen": 2410203136 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021806218655967901, + "loss": 1.2599, + "theoretical_loss": 3.4068620452875793, + "tokens_seen": 2410268672 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021804613841524572, + "loss": 1.1209, + "theoretical_loss": 3.406854664353548, + "tokens_seen": 2410334208 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021803009027081242, + "loss": 1.214, + "theoretical_loss": 3.4068472836763877, + "tokens_seen": 2410399744 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021801404212637917, + "loss": 1.2742, + "theoretical_loss": 3.406839903256084, + "tokens_seen": 2410465280 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021799799398194587, + "loss": 1.1978, + "theoretical_loss": 3.4068325230926195, + "tokens_seen": 2410530816 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021798194583751257, + "loss": 1.2035, + "theoretical_loss": 3.4068251431859795, + "tokens_seen": 2410596352 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021796589769307927, + "loss": 1.0771, + "theoretical_loss": 3.4068177635361474, + "tokens_seen": 2410661888 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021794984954864597, + "loss": 1.2823, + "theoretical_loss": 3.4068103841431077, + "tokens_seen": 2410727424 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021793380140421267, + "loss": 1.1852, + "theoretical_loss": 3.406803005006844, + "tokens_seen": 2410792960 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021791775325977937, + "loss": 1.28, + "theoretical_loss": 3.406795626127341, + "tokens_seen": 2410858496 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021790170511534608, + "loss": 1.258, + "theoretical_loss": 3.406788247504582, + "tokens_seen": 2410924032 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021788565697091278, + "loss": 1.1959, + "theoretical_loss": 3.4067808691385517, + "tokens_seen": 2410989568 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021786960882647948, + "loss": 1.143, + "theoretical_loss": 3.4067734910292344, + "tokens_seen": 2411055104 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021785356068204618, + "loss": 1.2556, + "theoretical_loss": 3.406766113176613, + "tokens_seen": 2411120640 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021783751253761288, + "loss": 1.2697, + "theoretical_loss": 3.4067587355806728, + "tokens_seen": 2411186176 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021782146439317955, + "loss": 1.2815, + "theoretical_loss": 3.4067513582413977, + "tokens_seen": 2411251712 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021780541624874625, + "loss": 1.2351, + "theoretical_loss": 3.4067439811587716, + "tokens_seen": 2411317248 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021778936810431295, + "loss": 1.2609, + "theoretical_loss": 3.4067366043327785, + "tokens_seen": 2411382784 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021777331995987965, + "loss": 1.2975, + "theoretical_loss": 3.4067292277634023, + "tokens_seen": 2411448320 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021775727181544635, + "loss": 1.2311, + "theoretical_loss": 3.4067218514506274, + "tokens_seen": 2411513856 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021774122367101305, + "loss": 1.2175, + "theoretical_loss": 3.4067144753944385, + "tokens_seen": 2411579392 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021772517552657976, + "loss": 1.2047, + "theoretical_loss": 3.4067070995948185, + "tokens_seen": 2411644928 + }, + { + "epoch": 1.11, + "objective/train/docs_used": 891365, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1605122089385986, + "objective/train/theoretical_loss": 3.4066997240517525, + "objective/train/tokens_used": 2432170464, + "theoretical_loss": 3.4066997240517525, + "tokens_seen": 2411710464 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021770912738214646, + "loss": 1.1605, + "theoretical_loss": 3.4066997240517525, + "tokens_seen": 2411710464 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021769307923771316, + "loss": 1.2165, + "theoretical_loss": 3.4066923487652234, + "tokens_seen": 2411776000 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021767703109327986, + "loss": 1.1673, + "theoretical_loss": 3.406684973735217, + "tokens_seen": 2411841536 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021766098294884656, + "loss": 1.2886, + "theoretical_loss": 3.406677598961716, + "tokens_seen": 2411907072 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021764493480441326, + "loss": 1.3409, + "theoretical_loss": 3.406670224444705, + "tokens_seen": 2411972608 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021762888665997996, + "loss": 1.2368, + "theoretical_loss": 3.4066628501841683, + "tokens_seen": 2412038144 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021761283851554666, + "loss": 1.3064, + "theoretical_loss": 3.40665547618009, + "tokens_seen": 2412103680 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021759679037111336, + "loss": 1.2525, + "theoretical_loss": 3.4066481024324538, + "tokens_seen": 2412169216 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021758074222668006, + "loss": 1.2039, + "theoretical_loss": 3.406640728941244, + "tokens_seen": 2412234752 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021756469408224676, + "loss": 1.1493, + "theoretical_loss": 3.406633355706445, + "tokens_seen": 2412300288 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021754864593781346, + "loss": 1.2263, + "theoretical_loss": 3.4066259827280407, + "tokens_seen": 2412365824 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021753259779338016, + "loss": 1.1727, + "theoretical_loss": 3.406618610006015, + "tokens_seen": 2412431360 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021751654964894686, + "loss": 1.1866, + "theoretical_loss": 3.4066112375403526, + "tokens_seen": 2412496896 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021750050150451356, + "loss": 1.21, + "theoretical_loss": 3.4066038653310367, + "tokens_seen": 2412562432 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021748445336008026, + "loss": 1.2122, + "theoretical_loss": 3.4065964933780526, + "tokens_seen": 2412627968 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021746840521564697, + "loss": 1.311, + "theoretical_loss": 3.4065891216813835, + "tokens_seen": 2412693504 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021745235707121367, + "loss": 1.2244, + "theoretical_loss": 3.406581750241014, + "tokens_seen": 2412759040 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021743630892678034, + "loss": 1.2192, + "theoretical_loss": 3.4065743790569276, + "tokens_seen": 2412824576 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021742026078234704, + "loss": 1.2736, + "theoretical_loss": 3.4065670081291097, + "tokens_seen": 2412890112 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021740421263791374, + "loss": 1.2323, + "theoretical_loss": 3.406559637457543, + "tokens_seen": 2412955648 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021738816449348044, + "loss": 1.1072, + "theoretical_loss": 3.4065522670422124, + "tokens_seen": 2413021184 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021737211634904714, + "loss": 1.171, + "theoretical_loss": 3.406544896883102, + "tokens_seen": 2413086720 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021735606820461384, + "loss": 1.2304, + "theoretical_loss": 3.4065375269801956, + "tokens_seen": 2413152256 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021734002006018054, + "loss": 1.1678, + "theoretical_loss": 3.406530157333478, + "tokens_seen": 2413217792 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021732397191574724, + "loss": 1.3402, + "theoretical_loss": 3.406522787942932, + "tokens_seen": 2413283328 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021730792377131394, + "loss": 1.2584, + "theoretical_loss": 3.406515418808543, + "tokens_seen": 2413348864 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021729187562688065, + "loss": 1.3243, + "theoretical_loss": 3.4065080499302955, + "tokens_seen": 2413414400 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021727582748244735, + "loss": 1.2346, + "theoretical_loss": 3.4065006813081724, + "tokens_seen": 2413479936 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021725977933801405, + "loss": 1.2755, + "theoretical_loss": 3.406493312942158, + "tokens_seen": 2413545472 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021724373119358075, + "loss": 1.1519, + "theoretical_loss": 3.4064859448322373, + "tokens_seen": 2413611008 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021722768304914745, + "loss": 1.2258, + "theoretical_loss": 3.406478576978394, + "tokens_seen": 2413676544 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021721163490471415, + "loss": 1.2539, + "theoretical_loss": 3.406471209380612, + "tokens_seen": 2413742080 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021719558676028085, + "loss": 1.2167, + "theoretical_loss": 3.4064638420388755, + "tokens_seen": 2413807616 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021717953861584755, + "loss": 1.2103, + "theoretical_loss": 3.406456474953169, + "tokens_seen": 2413873152 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021716349047141425, + "loss": 1.3098, + "theoretical_loss": 3.4064491081234767, + "tokens_seen": 2413938688 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021714744232698095, + "loss": 1.2368, + "theoretical_loss": 3.406441741549782, + "tokens_seen": 2414004224 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021713139418254765, + "loss": 1.207, + "theoretical_loss": 3.4064343752320703, + "tokens_seen": 2414069760 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021711534603811435, + "loss": 1.1929, + "theoretical_loss": 3.4064270091703244, + "tokens_seen": 2414135296 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021709929789368105, + "loss": 1.2832, + "theoretical_loss": 3.406419643364529, + "tokens_seen": 2414200832 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021708324974924775, + "loss": 1.0825, + "theoretical_loss": 3.4064122778146686, + "tokens_seen": 2414266368 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021706720160481445, + "loss": 1.226, + "theoretical_loss": 3.4064049125207267, + "tokens_seen": 2414331904 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021705115346038113, + "loss": 1.2906, + "theoretical_loss": 3.406397547482688, + "tokens_seen": 2414397440 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021703510531594783, + "loss": 1.1802, + "theoretical_loss": 3.406390182700537, + "tokens_seen": 2414462976 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021701905717151453, + "loss": 1.1955, + "theoretical_loss": 3.406382818174257, + "tokens_seen": 2414528512 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021700300902708123, + "loss": 1.2089, + "theoretical_loss": 3.4063754539038325, + "tokens_seen": 2414594048 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021698696088264793, + "loss": 1.2973, + "theoretical_loss": 3.406368089889248, + "tokens_seen": 2414659584 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021697091273821463, + "loss": 1.1776, + "theoretical_loss": 3.406360726130487, + "tokens_seen": 2414725120 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021695486459378133, + "loss": 1.0685, + "theoretical_loss": 3.4063533626275344, + "tokens_seen": 2414790656 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021693881644934803, + "loss": 1.2307, + "theoretical_loss": 3.406345999380374, + "tokens_seen": 2414856192 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021692276830491473, + "loss": 1.2278, + "theoretical_loss": 3.40633863638899, + "tokens_seen": 2414921728 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021690672016048143, + "loss": 1.2362, + "theoretical_loss": 3.4063312736533664, + "tokens_seen": 2414987264 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021689067201604813, + "loss": 1.1195, + "theoretical_loss": 3.4063239111734873, + "tokens_seen": 2415052800 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021687462387161483, + "loss": 1.1945, + "theoretical_loss": 3.4063165489493374, + "tokens_seen": 2415118336 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021685857572718154, + "loss": 1.1056, + "theoretical_loss": 3.406309186980901, + "tokens_seen": 2415183872 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021684252758274824, + "loss": 1.2861, + "theoretical_loss": 3.4063018252681614, + "tokens_seen": 2415249408 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021682647943831494, + "loss": 1.1941, + "theoretical_loss": 3.4062944638111032, + "tokens_seen": 2415314944 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021681043129388166, + "loss": 1.3023, + "theoretical_loss": 3.406287102609711, + "tokens_seen": 2415380480 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021679438314944836, + "loss": 1.2529, + "theoretical_loss": 3.406279741663968, + "tokens_seen": 2415446016 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021677833500501507, + "loss": 1.2143, + "theoretical_loss": 3.4062723809738595, + "tokens_seen": 2415511552 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021676228686058177, + "loss": 1.2053, + "theoretical_loss": 3.406265020539369, + "tokens_seen": 2415577088 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021674623871614847, + "loss": 1.1707, + "theoretical_loss": 3.406257660360481, + "tokens_seen": 2415642624 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021673019057171517, + "loss": 1.2323, + "theoretical_loss": 3.40625030043718, + "tokens_seen": 2415708160 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021671414242728187, + "loss": 1.2563, + "theoretical_loss": 3.4062429407694492, + "tokens_seen": 2415773696 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021669809428284857, + "loss": 1.2471, + "theoretical_loss": 3.4062355813572736, + "tokens_seen": 2415839232 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021668204613841527, + "loss": 1.2853, + "theoretical_loss": 3.406228222200637, + "tokens_seen": 2415904768 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021666599799398197, + "loss": 1.2155, + "theoretical_loss": 3.4062208632995237, + "tokens_seen": 2415970304 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021664994984954867, + "loss": 1.1442, + "theoretical_loss": 3.4062135046539184, + "tokens_seen": 2416035840 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021663390170511537, + "loss": 1.2936, + "theoretical_loss": 3.4062061462638042, + "tokens_seen": 2416101376 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021661785356068207, + "loss": 1.2318, + "theoretical_loss": 3.406198788129166, + "tokens_seen": 2416166912 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021660180541624877, + "loss": 1.2163, + "theoretical_loss": 3.4061914302499887, + "tokens_seen": 2416232448 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021658575727181547, + "loss": 1.1785, + "theoretical_loss": 3.4061840726262553, + "tokens_seen": 2416297984 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021656970912738217, + "loss": 1.1656, + "theoretical_loss": 3.4061767152579505, + "tokens_seen": 2416363520 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021655366098294887, + "loss": 1.2657, + "theoretical_loss": 3.4061693581450583, + "tokens_seen": 2416429056 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021653761283851557, + "loss": 1.2312, + "theoretical_loss": 3.406162001287563, + "tokens_seen": 2416494592 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021652156469408228, + "loss": 1.2074, + "theoretical_loss": 3.4061546446854494, + "tokens_seen": 2416560128 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021650551654964898, + "loss": 1.2214, + "theoretical_loss": 3.4061472883387007, + "tokens_seen": 2416625664 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021648946840521568, + "loss": 1.1665, + "theoretical_loss": 3.4061399322473016, + "tokens_seen": 2416691200 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021647342026078238, + "loss": 1.264, + "theoretical_loss": 3.4061325764112365, + "tokens_seen": 2416756736 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021645737211634908, + "loss": 1.1775, + "theoretical_loss": 3.4061252208304893, + "tokens_seen": 2416822272 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021644132397191578, + "loss": 1.2095, + "theoretical_loss": 3.4061178655050446, + "tokens_seen": 2416887808 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021642527582748245, + "loss": 1.16, + "theoretical_loss": 3.406110510434886, + "tokens_seen": 2416953344 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021640922768304915, + "loss": 1.2916, + "theoretical_loss": 3.406103155619998, + "tokens_seen": 2417018880 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021639317953861585, + "loss": 1.2123, + "theoretical_loss": 3.4060958010603652, + "tokens_seen": 2417084416 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021637713139418255, + "loss": 1.349, + "theoretical_loss": 3.4060884467559713, + "tokens_seen": 2417149952 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021636108324974925, + "loss": 1.2083, + "theoretical_loss": 3.406081092706801, + "tokens_seen": 2417215488 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021634503510531596, + "loss": 1.2866, + "theoretical_loss": 3.406073738912838, + "tokens_seen": 2417281024 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021632898696088266, + "loss": 1.2699, + "theoretical_loss": 3.406066385374067, + "tokens_seen": 2417346560 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021631293881644936, + "loss": 1.2439, + "theoretical_loss": 3.4060590320904716, + "tokens_seen": 2417412096 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021629689067201606, + "loss": 1.1284, + "theoretical_loss": 3.4060516790620365, + "tokens_seen": 2417477632 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021628084252758276, + "loss": 1.2487, + "theoretical_loss": 3.4060443262887463, + "tokens_seen": 2417543168 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021626479438314946, + "loss": 1.1844, + "theoretical_loss": 3.4060369737705845, + "tokens_seen": 2417608704 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021624874623871616, + "loss": 1.17, + "theoretical_loss": 3.406029621507536, + "tokens_seen": 2417674240 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021623269809428286, + "loss": 1.2426, + "theoretical_loss": 3.406022269499584, + "tokens_seen": 2417739776 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021621664994984956, + "loss": 1.3175, + "theoretical_loss": 3.406014917746714, + "tokens_seen": 2417805312 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021620060180541626, + "loss": 1.2204, + "theoretical_loss": 3.4060075662489093, + "tokens_seen": 2417870848 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021618455366098296, + "loss": 1.2819, + "theoretical_loss": 3.4060002150061544, + "tokens_seen": 2417936384 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021616850551654966, + "loss": 1.24, + "theoretical_loss": 3.4059928640184336, + "tokens_seen": 2418001920 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021615245737211636, + "loss": 1.116, + "theoretical_loss": 3.4059855132857315, + "tokens_seen": 2418067456 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021613640922768306, + "loss": 1.3005, + "theoretical_loss": 3.4059781628080317, + "tokens_seen": 2418132992 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021612036108324976, + "loss": 1.1926, + "theoretical_loss": 3.405970812585319, + "tokens_seen": 2418198528 + }, + { + "epoch": 1.11, + "objective/train/docs_used": 893565, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1807140111923218, + "objective/train/theoretical_loss": 3.405963462617577, + "objective/train/tokens_used": 2438724064, + "theoretical_loss": 3.405963462617577, + "tokens_seen": 2418264064 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021610431293881646, + "loss": 1.1807, + "theoretical_loss": 3.405963462617577, + "tokens_seen": 2418264064 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021608826479438317, + "loss": 1.2364, + "theoretical_loss": 3.40595611290479, + "tokens_seen": 2418329600 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021607221664994987, + "loss": 1.2706, + "theoretical_loss": 3.4059487634469434, + "tokens_seen": 2418395136 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021605616850551657, + "loss": 1.173, + "theoretical_loss": 3.4059414142440203, + "tokens_seen": 2418460672 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021604012036108324, + "loss": 1.2209, + "theoretical_loss": 3.4059340652960053, + "tokens_seen": 2418526208 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021602407221664994, + "loss": 1.2289, + "theoretical_loss": 3.4059267166028824, + "tokens_seen": 2418591744 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021600802407221664, + "loss": 1.1431, + "theoretical_loss": 3.4059193681646365, + "tokens_seen": 2418657280 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021599197592778334, + "loss": 1.1071, + "theoretical_loss": 3.4059120199812507, + "tokens_seen": 2418722816 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021597592778335004, + "loss": 1.2533, + "theoretical_loss": 3.4059046720527104, + "tokens_seen": 2418788352 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021595987963891674, + "loss": 1.2696, + "theoretical_loss": 3.4058973243789996, + "tokens_seen": 2418853888 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021594383149448344, + "loss": 1.2165, + "theoretical_loss": 3.4058899769601023, + "tokens_seen": 2418919424 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021592778335005014, + "loss": 1.263, + "theoretical_loss": 3.405882629796003, + "tokens_seen": 2418984960 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021591173520561685, + "loss": 1.1904, + "theoretical_loss": 3.4058752828866856, + "tokens_seen": 2419050496 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021589568706118355, + "loss": 1.1922, + "theoretical_loss": 3.4058679362321342, + "tokens_seen": 2419116032 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021587963891675025, + "loss": 1.2015, + "theoretical_loss": 3.405860589832334, + "tokens_seen": 2419181568 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021586359077231695, + "loss": 1.239, + "theoretical_loss": 3.4058532436872686, + "tokens_seen": 2419247104 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021584754262788365, + "loss": 1.1699, + "theoretical_loss": 3.4058458977969224, + "tokens_seen": 2419312640 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021583149448345035, + "loss": 1.2139, + "theoretical_loss": 3.4058385521612795, + "tokens_seen": 2419378176 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021581544633901705, + "loss": 1.2216, + "theoretical_loss": 3.4058312067803245, + "tokens_seen": 2419443712 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021579939819458375, + "loss": 1.174, + "theoretical_loss": 3.4058238616540413, + "tokens_seen": 2419509248 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021578335005015045, + "loss": 1.1961, + "theoretical_loss": 3.4058165167824144, + "tokens_seen": 2419574784 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021576730190571715, + "loss": 1.2043, + "theoretical_loss": 3.4058091721654282, + "tokens_seen": 2419640320 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021575125376128385, + "loss": 1.1976, + "theoretical_loss": 3.4058018278030664, + "tokens_seen": 2419705856 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021573520561685055, + "loss": 1.2561, + "theoretical_loss": 3.4057944836953142, + "tokens_seen": 2419771392 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021571915747241725, + "loss": 1.2635, + "theoretical_loss": 3.405787139842155, + "tokens_seen": 2419836928 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021570310932798395, + "loss": 1.2128, + "theoretical_loss": 3.4057797962435736, + "tokens_seen": 2419902464 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021568706118355065, + "loss": 1.2314, + "theoretical_loss": 3.405772452899554, + "tokens_seen": 2419968000 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021567101303911735, + "loss": 1.3401, + "theoretical_loss": 3.4057651098100807, + "tokens_seen": 2420033536 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021565496489468403, + "loss": 1.2037, + "theoretical_loss": 3.405757766975138, + "tokens_seen": 2420099072 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021563891675025073, + "loss": 1.2593, + "theoretical_loss": 3.40575042439471, + "tokens_seen": 2420164608 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021562286860581743, + "loss": 1.2892, + "theoretical_loss": 3.405743082068781, + "tokens_seen": 2420230144 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021560682046138418, + "loss": 1.29, + "theoretical_loss": 3.4057357399973354, + "tokens_seen": 2420295680 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021559077231695089, + "loss": 1.285, + "theoretical_loss": 3.405728398180357, + "tokens_seen": 2420361216 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021557472417251759, + "loss": 1.1924, + "theoretical_loss": 3.405721056617831, + "tokens_seen": 2420426752 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002155586760280843, + "loss": 1.3284, + "theoretical_loss": 3.405713715309741, + "tokens_seen": 2420492288 + }, + { + "epoch": 1.11, + "learning_rate": 0.000215542627883651, + "loss": 1.1651, + "theoretical_loss": 3.405706374256072, + "tokens_seen": 2420557824 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002155265797392177, + "loss": 1.2734, + "theoretical_loss": 3.4056990334568074, + "tokens_seen": 2420623360 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002155105315947844, + "loss": 1.3594, + "theoretical_loss": 3.405691692911932, + "tokens_seen": 2420688896 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002154944834503511, + "loss": 1.3422, + "theoretical_loss": 3.4056843526214298, + "tokens_seen": 2420754432 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002154784353059178, + "loss": 1.2628, + "theoretical_loss": 3.4056770125852855, + "tokens_seen": 2420819968 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002154623871614845, + "loss": 1.1852, + "theoretical_loss": 3.405669672803483, + "tokens_seen": 2420885504 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002154463390170512, + "loss": 1.3581, + "theoretical_loss": 3.405662333276007, + "tokens_seen": 2420951040 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002154302908726179, + "loss": 1.385, + "theoretical_loss": 3.4056549940028416, + "tokens_seen": 2421016576 + }, + { + "epoch": 1.11, + "learning_rate": 0.0002154142427281846, + "loss": 1.1772, + "theoretical_loss": 3.405647654983971, + "tokens_seen": 2421082112 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021539819458375127, + "loss": 1.2721, + "theoretical_loss": 3.4056403162193796, + "tokens_seen": 2421147648 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021538214643931797, + "loss": 1.1596, + "theoretical_loss": 3.405632977709052, + "tokens_seen": 2421213184 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021536609829488467, + "loss": 1.2585, + "theoretical_loss": 3.405625639452972, + "tokens_seen": 2421278720 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021535005015045137, + "loss": 1.1903, + "theoretical_loss": 3.405618301451124, + "tokens_seen": 2421344256 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021533400200601807, + "loss": 1.2984, + "theoretical_loss": 3.405610963703493, + "tokens_seen": 2421409792 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021531795386158477, + "loss": 1.1867, + "theoretical_loss": 3.405603626210062, + "tokens_seen": 2421475328 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021530190571715147, + "loss": 1.1924, + "theoretical_loss": 3.4055962889708167, + "tokens_seen": 2421540864 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021528585757271817, + "loss": 1.2661, + "theoretical_loss": 3.4055889519857407, + "tokens_seen": 2421606400 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021526980942828487, + "loss": 1.3054, + "theoretical_loss": 3.405581615254818, + "tokens_seen": 2421671936 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021525376128385157, + "loss": 1.3345, + "theoretical_loss": 3.4055742787780336, + "tokens_seen": 2421737472 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021523771313941827, + "loss": 1.2999, + "theoretical_loss": 3.405566942555372, + "tokens_seen": 2421803008 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021522166499498497, + "loss": 1.2062, + "theoretical_loss": 3.4055596065868166, + "tokens_seen": 2421868544 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021520561685055167, + "loss": 1.3362, + "theoretical_loss": 3.4055522708723522, + "tokens_seen": 2421934080 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021518956870611837, + "loss": 1.2438, + "theoretical_loss": 3.405544935411963, + "tokens_seen": 2421999616 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021517352056168507, + "loss": 1.2204, + "theoretical_loss": 3.4055376002056335, + "tokens_seen": 2422065152 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021515747241725178, + "loss": 1.309, + "theoretical_loss": 3.405530265253348, + "tokens_seen": 2422130688 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021514142427281848, + "loss": 1.3211, + "theoretical_loss": 3.405522930555091, + "tokens_seen": 2422196224 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021512537612838518, + "loss": 1.2339, + "theoretical_loss": 3.4055155961108463, + "tokens_seen": 2422261760 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021510932798395188, + "loss": 1.2348, + "theoretical_loss": 3.4055082619205987, + "tokens_seen": 2422327296 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021509327983951858, + "loss": 1.2401, + "theoretical_loss": 3.405500927984333, + "tokens_seen": 2422392832 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021507723169508528, + "loss": 1.2818, + "theoretical_loss": 3.4054935943020324, + "tokens_seen": 2422458368 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021506118355065198, + "loss": 1.3007, + "theoretical_loss": 3.4054862608736816, + "tokens_seen": 2422523904 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021504513540621868, + "loss": 1.2677, + "theoretical_loss": 3.405478927699265, + "tokens_seen": 2422589440 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021502908726178538, + "loss": 1.2068, + "theoretical_loss": 3.405471594778767, + "tokens_seen": 2422654976 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021501303911735205, + "loss": 1.1915, + "theoretical_loss": 3.4054642621121727, + "tokens_seen": 2422720512 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021499699097291875, + "loss": 1.2931, + "theoretical_loss": 3.405456929699465, + "tokens_seen": 2422786048 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021498094282848546, + "loss": 1.3749, + "theoretical_loss": 3.4054495975406294, + "tokens_seen": 2422851584 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021496489468405216, + "loss": 1.2814, + "theoretical_loss": 3.4054422656356493, + "tokens_seen": 2422917120 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021494884653961886, + "loss": 1.2515, + "theoretical_loss": 3.40543493398451, + "tokens_seen": 2422982656 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021493279839518556, + "loss": 1.2022, + "theoretical_loss": 3.4054276025871952, + "tokens_seen": 2423048192 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021491675025075226, + "loss": 1.2245, + "theoretical_loss": 3.4054202714436896, + "tokens_seen": 2423113728 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021490070210631896, + "loss": 1.2016, + "theoretical_loss": 3.405412940553977, + "tokens_seen": 2423179264 + }, + { + "epoch": 1.11, + "learning_rate": 0.00021488465396188566, + "loss": 1.2425, + "theoretical_loss": 3.4054056099180428, + "tokens_seen": 2423244800 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021486860581745236, + "loss": 1.1379, + "theoretical_loss": 3.4053982795358704, + "tokens_seen": 2423310336 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021485255767301906, + "loss": 1.2323, + "theoretical_loss": 3.405390949407444, + "tokens_seen": 2423375872 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021483650952858576, + "loss": 1.3344, + "theoretical_loss": 3.405383619532749, + "tokens_seen": 2423441408 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021482046138415246, + "loss": 1.2663, + "theoretical_loss": 3.4053762899117688, + "tokens_seen": 2423506944 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021480441323971916, + "loss": 1.2577, + "theoretical_loss": 3.405368960544488, + "tokens_seen": 2423572480 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021478836509528586, + "loss": 1.215, + "theoretical_loss": 3.4053616314308917, + "tokens_seen": 2423638016 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021477231695085256, + "loss": 1.2687, + "theoretical_loss": 3.405354302570963, + "tokens_seen": 2423703552 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021475626880641926, + "loss": 1.1816, + "theoretical_loss": 3.4053469739646873, + "tokens_seen": 2423769088 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021474022066198596, + "loss": 1.2585, + "theoretical_loss": 3.4053396456120484, + "tokens_seen": 2423834624 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021472417251755267, + "loss": 1.1422, + "theoretical_loss": 3.4053323175130306, + "tokens_seen": 2423900160 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021470812437311937, + "loss": 1.2885, + "theoretical_loss": 3.4053249896676188, + "tokens_seen": 2423965696 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021469207622868607, + "loss": 1.198, + "theoretical_loss": 3.405317662075797, + "tokens_seen": 2424031232 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021467602808425277, + "loss": 1.2728, + "theoretical_loss": 3.4053103347375497, + "tokens_seen": 2424096768 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021465997993981947, + "loss": 1.2522, + "theoretical_loss": 3.405303007652861, + "tokens_seen": 2424162304 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021464393179538617, + "loss": 1.1443, + "theoretical_loss": 3.405295680821715, + "tokens_seen": 2424227840 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021462788365095284, + "loss": 1.2504, + "theoretical_loss": 3.4052883542440977, + "tokens_seen": 2424293376 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021461183550651954, + "loss": 1.3022, + "theoretical_loss": 3.405281027919991, + "tokens_seen": 2424358912 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021459578736208624, + "loss": 1.3239, + "theoretical_loss": 3.4052737018493815, + "tokens_seen": 2424424448 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021457973921765294, + "loss": 1.2124, + "theoretical_loss": 3.4052663760322526, + "tokens_seen": 2424489984 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021456369107321964, + "loss": 1.2109, + "theoretical_loss": 3.4052590504685885, + "tokens_seen": 2424555520 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021454764292878635, + "loss": 1.2557, + "theoretical_loss": 3.405251725158374, + "tokens_seen": 2424621056 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021453159478435305, + "loss": 1.225, + "theoretical_loss": 3.405244400101593, + "tokens_seen": 2424686592 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021451554663991975, + "loss": 1.2053, + "theoretical_loss": 3.4052370752982304, + "tokens_seen": 2424752128 + }, + { + "epoch": 1.12, + "objective/train/docs_used": 895866, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1623787879943848, + "objective/train/theoretical_loss": 3.4052297507482705, + "objective/train/tokens_used": 2445277664, + "theoretical_loss": 3.4052297507482705, + "tokens_seen": 2424817664 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021449949849548645, + "loss": 1.1624, + "theoretical_loss": 3.4052297507482705, + "tokens_seen": 2424817664 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021448345035105315, + "loss": 1.2797, + "theoretical_loss": 3.4052224264516973, + "tokens_seen": 2424883200 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021446740220661985, + "loss": 1.3091, + "theoretical_loss": 3.4052151024084956, + "tokens_seen": 2424948736 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021445135406218655, + "loss": 1.194, + "theoretical_loss": 3.4052077786186494, + "tokens_seen": 2425014272 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021443530591775325, + "loss": 1.2102, + "theoretical_loss": 3.4052004550821433, + "tokens_seen": 2425079808 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021441925777331995, + "loss": 1.3169, + "theoretical_loss": 3.4051931317989617, + "tokens_seen": 2425145344 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002144032096288867, + "loss": 1.2948, + "theoretical_loss": 3.4051858087690894, + "tokens_seen": 2425210880 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021438716148445338, + "loss": 1.1307, + "theoretical_loss": 3.40517848599251, + "tokens_seen": 2425276416 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021437111334002008, + "loss": 1.2281, + "theoretical_loss": 3.4051711634692086, + "tokens_seen": 2425341952 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021435506519558678, + "loss": 1.2547, + "theoretical_loss": 3.405163841199169, + "tokens_seen": 2425407488 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021433901705115348, + "loss": 1.1787, + "theoretical_loss": 3.405156519182376, + "tokens_seen": 2425473024 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021432296890672018, + "loss": 1.1847, + "theoretical_loss": 3.405149197418814, + "tokens_seen": 2425538560 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021430692076228688, + "loss": 1.2533, + "theoretical_loss": 3.405141875908467, + "tokens_seen": 2425604096 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021429087261785358, + "loss": 1.3247, + "theoretical_loss": 3.40513455465132, + "tokens_seen": 2425669632 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021427482447342028, + "loss": 1.3095, + "theoretical_loss": 3.405127233647357, + "tokens_seen": 2425735168 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021425877632898698, + "loss": 1.2457, + "theoretical_loss": 3.4051199128965624, + "tokens_seen": 2425800704 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021424272818455368, + "loss": 1.2699, + "theoretical_loss": 3.4051125923989205, + "tokens_seen": 2425866240 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021422668004012038, + "loss": 1.3135, + "theoretical_loss": 3.405105272154416, + "tokens_seen": 2425931776 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021421063189568709, + "loss": 1.2346, + "theoretical_loss": 3.4050979521630333, + "tokens_seen": 2425997312 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021419458375125379, + "loss": 1.1451, + "theoretical_loss": 3.405090632424757, + "tokens_seen": 2426062848 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002141785356068205, + "loss": 1.2278, + "theoretical_loss": 3.405083312939571, + "tokens_seen": 2426128384 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002141624874623872, + "loss": 1.2684, + "theoretical_loss": 3.40507599370746, + "tokens_seen": 2426193920 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002141464393179539, + "loss": 1.2243, + "theoretical_loss": 3.4050686747284082, + "tokens_seen": 2426259456 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002141303911735206, + "loss": 1.3327, + "theoretical_loss": 3.4050613560024, + "tokens_seen": 2426324992 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002141143430290873, + "loss": 1.2488, + "theoretical_loss": 3.4050540375294203, + "tokens_seen": 2426390528 + }, + { + "epoch": 1.12, + "learning_rate": 0.000214098294884654, + "loss": 1.1064, + "theoretical_loss": 3.4050467193094534, + "tokens_seen": 2426456064 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002140822467402207, + "loss": 1.2224, + "theoretical_loss": 3.405039401342483, + "tokens_seen": 2426521600 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002140661985957874, + "loss": 1.238, + "theoretical_loss": 3.405032083628494, + "tokens_seen": 2426587136 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002140501504513541, + "loss": 1.2875, + "theoretical_loss": 3.4050247661674717, + "tokens_seen": 2426652672 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002140341023069208, + "loss": 1.2074, + "theoretical_loss": 3.405017448959399, + "tokens_seen": 2426718208 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002140180541624875, + "loss": 1.303, + "theoretical_loss": 3.405010132004261, + "tokens_seen": 2426783744 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021400200601805417, + "loss": 1.2197, + "theoretical_loss": 3.4050028153020424, + "tokens_seen": 2426849280 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021398595787362087, + "loss": 1.2983, + "theoretical_loss": 3.4049954988527276, + "tokens_seen": 2426914816 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021396990972918757, + "loss": 1.1522, + "theoretical_loss": 3.4049881826563, + "tokens_seen": 2426980352 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021395386158475427, + "loss": 1.2545, + "theoretical_loss": 3.4049808667127457, + "tokens_seen": 2427045888 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021393781344032097, + "loss": 1.2161, + "theoretical_loss": 3.404973551022048, + "tokens_seen": 2427111424 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021392176529588767, + "loss": 1.2478, + "theoretical_loss": 3.4049662355841916, + "tokens_seen": 2427176960 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021390571715145437, + "loss": 1.2877, + "theoretical_loss": 3.404958920399161, + "tokens_seen": 2427242496 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021388966900702107, + "loss": 1.2721, + "theoretical_loss": 3.40495160546694, + "tokens_seen": 2427308032 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021387362086258777, + "loss": 1.3182, + "theoretical_loss": 3.4049442907875145, + "tokens_seen": 2427373568 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021385757271815447, + "loss": 1.17, + "theoretical_loss": 3.4049369763608675, + "tokens_seen": 2427439104 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021384152457372117, + "loss": 1.2671, + "theoretical_loss": 3.404929662186984, + "tokens_seen": 2427504640 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021382547642928787, + "loss": 1.2268, + "theoretical_loss": 3.4049223482658486, + "tokens_seen": 2427570176 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021380942828485457, + "loss": 1.1854, + "theoretical_loss": 3.4049150345974453, + "tokens_seen": 2427635712 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021379338014042127, + "loss": 1.216, + "theoretical_loss": 3.4049077211817593, + "tokens_seen": 2427701248 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021377733199598798, + "loss": 1.2679, + "theoretical_loss": 3.404900408018774, + "tokens_seen": 2427766784 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021376128385155468, + "loss": 1.2541, + "theoretical_loss": 3.404893095108475, + "tokens_seen": 2427832320 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021374523570712138, + "loss": 1.2841, + "theoretical_loss": 3.4048857824508456, + "tokens_seen": 2427897856 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021372918756268808, + "loss": 1.3536, + "theoretical_loss": 3.404878470045871, + "tokens_seen": 2427963392 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021371313941825478, + "loss": 1.2838, + "theoretical_loss": 3.4048711578935356, + "tokens_seen": 2428028928 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021369709127382148, + "loss": 1.307, + "theoretical_loss": 3.4048638459938236, + "tokens_seen": 2428094464 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021368104312938818, + "loss": 1.2672, + "theoretical_loss": 3.4048565343467194, + "tokens_seen": 2428160000 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021366499498495488, + "loss": 1.2218, + "theoretical_loss": 3.4048492229522083, + "tokens_seen": 2428225536 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021364894684052158, + "loss": 1.2141, + "theoretical_loss": 3.4048419118102733, + "tokens_seen": 2428291072 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021363289869608828, + "loss": 1.1946, + "theoretical_loss": 3.4048346009209, + "tokens_seen": 2428356608 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021361685055165495, + "loss": 1.3355, + "theoretical_loss": 3.4048272902840724, + "tokens_seen": 2428422144 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021360080240722166, + "loss": 1.2392, + "theoretical_loss": 3.4048199798997754, + "tokens_seen": 2428487680 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021358475426278836, + "loss": 1.2119, + "theoretical_loss": 3.404812669767993, + "tokens_seen": 2428553216 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021356870611835506, + "loss": 1.2684, + "theoretical_loss": 3.404805359888709, + "tokens_seen": 2428618752 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021355265797392176, + "loss": 1.2568, + "theoretical_loss": 3.4047980502619097, + "tokens_seen": 2428684288 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021353660982948846, + "loss": 1.2522, + "theoretical_loss": 3.404790740887578, + "tokens_seen": 2428749824 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021352056168505516, + "loss": 1.2963, + "theoretical_loss": 3.404783431765699, + "tokens_seen": 2428815360 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021350451354062186, + "loss": 1.2508, + "theoretical_loss": 3.4047761228962568, + "tokens_seen": 2428880896 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021348846539618856, + "loss": 1.2702, + "theoretical_loss": 3.4047688142792367, + "tokens_seen": 2428946432 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021347241725175526, + "loss": 1.2837, + "theoretical_loss": 3.404761505914622, + "tokens_seen": 2429011968 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021345636910732196, + "loss": 1.147, + "theoretical_loss": 3.404754197802398, + "tokens_seen": 2429077504 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021344032096288866, + "loss": 1.243, + "theoretical_loss": 3.404746889942549, + "tokens_seen": 2429143040 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021342427281845536, + "loss": 1.2478, + "theoretical_loss": 3.4047395823350595, + "tokens_seen": 2429208576 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021340822467402206, + "loss": 1.1872, + "theoretical_loss": 3.4047322749799136, + "tokens_seen": 2429274112 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021339217652958876, + "loss": 1.1683, + "theoretical_loss": 3.404724967877096, + "tokens_seen": 2429339648 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021337612838515546, + "loss": 1.2197, + "theoretical_loss": 3.404717661026592, + "tokens_seen": 2429405184 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021336008024072216, + "loss": 1.1994, + "theoretical_loss": 3.4047103544283845, + "tokens_seen": 2429470720 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021334403209628887, + "loss": 1.2867, + "theoretical_loss": 3.4047030480824594, + "tokens_seen": 2429536256 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021332798395185557, + "loss": 1.2236, + "theoretical_loss": 3.4046957419888004, + "tokens_seen": 2429601792 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021331193580742227, + "loss": 1.2713, + "theoretical_loss": 3.404688436147392, + "tokens_seen": 2429667328 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021329588766298897, + "loss": 1.1926, + "theoretical_loss": 3.404681130558219, + "tokens_seen": 2429732864 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021327983951855567, + "loss": 1.2002, + "theoretical_loss": 3.404673825221266, + "tokens_seen": 2429798400 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021326379137412237, + "loss": 1.2694, + "theoretical_loss": 3.404666520136517, + "tokens_seen": 2429863936 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021324774322968907, + "loss": 1.1469, + "theoretical_loss": 3.4046592153039565, + "tokens_seen": 2429929472 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021323169508525574, + "loss": 1.2074, + "theoretical_loss": 3.4046519107235698, + "tokens_seen": 2429995008 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021321564694082244, + "loss": 1.1698, + "theoretical_loss": 3.4046446063953404, + "tokens_seen": 2430060544 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002131995987963892, + "loss": 1.2073, + "theoretical_loss": 3.4046373023192538, + "tokens_seen": 2430126080 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002131835506519559, + "loss": 1.1136, + "theoretical_loss": 3.4046299984952935, + "tokens_seen": 2430191616 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002131675025075226, + "loss": 1.165, + "theoretical_loss": 3.4046226949234444, + "tokens_seen": 2430257152 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002131514543630893, + "loss": 1.2187, + "theoretical_loss": 3.404615391603691, + "tokens_seen": 2430322688 + }, + { + "epoch": 1.12, + "learning_rate": 0.000213135406218656, + "loss": 1.2157, + "theoretical_loss": 3.404608088536018, + "tokens_seen": 2430388224 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002131193580742227, + "loss": 1.2133, + "theoretical_loss": 3.40460078572041, + "tokens_seen": 2430453760 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002131033099297894, + "loss": 1.2592, + "theoretical_loss": 3.4045934831568507, + "tokens_seen": 2430519296 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002130872617853561, + "loss": 1.2135, + "theoretical_loss": 3.4045861808453255, + "tokens_seen": 2430584832 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002130712136409228, + "loss": 1.0709, + "theoretical_loss": 3.404578878785818, + "tokens_seen": 2430650368 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002130551654964895, + "loss": 1.1574, + "theoretical_loss": 3.404571576978314, + "tokens_seen": 2430715904 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002130391173520562, + "loss": 1.2468, + "theoretical_loss": 3.4045642754227967, + "tokens_seen": 2430781440 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002130230692076229, + "loss": 1.2772, + "theoretical_loss": 3.4045569741192514, + "tokens_seen": 2430846976 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002130070210631896, + "loss": 1.1908, + "theoretical_loss": 3.4045496730676623, + "tokens_seen": 2430912512 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021299097291875628, + "loss": 1.2764, + "theoretical_loss": 3.404542372268014, + "tokens_seen": 2430978048 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021297492477432298, + "loss": 1.3855, + "theoretical_loss": 3.404535071720291, + "tokens_seen": 2431043584 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021295887662988968, + "loss": 1.2563, + "theoretical_loss": 3.404527771424478, + "tokens_seen": 2431109120 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021294282848545638, + "loss": 1.1832, + "theoretical_loss": 3.4045204713805592, + "tokens_seen": 2431174656 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021292678034102308, + "loss": 1.2173, + "theoretical_loss": 3.404513171588519, + "tokens_seen": 2431240192 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021291073219658978, + "loss": 1.2183, + "theoretical_loss": 3.4045058720483428, + "tokens_seen": 2431305728 + }, + { + "epoch": 1.12, + "objective/train/docs_used": 898079, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2422723770141602, + "objective/train/theoretical_loss": 3.404498572760014, + "objective/train/tokens_used": 2451831264, + "theoretical_loss": 3.404498572760014, + "tokens_seen": 2431371264 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021289468405215648, + "loss": 1.2423, + "theoretical_loss": 3.404498572760014, + "tokens_seen": 2431371264 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021287863590772318, + "loss": 1.2396, + "theoretical_loss": 3.404491273723518, + "tokens_seen": 2431436800 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021286258776328988, + "loss": 1.1809, + "theoretical_loss": 3.4044839749388385, + "tokens_seen": 2431502336 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021284653961885659, + "loss": 1.3545, + "theoretical_loss": 3.404476676405961, + "tokens_seen": 2431567872 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021283049147442329, + "loss": 1.2686, + "theoretical_loss": 3.404469378124869, + "tokens_seen": 2431633408 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021281444332999, + "loss": 1.2194, + "theoretical_loss": 3.4044620800955476, + "tokens_seen": 2431698944 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002127983951855567, + "loss": 1.3203, + "theoretical_loss": 3.4044547823179814, + "tokens_seen": 2431764480 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002127823470411234, + "loss": 1.2853, + "theoretical_loss": 3.404447484792155, + "tokens_seen": 2431830016 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002127662988966901, + "loss": 1.2608, + "theoretical_loss": 3.4044401875180523, + "tokens_seen": 2431895552 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002127502507522568, + "loss": 1.3729, + "theoretical_loss": 3.4044328904956584, + "tokens_seen": 2431961088 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002127342026078235, + "loss": 1.3054, + "theoretical_loss": 3.404425593724958, + "tokens_seen": 2432026624 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002127181544633902, + "loss": 1.2258, + "theoretical_loss": 3.404418297205935, + "tokens_seen": 2432092160 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002127021063189569, + "loss": 1.1406, + "theoretical_loss": 3.404411000938574, + "tokens_seen": 2432157696 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002126860581745236, + "loss": 1.1596, + "theoretical_loss": 3.4044037049228604, + "tokens_seen": 2432223232 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002126700100300903, + "loss": 1.1991, + "theoretical_loss": 3.404396409158778, + "tokens_seen": 2432288768 + }, + { + "epoch": 1.12, + "learning_rate": 0.000212653961885657, + "loss": 1.3153, + "theoretical_loss": 3.404389113646311, + "tokens_seen": 2432354304 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002126379137412237, + "loss": 1.2333, + "theoretical_loss": 3.404381818385445, + "tokens_seen": 2432419840 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002126218655967904, + "loss": 1.1882, + "theoretical_loss": 3.404374523376164, + "tokens_seen": 2432485376 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021260581745235707, + "loss": 1.1027, + "theoretical_loss": 3.4043672286184523, + "tokens_seen": 2432550912 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021258976930792377, + "loss": 1.2577, + "theoretical_loss": 3.4043599341122945, + "tokens_seen": 2432616448 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021257372116349047, + "loss": 1.204, + "theoretical_loss": 3.4043526398576756, + "tokens_seen": 2432681984 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021255767301905717, + "loss": 1.206, + "theoretical_loss": 3.40434534585458, + "tokens_seen": 2432747520 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021254162487462387, + "loss": 1.1891, + "theoretical_loss": 3.404338052102992, + "tokens_seen": 2432813056 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021252557673019057, + "loss": 1.1951, + "theoretical_loss": 3.404330758602896, + "tokens_seen": 2432878592 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021250952858575727, + "loss": 1.1786, + "theoretical_loss": 3.4043234653542767, + "tokens_seen": 2432944128 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021249348044132397, + "loss": 1.319, + "theoretical_loss": 3.4043161723571194, + "tokens_seen": 2433009664 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021247743229689067, + "loss": 1.2011, + "theoretical_loss": 3.404308879611407, + "tokens_seen": 2433075200 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021246138415245737, + "loss": 1.2875, + "theoretical_loss": 3.4043015871171263, + "tokens_seen": 2433140736 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021244533600802407, + "loss": 1.1832, + "theoretical_loss": 3.4042942948742603, + "tokens_seen": 2433206272 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021242928786359077, + "loss": 1.1825, + "theoretical_loss": 3.4042870028827936, + "tokens_seen": 2433271808 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021241323971915748, + "loss": 1.2921, + "theoretical_loss": 3.404279711142711, + "tokens_seen": 2433337344 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021239719157472418, + "loss": 1.2473, + "theoretical_loss": 3.4042724196539975, + "tokens_seen": 2433402880 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021238114343029088, + "loss": 1.2617, + "theoretical_loss": 3.4042651284166374, + "tokens_seen": 2433468416 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021236509528585758, + "loss": 1.2481, + "theoretical_loss": 3.404257837430615, + "tokens_seen": 2433533952 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021234904714142428, + "loss": 1.2079, + "theoretical_loss": 3.404250546695915, + "tokens_seen": 2433599488 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021233299899699098, + "loss": 1.3721, + "theoretical_loss": 3.404243256212522, + "tokens_seen": 2433665024 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021231695085255768, + "loss": 1.2233, + "theoretical_loss": 3.4042359659804204, + "tokens_seen": 2433730560 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021230090270812438, + "loss": 1.2345, + "theoretical_loss": 3.404228675999595, + "tokens_seen": 2433796096 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021228485456369108, + "loss": 1.2088, + "theoretical_loss": 3.404221386270031, + "tokens_seen": 2433861632 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021226880641925778, + "loss": 1.2666, + "theoretical_loss": 3.4042140967917116, + "tokens_seen": 2433927168 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021225275827482448, + "loss": 1.1691, + "theoretical_loss": 3.404206807564622, + "tokens_seen": 2433992704 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021223671013039118, + "loss": 1.2117, + "theoretical_loss": 3.4041995185887473, + "tokens_seen": 2434058240 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021222066198595786, + "loss": 1.3139, + "theoretical_loss": 3.4041922298640714, + "tokens_seen": 2434123776 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021220461384152456, + "loss": 1.1706, + "theoretical_loss": 3.404184941390579, + "tokens_seen": 2434189312 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021218856569709126, + "loss": 1.2828, + "theoretical_loss": 3.4041776531682553, + "tokens_seen": 2434254848 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021217251755265796, + "loss": 1.1937, + "theoretical_loss": 3.404170365197084, + "tokens_seen": 2434320384 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021215646940822466, + "loss": 1.2712, + "theoretical_loss": 3.40416307747705, + "tokens_seen": 2434385920 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021214042126379136, + "loss": 1.3018, + "theoretical_loss": 3.4041557900081383, + "tokens_seen": 2434451456 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021212437311935806, + "loss": 1.2583, + "theoretical_loss": 3.4041485027903327, + "tokens_seen": 2434516992 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021210832497492476, + "loss": 1.194, + "theoretical_loss": 3.4041412158236186, + "tokens_seen": 2434582528 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021209227683049146, + "loss": 1.2345, + "theoretical_loss": 3.4041339291079797, + "tokens_seen": 2434648064 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021207622868605816, + "loss": 1.2003, + "theoretical_loss": 3.4041266426434014, + "tokens_seen": 2434713600 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021206018054162486, + "loss": 1.1804, + "theoretical_loss": 3.4041193564298684, + "tokens_seen": 2434779136 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021204413239719156, + "loss": 1.1797, + "theoretical_loss": 3.4041120704673644, + "tokens_seen": 2434844672 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021202808425275826, + "loss": 1.2076, + "theoretical_loss": 3.4041047847558743, + "tokens_seen": 2434910208 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021201203610832496, + "loss": 1.2951, + "theoretical_loss": 3.4040974992953834, + "tokens_seen": 2434975744 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021199598796389172, + "loss": 1.3521, + "theoretical_loss": 3.4040902140858753, + "tokens_seen": 2435041280 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021197993981945842, + "loss": 1.2016, + "theoretical_loss": 3.4040829291273353, + "tokens_seen": 2435106816 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002119638916750251, + "loss": 1.2182, + "theoretical_loss": 3.404075644419748, + "tokens_seen": 2435172352 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002119478435305918, + "loss": 1.2908, + "theoretical_loss": 3.4040683599630976, + "tokens_seen": 2435237888 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002119317953861585, + "loss": 1.2577, + "theoretical_loss": 3.404061075757369, + "tokens_seen": 2435303424 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002119157472417252, + "loss": 1.2694, + "theoretical_loss": 3.4040537918025464, + "tokens_seen": 2435368960 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002118996990972919, + "loss": 1.1947, + "theoretical_loss": 3.404046508098615, + "tokens_seen": 2435434496 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002118836509528586, + "loss": 1.2778, + "theoretical_loss": 3.4040392246455586, + "tokens_seen": 2435500032 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002118676028084253, + "loss": 1.2883, + "theoretical_loss": 3.404031941443363, + "tokens_seen": 2435565568 + }, + { + "epoch": 1.12, + "learning_rate": 0.000211851554663992, + "loss": 1.245, + "theoretical_loss": 3.4040246584920117, + "tokens_seen": 2435631104 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002118355065195587, + "loss": 1.1466, + "theoretical_loss": 3.4040173757914896, + "tokens_seen": 2435696640 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002118194583751254, + "loss": 1.2005, + "theoretical_loss": 3.404010093341782, + "tokens_seen": 2435762176 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002118034102306921, + "loss": 1.0716, + "theoretical_loss": 3.404002811142872, + "tokens_seen": 2435827712 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002117873620862588, + "loss": 1.1266, + "theoretical_loss": 3.403995529194746, + "tokens_seen": 2435893248 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002117713139418255, + "loss": 1.2457, + "theoretical_loss": 3.403988247497388, + "tokens_seen": 2435958784 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002117552657973922, + "loss": 1.1861, + "theoretical_loss": 3.403980966050782, + "tokens_seen": 2436024320 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002117392176529589, + "loss": 1.3272, + "theoretical_loss": 3.403973684854913, + "tokens_seen": 2436089856 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002117231695085256, + "loss": 1.2783, + "theoretical_loss": 3.4039664039097652, + "tokens_seen": 2436155392 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002117071213640923, + "loss": 1.2172, + "theoretical_loss": 3.4039591232153246, + "tokens_seen": 2436220928 + }, + { + "epoch": 1.12, + "learning_rate": 0.000211691073219659, + "loss": 1.2773, + "theoretical_loss": 3.403951842771574, + "tokens_seen": 2436286464 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002116750250752257, + "loss": 1.0912, + "theoretical_loss": 3.4039445625784994, + "tokens_seen": 2436352000 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002116589769307924, + "loss": 1.3585, + "theoretical_loss": 3.403937282636085, + "tokens_seen": 2436417536 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002116429287863591, + "loss": 1.2625, + "theoretical_loss": 3.403930002944315, + "tokens_seen": 2436483072 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002116268806419258, + "loss": 1.2652, + "theoretical_loss": 3.403922723503175, + "tokens_seen": 2436548608 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002116108324974925, + "loss": 1.2416, + "theoretical_loss": 3.4039154443126485, + "tokens_seen": 2436614144 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002115947843530592, + "loss": 1.2272, + "theoretical_loss": 3.403908165372721, + "tokens_seen": 2436679680 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021157873620862588, + "loss": 1.2117, + "theoretical_loss": 3.4039008866833766, + "tokens_seen": 2436745216 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021156268806419258, + "loss": 1.2594, + "theoretical_loss": 3.4038936082446, + "tokens_seen": 2436810752 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021154663991975928, + "loss": 1.1839, + "theoretical_loss": 3.403886330056376, + "tokens_seen": 2436876288 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021153059177532598, + "loss": 1.1011, + "theoretical_loss": 3.4038790521186897, + "tokens_seen": 2436941824 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021151454363089268, + "loss": 1.2595, + "theoretical_loss": 3.4038717744315248, + "tokens_seen": 2437007360 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021149849548645938, + "loss": 1.1832, + "theoretical_loss": 3.4038644969948666, + "tokens_seen": 2437072896 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021148244734202609, + "loss": 1.2778, + "theoretical_loss": 3.403857219808699, + "tokens_seen": 2437138432 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021146639919759279, + "loss": 1.1602, + "theoretical_loss": 3.403849942873008, + "tokens_seen": 2437203968 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021145035105315949, + "loss": 1.2937, + "theoretical_loss": 3.403842666187777, + "tokens_seen": 2437269504 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002114343029087262, + "loss": 1.2275, + "theoretical_loss": 3.4038353897529907, + "tokens_seen": 2437335040 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002114182547642929, + "loss": 1.2034, + "theoretical_loss": 3.4038281135686344, + "tokens_seen": 2437400576 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002114022066198596, + "loss": 1.2352, + "theoretical_loss": 3.403820837634693, + "tokens_seen": 2437466112 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002113861584754263, + "loss": 1.2726, + "theoretical_loss": 3.4038135619511496, + "tokens_seen": 2437531648 + }, + { + "epoch": 1.12, + "learning_rate": 0.000211370110330993, + "loss": 1.2484, + "theoretical_loss": 3.40380628651799, + "tokens_seen": 2437597184 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002113540621865597, + "loss": 1.1777, + "theoretical_loss": 3.4037990113351992, + "tokens_seen": 2437662720 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002113380140421264, + "loss": 1.3249, + "theoretical_loss": 3.4037917364027614, + "tokens_seen": 2437728256 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002113219658976931, + "loss": 1.2096, + "theoretical_loss": 3.4037844617206607, + "tokens_seen": 2437793792 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002113059177532598, + "loss": 1.2147, + "theoretical_loss": 3.403777187288883, + "tokens_seen": 2437859328 + }, + { + "epoch": 1.12, + "objective/train/docs_used": 900483, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3135771751403809, + "objective/train/theoretical_loss": 3.4037699131074115, + "objective/train/tokens_used": 2458384864, + "theoretical_loss": 3.4037699131074115, + "tokens_seen": 2437924864 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002112898696088265, + "loss": 1.3136, + "theoretical_loss": 3.4037699131074115, + "tokens_seen": 2437924864 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002112738214643932, + "loss": 1.2658, + "theoretical_loss": 3.403762639176232, + "tokens_seen": 2437990400 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002112577733199599, + "loss": 1.1987, + "theoretical_loss": 3.4037553654953285, + "tokens_seen": 2438055936 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002112417251755266, + "loss": 1.1679, + "theoretical_loss": 3.403748092064686, + "tokens_seen": 2438121472 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002112256770310933, + "loss": 1.2836, + "theoretical_loss": 3.4037408188842893, + "tokens_seen": 2438187008 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021120962888666, + "loss": 1.2984, + "theoretical_loss": 3.4037335459541227, + "tokens_seen": 2438252544 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021119358074222667, + "loss": 1.2839, + "theoretical_loss": 3.4037262732741707, + "tokens_seen": 2438318080 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021117753259779337, + "loss": 1.2708, + "theoretical_loss": 3.4037190008444185, + "tokens_seen": 2438383616 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021116148445336007, + "loss": 1.3231, + "theoretical_loss": 3.4037117286648506, + "tokens_seen": 2438449152 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021114543630892677, + "loss": 1.2954, + "theoretical_loss": 3.4037044567354515, + "tokens_seen": 2438514688 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021112938816449347, + "loss": 1.2072, + "theoretical_loss": 3.4036971850562057, + "tokens_seen": 2438580224 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021111334002006017, + "loss": 1.149, + "theoretical_loss": 3.4036899136270984, + "tokens_seen": 2438645760 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021109729187562687, + "loss": 1.2799, + "theoretical_loss": 3.403682642448114, + "tokens_seen": 2438711296 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021108124373119357, + "loss": 1.3153, + "theoretical_loss": 3.4036753715192374, + "tokens_seen": 2438776832 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021106519558676027, + "loss": 1.3152, + "theoretical_loss": 3.403668100840453, + "tokens_seen": 2438842368 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021104914744232698, + "loss": 1.3013, + "theoretical_loss": 3.403660830411745, + "tokens_seen": 2438907904 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021103309929789368, + "loss": 1.1919, + "theoretical_loss": 3.4036535602330993, + "tokens_seen": 2438973440 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021101705115346038, + "loss": 1.1704, + "theoretical_loss": 3.4036462903044997, + "tokens_seen": 2439038976 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021100100300902708, + "loss": 1.2213, + "theoretical_loss": 3.4036390206259313, + "tokens_seen": 2439104512 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021098495486459378, + "loss": 1.1988, + "theoretical_loss": 3.403631751197378, + "tokens_seen": 2439170048 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021096890672016048, + "loss": 1.2238, + "theoretical_loss": 3.4036244820188255, + "tokens_seen": 2439235584 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021095285857572718, + "loss": 1.194, + "theoretical_loss": 3.403617213090258, + "tokens_seen": 2439301120 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021093681043129388, + "loss": 1.1611, + "theoretical_loss": 3.4036099444116603, + "tokens_seen": 2439366656 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021092076228686058, + "loss": 1.3156, + "theoretical_loss": 3.4036026759830165, + "tokens_seen": 2439432192 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021090471414242728, + "loss": 1.1857, + "theoretical_loss": 3.4035954078043122, + "tokens_seen": 2439497728 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021088866599799398, + "loss": 1.2573, + "theoretical_loss": 3.4035881398755317, + "tokens_seen": 2439563264 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021087261785356068, + "loss": 1.221, + "theoretical_loss": 3.4035808721966596, + "tokens_seen": 2439628800 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021085656970912738, + "loss": 1.2169, + "theoretical_loss": 3.403573604767681, + "tokens_seen": 2439694336 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021084052156469408, + "loss": 1.1789, + "theoretical_loss": 3.40356633758858, + "tokens_seen": 2439759872 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021082447342026078, + "loss": 1.1839, + "theoretical_loss": 3.4035590706593415, + "tokens_seen": 2439825408 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002108084252758275, + "loss": 1.271, + "theoretical_loss": 3.4035518039799504, + "tokens_seen": 2439890944 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002107923771313942, + "loss": 1.2016, + "theoretical_loss": 3.403544537550391, + "tokens_seen": 2439956480 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002107763289869609, + "loss": 1.1795, + "theoretical_loss": 3.4035372713706487, + "tokens_seen": 2440022016 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021076028084252761, + "loss": 1.1931, + "theoretical_loss": 3.403530005440708, + "tokens_seen": 2440087552 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021074423269809431, + "loss": 1.2132, + "theoretical_loss": 3.403522739760553, + "tokens_seen": 2440153088 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021072818455366101, + "loss": 1.1926, + "theoretical_loss": 3.4035154743301685, + "tokens_seen": 2440218624 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021071213640922772, + "loss": 1.2094, + "theoretical_loss": 3.40350820914954, + "tokens_seen": 2440284160 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021069608826479442, + "loss": 1.2138, + "theoretical_loss": 3.403500944218651, + "tokens_seen": 2440349696 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021068004012036112, + "loss": 1.163, + "theoretical_loss": 3.4034936795374877, + "tokens_seen": 2440415232 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021066399197592782, + "loss": 1.3031, + "theoretical_loss": 3.4034864151060336, + "tokens_seen": 2440480768 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021064794383149452, + "loss": 1.2148, + "theoretical_loss": 3.403479150924274, + "tokens_seen": 2440546304 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021063189568706122, + "loss": 1.2271, + "theoretical_loss": 3.403471886992193, + "tokens_seen": 2440611840 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021061584754262792, + "loss": 1.1519, + "theoretical_loss": 3.4034646233097763, + "tokens_seen": 2440677376 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021059979939819462, + "loss": 1.2079, + "theoretical_loss": 3.4034573598770077, + "tokens_seen": 2440742912 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021058375125376132, + "loss": 1.224, + "theoretical_loss": 3.4034500966938728, + "tokens_seen": 2440808448 + }, + { + "epoch": 1.12, + "learning_rate": 0.000210567703109328, + "loss": 1.1996, + "theoretical_loss": 3.4034428337603555, + "tokens_seen": 2440873984 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002105516549648947, + "loss": 1.2337, + "theoretical_loss": 3.4034355710764403, + "tokens_seen": 2440939520 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002105356068204614, + "loss": 1.2455, + "theoretical_loss": 3.403428308642113, + "tokens_seen": 2441005056 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002105195586760281, + "loss": 1.224, + "theoretical_loss": 3.4034210464573578, + "tokens_seen": 2441070592 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002105035105315948, + "loss": 1.1538, + "theoretical_loss": 3.4034137845221593, + "tokens_seen": 2441136128 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002104874623871615, + "loss": 1.1546, + "theoretical_loss": 3.4034065228365025, + "tokens_seen": 2441201664 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002104714142427282, + "loss": 1.2787, + "theoretical_loss": 3.403399261400372, + "tokens_seen": 2441267200 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002104553660982949, + "loss": 1.2367, + "theoretical_loss": 3.403392000213752, + "tokens_seen": 2441332736 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002104393179538616, + "loss": 1.2109, + "theoretical_loss": 3.403384739276628, + "tokens_seen": 2441398272 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002104232698094283, + "loss": 1.3096, + "theoretical_loss": 3.4033774785889843, + "tokens_seen": 2441463808 + }, + { + "epoch": 1.12, + "learning_rate": 0.000210407221664995, + "loss": 1.2355, + "theoretical_loss": 3.4033702181508056, + "tokens_seen": 2441529344 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002103911735205617, + "loss": 1.2373, + "theoretical_loss": 3.403362957962077, + "tokens_seen": 2441594880 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002103751253761284, + "loss": 1.1698, + "theoretical_loss": 3.403355698022783, + "tokens_seen": 2441660416 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002103590772316951, + "loss": 1.19, + "theoretical_loss": 3.4033484383329085, + "tokens_seen": 2441725952 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002103430290872618, + "loss": 1.1873, + "theoretical_loss": 3.403341178892438, + "tokens_seen": 2441791488 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002103269809428285, + "loss": 1.276, + "theoretical_loss": 3.4033339197013563, + "tokens_seen": 2441857024 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002103109327983952, + "loss": 1.1381, + "theoretical_loss": 3.4033266607596486, + "tokens_seen": 2441922560 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002102948846539619, + "loss": 1.2148, + "theoretical_loss": 3.4033194020672988, + "tokens_seen": 2441988096 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002102788365095286, + "loss": 1.2855, + "theoretical_loss": 3.4033121436242917, + "tokens_seen": 2442053632 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002102627883650953, + "loss": 1.1883, + "theoretical_loss": 3.4033048854306127, + "tokens_seen": 2442119168 + }, + { + "epoch": 1.12, + "learning_rate": 0.000210246740220662, + "loss": 1.207, + "theoretical_loss": 3.4032976274862468, + "tokens_seen": 2442184704 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002102306920762287, + "loss": 1.2726, + "theoretical_loss": 3.403290369791178, + "tokens_seen": 2442250240 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002102146439317954, + "loss": 1.2298, + "theoretical_loss": 3.403283112345391, + "tokens_seen": 2442315776 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002101985957873621, + "loss": 1.1407, + "theoretical_loss": 3.4032758551488707, + "tokens_seen": 2442381312 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021018254764292878, + "loss": 1.2681, + "theoretical_loss": 3.4032685982016018, + "tokens_seen": 2442446848 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021016649949849548, + "loss": 1.1961, + "theoretical_loss": 3.4032613415035695, + "tokens_seen": 2442512384 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021015045135406218, + "loss": 1.2019, + "theoretical_loss": 3.403254085054758, + "tokens_seen": 2442577920 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021013440320962888, + "loss": 1.231, + "theoretical_loss": 3.4032468288551527, + "tokens_seen": 2442643456 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021011835506519558, + "loss": 1.1916, + "theoretical_loss": 3.4032395729047376, + "tokens_seen": 2442708992 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021010230692076229, + "loss": 1.2095, + "theoretical_loss": 3.403232317203498, + "tokens_seen": 2442774528 + }, + { + "epoch": 1.12, + "learning_rate": 0.00021008625877632899, + "loss": 1.318, + "theoretical_loss": 3.403225061751418, + "tokens_seen": 2442840064 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002100702106318957, + "loss": 1.2542, + "theoretical_loss": 3.4032178065484837, + "tokens_seen": 2442905600 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002100541624874624, + "loss": 1.2131, + "theoretical_loss": 3.4032105515946784, + "tokens_seen": 2442971136 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002100381143430291, + "loss": 1.1741, + "theoretical_loss": 3.4032032968899877, + "tokens_seen": 2443036672 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002100220661985958, + "loss": 1.1519, + "theoretical_loss": 3.403196042434396, + "tokens_seen": 2443102208 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002100060180541625, + "loss": 1.2228, + "theoretical_loss": 3.4031887882278884, + "tokens_seen": 2443167744 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002099899699097292, + "loss": 1.2904, + "theoretical_loss": 3.4031815342704492, + "tokens_seen": 2443233280 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002099739217652959, + "loss": 1.2313, + "theoretical_loss": 3.4031742805620633, + "tokens_seen": 2443298816 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002099578736208626, + "loss": 1.2443, + "theoretical_loss": 3.403167027102716, + "tokens_seen": 2443364352 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002099418254764293, + "loss": 1.2474, + "theoretical_loss": 3.4031597738923915, + "tokens_seen": 2443429888 + }, + { + "epoch": 1.12, + "learning_rate": 0.000209925777331996, + "loss": 1.2256, + "theoretical_loss": 3.4031525209310747, + "tokens_seen": 2443495424 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002099097291875627, + "loss": 1.134, + "theoretical_loss": 3.40314526821875, + "tokens_seen": 2443560960 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002098936810431294, + "loss": 1.1542, + "theoretical_loss": 3.4031380157554034, + "tokens_seen": 2443626496 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002098776328986961, + "loss": 1.1871, + "theoretical_loss": 3.4031307635410184, + "tokens_seen": 2443692032 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002098615847542628, + "loss": 1.2226, + "theoretical_loss": 3.40312351157558, + "tokens_seen": 2443757568 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002098455366098295, + "loss": 1.2742, + "theoretical_loss": 3.4031162598590736, + "tokens_seen": 2443823104 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002098294884653962, + "loss": 1.2927, + "theoretical_loss": 3.4031090083914837, + "tokens_seen": 2443888640 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002098134403209629, + "loss": 1.2289, + "theoretical_loss": 3.4031017571727946, + "tokens_seen": 2443954176 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020979739217652957, + "loss": 1.2305, + "theoretical_loss": 3.403094506202992, + "tokens_seen": 2444019712 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020978134403209627, + "loss": 1.2778, + "theoretical_loss": 3.4030872554820597, + "tokens_seen": 2444085248 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020976529588766297, + "loss": 1.1789, + "theoretical_loss": 3.4030800050099828, + "tokens_seen": 2444150784 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020974924774322967, + "loss": 1.3124, + "theoretical_loss": 3.4030727547867463, + "tokens_seen": 2444216320 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020973319959879637, + "loss": 1.2394, + "theoretical_loss": 3.403065504812335, + "tokens_seen": 2444281856 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020971715145436307, + "loss": 1.2261, + "theoretical_loss": 3.4030582550867337, + "tokens_seen": 2444347392 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020970110330992977, + "loss": 1.1495, + "theoretical_loss": 3.403051005609927, + "tokens_seen": 2444412928 + }, + { + "epoch": 1.12, + "objective/train/docs_used": 903170, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.235598087310791, + "objective/train/theoretical_loss": 3.4030437563818996, + "objective/train/tokens_used": 2464938464, + "theoretical_loss": 3.4030437563818996, + "tokens_seen": 2444478464 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020968505516549647, + "loss": 1.2356, + "theoretical_loss": 3.4030437563818996, + "tokens_seen": 2444478464 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020966900702106318, + "loss": 1.1442, + "theoretical_loss": 3.4030365074026365, + "tokens_seen": 2444544000 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020965295887662988, + "loss": 1.2152, + "theoretical_loss": 3.4030292586721225, + "tokens_seen": 2444609536 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020963691073219658, + "loss": 1.2882, + "theoretical_loss": 3.4030220101903423, + "tokens_seen": 2444675072 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020962086258776328, + "loss": 1.1646, + "theoretical_loss": 3.403014761957281, + "tokens_seen": 2444740608 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020960481444333003, + "loss": 1.1798, + "theoretical_loss": 3.403007513972923, + "tokens_seen": 2444806144 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020958876629889673, + "loss": 1.1968, + "theoretical_loss": 3.403000266237253, + "tokens_seen": 2444871680 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020957271815446343, + "loss": 1.2423, + "theoretical_loss": 3.402993018750257, + "tokens_seen": 2444937216 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020955667001003013, + "loss": 1.2175, + "theoretical_loss": 3.4029857715119176, + "tokens_seen": 2445002752 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002095406218655968, + "loss": 1.2552, + "theoretical_loss": 3.4029785245222217, + "tokens_seen": 2445068288 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002095245737211635, + "loss": 1.2863, + "theoretical_loss": 3.4029712777811527, + "tokens_seen": 2445133824 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002095085255767302, + "loss": 1.1157, + "theoretical_loss": 3.4029640312886964, + "tokens_seen": 2445199360 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002094924774322969, + "loss": 1.3202, + "theoretical_loss": 3.4029567850448372, + "tokens_seen": 2445264896 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002094764292878636, + "loss": 1.2348, + "theoretical_loss": 3.4029495390495597, + "tokens_seen": 2445330432 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002094603811434303, + "loss": 1.2915, + "theoretical_loss": 3.402942293302849, + "tokens_seen": 2445395968 + }, + { + "epoch": 1.12, + "learning_rate": 0.000209444332998997, + "loss": 1.3116, + "theoretical_loss": 3.4029350478046894, + "tokens_seen": 2445461504 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002094282848545637, + "loss": 1.1621, + "theoretical_loss": 3.402927802555067, + "tokens_seen": 2445527040 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002094122367101304, + "loss": 1.1162, + "theoretical_loss": 3.4029205575539647, + "tokens_seen": 2445592576 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002093961885656971, + "loss": 1.2424, + "theoretical_loss": 3.402913312801369, + "tokens_seen": 2445658112 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020938014042126381, + "loss": 1.2358, + "theoretical_loss": 3.4029060682972636, + "tokens_seen": 2445723648 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020936409227683051, + "loss": 1.2682, + "theoretical_loss": 3.4028988240416345, + "tokens_seen": 2445789184 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020934804413239722, + "loss": 1.233, + "theoretical_loss": 3.4028915800344652, + "tokens_seen": 2445854720 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020933199598796392, + "loss": 1.2345, + "theoretical_loss": 3.4028843362757417, + "tokens_seen": 2445920256 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020931594784353062, + "loss": 1.198, + "theoretical_loss": 3.402877092765448, + "tokens_seen": 2445985792 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020929989969909732, + "loss": 1.3501, + "theoretical_loss": 3.4028698495035687, + "tokens_seen": 2446051328 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020928385155466402, + "loss": 1.2194, + "theoretical_loss": 3.40286260649009, + "tokens_seen": 2446116864 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020926780341023072, + "loss": 1.2955, + "theoretical_loss": 3.402855363724995, + "tokens_seen": 2446182400 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020925175526579742, + "loss": 1.1643, + "theoretical_loss": 3.4028481212082697, + "tokens_seen": 2446247936 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020923570712136412, + "loss": 1.1889, + "theoretical_loss": 3.4028408789398985, + "tokens_seen": 2446313472 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020921965897693082, + "loss": 1.1959, + "theoretical_loss": 3.4028336369198664, + "tokens_seen": 2446379008 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020920361083249752, + "loss": 1.113, + "theoretical_loss": 3.4028263951481583, + "tokens_seen": 2446444544 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020918756268806422, + "loss": 1.3006, + "theoretical_loss": 3.4028191536247587, + "tokens_seen": 2446510080 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020917151454363092, + "loss": 1.1561, + "theoretical_loss": 3.4028119123496525, + "tokens_seen": 2446575616 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002091554663991976, + "loss": 1.2759, + "theoretical_loss": 3.402804671322825, + "tokens_seen": 2446641152 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002091394182547643, + "loss": 1.2022, + "theoretical_loss": 3.4027974305442603, + "tokens_seen": 2446706688 + }, + { + "epoch": 1.12, + "learning_rate": 0.000209123370110331, + "loss": 1.2261, + "theoretical_loss": 3.402790190013944, + "tokens_seen": 2446772224 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002091073219658977, + "loss": 1.2223, + "theoretical_loss": 3.4027829497318605, + "tokens_seen": 2446837760 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002090912738214644, + "loss": 1.21, + "theoretical_loss": 3.4027757096979947, + "tokens_seen": 2446903296 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002090752256770311, + "loss": 1.2462, + "theoretical_loss": 3.4027684699123313, + "tokens_seen": 2446968832 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002090591775325978, + "loss": 1.3275, + "theoretical_loss": 3.402761230374855, + "tokens_seen": 2447034368 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002090431293881645, + "loss": 1.2528, + "theoretical_loss": 3.4027539910855515, + "tokens_seen": 2447099904 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002090270812437312, + "loss": 1.1621, + "theoretical_loss": 3.402746752044405, + "tokens_seen": 2447165440 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002090110330992979, + "loss": 1.2968, + "theoretical_loss": 3.4027395132514004, + "tokens_seen": 2447230976 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002089949849548646, + "loss": 1.1704, + "theoretical_loss": 3.4027322747065223, + "tokens_seen": 2447296512 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002089789368104313, + "loss": 1.1299, + "theoretical_loss": 3.4027250364097563, + "tokens_seen": 2447362048 + }, + { + "epoch": 1.12, + "learning_rate": 0.000208962888665998, + "loss": 1.2685, + "theoretical_loss": 3.4027177983610866, + "tokens_seen": 2447427584 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002089468405215647, + "loss": 1.2102, + "theoretical_loss": 3.4027105605604984, + "tokens_seen": 2447493120 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002089307923771314, + "loss": 1.257, + "theoretical_loss": 3.4027033230079757, + "tokens_seen": 2447558656 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002089147442326981, + "loss": 1.1788, + "theoretical_loss": 3.402696085703505, + "tokens_seen": 2447624192 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002088986960882648, + "loss": 1.1923, + "theoretical_loss": 3.4026888486470694, + "tokens_seen": 2447689728 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002088826479438315, + "loss": 1.22, + "theoretical_loss": 3.402681611838655, + "tokens_seen": 2447755264 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002088665997993982, + "loss": 1.1733, + "theoretical_loss": 3.402674375278246, + "tokens_seen": 2447820800 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002088505516549649, + "loss": 1.2292, + "theoretical_loss": 3.402667138965828, + "tokens_seen": 2447886336 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002088345035105316, + "loss": 1.1593, + "theoretical_loss": 3.402659902901385, + "tokens_seen": 2447951872 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002088184553660983, + "loss": 1.3333, + "theoretical_loss": 3.402652667084902, + "tokens_seen": 2448017408 + }, + { + "epoch": 1.12, + "learning_rate": 0.000208802407221665, + "loss": 1.1748, + "theoretical_loss": 3.4026454315163646, + "tokens_seen": 2448082944 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002087863590772317, + "loss": 1.2112, + "theoretical_loss": 3.402638196195757, + "tokens_seen": 2448148480 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020877031093279838, + "loss": 1.2598, + "theoretical_loss": 3.402630961123064, + "tokens_seen": 2448214016 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020875426278836508, + "loss": 1.3467, + "theoretical_loss": 3.4026237262982706, + "tokens_seen": 2448279552 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020873821464393179, + "loss": 1.3237, + "theoretical_loss": 3.402616491721362, + "tokens_seen": 2448345088 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020872216649949849, + "loss": 1.3268, + "theoretical_loss": 3.402609257392323, + "tokens_seen": 2448410624 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020870611835506519, + "loss": 1.2101, + "theoretical_loss": 3.402602023311138, + "tokens_seen": 2448476160 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002086900702106319, + "loss": 1.2311, + "theoretical_loss": 3.402594789477792, + "tokens_seen": 2448541696 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002086740220661986, + "loss": 1.2797, + "theoretical_loss": 3.4025875558922705, + "tokens_seen": 2448607232 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002086579739217653, + "loss": 1.196, + "theoretical_loss": 3.4025803225545577, + "tokens_seen": 2448672768 + }, + { + "epoch": 1.12, + "learning_rate": 0.000208641925777332, + "loss": 1.2063, + "theoretical_loss": 3.4025730894646387, + "tokens_seen": 2448738304 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002086258776328987, + "loss": 1.1421, + "theoretical_loss": 3.4025658566224983, + "tokens_seen": 2448803840 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002086098294884654, + "loss": 1.3207, + "theoretical_loss": 3.4025586240281216, + "tokens_seen": 2448869376 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002085937813440321, + "loss": 1.2437, + "theoretical_loss": 3.4025513916814933, + "tokens_seen": 2448934912 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002085777331995988, + "loss": 1.2118, + "theoretical_loss": 3.4025441595825985, + "tokens_seen": 2449000448 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002085616850551655, + "loss": 1.2276, + "theoretical_loss": 3.402536927731422, + "tokens_seen": 2449065984 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002085456369107322, + "loss": 1.2409, + "theoretical_loss": 3.402529696127948, + "tokens_seen": 2449131520 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002085295887662989, + "loss": 1.2858, + "theoretical_loss": 3.402522464772163, + "tokens_seen": 2449197056 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002085135406218656, + "loss": 1.2181, + "theoretical_loss": 3.40251523366405, + "tokens_seen": 2449262592 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002084974924774323, + "loss": 1.2488, + "theoretical_loss": 3.402508002803595, + "tokens_seen": 2449328128 + }, + { + "epoch": 1.12, + "learning_rate": 0.000208481444332999, + "loss": 1.2692, + "theoretical_loss": 3.4025007721907827, + "tokens_seen": 2449393664 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002084653961885657, + "loss": 1.2749, + "theoretical_loss": 3.4024935418255984, + "tokens_seen": 2449459200 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002084493480441324, + "loss": 1.1974, + "theoretical_loss": 3.4024863117080257, + "tokens_seen": 2449524736 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002084332998996991, + "loss": 1.2124, + "theoretical_loss": 3.402479081838051, + "tokens_seen": 2449590272 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002084172517552658, + "loss": 1.2484, + "theoretical_loss": 3.4024718522156583, + "tokens_seen": 2449655808 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020840120361083253, + "loss": 1.373, + "theoretical_loss": 3.402464622840833, + "tokens_seen": 2449721344 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020838515546639923, + "loss": 1.2777, + "theoretical_loss": 3.4024573937135596, + "tokens_seen": 2449786880 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020836910732196593, + "loss": 1.1956, + "theoretical_loss": 3.402450164833823, + "tokens_seen": 2449852416 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020835305917753263, + "loss": 1.2143, + "theoretical_loss": 3.4024429362016084, + "tokens_seen": 2449917952 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020833701103309933, + "loss": 1.2756, + "theoretical_loss": 3.4024357078169003, + "tokens_seen": 2449983488 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020832096288866603, + "loss": 1.2691, + "theoretical_loss": 3.402428479679684, + "tokens_seen": 2450049024 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020830491474423273, + "loss": 1.2499, + "theoretical_loss": 3.4024212517899444, + "tokens_seen": 2450114560 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020828886659979943, + "loss": 1.2827, + "theoretical_loss": 3.4024140241476664, + "tokens_seen": 2450180096 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020827281845536613, + "loss": 1.2326, + "theoretical_loss": 3.4024067967528344, + "tokens_seen": 2450245632 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020825677031093283, + "loss": 1.2086, + "theoretical_loss": 3.402399569605434, + "tokens_seen": 2450311168 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020824072216649953, + "loss": 1.2765, + "theoretical_loss": 3.4023923427054497, + "tokens_seen": 2450376704 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020822467402206623, + "loss": 1.2217, + "theoretical_loss": 3.4023851160528666, + "tokens_seen": 2450442240 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020820862587763293, + "loss": 1.2529, + "theoretical_loss": 3.4023778896476693, + "tokens_seen": 2450507776 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020819257773319963, + "loss": 1.327, + "theoretical_loss": 3.402370663489843, + "tokens_seen": 2450573312 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020817652958876633, + "loss": 1.2594, + "theoretical_loss": 3.4023634375793725, + "tokens_seen": 2450638848 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020816048144433304, + "loss": 1.1596, + "theoretical_loss": 3.402356211916243, + "tokens_seen": 2450704384 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002081444332998997, + "loss": 1.1956, + "theoretical_loss": 3.4023489865004395, + "tokens_seen": 2450769920 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002081283851554664, + "loss": 1.175, + "theoretical_loss": 3.4023417613319458, + "tokens_seen": 2450835456 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002081123370110331, + "loss": 1.3619, + "theoretical_loss": 3.4023345364107485, + "tokens_seen": 2450900992 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002080962888665998, + "loss": 1.2008, + "theoretical_loss": 3.4023273117368307, + "tokens_seen": 2450966528 + }, + { + "epoch": 1.12, + "objective/train/docs_used": 905643, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1318633556365967, + "objective/train/theoretical_loss": 3.402320087310179, + "objective/train/tokens_used": 2471492064, + "theoretical_loss": 3.402320087310179, + "tokens_seen": 2451032064 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002080802407221665, + "loss": 1.1319, + "theoretical_loss": 3.402320087310179, + "tokens_seen": 2451032064 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002080641925777332, + "loss": 1.2541, + "theoretical_loss": 3.4023128631307777, + "tokens_seen": 2451097600 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002080481444332999, + "loss": 1.2595, + "theoretical_loss": 3.4023056391986115, + "tokens_seen": 2451163136 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002080320962888666, + "loss": 1.2974, + "theoretical_loss": 3.4022984155136653, + "tokens_seen": 2451228672 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020801604814443331, + "loss": 1.2996, + "theoretical_loss": 3.402291192075924, + "tokens_seen": 2451294208 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020800000000000001, + "loss": 1.2538, + "theoretical_loss": 3.4022839688853734, + "tokens_seen": 2451359744 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020798395185556671, + "loss": 1.1103, + "theoretical_loss": 3.402276745941997, + "tokens_seen": 2451425280 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020796790371113342, + "loss": 1.1894, + "theoretical_loss": 3.4022695232457814, + "tokens_seen": 2451490816 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020795185556670012, + "loss": 1.321, + "theoretical_loss": 3.40226230079671, + "tokens_seen": 2451556352 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020793580742226682, + "loss": 1.1639, + "theoretical_loss": 3.4022550785947687, + "tokens_seen": 2451621888 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020791975927783352, + "loss": 1.2779, + "theoretical_loss": 3.4022478566399417, + "tokens_seen": 2451687424 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020790371113340022, + "loss": 1.1512, + "theoretical_loss": 3.4022406349322147, + "tokens_seen": 2451752960 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020788766298896692, + "loss": 1.1328, + "theoretical_loss": 3.4022334134715724, + "tokens_seen": 2451818496 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020787161484453362, + "loss": 1.2572, + "theoretical_loss": 3.4022261922579995, + "tokens_seen": 2451884032 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020785556670010032, + "loss": 1.2394, + "theoretical_loss": 3.4022189712914805, + "tokens_seen": 2451949568 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020783951855566702, + "loss": 1.1772, + "theoretical_loss": 3.4022117505720018, + "tokens_seen": 2452015104 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020782347041123372, + "loss": 1.3031, + "theoretical_loss": 3.402204530099547, + "tokens_seen": 2452080640 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020780742226680042, + "loss": 1.2225, + "theoretical_loss": 3.4021973098741016, + "tokens_seen": 2452146176 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020779137412236712, + "loss": 1.1508, + "theoretical_loss": 3.4021900898956505, + "tokens_seen": 2452211712 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020777532597793382, + "loss": 1.2077, + "theoretical_loss": 3.4021828701641788, + "tokens_seen": 2452277248 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002077592778335005, + "loss": 1.2376, + "theoretical_loss": 3.4021756506796708, + "tokens_seen": 2452342784 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002077432296890672, + "loss": 1.2601, + "theoretical_loss": 3.4021684314421123, + "tokens_seen": 2452408320 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002077271815446339, + "loss": 1.2792, + "theoretical_loss": 3.402161212451488, + "tokens_seen": 2452473856 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002077111334002006, + "loss": 1.2467, + "theoretical_loss": 3.4021539937077825, + "tokens_seen": 2452539392 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002076950852557673, + "loss": 1.2401, + "theoretical_loss": 3.402146775210981, + "tokens_seen": 2452604928 + }, + { + "epoch": 1.12, + "learning_rate": 0.000207679037111334, + "loss": 1.2804, + "theoretical_loss": 3.4021395569610684, + "tokens_seen": 2452670464 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002076629889669007, + "loss": 1.2319, + "theoretical_loss": 3.4021323389580296, + "tokens_seen": 2452736000 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002076469408224674, + "loss": 1.0951, + "theoretical_loss": 3.40212512120185, + "tokens_seen": 2452801536 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002076308926780341, + "loss": 1.2503, + "theoretical_loss": 3.402117903692514, + "tokens_seen": 2452867072 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002076148445336008, + "loss": 1.1761, + "theoretical_loss": 3.4021106864300066, + "tokens_seen": 2452932608 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002075987963891675, + "loss": 1.2266, + "theoretical_loss": 3.4021034694143135, + "tokens_seen": 2452998144 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002075827482447342, + "loss": 1.2005, + "theoretical_loss": 3.4020962526454186, + "tokens_seen": 2453063680 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002075667001003009, + "loss": 1.2335, + "theoretical_loss": 3.4020890361233076, + "tokens_seen": 2453129216 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002075506519558676, + "loss": 1.1961, + "theoretical_loss": 3.4020818198479654, + "tokens_seen": 2453194752 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002075346038114343, + "loss": 1.1812, + "theoretical_loss": 3.4020746038193765, + "tokens_seen": 2453260288 + }, + { + "epoch": 1.12, + "learning_rate": 0.000207518555667001, + "loss": 1.0897, + "theoretical_loss": 3.4020673880375267, + "tokens_seen": 2453325824 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002075025075225677, + "loss": 1.2664, + "theoretical_loss": 3.4020601725024, + "tokens_seen": 2453391360 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002074864593781344, + "loss": 1.2214, + "theoretical_loss": 3.402052957213982, + "tokens_seen": 2453456896 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002074704112337011, + "loss": 1.2835, + "theoretical_loss": 3.4020457421722576, + "tokens_seen": 2453522432 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002074543630892678, + "loss": 1.1695, + "theoretical_loss": 3.402038527377212, + "tokens_seen": 2453587968 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002074383149448345, + "loss": 1.2092, + "theoretical_loss": 3.402031312828829, + "tokens_seen": 2453653504 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002074222668004012, + "loss": 1.2547, + "theoretical_loss": 3.402024098527095, + "tokens_seen": 2453719040 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002074062186559679, + "loss": 1.0688, + "theoretical_loss": 3.4020168844719945, + "tokens_seen": 2453784576 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002073901705115346, + "loss": 1.1176, + "theoretical_loss": 3.4020096706635123, + "tokens_seen": 2453850112 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020737412236710128, + "loss": 1.1713, + "theoretical_loss": 3.4020024571016334, + "tokens_seen": 2453915648 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020735807422266799, + "loss": 1.1869, + "theoretical_loss": 3.401995243786343, + "tokens_seen": 2453981184 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020734202607823469, + "loss": 1.1467, + "theoretical_loss": 3.401988030717626, + "tokens_seen": 2454046720 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002073259779338014, + "loss": 1.3109, + "theoretical_loss": 3.401980817895467, + "tokens_seen": 2454112256 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002073099297893681, + "loss": 1.2392, + "theoretical_loss": 3.401973605319852, + "tokens_seen": 2454177792 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002072938816449348, + "loss": 1.1334, + "theoretical_loss": 3.4019663929907646, + "tokens_seen": 2454243328 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002072778335005015, + "loss": 1.1091, + "theoretical_loss": 3.4019591809081913, + "tokens_seen": 2454308864 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002072617853560682, + "loss": 1.3, + "theoretical_loss": 3.4019519690721154, + "tokens_seen": 2454374400 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002072457372116349, + "loss": 1.2435, + "theoretical_loss": 3.4019447574825232, + "tokens_seen": 2454439936 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002072296890672016, + "loss": 1.2269, + "theoretical_loss": 3.4019375461393997, + "tokens_seen": 2454505472 + }, + { + "epoch": 1.12, + "learning_rate": 0.0002072136409227683, + "loss": 1.1946, + "theoretical_loss": 3.4019303350427292, + "tokens_seen": 2454571008 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020719759277833505, + "loss": 1.2094, + "theoretical_loss": 3.401923124192497, + "tokens_seen": 2454636544 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020718154463390175, + "loss": 1.2476, + "theoretical_loss": 3.401915913588688, + "tokens_seen": 2454702080 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020716549648946845, + "loss": 1.1933, + "theoretical_loss": 3.4019087032312876, + "tokens_seen": 2454767616 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020714944834503515, + "loss": 1.1559, + "theoretical_loss": 3.4019014931202802, + "tokens_seen": 2454833152 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020713340020060182, + "loss": 1.1267, + "theoretical_loss": 3.4018942832556514, + "tokens_seen": 2454898688 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020711735205616852, + "loss": 1.2272, + "theoretical_loss": 3.4018870736373854, + "tokens_seen": 2454964224 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020710130391173522, + "loss": 1.159, + "theoretical_loss": 3.401879864265468, + "tokens_seen": 2455029760 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020708525576730192, + "loss": 1.3163, + "theoretical_loss": 3.4018726551398837, + "tokens_seen": 2455095296 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020706920762286862, + "loss": 1.2458, + "theoretical_loss": 3.401865446260618, + "tokens_seen": 2455160832 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020705315947843532, + "loss": 1.1639, + "theoretical_loss": 3.4018582376276556, + "tokens_seen": 2455226368 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020703711133400203, + "loss": 1.2178, + "theoretical_loss": 3.4018510292409814, + "tokens_seen": 2455291904 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020702106318956873, + "loss": 1.3153, + "theoretical_loss": 3.401843821100581, + "tokens_seen": 2455357440 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020700501504513543, + "loss": 1.209, + "theoretical_loss": 3.4018366132064384, + "tokens_seen": 2455422976 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020698896690070213, + "loss": 1.1518, + "theoretical_loss": 3.4018294055585394, + "tokens_seen": 2455488512 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020697291875626883, + "loss": 1.1543, + "theoretical_loss": 3.401822198156869, + "tokens_seen": 2455554048 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020695687061183553, + "loss": 1.2113, + "theoretical_loss": 3.4018149910014115, + "tokens_seen": 2455619584 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020694082246740223, + "loss": 1.1192, + "theoretical_loss": 3.401807784092153, + "tokens_seen": 2455685120 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020692477432296893, + "loss": 1.3149, + "theoretical_loss": 3.4018005774290776, + "tokens_seen": 2455750656 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020690872617853563, + "loss": 1.3335, + "theoretical_loss": 3.401793371012171, + "tokens_seen": 2455816192 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020689267803410233, + "loss": 1.1852, + "theoretical_loss": 3.4017861648414174, + "tokens_seen": 2455881728 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020687662988966903, + "loss": 1.2946, + "theoretical_loss": 3.4017789589168026, + "tokens_seen": 2455947264 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020686058174523573, + "loss": 1.2794, + "theoretical_loss": 3.4017717532383114, + "tokens_seen": 2456012800 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020684453360080243, + "loss": 1.2414, + "theoretical_loss": 3.4017645478059286, + "tokens_seen": 2456078336 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020682848545636913, + "loss": 1.2112, + "theoretical_loss": 3.4017573426196397, + "tokens_seen": 2456143872 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020681243731193583, + "loss": 1.263, + "theoretical_loss": 3.401750137679429, + "tokens_seen": 2456209408 + }, + { + "epoch": 1.12, + "learning_rate": 0.00020679638916750253, + "loss": 1.2026, + "theoretical_loss": 3.4017429329852824, + "tokens_seen": 2456274944 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020678034102306924, + "loss": 1.1556, + "theoretical_loss": 3.401735728537184, + "tokens_seen": 2456340480 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020676429287863594, + "loss": 1.3335, + "theoretical_loss": 3.4017285243351196, + "tokens_seen": 2456406016 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002067482447342026, + "loss": 1.2314, + "theoretical_loss": 3.4017213203790737, + "tokens_seen": 2456471552 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002067321965897693, + "loss": 1.2979, + "theoretical_loss": 3.401714116669032, + "tokens_seen": 2456537088 + }, + { + "epoch": 1.13, + "learning_rate": 0.000206716148445336, + "loss": 1.2003, + "theoretical_loss": 3.401706913204979, + "tokens_seen": 2456602624 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002067001003009027, + "loss": 1.1797, + "theoretical_loss": 3.4016997099869, + "tokens_seen": 2456668160 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002066840521564694, + "loss": 1.2545, + "theoretical_loss": 3.4016925070147797, + "tokens_seen": 2456733696 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002066680040120361, + "loss": 1.1576, + "theoretical_loss": 3.4016853042886033, + "tokens_seen": 2456799232 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002066519558676028, + "loss": 1.2599, + "theoretical_loss": 3.401678101808356, + "tokens_seen": 2456864768 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020663590772316951, + "loss": 1.3131, + "theoretical_loss": 3.4016708995740226, + "tokens_seen": 2456930304 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020661985957873621, + "loss": 1.1015, + "theoretical_loss": 3.4016636975855885, + "tokens_seen": 2456995840 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020660381143430292, + "loss": 1.2618, + "theoretical_loss": 3.4016564958430386, + "tokens_seen": 2457061376 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020658776328986962, + "loss": 1.239, + "theoretical_loss": 3.4016492943463574, + "tokens_seen": 2457126912 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020657171514543632, + "loss": 1.1866, + "theoretical_loss": 3.4016420930955307, + "tokens_seen": 2457192448 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020655566700100302, + "loss": 1.3017, + "theoretical_loss": 3.4016348920905433, + "tokens_seen": 2457257984 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020653961885656972, + "loss": 1.1846, + "theoretical_loss": 3.40162769133138, + "tokens_seen": 2457323520 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020652357071213642, + "loss": 1.2237, + "theoretical_loss": 3.4016204908180265, + "tokens_seen": 2457389056 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020650752256770312, + "loss": 1.1646, + "theoretical_loss": 3.401613290550467, + "tokens_seen": 2457454592 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020649147442326982, + "loss": 1.2898, + "theoretical_loss": 3.4016060905286873, + "tokens_seen": 2457520128 + }, + { + "epoch": 1.13, + "objective/train/docs_used": 907986, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.195603370666504, + "objective/train/theoretical_loss": 3.401598890752672, + "objective/train/tokens_used": 2478045664, + "theoretical_loss": 3.401598890752672, + "tokens_seen": 2457585664 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020647542627883652, + "loss": 1.1956, + "theoretical_loss": 3.401598890752672, + "tokens_seen": 2457585664 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020645937813440322, + "loss": 1.2201, + "theoretical_loss": 3.4015916912224062, + "tokens_seen": 2457651200 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020644332998996992, + "loss": 1.2695, + "theoretical_loss": 3.401584491937875, + "tokens_seen": 2457716736 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020642728184553662, + "loss": 1.0924, + "theoretical_loss": 3.4015772928990637, + "tokens_seen": 2457782272 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020641123370110332, + "loss": 1.1432, + "theoretical_loss": 3.4015700941059572, + "tokens_seen": 2457847808 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020639518555667002, + "loss": 1.2687, + "theoretical_loss": 3.4015628955585404, + "tokens_seen": 2457913344 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020637913741223672, + "loss": 1.242, + "theoretical_loss": 3.4015556972567986, + "tokens_seen": 2457978880 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002063630892678034, + "loss": 1.2542, + "theoretical_loss": 3.401548499200717, + "tokens_seen": 2458044416 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002063470411233701, + "loss": 1.2263, + "theoretical_loss": 3.4015413013902798, + "tokens_seen": 2458109952 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002063309929789368, + "loss": 1.24, + "theoretical_loss": 3.4015341038254734, + "tokens_seen": 2458175488 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002063149448345035, + "loss": 1.2276, + "theoretical_loss": 3.4015269065062816, + "tokens_seen": 2458241024 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002062988966900702, + "loss": 1.2041, + "theoretical_loss": 3.4015197094326903, + "tokens_seen": 2458306560 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002062828485456369, + "loss": 1.1465, + "theoretical_loss": 3.4015125126046843, + "tokens_seen": 2458372096 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002062668004012036, + "loss": 1.2026, + "theoretical_loss": 3.4015053160222486, + "tokens_seen": 2458437632 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002062507522567703, + "loss": 1.2127, + "theoretical_loss": 3.401498119685369, + "tokens_seen": 2458503168 + }, + { + "epoch": 1.13, + "learning_rate": 0.000206234704112337, + "loss": 1.3573, + "theoretical_loss": 3.401490923594029, + "tokens_seen": 2458568704 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002062186559679037, + "loss": 1.2968, + "theoretical_loss": 3.4014837277482153, + "tokens_seen": 2458634240 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002062026078234704, + "loss": 1.3042, + "theoretical_loss": 3.4014765321479117, + "tokens_seen": 2458699776 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002061865596790371, + "loss": 1.2276, + "theoretical_loss": 3.401469336793104, + "tokens_seen": 2458765312 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002061705115346038, + "loss": 1.3075, + "theoretical_loss": 3.4014621416837776, + "tokens_seen": 2458830848 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002061544633901705, + "loss": 1.2469, + "theoretical_loss": 3.401454946819917, + "tokens_seen": 2458896384 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002061384152457372, + "loss": 1.2615, + "theoretical_loss": 3.401447752201507, + "tokens_seen": 2458961920 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002061223671013039, + "loss": 1.287, + "theoretical_loss": 3.4014405578285336, + "tokens_seen": 2459027456 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002061063189568706, + "loss": 1.0639, + "theoretical_loss": 3.4014333637009813, + "tokens_seen": 2459092992 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002060902708124373, + "loss": 1.3128, + "theoretical_loss": 3.4014261698188353, + "tokens_seen": 2459158528 + }, + { + "epoch": 1.13, + "learning_rate": 0.000206074222668004, + "loss": 1.1751, + "theoretical_loss": 3.40141897618208, + "tokens_seen": 2459224064 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002060581745235707, + "loss": 1.222, + "theoretical_loss": 3.401411782790702, + "tokens_seen": 2459289600 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002060421263791374, + "loss": 1.276, + "theoretical_loss": 3.4014045896446854, + "tokens_seen": 2459355136 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002060260782347041, + "loss": 1.3548, + "theoretical_loss": 3.401397396744015, + "tokens_seen": 2459420672 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002060100300902708, + "loss": 1.2923, + "theoretical_loss": 3.4013902040886768, + "tokens_seen": 2459486208 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020599398194583754, + "loss": 1.1767, + "theoretical_loss": 3.4013830116786554, + "tokens_seen": 2459551744 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020597793380140424, + "loss": 1.1872, + "theoretical_loss": 3.4013758195139356, + "tokens_seen": 2459617280 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020596188565697094, + "loss": 1.1745, + "theoretical_loss": 3.401368627594503, + "tokens_seen": 2459682816 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020594583751253764, + "loss": 1.2295, + "theoretical_loss": 3.4013614359203426, + "tokens_seen": 2459748352 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020592978936810434, + "loss": 1.3087, + "theoretical_loss": 3.4013542444914395, + "tokens_seen": 2459813888 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020591374122367104, + "loss": 1.3066, + "theoretical_loss": 3.4013470533077785, + "tokens_seen": 2459879424 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020589769307923774, + "loss": 1.2092, + "theoretical_loss": 3.401339862369345, + "tokens_seen": 2459944960 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020588164493480444, + "loss": 1.2045, + "theoretical_loss": 3.4013326716761245, + "tokens_seen": 2460010496 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020586559679037114, + "loss": 1.2596, + "theoretical_loss": 3.401325481228101, + "tokens_seen": 2460076032 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020584954864593785, + "loss": 1.2475, + "theoretical_loss": 3.4013182910252606, + "tokens_seen": 2460141568 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020583350050150455, + "loss": 1.0748, + "theoretical_loss": 3.401311101067588, + "tokens_seen": 2460207104 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020581745235707125, + "loss": 1.265, + "theoretical_loss": 3.4013039113550683, + "tokens_seen": 2460272640 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020580140421263795, + "loss": 1.164, + "theoretical_loss": 3.401296721887687, + "tokens_seen": 2460338176 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020578535606820465, + "loss": 1.1365, + "theoretical_loss": 3.401289532665428, + "tokens_seen": 2460403712 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020576930792377135, + "loss": 1.2684, + "theoretical_loss": 3.401282343688278, + "tokens_seen": 2460469248 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020575325977933805, + "loss": 1.2302, + "theoretical_loss": 3.4012751549562212, + "tokens_seen": 2460534784 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020573721163490475, + "loss": 1.2225, + "theoretical_loss": 3.401267966469243, + "tokens_seen": 2460600320 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020572116349047142, + "loss": 1.2663, + "theoretical_loss": 3.401260778227329, + "tokens_seen": 2460665856 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020570511534603812, + "loss": 1.2095, + "theoretical_loss": 3.4012535902304633, + "tokens_seen": 2460731392 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020568906720160482, + "loss": 1.261, + "theoretical_loss": 3.4012464024786313, + "tokens_seen": 2460796928 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020567301905717153, + "loss": 1.2212, + "theoretical_loss": 3.4012392149718185, + "tokens_seen": 2460862464 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020565697091273823, + "loss": 1.3344, + "theoretical_loss": 3.40123202771001, + "tokens_seen": 2460928000 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020564092276830493, + "loss": 1.2528, + "theoretical_loss": 3.4012248406931906, + "tokens_seen": 2460993536 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020562487462387163, + "loss": 1.2947, + "theoretical_loss": 3.4012176539213455, + "tokens_seen": 2461059072 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020560882647943833, + "loss": 1.29, + "theoretical_loss": 3.4012104673944603, + "tokens_seen": 2461124608 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020559277833500503, + "loss": 1.312, + "theoretical_loss": 3.4012032811125197, + "tokens_seen": 2461190144 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020557673019057173, + "loss": 1.3655, + "theoretical_loss": 3.4011960950755085, + "tokens_seen": 2461255680 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020556068204613843, + "loss": 1.3491, + "theoretical_loss": 3.4011889092834124, + "tokens_seen": 2461321216 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020554463390170513, + "loss": 1.2466, + "theoretical_loss": 3.401181723736216, + "tokens_seen": 2461386752 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020552858575727183, + "loss": 1.1179, + "theoretical_loss": 3.401174538433905, + "tokens_seen": 2461452288 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020551253761283853, + "loss": 1.2531, + "theoretical_loss": 3.4011673533764646, + "tokens_seen": 2461517824 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020549648946840523, + "loss": 1.2651, + "theoretical_loss": 3.4011601685638797, + "tokens_seen": 2461583360 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020548044132397193, + "loss": 1.1771, + "theoretical_loss": 3.401152983996135, + "tokens_seen": 2461648896 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020546439317953863, + "loss": 1.3016, + "theoretical_loss": 3.401145799673216, + "tokens_seen": 2461714432 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020544834503510533, + "loss": 1.2121, + "theoretical_loss": 3.401138615595108, + "tokens_seen": 2461779968 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020543229689067203, + "loss": 1.2, + "theoretical_loss": 3.401131431761796, + "tokens_seen": 2461845504 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020541624874623874, + "loss": 1.1761, + "theoretical_loss": 3.4011242481732653, + "tokens_seen": 2461911040 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020540020060180544, + "loss": 1.1516, + "theoretical_loss": 3.4011170648295006, + "tokens_seen": 2461976576 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020538415245737214, + "loss": 1.2891, + "theoretical_loss": 3.4011098817304877, + "tokens_seen": 2462042112 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020536810431293884, + "loss": 1.2001, + "theoretical_loss": 3.401102698876211, + "tokens_seen": 2462107648 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020535205616850554, + "loss": 1.3076, + "theoretical_loss": 3.401095516266656, + "tokens_seen": 2462173184 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002053360080240722, + "loss": 1.2021, + "theoretical_loss": 3.401088333901808, + "tokens_seen": 2462238720 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002053199598796389, + "loss": 1.173, + "theoretical_loss": 3.401081151781652, + "tokens_seen": 2462304256 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002053039117352056, + "loss": 1.2941, + "theoretical_loss": 3.4010739699061734, + "tokens_seen": 2462369792 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002052878635907723, + "loss": 1.1402, + "theoretical_loss": 3.4010667882753567, + "tokens_seen": 2462435328 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020527181544633901, + "loss": 1.2025, + "theoretical_loss": 3.401059606889188, + "tokens_seen": 2462500864 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020525576730190571, + "loss": 1.2871, + "theoretical_loss": 3.4010524257476513, + "tokens_seen": 2462566400 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020523971915747242, + "loss": 1.2255, + "theoretical_loss": 3.401045244850733, + "tokens_seen": 2462631936 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020522367101303912, + "loss": 1.24, + "theoretical_loss": 3.401038064198417, + "tokens_seen": 2462697472 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020520762286860582, + "loss": 1.2069, + "theoretical_loss": 3.4010308837906895, + "tokens_seen": 2462763008 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020519157472417252, + "loss": 1.2456, + "theoretical_loss": 3.4010237036275353, + "tokens_seen": 2462828544 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020517552657973922, + "loss": 1.2357, + "theoretical_loss": 3.4010165237089396, + "tokens_seen": 2462894080 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020515947843530592, + "loss": 1.2311, + "theoretical_loss": 3.4010093440348874, + "tokens_seen": 2462959616 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020514343029087262, + "loss": 1.114, + "theoretical_loss": 3.401002164605364, + "tokens_seen": 2463025152 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020512738214643932, + "loss": 1.2185, + "theoretical_loss": 3.4009949854203545, + "tokens_seen": 2463090688 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020511133400200602, + "loss": 1.1131, + "theoretical_loss": 3.400987806479844, + "tokens_seen": 2463156224 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020509528585757272, + "loss": 1.2515, + "theoretical_loss": 3.400980627783818, + "tokens_seen": 2463221760 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020507923771313942, + "loss": 1.324, + "theoretical_loss": 3.4009734493322608, + "tokens_seen": 2463287296 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020506318956870612, + "loss": 1.3008, + "theoretical_loss": 3.400966271125159, + "tokens_seen": 2463352832 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020504714142427282, + "loss": 1.2498, + "theoretical_loss": 3.4009590931624967, + "tokens_seen": 2463418368 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020503109327983952, + "loss": 1.2911, + "theoretical_loss": 3.400951915444259, + "tokens_seen": 2463483904 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020501504513540622, + "loss": 1.1784, + "theoretical_loss": 3.400944737970432, + "tokens_seen": 2463549440 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020499899699097292, + "loss": 1.21, + "theoretical_loss": 3.4009375607409997, + "tokens_seen": 2463614976 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020498294884653963, + "loss": 1.2391, + "theoretical_loss": 3.400930383755948, + "tokens_seen": 2463680512 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020496690070210633, + "loss": 1.0911, + "theoretical_loss": 3.4009232070152624, + "tokens_seen": 2463746048 + }, + { + "epoch": 1.13, + "learning_rate": 0.000204950852557673, + "loss": 1.2741, + "theoretical_loss": 3.4009160305189274, + "tokens_seen": 2463811584 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002049348044132397, + "loss": 1.2665, + "theoretical_loss": 3.400908854266928, + "tokens_seen": 2463877120 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002049187562688064, + "loss": 1.1811, + "theoretical_loss": 3.4009016782592503, + "tokens_seen": 2463942656 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002049027081243731, + "loss": 1.2487, + "theoretical_loss": 3.4008945024958788, + "tokens_seen": 2464008192 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002048866599799398, + "loss": 1.134, + "theoretical_loss": 3.4008873269767985, + "tokens_seen": 2464073728 + }, + { + "epoch": 1.13, + "objective/train/docs_used": 910442, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2664554119110107, + "objective/train/theoretical_loss": 3.4008801517019958, + "objective/train/tokens_used": 2484599264, + "theoretical_loss": 3.4008801517019958, + "tokens_seen": 2464139264 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002048706118355065, + "loss": 1.2665, + "theoretical_loss": 3.4008801517019958, + "tokens_seen": 2464139264 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002048545636910732, + "loss": 1.2537, + "theoretical_loss": 3.4008729766714545, + "tokens_seen": 2464204800 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002048385155466399, + "loss": 1.2852, + "theoretical_loss": 3.4008658018851605, + "tokens_seen": 2464270336 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002048224674022066, + "loss": 1.3303, + "theoretical_loss": 3.4008586273430987, + "tokens_seen": 2464335872 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002048064192577733, + "loss": 1.1573, + "theoretical_loss": 3.400851453045254, + "tokens_seen": 2464401408 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020479037111334006, + "loss": 1.2797, + "theoretical_loss": 3.4008442789916127, + "tokens_seen": 2464466944 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020477432296890676, + "loss": 1.2548, + "theoretical_loss": 3.400837105182159, + "tokens_seen": 2464532480 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020475827482447346, + "loss": 1.2217, + "theoretical_loss": 3.400829931616878, + "tokens_seen": 2464598016 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020474222668004016, + "loss": 1.2345, + "theoretical_loss": 3.4008227582957558, + "tokens_seen": 2464663552 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020472617853560686, + "loss": 1.1364, + "theoretical_loss": 3.400815585218777, + "tokens_seen": 2464729088 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020471013039117354, + "loss": 1.1562, + "theoretical_loss": 3.4008084123859272, + "tokens_seen": 2464794624 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020469408224674024, + "loss": 1.2981, + "theoretical_loss": 3.400801239797191, + "tokens_seen": 2464860160 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020467803410230694, + "loss": 1.2657, + "theoretical_loss": 3.4007940674525536, + "tokens_seen": 2464925696 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020466198595787364, + "loss": 1.262, + "theoretical_loss": 3.4007868953520006, + "tokens_seen": 2464991232 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020464593781344034, + "loss": 1.2356, + "theoretical_loss": 3.4007797234955173, + "tokens_seen": 2465056768 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020462988966900704, + "loss": 1.2755, + "theoretical_loss": 3.4007725518830885, + "tokens_seen": 2465122304 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020461384152457374, + "loss": 1.205, + "theoretical_loss": 3.4007653805147, + "tokens_seen": 2465187840 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020459779338014044, + "loss": 1.2303, + "theoretical_loss": 3.400758209390336, + "tokens_seen": 2465253376 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020458174523570714, + "loss": 1.2338, + "theoretical_loss": 3.400751038509983, + "tokens_seen": 2465318912 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020456569709127384, + "loss": 1.1494, + "theoretical_loss": 3.4007438678736253, + "tokens_seen": 2465384448 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020454964894684054, + "loss": 1.1673, + "theoretical_loss": 3.400736697481248, + "tokens_seen": 2465449984 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020453360080240724, + "loss": 1.2328, + "theoretical_loss": 3.400729527332837, + "tokens_seen": 2465515520 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020451755265797394, + "loss": 1.2435, + "theoretical_loss": 3.400722357428377, + "tokens_seen": 2465581056 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020450150451354064, + "loss": 1.1738, + "theoretical_loss": 3.4007151877678536, + "tokens_seen": 2465646592 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020448545636910734, + "loss": 1.2677, + "theoretical_loss": 3.4007080183512515, + "tokens_seen": 2465712128 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020446940822467405, + "loss": 1.2651, + "theoretical_loss": 3.4007008491785564, + "tokens_seen": 2465777664 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020445336008024075, + "loss": 1.1938, + "theoretical_loss": 3.4006936802497534, + "tokens_seen": 2465843200 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020443731193580745, + "loss": 1.2368, + "theoretical_loss": 3.4006865115648277, + "tokens_seen": 2465908736 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020442126379137415, + "loss": 1.2119, + "theoretical_loss": 3.4006793431237643, + "tokens_seen": 2465974272 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020440521564694085, + "loss": 1.1012, + "theoretical_loss": 3.400672174926549, + "tokens_seen": 2466039808 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020438916750250755, + "loss": 1.1913, + "theoretical_loss": 3.400665006973166, + "tokens_seen": 2466105344 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020437311935807425, + "loss": 1.2418, + "theoretical_loss": 3.4006578392636015, + "tokens_seen": 2466170880 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020435707121364095, + "loss": 1.3104, + "theoretical_loss": 3.4006506717978406, + "tokens_seen": 2466236416 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020434102306920765, + "loss": 1.2367, + "theoretical_loss": 3.4006435045758683, + "tokens_seen": 2466301952 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020432497492477432, + "loss": 1.2311, + "theoretical_loss": 3.4006363375976694, + "tokens_seen": 2466367488 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020430892678034102, + "loss": 1.2801, + "theoretical_loss": 3.40062917086323, + "tokens_seen": 2466433024 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020429287863590773, + "loss": 1.203, + "theoretical_loss": 3.400622004372535, + "tokens_seen": 2466498560 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020427683049147443, + "loss": 1.2786, + "theoretical_loss": 3.4006148381255694, + "tokens_seen": 2466564096 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020426078234704113, + "loss": 1.2723, + "theoretical_loss": 3.4006076721223186, + "tokens_seen": 2466629632 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020424473420260783, + "loss": 1.2759, + "theoretical_loss": 3.4006005063627676, + "tokens_seen": 2466695168 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020422868605817453, + "loss": 1.1673, + "theoretical_loss": 3.400593340846902, + "tokens_seen": 2466760704 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020421263791374123, + "loss": 1.2519, + "theoretical_loss": 3.4005861755747073, + "tokens_seen": 2466826240 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020419658976930793, + "loss": 1.2361, + "theoretical_loss": 3.400579010546168, + "tokens_seen": 2466891776 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020418054162487463, + "loss": 1.2842, + "theoretical_loss": 3.4005718457612693, + "tokens_seen": 2466957312 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020416449348044133, + "loss": 1.226, + "theoretical_loss": 3.4005646812199974, + "tokens_seen": 2467022848 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020414844533600803, + "loss": 1.2668, + "theoretical_loss": 3.4005575169223365, + "tokens_seen": 2467088384 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020413239719157473, + "loss": 1.2607, + "theoretical_loss": 3.400550352868273, + "tokens_seen": 2467153920 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020411634904714143, + "loss": 1.2242, + "theoretical_loss": 3.4005431890577906, + "tokens_seen": 2467219456 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020410030090270813, + "loss": 1.242, + "theoretical_loss": 3.400536025490876, + "tokens_seen": 2467284992 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020408425275827483, + "loss": 1.2855, + "theoretical_loss": 3.400528862167514, + "tokens_seen": 2467350528 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020406820461384153, + "loss": 1.2116, + "theoretical_loss": 3.4005216990876894, + "tokens_seen": 2467416064 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020405215646940823, + "loss": 1.2185, + "theoretical_loss": 3.4005145362513876, + "tokens_seen": 2467481600 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020403610832497494, + "loss": 1.2247, + "theoretical_loss": 3.400507373658594, + "tokens_seen": 2467547136 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020402006018054164, + "loss": 1.1986, + "theoretical_loss": 3.4005002113092946, + "tokens_seen": 2467612672 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020400401203610834, + "loss": 1.2276, + "theoretical_loss": 3.4004930492034733, + "tokens_seen": 2467678208 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020398796389167504, + "loss": 1.1983, + "theoretical_loss": 3.400485887341116, + "tokens_seen": 2467743744 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020397191574724174, + "loss": 1.2286, + "theoretical_loss": 3.400478725722208, + "tokens_seen": 2467809280 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020395586760280844, + "loss": 1.2037, + "theoretical_loss": 3.4004715643467347, + "tokens_seen": 2467874816 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002039398194583751, + "loss": 1.1948, + "theoretical_loss": 3.4004644032146807, + "tokens_seen": 2467940352 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002039237713139418, + "loss": 1.2965, + "theoretical_loss": 3.4004572423260324, + "tokens_seen": 2468005888 + }, + { + "epoch": 1.13, + "learning_rate": 0.0002039077231695085, + "loss": 1.276, + "theoretical_loss": 3.400450081680774, + "tokens_seen": 2468071424 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020389167502507521, + "loss": 1.2029, + "theoretical_loss": 3.400442921278891, + "tokens_seen": 2468136960 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020387562688064191, + "loss": 1.2871, + "theoretical_loss": 3.4004357611203693, + "tokens_seen": 2468202496 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020385957873620862, + "loss": 1.2155, + "theoretical_loss": 3.400428601205193, + "tokens_seen": 2468268032 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020384353059177532, + "loss": 1.1903, + "theoretical_loss": 3.4004214415333487, + "tokens_seen": 2468333568 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020382748244734202, + "loss": 1.2198, + "theoretical_loss": 3.400414282104821, + "tokens_seen": 2468399104 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020381143430290872, + "loss": 1.2028, + "theoretical_loss": 3.4004071229195945, + "tokens_seen": 2468464640 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020379538615847542, + "loss": 1.0648, + "theoretical_loss": 3.4003999639776556, + "tokens_seen": 2468530176 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020377933801404212, + "loss": 1.2228, + "theoretical_loss": 3.4003928052789893, + "tokens_seen": 2468595712 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020376328986960882, + "loss": 1.2311, + "theoretical_loss": 3.4003856468235805, + "tokens_seen": 2468661248 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020374724172517552, + "loss": 1.3251, + "theoretical_loss": 3.4003784886114152, + "tokens_seen": 2468726784 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020373119358074222, + "loss": 1.1908, + "theoretical_loss": 3.4003713306424777, + "tokens_seen": 2468792320 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020371514543630892, + "loss": 1.2938, + "theoretical_loss": 3.4003641729167535, + "tokens_seen": 2468857856 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020369909729187562, + "loss": 1.2085, + "theoretical_loss": 3.4003570154342286, + "tokens_seen": 2468923392 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020368304914744232, + "loss": 1.3027, + "theoretical_loss": 3.400349858194888, + "tokens_seen": 2468988928 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020366700100300902, + "loss": 1.2041, + "theoretical_loss": 3.400342701198716, + "tokens_seen": 2469054464 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020365095285857572, + "loss": 1.2159, + "theoretical_loss": 3.4003355444456993, + "tokens_seen": 2469120000 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020363490471414242, + "loss": 1.1681, + "theoretical_loss": 3.4003283879358226, + "tokens_seen": 2469185536 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020361885656970912, + "loss": 1.2474, + "theoretical_loss": 3.400321231669071, + "tokens_seen": 2469251072 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020360280842527585, + "loss": 1.2264, + "theoretical_loss": 3.4003140756454298, + "tokens_seen": 2469316608 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020358676028084255, + "loss": 1.119, + "theoretical_loss": 3.4003069198648848, + "tokens_seen": 2469382144 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020357071213640925, + "loss": 1.18, + "theoretical_loss": 3.4002997643274204, + "tokens_seen": 2469447680 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020355466399197595, + "loss": 1.2103, + "theoretical_loss": 3.400292609033023, + "tokens_seen": 2469513216 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020353861584754266, + "loss": 1.1801, + "theoretical_loss": 3.400285453981677, + "tokens_seen": 2469578752 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020352256770310936, + "loss": 1.2219, + "theoretical_loss": 3.400278299173368, + "tokens_seen": 2469644288 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020350651955867606, + "loss": 1.321, + "theoretical_loss": 3.400271144608081, + "tokens_seen": 2469709824 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020349047141424276, + "loss": 1.3365, + "theoretical_loss": 3.400263990285802, + "tokens_seen": 2469775360 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020347442326980946, + "loss": 1.2634, + "theoretical_loss": 3.400256836206516, + "tokens_seen": 2469840896 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020345837512537616, + "loss": 1.2744, + "theoretical_loss": 3.400249682370208, + "tokens_seen": 2469906432 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020344232698094286, + "loss": 1.2333, + "theoretical_loss": 3.4002425287768636, + "tokens_seen": 2469971968 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020342627883650956, + "loss": 1.2227, + "theoretical_loss": 3.4002353754264676, + "tokens_seen": 2470037504 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020341023069207626, + "loss": 1.1862, + "theoretical_loss": 3.400228222319006, + "tokens_seen": 2470103040 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020339418254764296, + "loss": 1.2675, + "theoretical_loss": 3.4002210694544637, + "tokens_seen": 2470168576 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020337813440320966, + "loss": 1.303, + "theoretical_loss": 3.4002139168328265, + "tokens_seen": 2470234112 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020336208625877636, + "loss": 1.2569, + "theoretical_loss": 3.400206764454079, + "tokens_seen": 2470299648 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020334603811434306, + "loss": 1.1898, + "theoretical_loss": 3.4001996123182066, + "tokens_seen": 2470365184 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020332998996990976, + "loss": 1.142, + "theoretical_loss": 3.4001924604251954, + "tokens_seen": 2470430720 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020331394182547646, + "loss": 1.2095, + "theoretical_loss": 3.4001853087750296, + "tokens_seen": 2470496256 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020329789368104314, + "loss": 1.1826, + "theoretical_loss": 3.4001781573676952, + "tokens_seen": 2470561792 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020328184553660984, + "loss": 1.3406, + "theoretical_loss": 3.4001710062031774, + "tokens_seen": 2470627328 + }, + { + "epoch": 1.13, + "objective/train/docs_used": 913231, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2834594249725342, + "objective/train/theoretical_loss": 3.4001638552814617, + "objective/train/tokens_used": 2491152864, + "theoretical_loss": 3.4001638552814617, + "tokens_seen": 2470692864 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020326579739217654, + "loss": 1.2835, + "theoretical_loss": 3.4001638552814617, + "tokens_seen": 2470692864 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020324974924774324, + "loss": 1.1347, + "theoretical_loss": 3.4001567046025327, + "tokens_seen": 2470758400 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020323370110330994, + "loss": 1.138, + "theoretical_loss": 3.4001495541663767, + "tokens_seen": 2470823936 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020321765295887664, + "loss": 1.2295, + "theoretical_loss": 3.4001424039729784, + "tokens_seen": 2470889472 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020320160481444334, + "loss": 1.2383, + "theoretical_loss": 3.4001352540223233, + "tokens_seen": 2470955008 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020318555667001004, + "loss": 1.2532, + "theoretical_loss": 3.4001281043143967, + "tokens_seen": 2471020544 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020316950852557674, + "loss": 1.2041, + "theoretical_loss": 3.400120954849184, + "tokens_seen": 2471086080 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020315346038114344, + "loss": 1.2913, + "theoretical_loss": 3.40011380562667, + "tokens_seen": 2471151616 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020313741223671014, + "loss": 1.104, + "theoretical_loss": 3.400106656646841, + "tokens_seen": 2471217152 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020312136409227684, + "loss": 1.2625, + "theoretical_loss": 3.4000995079096814, + "tokens_seen": 2471282688 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020310531594784355, + "loss": 1.1825, + "theoretical_loss": 3.4000923594151766, + "tokens_seen": 2471348224 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020308926780341025, + "loss": 1.3205, + "theoretical_loss": 3.400085211163313, + "tokens_seen": 2471413760 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020307321965897695, + "loss": 1.2341, + "theoretical_loss": 3.4000780631540746, + "tokens_seen": 2471479296 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020305717151454365, + "loss": 1.2565, + "theoretical_loss": 3.4000709153874475, + "tokens_seen": 2471544832 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020304112337011035, + "loss": 1.3352, + "theoretical_loss": 3.4000637678634167, + "tokens_seen": 2471610368 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020302507522567705, + "loss": 1.1677, + "theoretical_loss": 3.400056620581968, + "tokens_seen": 2471675904 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020300902708124375, + "loss": 1.2582, + "theoretical_loss": 3.400049473543086, + "tokens_seen": 2471741440 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020299297893681045, + "loss": 1.2682, + "theoretical_loss": 3.4000423267467568, + "tokens_seen": 2471806976 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020297693079237715, + "loss": 1.2181, + "theoretical_loss": 3.400035180192965, + "tokens_seen": 2471872512 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020296088264794385, + "loss": 1.2419, + "theoretical_loss": 3.4000280338816964, + "tokens_seen": 2471938048 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020294483450351055, + "loss": 1.2886, + "theoretical_loss": 3.4000208878129365, + "tokens_seen": 2472003584 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020292878635907725, + "loss": 1.1632, + "theoretical_loss": 3.4000137419866703, + "tokens_seen": 2472069120 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020291273821464393, + "loss": 1.2848, + "theoretical_loss": 3.400006596402883, + "tokens_seen": 2472134656 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020289669007021063, + "loss": 1.2434, + "theoretical_loss": 3.3999994510615603, + "tokens_seen": 2472200192 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020288064192577733, + "loss": 1.279, + "theoretical_loss": 3.3999923059626873, + "tokens_seen": 2472265728 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020286459378134403, + "loss": 1.2768, + "theoretical_loss": 3.39998516110625, + "tokens_seen": 2472331264 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020284854563691073, + "loss": 1.3256, + "theoretical_loss": 3.399978016492233, + "tokens_seen": 2472396800 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020283249749247743, + "loss": 1.1522, + "theoretical_loss": 3.3999708721206217, + "tokens_seen": 2472462336 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020281644934804413, + "loss": 1.3189, + "theoretical_loss": 3.3999637279914015, + "tokens_seen": 2472527872 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020280040120361083, + "loss": 1.2081, + "theoretical_loss": 3.399956584104558, + "tokens_seen": 2472593408 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020278435305917753, + "loss": 1.1738, + "theoretical_loss": 3.3999494404600763, + "tokens_seen": 2472658944 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020276830491474423, + "loss": 1.2628, + "theoretical_loss": 3.399942297057942, + "tokens_seen": 2472724480 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020275225677031093, + "loss": 1.1181, + "theoretical_loss": 3.3999351538981406, + "tokens_seen": 2472790016 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020273620862587763, + "loss": 1.1652, + "theoretical_loss": 3.399928010980657, + "tokens_seen": 2472855552 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020272016048144433, + "loss": 1.1374, + "theoretical_loss": 3.3999208683054767, + "tokens_seen": 2472921088 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020270411233701103, + "loss": 1.27, + "theoretical_loss": 3.399913725872585, + "tokens_seen": 2472986624 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020268806419257773, + "loss": 1.2069, + "theoretical_loss": 3.3999065836819673, + "tokens_seen": 2473052160 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020267201604814444, + "loss": 1.2188, + "theoretical_loss": 3.3998994417336093, + "tokens_seen": 2473117696 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020265596790371114, + "loss": 1.3182, + "theoretical_loss": 3.399892300027496, + "tokens_seen": 2473183232 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020263991975927784, + "loss": 1.1635, + "theoretical_loss": 3.3998851585636127, + "tokens_seen": 2473248768 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020262387161484454, + "loss": 1.1486, + "theoretical_loss": 3.3998780173419454, + "tokens_seen": 2473314304 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020260782347041124, + "loss": 1.2505, + "theoretical_loss": 3.3998708763624785, + "tokens_seen": 2473379840 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020259177532597794, + "loss": 1.2679, + "theoretical_loss": 3.399863735625198, + "tokens_seen": 2473445376 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020257572718154464, + "loss": 1.1911, + "theoretical_loss": 3.3998565951300894, + "tokens_seen": 2473510912 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020255967903711134, + "loss": 1.2214, + "theoretical_loss": 3.3998494548771374, + "tokens_seen": 2473576448 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020254363089267804, + "loss": 1.0983, + "theoretical_loss": 3.399842314866328, + "tokens_seen": 2473641984 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020252758274824471, + "loss": 1.116, + "theoretical_loss": 3.399835175097646, + "tokens_seen": 2473707520 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020251153460381141, + "loss": 1.2509, + "theoretical_loss": 3.399828035571078, + "tokens_seen": 2473773056 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020249548645937812, + "loss": 1.1885, + "theoretical_loss": 3.3998208962866077, + "tokens_seen": 2473838592 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020247943831494482, + "loss": 1.2712, + "theoretical_loss": 3.3998137572442215, + "tokens_seen": 2473904128 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020246339017051152, + "loss": 1.189, + "theoretical_loss": 3.3998066184439044, + "tokens_seen": 2473969664 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020244734202607822, + "loss": 1.3096, + "theoretical_loss": 3.399799479885642, + "tokens_seen": 2474035200 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020243129388164492, + "loss": 1.2907, + "theoretical_loss": 3.39979234156942, + "tokens_seen": 2474100736 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020241524573721162, + "loss": 1.1754, + "theoretical_loss": 3.399785203495223, + "tokens_seen": 2474166272 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020239919759277837, + "loss": 1.2004, + "theoretical_loss": 3.399778065663037, + "tokens_seen": 2474231808 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020238314944834507, + "loss": 1.2572, + "theoretical_loss": 3.399770928072847, + "tokens_seen": 2474297344 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020236710130391177, + "loss": 1.206, + "theoretical_loss": 3.3997637907246387, + "tokens_seen": 2474362880 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020235105315947848, + "loss": 1.1789, + "theoretical_loss": 3.399756653618397, + "tokens_seen": 2474428416 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020233500501504518, + "loss": 1.1598, + "theoretical_loss": 3.399749516754108, + "tokens_seen": 2474493952 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020231895687061188, + "loss": 1.201, + "theoretical_loss": 3.3997423801317566, + "tokens_seen": 2474559488 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020230290872617858, + "loss": 1.1639, + "theoretical_loss": 3.399735243751328, + "tokens_seen": 2474625024 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020228686058174525, + "loss": 1.3371, + "theoretical_loss": 3.3997281076128085, + "tokens_seen": 2474690560 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020227081243731195, + "loss": 1.2044, + "theoretical_loss": 3.3997209717161825, + "tokens_seen": 2474756096 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020225476429287865, + "loss": 1.2242, + "theoretical_loss": 3.3997138360614363, + "tokens_seen": 2474821632 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020223871614844535, + "loss": 1.2276, + "theoretical_loss": 3.3997067006485544, + "tokens_seen": 2474887168 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020222266800401205, + "loss": 1.2012, + "theoretical_loss": 3.399699565477522, + "tokens_seen": 2474952704 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020220661985957875, + "loss": 1.2395, + "theoretical_loss": 3.399692430548326, + "tokens_seen": 2475018240 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020219057171514545, + "loss": 1.09, + "theoretical_loss": 3.3996852958609503, + "tokens_seen": 2475083776 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020217452357071215, + "loss": 1.2317, + "theoretical_loss": 3.3996781614153813, + "tokens_seen": 2475149312 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020215847542627886, + "loss": 1.3094, + "theoretical_loss": 3.3996710272116037, + "tokens_seen": 2475214848 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020214242728184556, + "loss": 1.1989, + "theoretical_loss": 3.3996638932496035, + "tokens_seen": 2475280384 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020212637913741226, + "loss": 1.1979, + "theoretical_loss": 3.3996567595293654, + "tokens_seen": 2475345920 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020211033099297896, + "loss": 1.2732, + "theoretical_loss": 3.3996496260508753, + "tokens_seen": 2475411456 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020209428284854566, + "loss": 1.2291, + "theoretical_loss": 3.399642492814119, + "tokens_seen": 2475476992 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020207823470411236, + "loss": 1.1129, + "theoretical_loss": 3.3996353598190807, + "tokens_seen": 2475542528 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020206218655967906, + "loss": 1.2045, + "theoretical_loss": 3.399628227065747, + "tokens_seen": 2475608064 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020204613841524576, + "loss": 1.2055, + "theoretical_loss": 3.3996210945541026, + "tokens_seen": 2475673600 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020203009027081246, + "loss": 1.1524, + "theoretical_loss": 3.3996139622841333, + "tokens_seen": 2475739136 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020201404212637916, + "loss": 1.307, + "theoretical_loss": 3.3996068302558244, + "tokens_seen": 2475804672 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020199799398194586, + "loss": 1.1869, + "theoretical_loss": 3.399599698469161, + "tokens_seen": 2475870208 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020198194583751256, + "loss": 1.2389, + "theoretical_loss": 3.399592566924129, + "tokens_seen": 2475935744 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020196589769307926, + "loss": 1.206, + "theoretical_loss": 3.399585435620714, + "tokens_seen": 2476001280 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020194984954864596, + "loss": 1.2313, + "theoretical_loss": 3.3995783045589003, + "tokens_seen": 2476066816 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020193380140421266, + "loss": 1.1828, + "theoretical_loss": 3.399571173738675, + "tokens_seen": 2476132352 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020191775325977937, + "loss": 1.2293, + "theoretical_loss": 3.3995640431600216, + "tokens_seen": 2476197888 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020190170511534604, + "loss": 1.1993, + "theoretical_loss": 3.399556912822927, + "tokens_seen": 2476263424 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020188565697091274, + "loss": 1.1864, + "theoretical_loss": 3.3995497827273757, + "tokens_seen": 2476328960 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020186960882647944, + "loss": 1.166, + "theoretical_loss": 3.3995426528733543, + "tokens_seen": 2476394496 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020185356068204614, + "loss": 1.2078, + "theoretical_loss": 3.399535523260847, + "tokens_seen": 2476460032 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020183751253761284, + "loss": 1.3183, + "theoretical_loss": 3.39952839388984, + "tokens_seen": 2476525568 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020182146439317954, + "loss": 1.2024, + "theoretical_loss": 3.3995212647603177, + "tokens_seen": 2476591104 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020180541624874624, + "loss": 1.2895, + "theoretical_loss": 3.399514135872267, + "tokens_seen": 2476656640 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020178936810431294, + "loss": 1.2702, + "theoretical_loss": 3.3995070072256723, + "tokens_seen": 2476722176 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020177331995987964, + "loss": 1.1489, + "theoretical_loss": 3.3994998788205195, + "tokens_seen": 2476787712 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020175727181544634, + "loss": 1.1909, + "theoretical_loss": 3.399492750656794, + "tokens_seen": 2476853248 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020174122367101304, + "loss": 1.2078, + "theoretical_loss": 3.3994856227344807, + "tokens_seen": 2476918784 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020172517552657975, + "loss": 1.2257, + "theoretical_loss": 3.3994784950535655, + "tokens_seen": 2476984320 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020170912738214645, + "loss": 1.2055, + "theoretical_loss": 3.399471367614034, + "tokens_seen": 2477049856 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020169307923771315, + "loss": 1.267, + "theoretical_loss": 3.3994642404158713, + "tokens_seen": 2477115392 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020167703109327985, + "loss": 1.2851, + "theoretical_loss": 3.399457113459063, + "tokens_seen": 2477180928 + }, + { + "epoch": 1.13, + "objective/train/docs_used": 915474, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1075177192687988, + "objective/train/theoretical_loss": 3.3994499867435946, + "objective/train/tokens_used": 2497706464, + "theoretical_loss": 3.3994499867435946, + "tokens_seen": 2477246464 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020166098294884655, + "loss": 1.1075, + "theoretical_loss": 3.3994499867435946, + "tokens_seen": 2477246464 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020164493480441325, + "loss": 1.0787, + "theoretical_loss": 3.399442860269451, + "tokens_seen": 2477312000 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020162888665997995, + "loss": 1.1647, + "theoretical_loss": 3.3994357340366186, + "tokens_seen": 2477377536 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020161283851554665, + "loss": 1.2214, + "theoretical_loss": 3.399428608045082, + "tokens_seen": 2477443072 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020159679037111335, + "loss": 1.274, + "theoretical_loss": 3.399421482294827, + "tokens_seen": 2477508608 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020158074222668005, + "loss": 1.168, + "theoretical_loss": 3.3994143567858393, + "tokens_seen": 2477574144 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020156469408224675, + "loss": 1.2151, + "theoretical_loss": 3.399407231518104, + "tokens_seen": 2477639680 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020154864593781345, + "loss": 1.2192, + "theoretical_loss": 3.3994001064916066, + "tokens_seen": 2477705216 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020153259779338015, + "loss": 1.2763, + "theoretical_loss": 3.3993929817063324, + "tokens_seen": 2477770752 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020151654964894683, + "loss": 1.2121, + "theoretical_loss": 3.399385857162267, + "tokens_seen": 2477836288 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020150050150451353, + "loss": 1.3102, + "theoretical_loss": 3.3993787328593963, + "tokens_seen": 2477901824 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020148445336008023, + "loss": 1.2844, + "theoretical_loss": 3.399371608797705, + "tokens_seen": 2477967360 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020146840521564693, + "loss": 1.2276, + "theoretical_loss": 3.3993644849771787, + "tokens_seen": 2478032896 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020145235707121363, + "loss": 1.3044, + "theoretical_loss": 3.3993573613978034, + "tokens_seen": 2478098432 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020143630892678033, + "loss": 1.182, + "theoretical_loss": 3.399350238059564, + "tokens_seen": 2478163968 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020142026078234703, + "loss": 1.2323, + "theoretical_loss": 3.3993431149624467, + "tokens_seen": 2478229504 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020140421263791373, + "loss": 1.1767, + "theoretical_loss": 3.399335992106436, + "tokens_seen": 2478295040 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020138816449348043, + "loss": 1.1999, + "theoretical_loss": 3.3993288694915176, + "tokens_seen": 2478360576 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020137211634904713, + "loss": 1.2299, + "theoretical_loss": 3.3993217471176775, + "tokens_seen": 2478426112 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020135606820461383, + "loss": 1.1727, + "theoretical_loss": 3.3993146249849007, + "tokens_seen": 2478491648 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020134002006018053, + "loss": 1.2003, + "theoretical_loss": 3.399307503093173, + "tokens_seen": 2478557184 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020132397191574723, + "loss": 1.2062, + "theoretical_loss": 3.3993003814424796, + "tokens_seen": 2478622720 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020130792377131393, + "loss": 1.1756, + "theoretical_loss": 3.399293260032806, + "tokens_seen": 2478688256 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020129187562688064, + "loss": 1.1938, + "theoretical_loss": 3.399286138864138, + "tokens_seen": 2478753792 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020127582748244734, + "loss": 1.2812, + "theoretical_loss": 3.3992790179364603, + "tokens_seen": 2478819328 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020125977933801404, + "loss": 1.3146, + "theoretical_loss": 3.399271897249759, + "tokens_seen": 2478884864 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020124373119358074, + "loss": 1.1514, + "theoretical_loss": 3.3992647768040194, + "tokens_seen": 2478950400 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020122768304914744, + "loss": 1.3465, + "theoretical_loss": 3.399257656599227, + "tokens_seen": 2479015936 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020121163490471414, + "loss": 1.1674, + "theoretical_loss": 3.3992505366353676, + "tokens_seen": 2479081472 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020119558676028087, + "loss": 1.2274, + "theoretical_loss": 3.399243416912426, + "tokens_seen": 2479147008 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020117953861584757, + "loss": 1.2147, + "theoretical_loss": 3.399236297430388, + "tokens_seen": 2479212544 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020116349047141427, + "loss": 1.292, + "theoretical_loss": 3.3992291781892394, + "tokens_seen": 2479278080 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020114744232698097, + "loss": 1.1694, + "theoretical_loss": 3.3992220591889657, + "tokens_seen": 2479343616 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020113139418254767, + "loss": 1.1823, + "theoretical_loss": 3.3992149404295513, + "tokens_seen": 2479409152 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020111534603811437, + "loss": 1.2778, + "theoretical_loss": 3.399207821910983, + "tokens_seen": 2479474688 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020109929789368107, + "loss": 1.2035, + "theoretical_loss": 3.3992007036332454, + "tokens_seen": 2479540224 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020108324974924777, + "loss": 1.2055, + "theoretical_loss": 3.3991935855963247, + "tokens_seen": 2479605760 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020106720160481447, + "loss": 1.1347, + "theoretical_loss": 3.399186467800206, + "tokens_seen": 2479671296 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020105115346038117, + "loss": 1.1613, + "theoretical_loss": 3.3991793502448746, + "tokens_seen": 2479736832 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020103510531594787, + "loss": 1.187, + "theoretical_loss": 3.3991722329303165, + "tokens_seen": 2479802368 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020101905717151457, + "loss": 1.2031, + "theoretical_loss": 3.399165115856517, + "tokens_seen": 2479867904 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020100300902708127, + "loss": 1.2486, + "theoretical_loss": 3.399157999023461, + "tokens_seen": 2479933440 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020098696088264797, + "loss": 1.2179, + "theoretical_loss": 3.3991508824311345, + "tokens_seen": 2479998976 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020097091273821468, + "loss": 1.1376, + "theoretical_loss": 3.3991437660795234, + "tokens_seen": 2480064512 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020095486459378138, + "loss": 1.2416, + "theoretical_loss": 3.3991366499686126, + "tokens_seen": 2480130048 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020093881644934808, + "loss": 1.1025, + "theoretical_loss": 3.399129534098388, + "tokens_seen": 2480195584 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020092276830491478, + "loss": 1.2194, + "theoretical_loss": 3.399122418468835, + "tokens_seen": 2480261120 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020090672016048148, + "loss": 1.1696, + "theoretical_loss": 3.399115303079938, + "tokens_seen": 2480326656 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020089067201604815, + "loss": 1.2536, + "theoretical_loss": 3.3991081879316845, + "tokens_seen": 2480392192 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020087462387161485, + "loss": 1.1572, + "theoretical_loss": 3.3991010730240587, + "tokens_seen": 2480457728 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020085857572718155, + "loss": 1.1233, + "theoretical_loss": 3.3990939583570463, + "tokens_seen": 2480523264 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020084252758274825, + "loss": 1.2871, + "theoretical_loss": 3.399086843930633, + "tokens_seen": 2480588800 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020082647943831495, + "loss": 1.1645, + "theoretical_loss": 3.3990797297448045, + "tokens_seen": 2480654336 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020081043129388165, + "loss": 1.2453, + "theoretical_loss": 3.399072615799546, + "tokens_seen": 2480719872 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020079438314944836, + "loss": 1.2037, + "theoretical_loss": 3.3990655020948424, + "tokens_seen": 2480785408 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020077833500501506, + "loss": 1.1703, + "theoretical_loss": 3.39905838863068, + "tokens_seen": 2480850944 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020076228686058176, + "loss": 1.0778, + "theoretical_loss": 3.3990512754070448, + "tokens_seen": 2480916480 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020074623871614846, + "loss": 1.1159, + "theoretical_loss": 3.399044162423921, + "tokens_seen": 2480982016 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020073019057171516, + "loss": 1.2765, + "theoretical_loss": 3.399037049681295, + "tokens_seen": 2481047552 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020071414242728186, + "loss": 1.2162, + "theoretical_loss": 3.399029937179152, + "tokens_seen": 2481113088 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020069809428284856, + "loss": 1.1207, + "theoretical_loss": 3.399022824917478, + "tokens_seen": 2481178624 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020068204613841526, + "loss": 1.0686, + "theoretical_loss": 3.399015712896258, + "tokens_seen": 2481244160 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020066599799398196, + "loss": 1.1519, + "theoretical_loss": 3.3990086011154776, + "tokens_seen": 2481309696 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020064994984954866, + "loss": 1.1485, + "theoretical_loss": 3.399001489575122, + "tokens_seen": 2481375232 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020063390170511536, + "loss": 1.2211, + "theoretical_loss": 3.3989943782751775, + "tokens_seen": 2481440768 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020061785356068206, + "loss": 1.1677, + "theoretical_loss": 3.398987267215629, + "tokens_seen": 2481506304 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020060180541624876, + "loss": 1.2703, + "theoretical_loss": 3.398980156396463, + "tokens_seen": 2481571840 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020058575727181546, + "loss": 1.2764, + "theoretical_loss": 3.3989730458176632, + "tokens_seen": 2481637376 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020056970912738216, + "loss": 1.2625, + "theoretical_loss": 3.3989659354792168, + "tokens_seen": 2481702912 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020055366098294886, + "loss": 1.2736, + "theoretical_loss": 3.3989588253811087, + "tokens_seen": 2481768448 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020053761283851557, + "loss": 1.2465, + "theoretical_loss": 3.3989517155233244, + "tokens_seen": 2481833984 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020052156469408227, + "loss": 1.2451, + "theoretical_loss": 3.3989446059058492, + "tokens_seen": 2481899520 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020050551654964894, + "loss": 1.1843, + "theoretical_loss": 3.3989374965286694, + "tokens_seen": 2481965056 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020048946840521564, + "loss": 1.2273, + "theoretical_loss": 3.39893038739177, + "tokens_seen": 2482030592 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020047342026078234, + "loss": 1.33, + "theoretical_loss": 3.398923278495136, + "tokens_seen": 2482096128 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020045737211634904, + "loss": 1.2076, + "theoretical_loss": 3.398916169838754, + "tokens_seen": 2482161664 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020044132397191574, + "loss": 1.1361, + "theoretical_loss": 3.3989090614226094, + "tokens_seen": 2482227200 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020042527582748244, + "loss": 1.2481, + "theoretical_loss": 3.398901953246687, + "tokens_seen": 2482292736 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020040922768304914, + "loss": 1.2122, + "theoretical_loss": 3.3988948453109726, + "tokens_seen": 2482358272 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020039317953861584, + "loss": 1.1019, + "theoretical_loss": 3.3988877376154525, + "tokens_seen": 2482423808 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020037713139418254, + "loss": 1.2501, + "theoretical_loss": 3.398880630160111, + "tokens_seen": 2482489344 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020036108324974925, + "loss": 1.2226, + "theoretical_loss": 3.3988735229449345, + "tokens_seen": 2482554880 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020034503510531595, + "loss": 1.1653, + "theoretical_loss": 3.3988664159699082, + "tokens_seen": 2482620416 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020032898696088265, + "loss": 1.1188, + "theoretical_loss": 3.398859309235018, + "tokens_seen": 2482685952 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020031293881644935, + "loss": 1.1344, + "theoretical_loss": 3.3988522027402492, + "tokens_seen": 2482751488 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020029689067201605, + "loss": 1.1502, + "theoretical_loss": 3.398845096485587, + "tokens_seen": 2482817024 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020028084252758275, + "loss": 1.2845, + "theoretical_loss": 3.3988379904710175, + "tokens_seen": 2482882560 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020026479438314945, + "loss": 1.2072, + "theoretical_loss": 3.398830884696526, + "tokens_seen": 2482948096 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020024874623871615, + "loss": 1.2519, + "theoretical_loss": 3.3988237791620985, + "tokens_seen": 2483013632 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020023269809428285, + "loss": 1.1075, + "theoretical_loss": 3.39881667386772, + "tokens_seen": 2483079168 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020021664994984955, + "loss": 1.2645, + "theoretical_loss": 3.398809568813376, + "tokens_seen": 2483144704 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020020060180541625, + "loss": 1.2536, + "theoretical_loss": 3.3988024639990524, + "tokens_seen": 2483210240 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020018455366098295, + "loss": 1.2228, + "theoretical_loss": 3.3987953594247347, + "tokens_seen": 2483275776 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020016850551654965, + "loss": 1.2199, + "theoretical_loss": 3.3987882550904085, + "tokens_seen": 2483341312 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020015245737211635, + "loss": 1.2286, + "theoretical_loss": 3.3987811509960593, + "tokens_seen": 2483406848 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020013640922768305, + "loss": 1.1486, + "theoretical_loss": 3.3987740471416723, + "tokens_seen": 2483472384 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020012036108324973, + "loss": 1.1705, + "theoretical_loss": 3.3987669435272334, + "tokens_seen": 2483537920 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020010431293881643, + "loss": 1.2293, + "theoretical_loss": 3.3987598401527284, + "tokens_seen": 2483603456 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020008826479438313, + "loss": 1.0816, + "theoretical_loss": 3.3987527370181425, + "tokens_seen": 2483668992 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020007221664994983, + "loss": 1.1381, + "theoretical_loss": 3.398745634123461, + "tokens_seen": 2483734528 + }, + { + "epoch": 1.13, + "objective/train/docs_used": 917664, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2696635723114014, + "objective/train/theoretical_loss": 3.3987385314686707, + "objective/train/tokens_used": 2504260064, + "theoretical_loss": 3.3987385314686707, + "tokens_seen": 2483800064 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020005616850551653, + "loss": 1.2697, + "theoretical_loss": 3.3987385314686707, + "tokens_seen": 2483800064 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020004012036108323, + "loss": 1.1802, + "theoretical_loss": 3.398731429053756, + "tokens_seen": 2483865600 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020002407221664993, + "loss": 1.2241, + "theoretical_loss": 3.3987243268787024, + "tokens_seen": 2483931136 + }, + { + "epoch": 1.13, + "learning_rate": 0.00020000802407221663, + "loss": 1.1743, + "theoretical_loss": 3.398717224943496, + "tokens_seen": 2483996672 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019999197592778336, + "loss": 1.2093, + "theoretical_loss": 3.3987101232481223, + "tokens_seen": 2484062208 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019997592778335006, + "loss": 1.1787, + "theoretical_loss": 3.3987030217925667, + "tokens_seen": 2484127744 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019995987963891676, + "loss": 1.2528, + "theoretical_loss": 3.398695920576815, + "tokens_seen": 2484193280 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019994383149448346, + "loss": 1.1732, + "theoretical_loss": 3.3986888196008525, + "tokens_seen": 2484258816 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019992778335005016, + "loss": 1.1963, + "theoretical_loss": 3.398681718864665, + "tokens_seen": 2484324352 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019991173520561686, + "loss": 1.0972, + "theoretical_loss": 3.398674618368238, + "tokens_seen": 2484389888 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019989568706118356, + "loss": 1.2106, + "theoretical_loss": 3.398667518111557, + "tokens_seen": 2484455424 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019987963891675026, + "loss": 1.1505, + "theoretical_loss": 3.398660418094608, + "tokens_seen": 2484520960 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019986359077231696, + "loss": 1.2621, + "theoretical_loss": 3.398653318317376, + "tokens_seen": 2484586496 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019984754262788367, + "loss": 1.1758, + "theoretical_loss": 3.398646218779847, + "tokens_seen": 2484652032 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019983149448345037, + "loss": 1.2007, + "theoretical_loss": 3.3986391194820063, + "tokens_seen": 2484717568 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019981544633901707, + "loss": 1.2467, + "theoretical_loss": 3.3986320204238396, + "tokens_seen": 2484783104 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019979939819458374, + "loss": 1.2267, + "theoretical_loss": 3.3986249216053324, + "tokens_seen": 2484848640 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019978335005015044, + "loss": 1.168, + "theoretical_loss": 3.3986178230264708, + "tokens_seen": 2484914176 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019976730190571714, + "loss": 1.2514, + "theoretical_loss": 3.3986107246872392, + "tokens_seen": 2484979712 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019975125376128384, + "loss": 1.2259, + "theoretical_loss": 3.3986036265876245, + "tokens_seen": 2485045248 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019973520561685054, + "loss": 1.1794, + "theoretical_loss": 3.3985965287276114, + "tokens_seen": 2485110784 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019971915747241724, + "loss": 1.2419, + "theoretical_loss": 3.3985894311071867, + "tokens_seen": 2485176320 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019970310932798394, + "loss": 1.1743, + "theoretical_loss": 3.3985823337263343, + "tokens_seen": 2485241856 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019968706118355067, + "loss": 1.116, + "theoretical_loss": 3.398575236585041, + "tokens_seen": 2485307392 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019967101303911737, + "loss": 1.3479, + "theoretical_loss": 3.3985681396832916, + "tokens_seen": 2485372928 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019965496489468407, + "loss": 1.2668, + "theoretical_loss": 3.398561043021073, + "tokens_seen": 2485438464 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019963891675025077, + "loss": 1.2408, + "theoretical_loss": 3.398553946598369, + "tokens_seen": 2485504000 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019962286860581747, + "loss": 1.2564, + "theoretical_loss": 3.398546850415167, + "tokens_seen": 2485569536 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019960682046138418, + "loss": 1.1826, + "theoretical_loss": 3.398539754471451, + "tokens_seen": 2485635072 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019959077231695088, + "loss": 1.2064, + "theoretical_loss": 3.398532658767208, + "tokens_seen": 2485700608 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019957472417251758, + "loss": 1.2057, + "theoretical_loss": 3.3985255633024223, + "tokens_seen": 2485766144 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019955867602808428, + "loss": 1.1839, + "theoretical_loss": 3.3985184680770804, + "tokens_seen": 2485831680 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019954262788365098, + "loss": 1.2527, + "theoretical_loss": 3.398511373091168, + "tokens_seen": 2485897216 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019952657973921768, + "loss": 1.1428, + "theoretical_loss": 3.39850427834467, + "tokens_seen": 2485962752 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019951053159478438, + "loss": 1.2293, + "theoretical_loss": 3.398497183837572, + "tokens_seen": 2486028288 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019949448345035108, + "loss": 1.194, + "theoretical_loss": 3.3984900895698607, + "tokens_seen": 2486093824 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019947843530591775, + "loss": 1.1075, + "theoretical_loss": 3.398482995541521, + "tokens_seen": 2486159360 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019946238716148445, + "loss": 1.1834, + "theoretical_loss": 3.398475901752538, + "tokens_seen": 2486224896 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019944633901705115, + "loss": 1.1293, + "theoretical_loss": 3.398468808202898, + "tokens_seen": 2486290432 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019943029087261786, + "loss": 1.1054, + "theoretical_loss": 3.3984617148925866, + "tokens_seen": 2486355968 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019941424272818456, + "loss": 1.1299, + "theoretical_loss": 3.3984546218215894, + "tokens_seen": 2486421504 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019939819458375126, + "loss": 1.1433, + "theoretical_loss": 3.398447528989892, + "tokens_seen": 2486487040 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019938214643931796, + "loss": 1.264, + "theoretical_loss": 3.3984404363974794, + "tokens_seen": 2486552576 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019936609829488466, + "loss": 1.1525, + "theoretical_loss": 3.3984333440443377, + "tokens_seen": 2486618112 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019935005015045136, + "loss": 1.1928, + "theoretical_loss": 3.398426251930453, + "tokens_seen": 2486683648 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019933400200601806, + "loss": 1.2548, + "theoretical_loss": 3.39841916005581, + "tokens_seen": 2486749184 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019931795386158476, + "loss": 1.1125, + "theoretical_loss": 3.398412068420395, + "tokens_seen": 2486814720 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019930190571715146, + "loss": 1.2282, + "theoretical_loss": 3.3984049770241938, + "tokens_seen": 2486880256 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019928585757271816, + "loss": 1.1973, + "theoretical_loss": 3.3983978858671913, + "tokens_seen": 2486945792 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019926980942828486, + "loss": 1.2556, + "theoretical_loss": 3.3983907949493735, + "tokens_seen": 2487011328 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019925376128385156, + "loss": 1.1663, + "theoretical_loss": 3.398383704270726, + "tokens_seen": 2487076864 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019923771313941826, + "loss": 1.2137, + "theoretical_loss": 3.3983766138312346, + "tokens_seen": 2487142400 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019922166499498496, + "loss": 1.23, + "theoretical_loss": 3.3983695236308846, + "tokens_seen": 2487207936 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019920561685055166, + "loss": 1.1977, + "theoretical_loss": 3.3983624336696616, + "tokens_seen": 2487273472 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019918956870611836, + "loss": 1.2948, + "theoretical_loss": 3.398355343947552, + "tokens_seen": 2487339008 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019917352056168507, + "loss": 1.2313, + "theoretical_loss": 3.3983482544645405, + "tokens_seen": 2487404544 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019915747241725177, + "loss": 1.2063, + "theoretical_loss": 3.3983411652206135, + "tokens_seen": 2487470080 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019914142427281847, + "loss": 1.149, + "theoretical_loss": 3.3983340762157557, + "tokens_seen": 2487535616 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019912537612838517, + "loss": 1.2491, + "theoretical_loss": 3.3983269874499533, + "tokens_seen": 2487601152 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019910932798395187, + "loss": 1.2112, + "theoretical_loss": 3.398319898923192, + "tokens_seen": 2487666688 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019909327983951857, + "loss": 1.235, + "theoretical_loss": 3.398312810635458, + "tokens_seen": 2487732224 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019907723169508527, + "loss": 1.229, + "theoretical_loss": 3.398305722586736, + "tokens_seen": 2487797760 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019906118355065197, + "loss": 1.2028, + "theoretical_loss": 3.3982986347770114, + "tokens_seen": 2487863296 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019904513540621867, + "loss": 1.1397, + "theoretical_loss": 3.398291547206271, + "tokens_seen": 2487928832 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019902908726178537, + "loss": 1.2771, + "theoretical_loss": 3.3982844598744997, + "tokens_seen": 2487994368 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019901303911735207, + "loss": 1.3354, + "theoretical_loss": 3.398277372781683, + "tokens_seen": 2488059904 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019899699097291877, + "loss": 1.1282, + "theoretical_loss": 3.3982702859278073, + "tokens_seen": 2488125440 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019898094282848547, + "loss": 1.1107, + "theoretical_loss": 3.398263199312858, + "tokens_seen": 2488190976 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019896489468405217, + "loss": 1.2414, + "theoretical_loss": 3.39825611293682, + "tokens_seen": 2488256512 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019894884653961887, + "loss": 1.238, + "theoretical_loss": 3.3982490267996797, + "tokens_seen": 2488322048 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019893279839518557, + "loss": 1.1414, + "theoretical_loss": 3.3982419409014226, + "tokens_seen": 2488387584 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019891675025075228, + "loss": 1.1088, + "theoretical_loss": 3.398234855242034, + "tokens_seen": 2488453120 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019890070210631898, + "loss": 1.1744, + "theoretical_loss": 3.3982277698215, + "tokens_seen": 2488518656 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019888465396188568, + "loss": 1.1944, + "theoretical_loss": 3.3982206846398064, + "tokens_seen": 2488584192 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019886860581745238, + "loss": 1.2101, + "theoretical_loss": 3.398213599696939, + "tokens_seen": 2488649728 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019885255767301908, + "loss": 1.1626, + "theoretical_loss": 3.398206514992882, + "tokens_seen": 2488715264 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019883650952858578, + "loss": 1.193, + "theoretical_loss": 3.398199430527623, + "tokens_seen": 2488780800 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019882046138415248, + "loss": 1.2037, + "theoretical_loss": 3.3981923463011467, + "tokens_seen": 2488846336 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019880441323971918, + "loss": 1.1752, + "theoretical_loss": 3.3981852623134383, + "tokens_seen": 2488911872 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019878836509528585, + "loss": 1.158, + "theoretical_loss": 3.3981781785644847, + "tokens_seen": 2488977408 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019877231695085255, + "loss": 1.1991, + "theoretical_loss": 3.3981710950542703, + "tokens_seen": 2489042944 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019875626880641925, + "loss": 1.1883, + "theoretical_loss": 3.3981640117827814, + "tokens_seen": 2489108480 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019874022066198596, + "loss": 1.1726, + "theoretical_loss": 3.3981569287500037, + "tokens_seen": 2489174016 + }, + { + "epoch": 1.13, + "learning_rate": 0.00019872417251755266, + "loss": 1.1967, + "theoretical_loss": 3.398149845955923, + "tokens_seen": 2489239552 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019870812437311936, + "loss": 1.2246, + "theoretical_loss": 3.3981427634005246, + "tokens_seen": 2489305088 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019869207622868606, + "loss": 1.2731, + "theoretical_loss": 3.3981356810837946, + "tokens_seen": 2489370624 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019867602808425276, + "loss": 1.1811, + "theoretical_loss": 3.398128599005718, + "tokens_seen": 2489436160 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019865997993981946, + "loss": 1.2824, + "theoretical_loss": 3.398121517166281, + "tokens_seen": 2489501696 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019864393179538616, + "loss": 1.2131, + "theoretical_loss": 3.3981144355654695, + "tokens_seen": 2489567232 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019862788365095286, + "loss": 1.2427, + "theoretical_loss": 3.3981073542032685, + "tokens_seen": 2489632768 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019861183550651956, + "loss": 1.0486, + "theoretical_loss": 3.398100273079664, + "tokens_seen": 2489698304 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019859578736208626, + "loss": 1.1952, + "theoretical_loss": 3.398093192194642, + "tokens_seen": 2489763840 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019857973921765296, + "loss": 1.188, + "theoretical_loss": 3.3980861115481873, + "tokens_seen": 2489829376 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019856369107321966, + "loss": 1.2485, + "theoretical_loss": 3.3980790311402864, + "tokens_seen": 2489894912 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019854764292878636, + "loss": 1.3403, + "theoretical_loss": 3.398071950970925, + "tokens_seen": 2489960448 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019853159478435306, + "loss": 1.2101, + "theoretical_loss": 3.3980648710400883, + "tokens_seen": 2490025984 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019851554663991976, + "loss": 1.1297, + "theoretical_loss": 3.3980577913477625, + "tokens_seen": 2490091520 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019849949849548646, + "loss": 1.1707, + "theoretical_loss": 3.3980507118939327, + "tokens_seen": 2490157056 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001984834503510532, + "loss": 1.25, + "theoretical_loss": 3.398043632678585, + "tokens_seen": 2490222592 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019846740220661987, + "loss": 1.1858, + "theoretical_loss": 3.398036553701705, + "tokens_seen": 2490288128 + }, + { + "epoch": 1.14, + "objective/train/docs_used": 920029, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2270176410675049, + "objective/train/theoretical_loss": 3.3980294749632787, + "objective/train/tokens_used": 2510813664, + "theoretical_loss": 3.3980294749632787, + "tokens_seen": 2490353664 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019845135406218657, + "loss": 1.227, + "theoretical_loss": 3.3980294749632787, + "tokens_seen": 2490353664 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019843530591775327, + "loss": 1.2146, + "theoretical_loss": 3.3980223964632907, + "tokens_seen": 2490419200 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019841925777331997, + "loss": 1.206, + "theoretical_loss": 3.398015318201728, + "tokens_seen": 2490484736 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019840320962888667, + "loss": 1.2227, + "theoretical_loss": 3.3980082401785756, + "tokens_seen": 2490550272 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019838716148445337, + "loss": 1.1892, + "theoretical_loss": 3.3980011623938196, + "tokens_seen": 2490615808 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019837111334002007, + "loss": 1.1907, + "theoretical_loss": 3.3979940848474453, + "tokens_seen": 2490681344 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019835506519558677, + "loss": 1.1972, + "theoretical_loss": 3.3979870075394385, + "tokens_seen": 2490746880 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019833901705115347, + "loss": 1.2236, + "theoretical_loss": 3.397979930469785, + "tokens_seen": 2490812416 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019832296890672017, + "loss": 1.1836, + "theoretical_loss": 3.3979728536384703, + "tokens_seen": 2490877952 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019830692076228687, + "loss": 1.2291, + "theoretical_loss": 3.3979657770454805, + "tokens_seen": 2490943488 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019829087261785357, + "loss": 1.1923, + "theoretical_loss": 3.397958700690801, + "tokens_seen": 2491009024 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019827482447342027, + "loss": 1.2311, + "theoretical_loss": 3.397951624574418, + "tokens_seen": 2491074560 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019825877632898697, + "loss": 1.1973, + "theoretical_loss": 3.397944548696316, + "tokens_seen": 2491140096 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019824272818455367, + "loss": 1.256, + "theoretical_loss": 3.3979374730564817, + "tokens_seen": 2491205632 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019822668004012038, + "loss": 1.2411, + "theoretical_loss": 3.3979303976549007, + "tokens_seen": 2491271168 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019821063189568708, + "loss": 1.2068, + "theoretical_loss": 3.397923322491559, + "tokens_seen": 2491336704 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019819458375125378, + "loss": 1.2155, + "theoretical_loss": 3.397916247566441, + "tokens_seen": 2491402240 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019817853560682048, + "loss": 1.2275, + "theoretical_loss": 3.397909172879534, + "tokens_seen": 2491467776 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019816248746238718, + "loss": 1.1657, + "theoretical_loss": 3.397902098430823, + "tokens_seen": 2491533312 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019814643931795388, + "loss": 1.146, + "theoretical_loss": 3.3978950242202934, + "tokens_seen": 2491598848 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019813039117352058, + "loss": 1.1396, + "theoretical_loss": 3.3978879502479318, + "tokens_seen": 2491664384 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019811434302908728, + "loss": 1.3036, + "theoretical_loss": 3.397880876513723, + "tokens_seen": 2491729920 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019809829488465398, + "loss": 1.2515, + "theoretical_loss": 3.3978738030176534, + "tokens_seen": 2491795456 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019808224674022065, + "loss": 1.1861, + "theoretical_loss": 3.3978667297597083, + "tokens_seen": 2491860992 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019806619859578735, + "loss": 1.1364, + "theoretical_loss": 3.3978596567398736, + "tokens_seen": 2491926528 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019805015045135406, + "loss": 1.2437, + "theoretical_loss": 3.3978525839581346, + "tokens_seen": 2491992064 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019803410230692076, + "loss": 1.1699, + "theoretical_loss": 3.3978455114144777, + "tokens_seen": 2492057600 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019801805416248746, + "loss": 1.2625, + "theoretical_loss": 3.3978384391088885, + "tokens_seen": 2492123136 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019800200601805416, + "loss": 1.2482, + "theoretical_loss": 3.397831367041352, + "tokens_seen": 2492188672 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019798595787362086, + "loss": 1.3052, + "theoretical_loss": 3.397824295211855, + "tokens_seen": 2492254208 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019796990972918756, + "loss": 1.2985, + "theoretical_loss": 3.3978172236203825, + "tokens_seen": 2492319744 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019795386158475426, + "loss": 1.2178, + "theoretical_loss": 3.3978101522669206, + "tokens_seen": 2492385280 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019793781344032096, + "loss": 1.2254, + "theoretical_loss": 3.3978030811514546, + "tokens_seen": 2492450816 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019792176529588766, + "loss": 1.2711, + "theoretical_loss": 3.3977960102739706, + "tokens_seen": 2492516352 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019790571715145436, + "loss": 1.2328, + "theoretical_loss": 3.397788939634454, + "tokens_seen": 2492581888 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001978896690070211, + "loss": 1.2774, + "theoretical_loss": 3.397781869232891, + "tokens_seen": 2492647424 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001978736208625878, + "loss": 1.2379, + "theoretical_loss": 3.3977747990692677, + "tokens_seen": 2492712960 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001978575727181545, + "loss": 1.1583, + "theoretical_loss": 3.3977677291435686, + "tokens_seen": 2492778496 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001978415245737212, + "loss": 1.29, + "theoretical_loss": 3.39776065945578, + "tokens_seen": 2492844032 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001978254764292879, + "loss": 1.264, + "theoretical_loss": 3.397753590005888, + "tokens_seen": 2492909568 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001978094282848546, + "loss": 1.2106, + "theoretical_loss": 3.397746520793878, + "tokens_seen": 2492975104 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001977933801404213, + "loss": 1.1885, + "theoretical_loss": 3.3977394518197355, + "tokens_seen": 2493040640 + }, + { + "epoch": 1.14, + "learning_rate": 0.000197777331995988, + "loss": 1.2382, + "theoretical_loss": 3.3977323830834467, + "tokens_seen": 2493106176 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019776128385155467, + "loss": 1.1771, + "theoretical_loss": 3.3977253145849975, + "tokens_seen": 2493171712 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019774523570712137, + "loss": 1.1843, + "theoretical_loss": 3.3977182463243727, + "tokens_seen": 2493237248 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019772918756268807, + "loss": 1.2379, + "theoretical_loss": 3.3977111783015594, + "tokens_seen": 2493302784 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019771313941825477, + "loss": 1.2275, + "theoretical_loss": 3.397704110516542, + "tokens_seen": 2493368320 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019769709127382147, + "loss": 1.2041, + "theoretical_loss": 3.397697042969307, + "tokens_seen": 2493433856 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019768104312938817, + "loss": 1.2379, + "theoretical_loss": 3.3976899756598407, + "tokens_seen": 2493499392 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019766499498495487, + "loss": 1.2242, + "theoretical_loss": 3.3976829085881275, + "tokens_seen": 2493564928 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019764894684052157, + "loss": 1.224, + "theoretical_loss": 3.397675841754154, + "tokens_seen": 2493630464 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019763289869608827, + "loss": 1.3116, + "theoretical_loss": 3.397668775157906, + "tokens_seen": 2493696000 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019761685055165497, + "loss": 1.3067, + "theoretical_loss": 3.397661708799369, + "tokens_seen": 2493761536 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019760080240722167, + "loss": 1.1906, + "theoretical_loss": 3.3976546426785283, + "tokens_seen": 2493827072 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019758475426278837, + "loss": 1.2753, + "theoretical_loss": 3.3976475767953707, + "tokens_seen": 2493892608 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019756870611835507, + "loss": 1.2069, + "theoretical_loss": 3.3976405111498815, + "tokens_seen": 2493958144 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019755265797392178, + "loss": 1.2049, + "theoretical_loss": 3.397633445742046, + "tokens_seen": 2494023680 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019753660982948848, + "loss": 1.2836, + "theoretical_loss": 3.3976263805718503, + "tokens_seen": 2494089216 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019752056168505518, + "loss": 1.1961, + "theoretical_loss": 3.3976193156392807, + "tokens_seen": 2494154752 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019750451354062188, + "loss": 1.2569, + "theoretical_loss": 3.397612250944322, + "tokens_seen": 2494220288 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019748846539618858, + "loss": 1.1881, + "theoretical_loss": 3.3976051864869605, + "tokens_seen": 2494285824 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019747241725175528, + "loss": 1.3121, + "theoretical_loss": 3.397598122267182, + "tokens_seen": 2494351360 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019745636910732198, + "loss": 1.2305, + "theoretical_loss": 3.3975910582849727, + "tokens_seen": 2494416896 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019744032096288868, + "loss": 1.2129, + "theoretical_loss": 3.397583994540317, + "tokens_seen": 2494482432 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019742427281845538, + "loss": 1.2725, + "theoretical_loss": 3.397576931033202, + "tokens_seen": 2494547968 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019740822467402208, + "loss": 1.2191, + "theoretical_loss": 3.397569867763613, + "tokens_seen": 2494613504 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019739217652958878, + "loss": 1.128, + "theoretical_loss": 3.3975628047315354, + "tokens_seen": 2494679040 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019737612838515545, + "loss": 1.1642, + "theoretical_loss": 3.3975557419369555, + "tokens_seen": 2494744576 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019736008024072216, + "loss": 1.2097, + "theoretical_loss": 3.3975486793798586, + "tokens_seen": 2494810112 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019734403209628886, + "loss": 1.2652, + "theoretical_loss": 3.3975416170602313, + "tokens_seen": 2494875648 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019732798395185556, + "loss": 1.03, + "theoretical_loss": 3.3975345549780585, + "tokens_seen": 2494941184 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019731193580742226, + "loss": 1.1806, + "theoretical_loss": 3.397527493133327, + "tokens_seen": 2495006720 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019729588766298899, + "loss": 1.2737, + "theoretical_loss": 3.3975204315260212, + "tokens_seen": 2495072256 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019727983951855569, + "loss": 1.2028, + "theoretical_loss": 3.397513370156128, + "tokens_seen": 2495137792 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001972637913741224, + "loss": 1.1446, + "theoretical_loss": 3.3975063090236324, + "tokens_seen": 2495203328 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001972477432296891, + "loss": 1.151, + "theoretical_loss": 3.3974992481285207, + "tokens_seen": 2495268864 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001972316950852558, + "loss": 1.1798, + "theoretical_loss": 3.3974921874707786, + "tokens_seen": 2495334400 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001972156469408225, + "loss": 1.2533, + "theoretical_loss": 3.397485127050392, + "tokens_seen": 2495399936 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001971995987963892, + "loss": 1.2372, + "theoretical_loss": 3.3974780668673463, + "tokens_seen": 2495465472 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001971835506519559, + "loss": 1.172, + "theoretical_loss": 3.3974710069216276, + "tokens_seen": 2495531008 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001971675025075226, + "loss": 1.1914, + "theoretical_loss": 3.3974639472132218, + "tokens_seen": 2495596544 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001971514543630893, + "loss": 1.1919, + "theoretical_loss": 3.397456887742114, + "tokens_seen": 2495662080 + }, + { + "epoch": 1.14, + "learning_rate": 0.000197135406218656, + "loss": 1.1759, + "theoretical_loss": 3.3974498285082912, + "tokens_seen": 2495727616 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001971193580742227, + "loss": 1.0475, + "theoretical_loss": 3.3974427695117377, + "tokens_seen": 2495793152 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001971033099297894, + "loss": 1.2283, + "theoretical_loss": 3.3974357107524407, + "tokens_seen": 2495858688 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001970872617853561, + "loss": 1.239, + "theoretical_loss": 3.397428652230385, + "tokens_seen": 2495924224 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019707121364092277, + "loss": 1.3, + "theoretical_loss": 3.397421593945557, + "tokens_seen": 2495989760 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019705516549648947, + "loss": 1.3274, + "theoretical_loss": 3.3974145358979424, + "tokens_seen": 2496055296 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019703911735205617, + "loss": 1.3172, + "theoretical_loss": 3.3974074780875263, + "tokens_seen": 2496120832 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019702306920762287, + "loss": 1.1605, + "theoretical_loss": 3.3974004205142956, + "tokens_seen": 2496186368 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019700702106318957, + "loss": 1.2118, + "theoretical_loss": 3.3973933631782356, + "tokens_seen": 2496251904 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019699097291875627, + "loss": 1.1635, + "theoretical_loss": 3.3973863060793317, + "tokens_seen": 2496317440 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019697492477432297, + "loss": 1.2419, + "theoretical_loss": 3.3973792492175705, + "tokens_seen": 2496382976 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019695887662988967, + "loss": 1.118, + "theoretical_loss": 3.397372192592937, + "tokens_seen": 2496448512 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019694282848545637, + "loss": 1.2279, + "theoretical_loss": 3.397365136205418, + "tokens_seen": 2496514048 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019692678034102307, + "loss": 1.2562, + "theoretical_loss": 3.397358080054998, + "tokens_seen": 2496579584 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019691073219658977, + "loss": 1.1804, + "theoretical_loss": 3.397351024141664, + "tokens_seen": 2496645120 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019689468405215647, + "loss": 1.2185, + "theoretical_loss": 3.3973439684654005, + "tokens_seen": 2496710656 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019687863590772317, + "loss": 1.1426, + "theoretical_loss": 3.3973369130261952, + "tokens_seen": 2496776192 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019686258776328988, + "loss": 1.2999, + "theoretical_loss": 3.3973298578240323, + "tokens_seen": 2496841728 + }, + { + "epoch": 1.14, + "objective/train/docs_used": 922178, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1637041568756104, + "objective/train/theoretical_loss": 3.3973228028588984, + "objective/train/tokens_used": 2517367264, + "theoretical_loss": 3.3973228028588984, + "tokens_seen": 2496907264 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019684653961885658, + "loss": 1.1637, + "theoretical_loss": 3.3973228028588984, + "tokens_seen": 2496907264 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019683049147442328, + "loss": 1.2351, + "theoretical_loss": 3.3973157481307785, + "tokens_seen": 2496972800 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019681444332998998, + "loss": 1.2124, + "theoretical_loss": 3.3973086936396597, + "tokens_seen": 2497038336 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019679839518555668, + "loss": 1.1575, + "theoretical_loss": 3.397301639385527, + "tokens_seen": 2497103872 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019678234704112338, + "loss": 1.209, + "theoretical_loss": 3.397294585368366, + "tokens_seen": 2497169408 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019676629889669008, + "loss": 1.305, + "theoretical_loss": 3.397287531588163, + "tokens_seen": 2497234944 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019675025075225678, + "loss": 1.2301, + "theoretical_loss": 3.3972804780449035, + "tokens_seen": 2497300480 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019673420260782348, + "loss": 1.2471, + "theoretical_loss": 3.397273424738574, + "tokens_seen": 2497366016 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019671815446339018, + "loss": 1.157, + "theoretical_loss": 3.3972663716691596, + "tokens_seen": 2497431552 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019670210631895688, + "loss": 1.3052, + "theoretical_loss": 3.397259318836646, + "tokens_seen": 2497497088 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019668605817452358, + "loss": 1.0977, + "theoretical_loss": 3.39725226624102, + "tokens_seen": 2497562624 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019667001003009028, + "loss": 1.238, + "theoretical_loss": 3.3972452138822664, + "tokens_seen": 2497628160 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019665396188565698, + "loss": 1.153, + "theoretical_loss": 3.3972381617603715, + "tokens_seen": 2497693696 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019663791374122368, + "loss": 1.203, + "theoretical_loss": 3.397231109875321, + "tokens_seen": 2497759232 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019662186559679038, + "loss": 1.1814, + "theoretical_loss": 3.397224058227101, + "tokens_seen": 2497824768 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019660581745235709, + "loss": 1.1912, + "theoretical_loss": 3.397217006815697, + "tokens_seen": 2497890304 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019658976930792379, + "loss": 1.1209, + "theoretical_loss": 3.397209955641095, + "tokens_seen": 2497955840 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001965737211634905, + "loss": 1.2033, + "theoretical_loss": 3.397202904703281, + "tokens_seen": 2498021376 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001965576730190572, + "loss": 1.2214, + "theoretical_loss": 3.39719585400224, + "tokens_seen": 2498086912 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001965416248746239, + "loss": 1.1667, + "theoretical_loss": 3.397188803537959, + "tokens_seen": 2498152448 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001965255767301906, + "loss": 1.2568, + "theoretical_loss": 3.397181753310423, + "tokens_seen": 2498217984 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001965095285857573, + "loss": 1.1625, + "theoretical_loss": 3.3971747033196187, + "tokens_seen": 2498283520 + }, + { + "epoch": 1.14, + "learning_rate": 0.000196493480441324, + "loss": 1.3138, + "theoretical_loss": 3.397167653565531, + "tokens_seen": 2498349056 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001964774322968907, + "loss": 1.1272, + "theoretical_loss": 3.397160604048146, + "tokens_seen": 2498414592 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001964613841524574, + "loss": 1.2209, + "theoretical_loss": 3.39715355476745, + "tokens_seen": 2498480128 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001964453360080241, + "loss": 1.2653, + "theoretical_loss": 3.3971465057234282, + "tokens_seen": 2498545664 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001964292878635908, + "loss": 1.1555, + "theoretical_loss": 3.397139456916067, + "tokens_seen": 2498611200 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001964132397191575, + "loss": 1.2205, + "theoretical_loss": 3.3971324083453522, + "tokens_seen": 2498676736 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001963971915747242, + "loss": 1.1572, + "theoretical_loss": 3.3971253600112687, + "tokens_seen": 2498742272 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001963811434302909, + "loss": 1.2159, + "theoretical_loss": 3.397118311913804, + "tokens_seen": 2498807808 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019636509528585757, + "loss": 1.2207, + "theoretical_loss": 3.397111264052943, + "tokens_seen": 2498873344 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019634904714142427, + "loss": 1.1971, + "theoretical_loss": 3.397104216428671, + "tokens_seen": 2498938880 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019633299899699097, + "loss": 1.2623, + "theoretical_loss": 3.3970971690409746, + "tokens_seen": 2499004416 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019631695085255767, + "loss": 1.301, + "theoretical_loss": 3.39709012188984, + "tokens_seen": 2499069952 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019630090270812437, + "loss": 1.1695, + "theoretical_loss": 3.3970830749752525, + "tokens_seen": 2499135488 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019628485456369107, + "loss": 1.2057, + "theoretical_loss": 3.3970760282971977, + "tokens_seen": 2499201024 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019626880641925777, + "loss": 1.3007, + "theoretical_loss": 3.397068981855662, + "tokens_seen": 2499266560 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019625275827482447, + "loss": 1.2195, + "theoretical_loss": 3.397061935650631, + "tokens_seen": 2499332096 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019623671013039117, + "loss": 1.1851, + "theoretical_loss": 3.3970548896820905, + "tokens_seen": 2499397632 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019622066198595787, + "loss": 1.2114, + "theoretical_loss": 3.397047843950027, + "tokens_seen": 2499463168 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019620461384152457, + "loss": 1.2237, + "theoretical_loss": 3.3970407984544253, + "tokens_seen": 2499528704 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019618856569709127, + "loss": 1.1053, + "theoretical_loss": 3.397033753195272, + "tokens_seen": 2499594240 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019617251755265798, + "loss": 1.2642, + "theoretical_loss": 3.397026708172553, + "tokens_seen": 2499659776 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019615646940822468, + "loss": 1.2152, + "theoretical_loss": 3.3970196633862537, + "tokens_seen": 2499725312 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019614042126379138, + "loss": 1.1751, + "theoretical_loss": 3.39701261883636, + "tokens_seen": 2499790848 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019612437311935808, + "loss": 1.2449, + "theoretical_loss": 3.3970055745228582, + "tokens_seen": 2499856384 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019610832497492478, + "loss": 1.3109, + "theoretical_loss": 3.396998530445734, + "tokens_seen": 2499921920 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001960922768304915, + "loss": 1.2019, + "theoretical_loss": 3.396991486604973, + "tokens_seen": 2499987456 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001960762286860582, + "loss": 1.2329, + "theoretical_loss": 3.396984443000562, + "tokens_seen": 2500052992 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001960601805416249, + "loss": 1.2769, + "theoretical_loss": 3.3969773996324855, + "tokens_seen": 2500118528 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019604413239719158, + "loss": 1.2156, + "theoretical_loss": 3.39697035650073, + "tokens_seen": 2500184064 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019602808425275828, + "loss": 1.2584, + "theoretical_loss": 3.3969633136052817, + "tokens_seen": 2500249600 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019601203610832498, + "loss": 1.1627, + "theoretical_loss": 3.3969562709461263, + "tokens_seen": 2500315136 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019599598796389168, + "loss": 1.3134, + "theoretical_loss": 3.396949228523249, + "tokens_seen": 2500380672 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019597993981945838, + "loss": 1.107, + "theoretical_loss": 3.396942186336637, + "tokens_seen": 2500446208 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019596389167502508, + "loss": 1.1285, + "theoretical_loss": 3.396935144386275, + "tokens_seen": 2500511744 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019594784353059178, + "loss": 1.265, + "theoretical_loss": 3.3969281026721494, + "tokens_seen": 2500577280 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019593179538615848, + "loss": 1.1741, + "theoretical_loss": 3.396921061194246, + "tokens_seen": 2500642816 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019591574724172519, + "loss": 1.1451, + "theoretical_loss": 3.3969140199525505, + "tokens_seen": 2500708352 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019589969909729189, + "loss": 1.1701, + "theoretical_loss": 3.3969069789470496, + "tokens_seen": 2500773888 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001958836509528586, + "loss": 1.2104, + "theoretical_loss": 3.396899938177728, + "tokens_seen": 2500839424 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001958676028084253, + "loss": 1.2463, + "theoretical_loss": 3.3968928976445723, + "tokens_seen": 2500904960 + }, + { + "epoch": 1.14, + "learning_rate": 0.000195851554663992, + "loss": 1.2125, + "theoretical_loss": 3.396885857347568, + "tokens_seen": 2500970496 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001958355065195587, + "loss": 1.185, + "theoretical_loss": 3.396878817286702, + "tokens_seen": 2501036032 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001958194583751254, + "loss": 1.289, + "theoretical_loss": 3.396871777461959, + "tokens_seen": 2501101568 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001958034102306921, + "loss": 1.1525, + "theoretical_loss": 3.396864737873325, + "tokens_seen": 2501167104 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001957873620862588, + "loss": 1.1445, + "theoretical_loss": 3.3968576985207863, + "tokens_seen": 2501232640 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001957713139418255, + "loss": 1.1461, + "theoretical_loss": 3.396850659404329, + "tokens_seen": 2501298176 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001957552657973922, + "loss": 1.1792, + "theoretical_loss": 3.3968436205239385, + "tokens_seen": 2501363712 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001957392176529589, + "loss": 1.1943, + "theoretical_loss": 3.3968365818796014, + "tokens_seen": 2501429248 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001957231695085256, + "loss": 1.2387, + "theoretical_loss": 3.3968295434713025, + "tokens_seen": 2501494784 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001957071213640923, + "loss": 1.155, + "theoretical_loss": 3.3968225052990286, + "tokens_seen": 2501560320 + }, + { + "epoch": 1.14, + "learning_rate": 0.000195691073219659, + "loss": 1.1766, + "theoretical_loss": 3.3968154673627655, + "tokens_seen": 2501625856 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001956750250752257, + "loss": 1.2452, + "theoretical_loss": 3.3968084296624985, + "tokens_seen": 2501691392 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019565897693079237, + "loss": 1.2164, + "theoretical_loss": 3.3968013921982143, + "tokens_seen": 2501756928 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019564292878635907, + "loss": 1.2361, + "theoretical_loss": 3.396794354969898, + "tokens_seen": 2501822464 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019562688064192577, + "loss": 1.2669, + "theoretical_loss": 3.396787317977536, + "tokens_seen": 2501888000 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019561083249749247, + "loss": 1.1848, + "theoretical_loss": 3.3967802812211145, + "tokens_seen": 2501953536 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019559478435305917, + "loss": 1.2361, + "theoretical_loss": 3.396773244700619, + "tokens_seen": 2502019072 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019557873620862587, + "loss": 1.1809, + "theoretical_loss": 3.396766208416035, + "tokens_seen": 2502084608 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019556268806419257, + "loss": 1.2236, + "theoretical_loss": 3.396759172367349, + "tokens_seen": 2502150144 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019554663991975927, + "loss": 1.1384, + "theoretical_loss": 3.3967521365545474, + "tokens_seen": 2502215680 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019553059177532597, + "loss": 1.138, + "theoretical_loss": 3.396745100977615, + "tokens_seen": 2502281216 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019551454363089267, + "loss": 1.2245, + "theoretical_loss": 3.396738065636538, + "tokens_seen": 2502346752 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019549849548645937, + "loss": 1.2066, + "theoretical_loss": 3.3967310305313028, + "tokens_seen": 2502412288 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001954824473420261, + "loss": 1.2788, + "theoretical_loss": 3.396723995661895, + "tokens_seen": 2502477824 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001954663991975928, + "loss": 1.1543, + "theoretical_loss": 3.396716961028301, + "tokens_seen": 2502543360 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001954503510531595, + "loss": 1.2046, + "theoretical_loss": 3.3967099266305056, + "tokens_seen": 2502608896 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001954343029087262, + "loss": 1.2028, + "theoretical_loss": 3.396702892468496, + "tokens_seen": 2502674432 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001954182547642929, + "loss": 1.3102, + "theoretical_loss": 3.396695858542257, + "tokens_seen": 2502739968 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001954022066198596, + "loss": 1.1932, + "theoretical_loss": 3.3966888248517755, + "tokens_seen": 2502805504 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001953861584754263, + "loss": 1.2207, + "theoretical_loss": 3.396681791397037, + "tokens_seen": 2502871040 + }, + { + "epoch": 1.14, + "learning_rate": 0.000195370110330993, + "loss": 1.205, + "theoretical_loss": 3.396674758178027, + "tokens_seen": 2502936576 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001953540621865597, + "loss": 1.2307, + "theoretical_loss": 3.396667725194732, + "tokens_seen": 2503002112 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019533801404212638, + "loss": 1.2221, + "theoretical_loss": 3.3966606924471376, + "tokens_seen": 2503067648 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019532196589769308, + "loss": 1.1992, + "theoretical_loss": 3.39665365993523, + "tokens_seen": 2503133184 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019530591775325978, + "loss": 1.204, + "theoretical_loss": 3.3966466276589955, + "tokens_seen": 2503198720 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019528986960882648, + "loss": 1.2138, + "theoretical_loss": 3.396639595618419, + "tokens_seen": 2503264256 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019527382146439318, + "loss": 1.2258, + "theoretical_loss": 3.3966325638134873, + "tokens_seen": 2503329792 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019525777331995988, + "loss": 1.2362, + "theoretical_loss": 3.396625532244186, + "tokens_seen": 2503395328 + }, + { + "epoch": 1.14, + "objective/train/docs_used": 924514, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2182332277297974, + "objective/train/theoretical_loss": 3.3966185009105008, + "objective/train/tokens_used": 2523920864, + "theoretical_loss": 3.3966185009105008, + "tokens_seen": 2503460864 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019524172517552659, + "loss": 1.2182, + "theoretical_loss": 3.3966185009105008, + "tokens_seen": 2503460864 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019522567703109329, + "loss": 1.1852, + "theoretical_loss": 3.3966114698124183, + "tokens_seen": 2503526400 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019520962888665999, + "loss": 1.197, + "theoretical_loss": 3.3966044389499235, + "tokens_seen": 2503591936 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001951935807422267, + "loss": 1.1664, + "theoretical_loss": 3.3965974083230033, + "tokens_seen": 2503657472 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001951775325977934, + "loss": 1.1803, + "theoretical_loss": 3.396590377931643, + "tokens_seen": 2503723008 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001951614844533601, + "loss": 1.3158, + "theoretical_loss": 3.396583347775829, + "tokens_seen": 2503788544 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001951454363089268, + "loss": 1.2195, + "theoretical_loss": 3.396576317855547, + "tokens_seen": 2503854080 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001951293881644935, + "loss": 1.2456, + "theoretical_loss": 3.3965692881707827, + "tokens_seen": 2503919616 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001951133400200602, + "loss": 1.2845, + "theoretical_loss": 3.3965622587215227, + "tokens_seen": 2503985152 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001950972918756269, + "loss": 1.1854, + "theoretical_loss": 3.396555229507752, + "tokens_seen": 2504050688 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001950812437311936, + "loss": 1.22, + "theoretical_loss": 3.3965482005294576, + "tokens_seen": 2504116224 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001950651955867603, + "loss": 1.2635, + "theoretical_loss": 3.3965411717866245, + "tokens_seen": 2504181760 + }, + { + "epoch": 1.14, + "learning_rate": 0.000195049147442327, + "loss": 1.227, + "theoretical_loss": 3.396534143279239, + "tokens_seen": 2504247296 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001950330992978937, + "loss": 1.2079, + "theoretical_loss": 3.396527115007288, + "tokens_seen": 2504312832 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001950170511534604, + "loss": 1.1875, + "theoretical_loss": 3.3965200869707557, + "tokens_seen": 2504378368 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001950010030090271, + "loss": 1.2218, + "theoretical_loss": 3.3965130591696293, + "tokens_seen": 2504443904 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001949849548645938, + "loss": 1.2445, + "theoretical_loss": 3.3965060316038946, + "tokens_seen": 2504509440 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001949689067201605, + "loss": 1.1779, + "theoretical_loss": 3.396499004273537, + "tokens_seen": 2504574976 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019495285857572717, + "loss": 1.1821, + "theoretical_loss": 3.3964919771785427, + "tokens_seen": 2504640512 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019493681043129387, + "loss": 1.1785, + "theoretical_loss": 3.3964849503188983, + "tokens_seen": 2504706048 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019492076228686057, + "loss": 1.1943, + "theoretical_loss": 3.396477923694589, + "tokens_seen": 2504771584 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019490471414242727, + "loss": 1.1798, + "theoretical_loss": 3.396470897305601, + "tokens_seen": 2504837120 + }, + { + "epoch": 1.14, + "learning_rate": 0.000194888665997994, + "loss": 1.2754, + "theoretical_loss": 3.39646387115192, + "tokens_seen": 2504902656 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001948726178535607, + "loss": 1.1817, + "theoretical_loss": 3.3964568452335326, + "tokens_seen": 2504968192 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001948565697091274, + "loss": 1.1375, + "theoretical_loss": 3.396449819550424, + "tokens_seen": 2505033728 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001948405215646941, + "loss": 1.1321, + "theoretical_loss": 3.3964427941025805, + "tokens_seen": 2505099264 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001948244734202608, + "loss": 1.2014, + "theoretical_loss": 3.396435768889989, + "tokens_seen": 2505164800 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001948084252758275, + "loss": 1.1234, + "theoretical_loss": 3.396428743912634, + "tokens_seen": 2505230336 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001947923771313942, + "loss": 1.2306, + "theoretical_loss": 3.396421719170502, + "tokens_seen": 2505295872 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001947763289869609, + "loss": 1.134, + "theoretical_loss": 3.3964146946635787, + "tokens_seen": 2505361408 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001947602808425276, + "loss": 1.1024, + "theoretical_loss": 3.396407670391851, + "tokens_seen": 2505426944 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001947442326980943, + "loss": 1.2032, + "theoretical_loss": 3.3964006463553043, + "tokens_seen": 2505492480 + }, + { + "epoch": 1.14, + "learning_rate": 0.000194728184553661, + "loss": 1.3158, + "theoretical_loss": 3.396393622553924, + "tokens_seen": 2505558016 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001947121364092277, + "loss": 1.1409, + "theoretical_loss": 3.396386598987697, + "tokens_seen": 2505623552 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001946960882647944, + "loss": 1.281, + "theoretical_loss": 3.3963795756566086, + "tokens_seen": 2505689088 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001946800401203611, + "loss": 1.2238, + "theoretical_loss": 3.3963725525606456, + "tokens_seen": 2505754624 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001946639919759278, + "loss": 1.2102, + "theoretical_loss": 3.396365529699793, + "tokens_seen": 2505820160 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019464794383149448, + "loss": 1.2085, + "theoretical_loss": 3.3963585070740376, + "tokens_seen": 2505885696 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019463189568706118, + "loss": 1.2328, + "theoretical_loss": 3.3963514846833647, + "tokens_seen": 2505951232 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019461584754262788, + "loss": 1.1942, + "theoretical_loss": 3.396344462527761, + "tokens_seen": 2506016768 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019459979939819458, + "loss": 1.1755, + "theoretical_loss": 3.396337440607212, + "tokens_seen": 2506082304 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019458375125376128, + "loss": 1.2148, + "theoretical_loss": 3.3963304189217034, + "tokens_seen": 2506147840 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019456770310932798, + "loss": 1.2021, + "theoretical_loss": 3.396323397471222, + "tokens_seen": 2506213376 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019455165496489469, + "loss": 1.2223, + "theoretical_loss": 3.396316376255753, + "tokens_seen": 2506278912 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019453560682046139, + "loss": 1.3094, + "theoretical_loss": 3.3963093552752825, + "tokens_seen": 2506344448 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001945195586760281, + "loss": 1.252, + "theoretical_loss": 3.396302334529797, + "tokens_seen": 2506409984 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001945035105315948, + "loss": 1.2386, + "theoretical_loss": 3.396295314019283, + "tokens_seen": 2506475520 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001944874623871615, + "loss": 1.2279, + "theoretical_loss": 3.3962882937437247, + "tokens_seen": 2506541056 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001944714142427282, + "loss": 1.2606, + "theoretical_loss": 3.3962812737031096, + "tokens_seen": 2506606592 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001944553660982949, + "loss": 1.3213, + "theoretical_loss": 3.396274253897423, + "tokens_seen": 2506672128 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001944393179538616, + "loss": 1.2106, + "theoretical_loss": 3.3962672343266513, + "tokens_seen": 2506737664 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001944232698094283, + "loss": 1.2121, + "theoretical_loss": 3.39626021499078, + "tokens_seen": 2506803200 + }, + { + "epoch": 1.14, + "learning_rate": 0.000194407221664995, + "loss": 1.1448, + "theoretical_loss": 3.3962531958897957, + "tokens_seen": 2506868736 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001943911735205617, + "loss": 1.2402, + "theoretical_loss": 3.396246177023684, + "tokens_seen": 2506934272 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001943751253761284, + "loss": 1.2609, + "theoretical_loss": 3.3962391583924307, + "tokens_seen": 2506999808 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001943590772316951, + "loss": 1.1446, + "theoretical_loss": 3.3962321399960222, + "tokens_seen": 2507065344 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001943430290872618, + "loss": 1.1675, + "theoretical_loss": 3.396225121834445, + "tokens_seen": 2507130880 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001943269809428285, + "loss": 1.2429, + "theoretical_loss": 3.3962181039076835, + "tokens_seen": 2507196416 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001943109327983952, + "loss": 1.189, + "theoretical_loss": 3.3962110862157253, + "tokens_seen": 2507261952 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001942948846539619, + "loss": 1.244, + "theoretical_loss": 3.396204068758556, + "tokens_seen": 2507327488 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001942788365095286, + "loss": 1.2194, + "theoretical_loss": 3.396197051536161, + "tokens_seen": 2507393024 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001942627883650953, + "loss": 1.2261, + "theoretical_loss": 3.3961900345485274, + "tokens_seen": 2507458560 + }, + { + "epoch": 1.14, + "learning_rate": 0.000194246740220662, + "loss": 1.2402, + "theoretical_loss": 3.39618301779564, + "tokens_seen": 2507524096 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001942306920762287, + "loss": 1.1735, + "theoretical_loss": 3.3961760012774853, + "tokens_seen": 2507589632 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001942146439317954, + "loss": 1.09, + "theoretical_loss": 3.3961689849940493, + "tokens_seen": 2507655168 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001941985957873621, + "loss": 1.2823, + "theoretical_loss": 3.3961619689453184, + "tokens_seen": 2507720704 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001941825476429288, + "loss": 1.1484, + "theoretical_loss": 3.3961549531312785, + "tokens_seen": 2507786240 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001941664994984955, + "loss": 1.1979, + "theoretical_loss": 3.396147937551915, + "tokens_seen": 2507851776 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001941504513540622, + "loss": 1.1907, + "theoretical_loss": 3.3961409222072145, + "tokens_seen": 2507917312 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001941344032096289, + "loss": 1.1756, + "theoretical_loss": 3.3961339070971626, + "tokens_seen": 2507982848 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001941183550651956, + "loss": 1.2525, + "theoretical_loss": 3.396126892221746, + "tokens_seen": 2508048384 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001941023069207623, + "loss": 1.1279, + "theoretical_loss": 3.39611987758095, + "tokens_seen": 2508113920 + }, + { + "epoch": 1.14, + "learning_rate": 0.000194086258776329, + "loss": 1.153, + "theoretical_loss": 3.396112863174761, + "tokens_seen": 2508179456 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001940702106318957, + "loss": 1.1226, + "theoretical_loss": 3.396105849003165, + "tokens_seen": 2508244992 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001940541624874624, + "loss": 1.2318, + "theoretical_loss": 3.396098835066148, + "tokens_seen": 2508310528 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001940381143430291, + "loss": 1.2315, + "theoretical_loss": 3.396091821363696, + "tokens_seen": 2508376064 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001940220661985958, + "loss": 1.2928, + "theoretical_loss": 3.396084807895795, + "tokens_seen": 2508441600 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001940060180541625, + "loss": 1.1544, + "theoretical_loss": 3.3960777946624305, + "tokens_seen": 2508507136 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001939899699097292, + "loss": 1.1587, + "theoretical_loss": 3.3960707816635898, + "tokens_seen": 2508572672 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001939739217652959, + "loss": 1.2433, + "theoretical_loss": 3.396063768899258, + "tokens_seen": 2508638208 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001939578736208626, + "loss": 1.0396, + "theoretical_loss": 3.396056756369421, + "tokens_seen": 2508703744 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019394182547642928, + "loss": 1.1524, + "theoretical_loss": 3.3960497440740656, + "tokens_seen": 2508769280 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019392577733199598, + "loss": 1.2799, + "theoretical_loss": 3.396042732013177, + "tokens_seen": 2508834816 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019390972918756268, + "loss": 1.2588, + "theoretical_loss": 3.396035720186742, + "tokens_seen": 2508900352 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019389368104312938, + "loss": 1.1912, + "theoretical_loss": 3.3960287085947467, + "tokens_seen": 2508965888 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019387763289869608, + "loss": 1.0958, + "theoretical_loss": 3.396021697237176, + "tokens_seen": 2509031424 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019386158475426279, + "loss": 1.1204, + "theoretical_loss": 3.3960146861140172, + "tokens_seen": 2509096960 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019384553660982949, + "loss": 1.2421, + "theoretical_loss": 3.3960076752252553, + "tokens_seen": 2509162496 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001938294884653962, + "loss": 1.2203, + "theoretical_loss": 3.3960006645708773, + "tokens_seen": 2509228032 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001938134403209629, + "loss": 1.111, + "theoretical_loss": 3.3959936541508684, + "tokens_seen": 2509293568 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001937973921765296, + "loss": 1.2943, + "theoretical_loss": 3.395986643965215, + "tokens_seen": 2509359104 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001937813440320963, + "loss": 1.197, + "theoretical_loss": 3.3959796340139032, + "tokens_seen": 2509424640 + }, + { + "epoch": 1.14, + "learning_rate": 0.000193765295887663, + "loss": 1.1201, + "theoretical_loss": 3.3959726242969195, + "tokens_seen": 2509490176 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001937492477432297, + "loss": 1.2667, + "theoretical_loss": 3.395965614814249, + "tokens_seen": 2509555712 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001937331995987964, + "loss": 1.1938, + "theoretical_loss": 3.395958605565878, + "tokens_seen": 2509621248 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001937171514543631, + "loss": 1.1894, + "theoretical_loss": 3.3959515965517935, + "tokens_seen": 2509686784 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001937011033099298, + "loss": 1.2208, + "theoretical_loss": 3.3959445877719805, + "tokens_seen": 2509752320 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019368505516549652, + "loss": 1.1156, + "theoretical_loss": 3.395937579226425, + "tokens_seen": 2509817856 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019366900702106322, + "loss": 1.2058, + "theoretical_loss": 3.395930570915114, + "tokens_seen": 2509883392 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019365295887662992, + "loss": 1.2556, + "theoretical_loss": 3.395923562838033, + "tokens_seen": 2509948928 + }, + { + "epoch": 1.14, + "objective/train/docs_used": 927348, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1422730684280396, + "objective/train/theoretical_loss": 3.3959165549951678, + "objective/train/tokens_used": 2530474464, + "theoretical_loss": 3.3959165549951678, + "tokens_seen": 2510014464 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019363691073219662, + "loss": 1.1423, + "theoretical_loss": 3.3959165549951678, + "tokens_seen": 2510014464 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001936208625877633, + "loss": 1.0235, + "theoretical_loss": 3.395909547386504, + "tokens_seen": 2510080000 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019360481444333, + "loss": 1.1854, + "theoretical_loss": 3.3959025400120293, + "tokens_seen": 2510145536 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001935887662988967, + "loss": 1.1812, + "theoretical_loss": 3.3958955328717284, + "tokens_seen": 2510211072 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001935727181544634, + "loss": 1.1381, + "theoretical_loss": 3.3958885259655878, + "tokens_seen": 2510276608 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001935566700100301, + "loss": 1.2204, + "theoretical_loss": 3.395881519293594, + "tokens_seen": 2510342144 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001935406218655968, + "loss": 1.2019, + "theoretical_loss": 3.395874512855732, + "tokens_seen": 2510407680 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001935245737211635, + "loss": 1.2274, + "theoretical_loss": 3.3958675066519888, + "tokens_seen": 2510473216 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001935085255767302, + "loss": 1.2519, + "theoretical_loss": 3.39586050068235, + "tokens_seen": 2510538752 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001934924774322969, + "loss": 1.1507, + "theoretical_loss": 3.395853494946802, + "tokens_seen": 2510604288 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001934764292878636, + "loss": 1.2429, + "theoretical_loss": 3.39584648944533, + "tokens_seen": 2510669824 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001934603811434303, + "loss": 1.2347, + "theoretical_loss": 3.3958394841779214, + "tokens_seen": 2510735360 + }, + { + "epoch": 1.14, + "learning_rate": 0.000193444332998997, + "loss": 1.1579, + "theoretical_loss": 3.3958324791445613, + "tokens_seen": 2510800896 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001934282848545637, + "loss": 1.2292, + "theoretical_loss": 3.3958254743452363, + "tokens_seen": 2510866432 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001934122367101304, + "loss": 1.1634, + "theoretical_loss": 3.395818469779932, + "tokens_seen": 2510931968 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001933961885656971, + "loss": 1.2206, + "theoretical_loss": 3.3958114654486353, + "tokens_seen": 2510997504 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001933801404212638, + "loss": 1.162, + "theoretical_loss": 3.395804461351331, + "tokens_seen": 2511063040 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001933640922768305, + "loss": 1.2803, + "theoretical_loss": 3.395797457488006, + "tokens_seen": 2511128576 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001933480441323972, + "loss": 1.1692, + "theoretical_loss": 3.3957904538586465, + "tokens_seen": 2511194112 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001933319959879639, + "loss": 1.1215, + "theoretical_loss": 3.395783450463238, + "tokens_seen": 2511259648 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001933159478435306, + "loss": 1.1995, + "theoretical_loss": 3.3957764473017673, + "tokens_seen": 2511325184 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001932998996990973, + "loss": 1.2744, + "theoretical_loss": 3.39576944437422, + "tokens_seen": 2511390720 + }, + { + "epoch": 1.14, + "learning_rate": 0.000193283851554664, + "loss": 1.1838, + "theoretical_loss": 3.3957624416805823, + "tokens_seen": 2511456256 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001932678034102307, + "loss": 1.2825, + "theoretical_loss": 3.39575543922084, + "tokens_seen": 2511521792 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001932517552657974, + "loss": 1.1562, + "theoretical_loss": 3.3957484369949795, + "tokens_seen": 2511587328 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019323570712136408, + "loss": 1.2174, + "theoretical_loss": 3.395741435002987, + "tokens_seen": 2511652864 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019321965897693078, + "loss": 1.2437, + "theoretical_loss": 3.395734433244848, + "tokens_seen": 2511718400 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019320361083249748, + "loss": 1.1826, + "theoretical_loss": 3.3957274317205495, + "tokens_seen": 2511783936 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019318756268806419, + "loss": 1.2125, + "theoretical_loss": 3.395720430430077, + "tokens_seen": 2511849472 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019317151454363089, + "loss": 1.2041, + "theoretical_loss": 3.3957134293734166, + "tokens_seen": 2511915008 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019315546639919759, + "loss": 1.1746, + "theoretical_loss": 3.3957064285505543, + "tokens_seen": 2511980544 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001931394182547643, + "loss": 1.1461, + "theoretical_loss": 3.3956994279614765, + "tokens_seen": 2512046080 + }, + { + "epoch": 1.14, + "learning_rate": 0.000193123370110331, + "loss": 1.2976, + "theoretical_loss": 3.3956924276061695, + "tokens_seen": 2512111616 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001931073219658977, + "loss": 1.2739, + "theoretical_loss": 3.3956854274846187, + "tokens_seen": 2512177152 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019309127382146442, + "loss": 1.2431, + "theoretical_loss": 3.3956784275968106, + "tokens_seen": 2512242688 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019307522567703112, + "loss": 1.2629, + "theoretical_loss": 3.395671427942731, + "tokens_seen": 2512308224 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019305917753259782, + "loss": 1.1684, + "theoretical_loss": 3.3956644285223665, + "tokens_seen": 2512373760 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019304312938816452, + "loss": 1.1827, + "theoretical_loss": 3.395657429335703, + "tokens_seen": 2512439296 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019302708124373122, + "loss": 1.0845, + "theoretical_loss": 3.3956504303827266, + "tokens_seen": 2512504832 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019301103309929792, + "loss": 1.192, + "theoretical_loss": 3.395643431663423, + "tokens_seen": 2512570368 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019299498495486462, + "loss": 1.1807, + "theoretical_loss": 3.395636433177779, + "tokens_seen": 2512635904 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019297893681043132, + "loss": 1.2627, + "theoretical_loss": 3.3956294349257803, + "tokens_seen": 2512701440 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019296288866599802, + "loss": 1.0778, + "theoretical_loss": 3.3956224369074133, + "tokens_seen": 2512766976 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019294684052156472, + "loss": 1.1684, + "theoretical_loss": 3.3956154391226634, + "tokens_seen": 2512832512 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001929307923771314, + "loss": 1.1906, + "theoretical_loss": 3.3956084415715173, + "tokens_seen": 2512898048 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001929147442326981, + "loss": 1.2668, + "theoretical_loss": 3.3956014442539613, + "tokens_seen": 2512963584 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001928986960882648, + "loss": 1.1933, + "theoretical_loss": 3.395594447169981, + "tokens_seen": 2513029120 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001928826479438315, + "loss": 1.2166, + "theoretical_loss": 3.395587450319563, + "tokens_seen": 2513094656 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001928665997993982, + "loss": 1.2139, + "theoretical_loss": 3.3955804537026926, + "tokens_seen": 2513160192 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001928505516549649, + "loss": 1.2978, + "theoretical_loss": 3.395573457319357, + "tokens_seen": 2513225728 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001928345035105316, + "loss": 1.2405, + "theoretical_loss": 3.3955664611695413, + "tokens_seen": 2513291264 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001928184553660983, + "loss": 1.1382, + "theoretical_loss": 3.395559465253232, + "tokens_seen": 2513356800 + }, + { + "epoch": 1.14, + "learning_rate": 0.000192802407221665, + "loss": 1.3533, + "theoretical_loss": 3.395552469570416, + "tokens_seen": 2513422336 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001927863590772317, + "loss": 1.2409, + "theoretical_loss": 3.3955454741210778, + "tokens_seen": 2513487872 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001927703109327984, + "loss": 1.2898, + "theoretical_loss": 3.395538478905205, + "tokens_seen": 2513553408 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001927542627883651, + "loss": 1.1573, + "theoretical_loss": 3.395531483922783, + "tokens_seen": 2513618944 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001927382146439318, + "loss": 1.2325, + "theoretical_loss": 3.395524489173798, + "tokens_seen": 2513684480 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001927221664994985, + "loss": 1.1676, + "theoretical_loss": 3.3955174946582365, + "tokens_seen": 2513750016 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001927061183550652, + "loss": 1.1508, + "theoretical_loss": 3.395510500376084, + "tokens_seen": 2513815552 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001926900702106319, + "loss": 1.2347, + "theoretical_loss": 3.3955035063273273, + "tokens_seen": 2513881088 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001926740220661986, + "loss": 1.172, + "theoretical_loss": 3.3954965125119516, + "tokens_seen": 2513946624 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001926579739217653, + "loss": 1.2229, + "theoretical_loss": 3.3954895189299443, + "tokens_seen": 2514012160 + }, + { + "epoch": 1.14, + "learning_rate": 0.000192641925777332, + "loss": 1.2693, + "theoretical_loss": 3.3954825255812904, + "tokens_seen": 2514077696 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001926258776328987, + "loss": 1.1907, + "theoretical_loss": 3.3954755324659764, + "tokens_seen": 2514143232 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001926098294884654, + "loss": 1.2981, + "theoretical_loss": 3.3954685395839888, + "tokens_seen": 2514208768 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001925937813440321, + "loss": 1.2392, + "theoretical_loss": 3.395461546935313, + "tokens_seen": 2514274304 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001925777331995988, + "loss": 1.3123, + "theoretical_loss": 3.3954545545199366, + "tokens_seen": 2514339840 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001925616850551655, + "loss": 1.2243, + "theoretical_loss": 3.3954475623378437, + "tokens_seen": 2514405376 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019254563691073218, + "loss": 1.1472, + "theoretical_loss": 3.395440570389022, + "tokens_seen": 2514470912 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019252958876629888, + "loss": 1.2336, + "theoretical_loss": 3.3954335786734564, + "tokens_seen": 2514536448 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019251354062186558, + "loss": 1.2238, + "theoretical_loss": 3.3954265871911344, + "tokens_seen": 2514601984 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019249749247743229, + "loss": 1.2177, + "theoretical_loss": 3.3954195959420415, + "tokens_seen": 2514667520 + }, + { + "epoch": 1.14, + "learning_rate": 0.000192481444332999, + "loss": 1.2429, + "theoretical_loss": 3.3954126049261633, + "tokens_seen": 2514733056 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019246539618856571, + "loss": 1.1449, + "theoretical_loss": 3.3954056141434865, + "tokens_seen": 2514798592 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019244934804413241, + "loss": 1.2255, + "theoretical_loss": 3.3953986235939975, + "tokens_seen": 2514864128 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019243329989969911, + "loss": 1.1855, + "theoretical_loss": 3.395391633277682, + "tokens_seen": 2514929664 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019241725175526582, + "loss": 1.3341, + "theoretical_loss": 3.3953846431945265, + "tokens_seen": 2514995200 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019240120361083252, + "loss": 1.1443, + "theoretical_loss": 3.3953776533445166, + "tokens_seen": 2515060736 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019238515546639922, + "loss": 1.2355, + "theoretical_loss": 3.395370663727639, + "tokens_seen": 2515126272 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019236910732196592, + "loss": 1.2407, + "theoretical_loss": 3.3953636743438795, + "tokens_seen": 2515191808 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019235305917753262, + "loss": 1.2408, + "theoretical_loss": 3.3953566851932244, + "tokens_seen": 2515257344 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019233701103309932, + "loss": 1.2491, + "theoretical_loss": 3.3953496962756597, + "tokens_seen": 2515322880 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019232096288866602, + "loss": 1.1762, + "theoretical_loss": 3.3953427075911717, + "tokens_seen": 2515388416 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019230491474423272, + "loss": 1.1595, + "theoretical_loss": 3.395335719139747, + "tokens_seen": 2515453952 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019228886659979942, + "loss": 1.1813, + "theoretical_loss": 3.3953287309213707, + "tokens_seen": 2515519488 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019227281845536612, + "loss": 1.1299, + "theoretical_loss": 3.39532174293603, + "tokens_seen": 2515585024 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019225677031093282, + "loss": 1.23, + "theoretical_loss": 3.39531475518371, + "tokens_seen": 2515650560 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019224072216649952, + "loss": 1.1415, + "theoretical_loss": 3.3953077676643986, + "tokens_seen": 2515716096 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001922246740220662, + "loss": 1.1609, + "theoretical_loss": 3.39530078037808, + "tokens_seen": 2515781632 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001922086258776329, + "loss": 1.1281, + "theoretical_loss": 3.395293793324741, + "tokens_seen": 2515847168 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001921925777331996, + "loss": 1.194, + "theoretical_loss": 3.3952868065043686, + "tokens_seen": 2515912704 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001921765295887663, + "loss": 1.127, + "theoretical_loss": 3.395279819916948, + "tokens_seen": 2515978240 + }, + { + "epoch": 1.14, + "learning_rate": 0.000192160481444333, + "loss": 1.1985, + "theoretical_loss": 3.3952728335624656, + "tokens_seen": 2516043776 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001921444332998997, + "loss": 1.3145, + "theoretical_loss": 3.395265847440908, + "tokens_seen": 2516109312 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001921283851554664, + "loss": 1.2032, + "theoretical_loss": 3.3952588615522608, + "tokens_seen": 2516174848 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001921123370110331, + "loss": 1.2005, + "theoretical_loss": 3.39525187589651, + "tokens_seen": 2516240384 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001920962888665998, + "loss": 1.1283, + "theoretical_loss": 3.3952448904736428, + "tokens_seen": 2516305920 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001920802407221665, + "loss": 1.1161, + "theoretical_loss": 3.3952379052836443, + "tokens_seen": 2516371456 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001920641925777332, + "loss": 1.2559, + "theoretical_loss": 3.3952309203265014, + "tokens_seen": 2516436992 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001920481444332999, + "loss": 1.2158, + "theoretical_loss": 3.3952239356022, + "tokens_seen": 2516502528 + }, + { + "epoch": 1.14, + "objective/train/docs_used": 929759, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.258682131767273, + "objective/train/theoretical_loss": 3.3952169511107257, + "objective/train/tokens_used": 2537028064, + "theoretical_loss": 3.3952169511107257, + "tokens_seen": 2516568064 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001920320962888666, + "loss": 1.2587, + "theoretical_loss": 3.3952169511107257, + "tokens_seen": 2516568064 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001920160481444333, + "loss": 1.1598, + "theoretical_loss": 3.395209966852066, + "tokens_seen": 2516633600 + }, + { + "epoch": 1.14, + "learning_rate": 0.000192, + "loss": 1.1301, + "theoretical_loss": 3.3952029828262056, + "tokens_seen": 2516699136 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001919839518555667, + "loss": 1.2545, + "theoretical_loss": 3.395195999033132, + "tokens_seen": 2516764672 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001919679037111334, + "loss": 1.2395, + "theoretical_loss": 3.3951890154728304, + "tokens_seen": 2516830208 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001919518555667001, + "loss": 1.2473, + "theoretical_loss": 3.3951820321452875, + "tokens_seen": 2516895744 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001919358074222668, + "loss": 1.1312, + "theoretical_loss": 3.395175049050489, + "tokens_seen": 2516961280 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001919197592778335, + "loss": 1.1451, + "theoretical_loss": 3.395168066188422, + "tokens_seen": 2517026816 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001919037111334002, + "loss": 1.1813, + "theoretical_loss": 3.3951610835590715, + "tokens_seen": 2517092352 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001918876629889669, + "loss": 1.2284, + "theoretical_loss": 3.3951541011624244, + "tokens_seen": 2517157888 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001918716148445336, + "loss": 1.2049, + "theoretical_loss": 3.3951471189984668, + "tokens_seen": 2517223424 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001918555667001003, + "loss": 1.1432, + "theoretical_loss": 3.395140137067185, + "tokens_seen": 2517288960 + }, + { + "epoch": 1.14, + "learning_rate": 0.000191839518555667, + "loss": 1.3042, + "theoretical_loss": 3.395133155368565, + "tokens_seen": 2517354496 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001918234704112337, + "loss": 1.2215, + "theoretical_loss": 3.395126173902593, + "tokens_seen": 2517420032 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001918074222668004, + "loss": 1.2022, + "theoretical_loss": 3.3951191926692554, + "tokens_seen": 2517485568 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001917913741223671, + "loss": 1.2504, + "theoretical_loss": 3.395112211668538, + "tokens_seen": 2517551104 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019177532597793381, + "loss": 1.0752, + "theoretical_loss": 3.3951052309004273, + "tokens_seen": 2517616640 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019175927783350051, + "loss": 1.1136, + "theoretical_loss": 3.395098250364909, + "tokens_seen": 2517682176 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019174322968906722, + "loss": 1.118, + "theoretical_loss": 3.3950912700619704, + "tokens_seen": 2517747712 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019172718154463392, + "loss": 1.4339, + "theoretical_loss": 3.3950842899915967, + "tokens_seen": 2517813248 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019171113340020062, + "loss": 1.2459, + "theoretical_loss": 3.3950773101537743, + "tokens_seen": 2517878784 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019169508525576732, + "loss": 1.2539, + "theoretical_loss": 3.3950703305484895, + "tokens_seen": 2517944320 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019167903711133402, + "loss": 1.1881, + "theoretical_loss": 3.395063351175729, + "tokens_seen": 2518009856 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019166298896690072, + "loss": 1.1265, + "theoretical_loss": 3.395056372035478, + "tokens_seen": 2518075392 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019164694082246742, + "loss": 1.3772, + "theoretical_loss": 3.395049393127723, + "tokens_seen": 2518140928 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019163089267803412, + "loss": 1.1926, + "theoretical_loss": 3.3950424144524507, + "tokens_seen": 2518206464 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019161484453360082, + "loss": 1.2355, + "theoretical_loss": 3.395035436009647, + "tokens_seen": 2518272000 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019159879638916752, + "loss": 1.3364, + "theoretical_loss": 3.395028457799298, + "tokens_seen": 2518337536 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019158274824473422, + "loss": 1.3784, + "theoretical_loss": 3.3950214798213905, + "tokens_seen": 2518403072 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019156670010030092, + "loss": 1.2379, + "theoretical_loss": 3.3950145020759095, + "tokens_seen": 2518468608 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019155065195586762, + "loss": 1.1712, + "theoretical_loss": 3.3950075245628426, + "tokens_seen": 2518534144 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019153460381143432, + "loss": 1.3244, + "theoretical_loss": 3.395000547282175, + "tokens_seen": 2518599680 + }, + { + "epoch": 1.14, + "learning_rate": 0.000191518555667001, + "loss": 1.1796, + "theoretical_loss": 3.3949935702338934, + "tokens_seen": 2518665216 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001915025075225677, + "loss": 1.339, + "theoretical_loss": 3.3949865934179835, + "tokens_seen": 2518730752 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001914864593781344, + "loss": 1.2049, + "theoretical_loss": 3.3949796168344326, + "tokens_seen": 2518796288 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001914704112337011, + "loss": 1.1261, + "theoretical_loss": 3.3949726404832257, + "tokens_seen": 2518861824 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001914543630892678, + "loss": 1.2499, + "theoretical_loss": 3.3949656643643498, + "tokens_seen": 2518927360 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001914383149448345, + "loss": 1.2933, + "theoretical_loss": 3.3949586884777907, + "tokens_seen": 2518992896 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001914222668004012, + "loss": 1.1798, + "theoretical_loss": 3.3949517128235347, + "tokens_seen": 2519058432 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001914062186559679, + "loss": 1.1644, + "theoretical_loss": 3.394944737401568, + "tokens_seen": 2519123968 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001913901705115346, + "loss": 1.2586, + "theoretical_loss": 3.3949377622118773, + "tokens_seen": 2519189504 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001913741223671013, + "loss": 1.3134, + "theoretical_loss": 3.3949307872544483, + "tokens_seen": 2519255040 + }, + { + "epoch": 1.14, + "learning_rate": 0.000191358074222668, + "loss": 1.2676, + "theoretical_loss": 3.3949238125292673, + "tokens_seen": 2519320576 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001913420260782347, + "loss": 1.2848, + "theoretical_loss": 3.3949168380363206, + "tokens_seen": 2519386112 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001913259779338014, + "loss": 1.2028, + "theoretical_loss": 3.3949098637755943, + "tokens_seen": 2519451648 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001913099297893681, + "loss": 1.2644, + "theoretical_loss": 3.3949028897470748, + "tokens_seen": 2519517184 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001912938816449348, + "loss": 1.242, + "theoretical_loss": 3.394895915950748, + "tokens_seen": 2519582720 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019127783350050153, + "loss": 1.2627, + "theoretical_loss": 3.394888942386601, + "tokens_seen": 2519648256 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019126178535606823, + "loss": 1.1993, + "theoretical_loss": 3.394881969054619, + "tokens_seen": 2519713792 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019124573721163493, + "loss": 1.3213, + "theoretical_loss": 3.394874995954789, + "tokens_seen": 2519779328 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019122968906720164, + "loss": 1.2614, + "theoretical_loss": 3.3948680230870965, + "tokens_seen": 2519844864 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001912136409227683, + "loss": 1.1366, + "theoretical_loss": 3.3948610504515284, + "tokens_seen": 2519910400 + }, + { + "epoch": 1.14, + "learning_rate": 0.000191197592778335, + "loss": 1.2543, + "theoretical_loss": 3.3948540780480707, + "tokens_seen": 2519975936 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001911815446339017, + "loss": 1.2515, + "theoretical_loss": 3.3948471058767096, + "tokens_seen": 2520041472 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001911654964894684, + "loss": 1.1972, + "theoretical_loss": 3.3948401339374312, + "tokens_seen": 2520107008 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001911494483450351, + "loss": 1.2135, + "theoretical_loss": 3.394833162230222, + "tokens_seen": 2520172544 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001911334002006018, + "loss": 1.1194, + "theoretical_loss": 3.394826190755068, + "tokens_seen": 2520238080 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001911173520561685, + "loss": 1.1807, + "theoretical_loss": 3.3948192195119553, + "tokens_seen": 2520303616 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001911013039117352, + "loss": 1.2916, + "theoretical_loss": 3.394812248500871, + "tokens_seen": 2520369152 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019108525576730191, + "loss": 1.2846, + "theoretical_loss": 3.3948052777218, + "tokens_seen": 2520434688 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019106920762286861, + "loss": 1.1531, + "theoretical_loss": 3.39479830717473, + "tokens_seen": 2520500224 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019105315947843532, + "loss": 1.1595, + "theoretical_loss": 3.3947913368596465, + "tokens_seen": 2520565760 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019103711133400202, + "loss": 1.1563, + "theoretical_loss": 3.3947843667765354, + "tokens_seen": 2520631296 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019102106318956872, + "loss": 1.0953, + "theoretical_loss": 3.3947773969253836, + "tokens_seen": 2520696832 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019100501504513542, + "loss": 1.2629, + "theoretical_loss": 3.394770427306177, + "tokens_seen": 2520762368 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019098896690070212, + "loss": 1.2511, + "theoretical_loss": 3.394763457918902, + "tokens_seen": 2520827904 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019097291875626882, + "loss": 1.3802, + "theoretical_loss": 3.394756488763545, + "tokens_seen": 2520893440 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019095687061183552, + "loss": 1.2701, + "theoretical_loss": 3.394749519840092, + "tokens_seen": 2520958976 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019094082246740222, + "loss": 1.2583, + "theoretical_loss": 3.394742551148529, + "tokens_seen": 2521024512 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019092477432296892, + "loss": 1.2296, + "theoretical_loss": 3.394735582688843, + "tokens_seen": 2521090048 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019090872617853562, + "loss": 1.1924, + "theoretical_loss": 3.3947286144610187, + "tokens_seen": 2521155584 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019089267803410232, + "loss": 1.1716, + "theoretical_loss": 3.3947216464650447, + "tokens_seen": 2521221120 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019087662988966902, + "loss": 1.1554, + "theoretical_loss": 3.3947146787009057, + "tokens_seen": 2521286656 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019086058174523572, + "loss": 1.2868, + "theoretical_loss": 3.394707711168588, + "tokens_seen": 2521352192 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019084453360080242, + "loss": 1.2074, + "theoretical_loss": 3.3947007438680785, + "tokens_seen": 2521417728 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001908284854563691, + "loss": 1.1826, + "theoretical_loss": 3.394693776799363, + "tokens_seen": 2521483264 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001908124373119358, + "loss": 1.1376, + "theoretical_loss": 3.394686809962428, + "tokens_seen": 2521548800 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001907963891675025, + "loss": 1.1918, + "theoretical_loss": 3.394679843357259, + "tokens_seen": 2521614336 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001907803410230692, + "loss": 1.2137, + "theoretical_loss": 3.3946728769838437, + "tokens_seen": 2521679872 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001907642928786359, + "loss": 1.2783, + "theoretical_loss": 3.3946659108421673, + "tokens_seen": 2521745408 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001907482447342026, + "loss": 1.288, + "theoretical_loss": 3.3946589449322166, + "tokens_seen": 2521810944 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001907321965897693, + "loss": 1.186, + "theoretical_loss": 3.394651979253977, + "tokens_seen": 2521876480 + }, + { + "epoch": 1.14, + "learning_rate": 0.000190716148445336, + "loss": 1.2176, + "theoretical_loss": 3.394645013807436, + "tokens_seen": 2521942016 + }, + { + "epoch": 1.14, + "learning_rate": 0.0001907001003009027, + "loss": 1.2246, + "theoretical_loss": 3.3946380485925793, + "tokens_seen": 2522007552 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019068405215646943, + "loss": 1.2808, + "theoretical_loss": 3.3946310836093927, + "tokens_seen": 2522073088 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019066800401203613, + "loss": 1.306, + "theoretical_loss": 3.3946241188578634, + "tokens_seen": 2522138624 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019065195586760283, + "loss": 1.13, + "theoretical_loss": 3.394617154337977, + "tokens_seen": 2522204160 + }, + { + "epoch": 1.14, + "learning_rate": 0.00019063590772316953, + "loss": 1.2245, + "theoretical_loss": 3.3946101900497196, + "tokens_seen": 2522269696 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019061985957873623, + "loss": 1.21, + "theoretical_loss": 3.394603225993078, + "tokens_seen": 2522335232 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019060381143430293, + "loss": 1.2217, + "theoretical_loss": 3.3945962621680383, + "tokens_seen": 2522400768 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019058776328986963, + "loss": 1.1562, + "theoretical_loss": 3.394589298574587, + "tokens_seen": 2522466304 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019057171514543633, + "loss": 1.2959, + "theoretical_loss": 3.3945823352127107, + "tokens_seen": 2522531840 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019055566700100303, + "loss": 1.293, + "theoretical_loss": 3.3945753720823943, + "tokens_seen": 2522597376 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019053961885656974, + "loss": 1.1848, + "theoretical_loss": 3.3945684091836252, + "tokens_seen": 2522662912 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019052357071213644, + "loss": 1.1572, + "theoretical_loss": 3.3945614465163896, + "tokens_seen": 2522728448 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001905075225677031, + "loss": 1.1846, + "theoretical_loss": 3.3945544840806736, + "tokens_seen": 2522793984 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001904914744232698, + "loss": 1.2164, + "theoretical_loss": 3.3945475218764636, + "tokens_seen": 2522859520 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001904754262788365, + "loss": 1.1908, + "theoretical_loss": 3.3945405599037457, + "tokens_seen": 2522925056 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001904593781344032, + "loss": 1.3277, + "theoretical_loss": 3.394533598162506, + "tokens_seen": 2522990592 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001904433299899699, + "loss": 1.2605, + "theoretical_loss": 3.3945266366527314, + "tokens_seen": 2523056128 + }, + { + "epoch": 1.15, + "objective/train/docs_used": 932178, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.0781424045562744, + "objective/train/theoretical_loss": 3.394519675374408, + "objective/train/tokens_used": 2543581664, + "theoretical_loss": 3.394519675374408, + "tokens_seen": 2523121664 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001904272818455366, + "loss": 1.0781, + "theoretical_loss": 3.394519675374408, + "tokens_seen": 2523121664 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001904112337011033, + "loss": 1.2916, + "theoretical_loss": 3.3945127143275218, + "tokens_seen": 2523187200 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019039518555667001, + "loss": 1.1741, + "theoretical_loss": 3.3945057535120595, + "tokens_seen": 2523252736 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019037913741223671, + "loss": 1.2748, + "theoretical_loss": 3.3944987929280064, + "tokens_seen": 2523318272 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019036308926780342, + "loss": 1.2333, + "theoretical_loss": 3.3944918325753504, + "tokens_seen": 2523383808 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019034704112337012, + "loss": 1.2578, + "theoretical_loss": 3.3944848724540764, + "tokens_seen": 2523449344 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019033099297893682, + "loss": 1.2039, + "theoretical_loss": 3.394477912564172, + "tokens_seen": 2523514880 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019031494483450352, + "loss": 1.1169, + "theoretical_loss": 3.394470952905622, + "tokens_seen": 2523580416 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019029889669007022, + "loss": 1.3005, + "theoretical_loss": 3.3944639934784138, + "tokens_seen": 2523645952 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019028284854563692, + "loss": 1.1505, + "theoretical_loss": 3.3944570342825333, + "tokens_seen": 2523711488 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019026680040120362, + "loss": 1.2469, + "theoretical_loss": 3.394450075317967, + "tokens_seen": 2523777024 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019025075225677032, + "loss": 1.1397, + "theoretical_loss": 3.3944431165847004, + "tokens_seen": 2523842560 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019023470411233702, + "loss": 1.1623, + "theoretical_loss": 3.394436158082721, + "tokens_seen": 2523908096 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019021865596790372, + "loss": 1.2045, + "theoretical_loss": 3.3944291998120146, + "tokens_seen": 2523973632 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019020260782347042, + "loss": 1.2624, + "theoretical_loss": 3.394422241772568, + "tokens_seen": 2524039168 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019018655967903712, + "loss": 1.2457, + "theoretical_loss": 3.394415283964366, + "tokens_seen": 2524104704 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019017051153460382, + "loss": 1.2442, + "theoretical_loss": 3.3944083263873965, + "tokens_seen": 2524170240 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019015446339017052, + "loss": 1.2687, + "theoretical_loss": 3.394401369041645, + "tokens_seen": 2524235776 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019013841524573722, + "loss": 1.2256, + "theoretical_loss": 3.394394411927098, + "tokens_seen": 2524301312 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001901223671013039, + "loss": 1.2577, + "theoretical_loss": 3.3943874550437423, + "tokens_seen": 2524366848 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001901063189568706, + "loss": 1.2581, + "theoretical_loss": 3.394380498391563, + "tokens_seen": 2524432384 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001900902708124373, + "loss": 1.1954, + "theoretical_loss": 3.394373541970548, + "tokens_seen": 2524497920 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019007422266800403, + "loss": 1.2347, + "theoretical_loss": 3.3943665857806824, + "tokens_seen": 2524563456 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019005817452357073, + "loss": 1.1463, + "theoretical_loss": 3.3943596298219525, + "tokens_seen": 2524628992 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019004212637913743, + "loss": 1.2312, + "theoretical_loss": 3.3943526740943457, + "tokens_seen": 2524694528 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019002607823470413, + "loss": 1.2363, + "theoretical_loss": 3.3943457185978474, + "tokens_seen": 2524760064 + }, + { + "epoch": 1.15, + "learning_rate": 0.00019001003009027083, + "loss": 1.174, + "theoretical_loss": 3.394338763332444, + "tokens_seen": 2524825600 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018999398194583753, + "loss": 1.1077, + "theoretical_loss": 3.3943318082981224, + "tokens_seen": 2524891136 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018997793380140423, + "loss": 1.3406, + "theoretical_loss": 3.3943248534948687, + "tokens_seen": 2524956672 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018996188565697093, + "loss": 1.3478, + "theoretical_loss": 3.3943178989226688, + "tokens_seen": 2525022208 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018994583751253763, + "loss": 1.1919, + "theoretical_loss": 3.394310944581509, + "tokens_seen": 2525087744 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018992978936810433, + "loss": 1.2592, + "theoretical_loss": 3.3943039904713763, + "tokens_seen": 2525153280 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018991374122367103, + "loss": 1.1913, + "theoretical_loss": 3.3942970365922562, + "tokens_seen": 2525218816 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018989769307923773, + "loss": 1.1372, + "theoretical_loss": 3.394290082944136, + "tokens_seen": 2525284352 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018988164493480443, + "loss": 1.2963, + "theoretical_loss": 3.3942831295270013, + "tokens_seen": 2525349888 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018986559679037114, + "loss": 1.2751, + "theoretical_loss": 3.3942761763408384, + "tokens_seen": 2525415424 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018984954864593784, + "loss": 1.2834, + "theoretical_loss": 3.3942692233856344, + "tokens_seen": 2525480960 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018983350050150454, + "loss": 1.2255, + "theoretical_loss": 3.3942622706613745, + "tokens_seen": 2525546496 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018981745235707124, + "loss": 1.2323, + "theoretical_loss": 3.394255318168046, + "tokens_seen": 2525612032 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001898014042126379, + "loss": 1.2233, + "theoretical_loss": 3.394248365905635, + "tokens_seen": 2525677568 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001897853560682046, + "loss": 1.2529, + "theoretical_loss": 3.3942414138741275, + "tokens_seen": 2525743104 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001897693079237713, + "loss": 1.3214, + "theoretical_loss": 3.39423446207351, + "tokens_seen": 2525808640 + }, + { + "epoch": 1.15, + "learning_rate": 0.000189753259779338, + "loss": 1.2306, + "theoretical_loss": 3.394227510503769, + "tokens_seen": 2525874176 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001897372116349047, + "loss": 1.221, + "theoretical_loss": 3.394220559164891, + "tokens_seen": 2525939712 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018972116349047141, + "loss": 1.215, + "theoretical_loss": 3.394213608056862, + "tokens_seen": 2526005248 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018970511534603811, + "loss": 1.2808, + "theoretical_loss": 3.394206657179668, + "tokens_seen": 2526070784 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018968906720160481, + "loss": 1.1928, + "theoretical_loss": 3.3941997065332963, + "tokens_seen": 2526136320 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018967301905717152, + "loss": 1.1675, + "theoretical_loss": 3.3941927561177323, + "tokens_seen": 2526201856 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018965697091273822, + "loss": 1.1763, + "theoretical_loss": 3.3941858059329633, + "tokens_seen": 2526267392 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018964092276830492, + "loss": 1.2249, + "theoretical_loss": 3.3941788559789745, + "tokens_seen": 2526332928 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018962487462387162, + "loss": 1.177, + "theoretical_loss": 3.394171906255753, + "tokens_seen": 2526398464 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018960882647943832, + "loss": 1.2482, + "theoretical_loss": 3.3941649567632854, + "tokens_seen": 2526464000 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018959277833500502, + "loss": 1.154, + "theoretical_loss": 3.3941580075015576, + "tokens_seen": 2526529536 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018957673019057172, + "loss": 1.2873, + "theoretical_loss": 3.394151058470556, + "tokens_seen": 2526595072 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018956068204613842, + "loss": 1.3024, + "theoretical_loss": 3.394144109670267, + "tokens_seen": 2526660608 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018954463390170512, + "loss": 1.2155, + "theoretical_loss": 3.394137161100677, + "tokens_seen": 2526726144 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018952858575727182, + "loss": 1.2942, + "theoretical_loss": 3.3941302127617723, + "tokens_seen": 2526791680 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018951253761283852, + "loss": 1.2682, + "theoretical_loss": 3.394123264653539, + "tokens_seen": 2526857216 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018949648946840522, + "loss": 1.2194, + "theoretical_loss": 3.394116316775964, + "tokens_seen": 2526922752 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018948044132397192, + "loss": 1.1681, + "theoretical_loss": 3.394109369129033, + "tokens_seen": 2526988288 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018946439317953862, + "loss": 1.2174, + "theoretical_loss": 3.394102421712733, + "tokens_seen": 2527053824 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018944834503510532, + "loss": 1.1839, + "theoretical_loss": 3.39409547452705, + "tokens_seen": 2527119360 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018943229689067203, + "loss": 1.2796, + "theoretical_loss": 3.394088527571971, + "tokens_seen": 2527184896 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018941624874623873, + "loss": 1.1954, + "theoretical_loss": 3.394081580847481, + "tokens_seen": 2527250432 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018940020060180543, + "loss": 1.2093, + "theoretical_loss": 3.3940746343535677, + "tokens_seen": 2527315968 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018938415245737213, + "loss": 1.2127, + "theoretical_loss": 3.394067688090217, + "tokens_seen": 2527381504 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018936810431293883, + "loss": 1.2791, + "theoretical_loss": 3.394060742057415, + "tokens_seen": 2527447040 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018935205616850553, + "loss": 1.2193, + "theoretical_loss": 3.3940537962551485, + "tokens_seen": 2527512576 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018933600802407223, + "loss": 1.2439, + "theoretical_loss": 3.3940468506834036, + "tokens_seen": 2527578112 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018931995987963893, + "loss": 1.1707, + "theoretical_loss": 3.394039905342167, + "tokens_seen": 2527643648 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018930391173520563, + "loss": 1.2777, + "theoretical_loss": 3.3940329602314243, + "tokens_seen": 2527709184 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018928786359077233, + "loss": 1.2039, + "theoretical_loss": 3.3940260153511628, + "tokens_seen": 2527774720 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018927181544633903, + "loss": 1.27, + "theoretical_loss": 3.3940190707013684, + "tokens_seen": 2527840256 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018925576730190573, + "loss": 1.1938, + "theoretical_loss": 3.3940121262820275, + "tokens_seen": 2527905792 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018923971915747243, + "loss": 1.222, + "theoretical_loss": 3.3940051820931267, + "tokens_seen": 2527971328 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018922367101303913, + "loss": 1.2542, + "theoretical_loss": 3.3939982381346523, + "tokens_seen": 2528036864 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018920762286860583, + "loss": 1.3099, + "theoretical_loss": 3.3939912944065904, + "tokens_seen": 2528102400 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018919157472417253, + "loss": 1.2746, + "theoretical_loss": 3.3939843509089274, + "tokens_seen": 2528167936 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018917552657973924, + "loss": 1.2123, + "theoretical_loss": 3.3939774076416502, + "tokens_seen": 2528233472 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018915947843530594, + "loss": 1.2418, + "theoretical_loss": 3.3939704646047444, + "tokens_seen": 2528299008 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018914343029087264, + "loss": 1.2828, + "theoretical_loss": 3.3939635217981974, + "tokens_seen": 2528364544 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018912738214643934, + "loss": 1.1795, + "theoretical_loss": 3.393956579221995, + "tokens_seen": 2528430080 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018911133400200604, + "loss": 1.2701, + "theoretical_loss": 3.393949636876123, + "tokens_seen": 2528495616 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001890952858575727, + "loss": 1.1974, + "theoretical_loss": 3.393942694760569, + "tokens_seen": 2528561152 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001890792377131394, + "loss": 1.2113, + "theoretical_loss": 3.3939357528753185, + "tokens_seen": 2528626688 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001890631895687061, + "loss": 1.2089, + "theoretical_loss": 3.3939288112203583, + "tokens_seen": 2528692224 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001890471414242728, + "loss": 1.2722, + "theoretical_loss": 3.3939218697956743, + "tokens_seen": 2528757760 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018903109327983951, + "loss": 1.2029, + "theoretical_loss": 3.3939149286012538, + "tokens_seen": 2528823296 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018901504513540621, + "loss": 1.2328, + "theoretical_loss": 3.3939079876370823, + "tokens_seen": 2528888832 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018899899699097292, + "loss": 1.1899, + "theoretical_loss": 3.3939010469031468, + "tokens_seen": 2528954368 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018898294884653962, + "loss": 1.2605, + "theoretical_loss": 3.3938941063994332, + "tokens_seen": 2529019904 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018896690070210632, + "loss": 1.2371, + "theoretical_loss": 3.3938871661259284, + "tokens_seen": 2529085440 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018895085255767302, + "loss": 1.2749, + "theoretical_loss": 3.393880226082618, + "tokens_seen": 2529150976 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018893480441323972, + "loss": 1.2288, + "theoretical_loss": 3.3938732862694896, + "tokens_seen": 2529216512 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018891875626880642, + "loss": 1.2985, + "theoretical_loss": 3.3938663466865284, + "tokens_seen": 2529282048 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018890270812437312, + "loss": 1.3055, + "theoretical_loss": 3.393859407333722, + "tokens_seen": 2529347584 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018888665997993985, + "loss": 1.272, + "theoretical_loss": 3.3938524682110556, + "tokens_seen": 2529413120 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018887061183550655, + "loss": 1.2037, + "theoretical_loss": 3.3938455293185164, + "tokens_seen": 2529478656 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018885456369107325, + "loss": 1.1996, + "theoretical_loss": 3.3938385906560904, + "tokens_seen": 2529544192 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018883851554663995, + "loss": 1.1537, + "theoretical_loss": 3.3938316522237644, + "tokens_seen": 2529609728 + }, + { + "epoch": 1.15, + "objective/train/docs_used": 934425, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2042407989501953, + "objective/train/theoretical_loss": 3.393824714021524, + "objective/train/tokens_used": 2550135264, + "theoretical_loss": 3.393824714021524, + "tokens_seen": 2529675264 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018882246740220665, + "loss": 1.2042, + "theoretical_loss": 3.393824714021524, + "tokens_seen": 2529675264 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018880641925777335, + "loss": 1.3052, + "theoretical_loss": 3.393817776049357, + "tokens_seen": 2529740800 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018879037111334002, + "loss": 1.2508, + "theoretical_loss": 3.3938108383072487, + "tokens_seen": 2529806336 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018877432296890672, + "loss": 1.2065, + "theoretical_loss": 3.3938039007951857, + "tokens_seen": 2529871872 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018875827482447342, + "loss": 1.2585, + "theoretical_loss": 3.3937969635131546, + "tokens_seen": 2529937408 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018874222668004013, + "loss": 1.1606, + "theoretical_loss": 3.393790026461142, + "tokens_seen": 2530002944 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018872617853560683, + "loss": 1.2476, + "theoretical_loss": 3.3937830896391334, + "tokens_seen": 2530068480 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018871013039117353, + "loss": 1.1407, + "theoretical_loss": 3.393776153047116, + "tokens_seen": 2530134016 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018869408224674023, + "loss": 1.2567, + "theoretical_loss": 3.393769216685077, + "tokens_seen": 2530199552 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018867803410230693, + "loss": 1.1004, + "theoretical_loss": 3.393762280553001, + "tokens_seen": 2530265088 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018866198595787363, + "loss": 1.1765, + "theoretical_loss": 3.3937553446508755, + "tokens_seen": 2530330624 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018864593781344033, + "loss": 1.2806, + "theoretical_loss": 3.393748408978687, + "tokens_seen": 2530396160 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018862988966900703, + "loss": 1.2577, + "theoretical_loss": 3.393741473536421, + "tokens_seen": 2530461696 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018861384152457373, + "loss": 1.3009, + "theoretical_loss": 3.393734538324065, + "tokens_seen": 2530527232 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018859779338014043, + "loss": 1.2308, + "theoretical_loss": 3.3937276033416053, + "tokens_seen": 2530592768 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018858174523570713, + "loss": 1.2284, + "theoretical_loss": 3.3937206685890278, + "tokens_seen": 2530658304 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018856569709127383, + "loss": 1.2293, + "theoretical_loss": 3.3937137340663193, + "tokens_seen": 2530723840 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018854964894684053, + "loss": 1.1847, + "theoretical_loss": 3.393706799773466, + "tokens_seen": 2530789376 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018853360080240723, + "loss": 1.1487, + "theoretical_loss": 3.3936998657104547, + "tokens_seen": 2530854912 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018851755265797393, + "loss": 1.2881, + "theoretical_loss": 3.393692931877271, + "tokens_seen": 2530920448 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018850150451354063, + "loss": 1.3294, + "theoretical_loss": 3.3936859982739023, + "tokens_seen": 2530985984 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018848545636910734, + "loss": 1.159, + "theoretical_loss": 3.3936790649003346, + "tokens_seen": 2531051520 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018846940822467404, + "loss": 1.273, + "theoretical_loss": 3.393672131756554, + "tokens_seen": 2531117056 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018845336008024074, + "loss": 1.1234, + "theoretical_loss": 3.3936651988425477, + "tokens_seen": 2531182592 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018843731193580744, + "loss": 1.1676, + "theoretical_loss": 3.3936582661583015, + "tokens_seen": 2531248128 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018842126379137414, + "loss": 1.1587, + "theoretical_loss": 3.393651333703802, + "tokens_seen": 2531313664 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001884052156469408, + "loss": 1.2497, + "theoretical_loss": 3.393644401479036, + "tokens_seen": 2531379200 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001883891675025075, + "loss": 1.1596, + "theoretical_loss": 3.3936374694839895, + "tokens_seen": 2531444736 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001883731193580742, + "loss": 1.1396, + "theoretical_loss": 3.393630537718649, + "tokens_seen": 2531510272 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001883570712136409, + "loss": 1.2096, + "theoretical_loss": 3.3936236061830014, + "tokens_seen": 2531575808 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018834102306920761, + "loss": 1.1955, + "theoretical_loss": 3.3936166748770322, + "tokens_seen": 2531641344 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018832497492477431, + "loss": 1.2013, + "theoretical_loss": 3.393609743800729, + "tokens_seen": 2531706880 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018830892678034102, + "loss": 1.1811, + "theoretical_loss": 3.3936028129540774, + "tokens_seen": 2531772416 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018829287863590772, + "loss": 1.1297, + "theoretical_loss": 3.393595882337064, + "tokens_seen": 2531837952 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018827683049147444, + "loss": 1.2888, + "theoretical_loss": 3.3935889519496754, + "tokens_seen": 2531903488 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018826078234704114, + "loss": 1.2317, + "theoretical_loss": 3.393582021791898, + "tokens_seen": 2531969024 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018824473420260784, + "loss": 1.2485, + "theoretical_loss": 3.393575091863718, + "tokens_seen": 2532034560 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018822868605817455, + "loss": 1.2864, + "theoretical_loss": 3.3935681621651224, + "tokens_seen": 2532100096 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018821263791374125, + "loss": 1.1243, + "theoretical_loss": 3.3935612326960976, + "tokens_seen": 2532165632 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018819658976930795, + "loss": 1.1082, + "theoretical_loss": 3.3935543034566296, + "tokens_seen": 2532231168 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018818054162487465, + "loss": 1.2896, + "theoretical_loss": 3.3935473744467046, + "tokens_seen": 2532296704 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018816449348044135, + "loss": 1.1502, + "theoretical_loss": 3.39354044566631, + "tokens_seen": 2532362240 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018814844533600805, + "loss": 1.2519, + "theoretical_loss": 3.393533517115432, + "tokens_seen": 2532427776 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018813239719157475, + "loss": 1.1695, + "theoretical_loss": 3.393526588794056, + "tokens_seen": 2532493312 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018811634904714145, + "loss": 1.2836, + "theoretical_loss": 3.39351966070217, + "tokens_seen": 2532558848 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018810030090270815, + "loss": 1.249, + "theoretical_loss": 3.39351273283976, + "tokens_seen": 2532624384 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018808425275827482, + "loss": 1.2373, + "theoretical_loss": 3.393505805206811, + "tokens_seen": 2532689920 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018806820461384152, + "loss": 1.1966, + "theoretical_loss": 3.393498877803312, + "tokens_seen": 2532755456 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018805215646940823, + "loss": 1.241, + "theoretical_loss": 3.3934919506292474, + "tokens_seen": 2532820992 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018803610832497493, + "loss": 1.2192, + "theoretical_loss": 3.3934850236846046, + "tokens_seen": 2532886528 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018802006018054163, + "loss": 1.2415, + "theoretical_loss": 3.39347809696937, + "tokens_seen": 2532952064 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018800401203610833, + "loss": 1.2453, + "theoretical_loss": 3.3934711704835294, + "tokens_seen": 2533017600 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018798796389167503, + "loss": 1.1805, + "theoretical_loss": 3.3934642442270704, + "tokens_seen": 2533083136 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018797191574724173, + "loss": 1.1912, + "theoretical_loss": 3.393457318199979, + "tokens_seen": 2533148672 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018795586760280843, + "loss": 1.1658, + "theoretical_loss": 3.393450392402241, + "tokens_seen": 2533214208 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018793981945837513, + "loss": 1.1637, + "theoretical_loss": 3.3934434668338436, + "tokens_seen": 2533279744 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018792377131394183, + "loss": 1.2449, + "theoretical_loss": 3.393436541494773, + "tokens_seen": 2533345280 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018790772316950853, + "loss": 1.1327, + "theoretical_loss": 3.393429616385016, + "tokens_seen": 2533410816 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018789167502507523, + "loss": 1.1938, + "theoretical_loss": 3.3934226915045587, + "tokens_seen": 2533476352 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018787562688064193, + "loss": 1.2082, + "theoretical_loss": 3.393415766853388, + "tokens_seen": 2533541888 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018785957873620863, + "loss": 1.2216, + "theoretical_loss": 3.39340884243149, + "tokens_seen": 2533607424 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018784353059177533, + "loss": 1.3161, + "theoretical_loss": 3.3934019182388506, + "tokens_seen": 2533672960 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018782748244734203, + "loss": 1.1394, + "theoretical_loss": 3.3933949942754573, + "tokens_seen": 2533738496 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018781143430290873, + "loss": 1.2157, + "theoretical_loss": 3.3933880705412967, + "tokens_seen": 2533804032 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018779538615847544, + "loss": 1.1745, + "theoretical_loss": 3.3933811470363544, + "tokens_seen": 2533869568 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018777933801404214, + "loss": 1.2789, + "theoretical_loss": 3.3933742237606177, + "tokens_seen": 2533935104 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018776328986960884, + "loss": 1.247, + "theoretical_loss": 3.3933673007140723, + "tokens_seen": 2534000640 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018774724172517554, + "loss": 1.1665, + "theoretical_loss": 3.393360377896705, + "tokens_seen": 2534066176 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018773119358074224, + "loss": 1.1989, + "theoretical_loss": 3.3933534553085023, + "tokens_seen": 2534131712 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018771514543630894, + "loss": 1.2737, + "theoretical_loss": 3.393346532949451, + "tokens_seen": 2534197248 + }, + { + "epoch": 1.15, + "learning_rate": 0.0001876990972918756, + "loss": 1.11, + "theoretical_loss": 3.3933396108195373, + "tokens_seen": 2534262784 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018768304914744234, + "loss": 1.1267, + "theoretical_loss": 3.393332688918748, + "tokens_seen": 2534328320 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018766700100300904, + "loss": 1.252, + "theoretical_loss": 3.393325767247069, + "tokens_seen": 2534393856 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018765095285857574, + "loss": 1.3034, + "theoretical_loss": 3.393318845804487, + "tokens_seen": 2534459392 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018763490471414244, + "loss": 1.2126, + "theoretical_loss": 3.393311924590989, + "tokens_seen": 2534524928 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018761885656970914, + "loss": 1.2546, + "theoretical_loss": 3.3933050036065606, + "tokens_seen": 2534590464 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018760280842527584, + "loss": 1.1506, + "theoretical_loss": 3.393298082851189, + "tokens_seen": 2534656000 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018758676028084254, + "loss": 1.2371, + "theoretical_loss": 3.3932911623248607, + "tokens_seen": 2534721536 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018757071213640924, + "loss": 1.1732, + "theoretical_loss": 3.3932842420275615, + "tokens_seen": 2534787072 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018755466399197595, + "loss": 1.2282, + "theoretical_loss": 3.3932773219592787, + "tokens_seen": 2534852608 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018753861584754265, + "loss": 1.17, + "theoretical_loss": 3.3932704021199984, + "tokens_seen": 2534918144 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018752256770310935, + "loss": 1.2959, + "theoretical_loss": 3.3932634825097074, + "tokens_seen": 2534983680 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018750651955867605, + "loss": 1.2753, + "theoretical_loss": 3.393256563128392, + "tokens_seen": 2535049216 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018749047141424275, + "loss": 1.1849, + "theoretical_loss": 3.3932496439760382, + "tokens_seen": 2535114752 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018747442326980945, + "loss": 1.1968, + "theoretical_loss": 3.3932427250526334, + "tokens_seen": 2535180288 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018745837512537615, + "loss": 1.2198, + "theoretical_loss": 3.3932358063581636, + "tokens_seen": 2535245824 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018744232698094285, + "loss": 1.2128, + "theoretical_loss": 3.3932288878926156, + "tokens_seen": 2535311360 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018742627883650955, + "loss": 1.0965, + "theoretical_loss": 3.3932219696559756, + "tokens_seen": 2535376896 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018741023069207625, + "loss": 1.1527, + "theoretical_loss": 3.39321505164823, + "tokens_seen": 2535442432 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018739418254764295, + "loss": 1.2363, + "theoretical_loss": 3.393208133869366, + "tokens_seen": 2535507968 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018737813440320962, + "loss": 1.1264, + "theoretical_loss": 3.393201216319369, + "tokens_seen": 2535573504 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018736208625877633, + "loss": 1.2779, + "theoretical_loss": 3.3931942989982264, + "tokens_seen": 2535639040 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018734603811434303, + "loss": 1.1939, + "theoretical_loss": 3.393187381905925, + "tokens_seen": 2535704576 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018732998996990973, + "loss": 1.2204, + "theoretical_loss": 3.39318046504245, + "tokens_seen": 2535770112 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018731394182547643, + "loss": 1.2297, + "theoretical_loss": 3.3931735484077894, + "tokens_seen": 2535835648 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018729789368104313, + "loss": 1.2031, + "theoretical_loss": 3.3931666320019285, + "tokens_seen": 2535901184 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018728184553660983, + "loss": 1.201, + "theoretical_loss": 3.3931597158248548, + "tokens_seen": 2535966720 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018726579739217653, + "loss": 1.1238, + "theoretical_loss": 3.393152799876554, + "tokens_seen": 2536032256 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018724974924774323, + "loss": 1.2037, + "theoretical_loss": 3.393145884157013, + "tokens_seen": 2536097792 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018723370110330993, + "loss": 1.1616, + "theoretical_loss": 3.3931389686662183, + "tokens_seen": 2536163328 + }, + { + "epoch": 1.15, + "objective/train/docs_used": 936774, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.0841388702392578, + "objective/train/theoretical_loss": 3.3931320534041567, + "objective/train/tokens_used": 2556688864, + "theoretical_loss": 3.3931320534041567, + "tokens_seen": 2536228864 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018721765295887663, + "loss": 1.0841, + "theoretical_loss": 3.3931320534041567, + "tokens_seen": 2536228864 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018720160481444333, + "loss": 1.2284, + "theoretical_loss": 3.3931251383708143, + "tokens_seen": 2536294400 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018718555667001003, + "loss": 1.2142, + "theoretical_loss": 3.393118223566178, + "tokens_seen": 2536359936 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018716950852557673, + "loss": 1.2883, + "theoretical_loss": 3.3931113089902336, + "tokens_seen": 2536425472 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018715346038114343, + "loss": 1.1768, + "theoretical_loss": 3.3931043946429686, + "tokens_seen": 2536491008 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018713741223671013, + "loss": 1.2148, + "theoretical_loss": 3.3930974805243688, + "tokens_seen": 2536556544 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018712136409227684, + "loss": 1.2113, + "theoretical_loss": 3.393090566634421, + "tokens_seen": 2536622080 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018710531594784354, + "loss": 1.2031, + "theoretical_loss": 3.393083652973112, + "tokens_seen": 2536687616 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018708926780341024, + "loss": 1.2168, + "theoretical_loss": 3.3930767395404278, + "tokens_seen": 2536753152 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018707321965897694, + "loss": 1.3174, + "theoretical_loss": 3.3930698263363555, + "tokens_seen": 2536818688 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018705717151454364, + "loss": 1.3043, + "theoretical_loss": 3.393062913360881, + "tokens_seen": 2536884224 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018704112337011034, + "loss": 1.132, + "theoretical_loss": 3.393056000613991, + "tokens_seen": 2536949760 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018702507522567704, + "loss": 1.1597, + "theoretical_loss": 3.393049088095673, + "tokens_seen": 2537015296 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018700902708124374, + "loss": 1.1853, + "theoretical_loss": 3.393042175805912, + "tokens_seen": 2537080832 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018699297893681044, + "loss": 1.2753, + "theoretical_loss": 3.3930352637446957, + "tokens_seen": 2537146368 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018697693079237714, + "loss": 1.1522, + "theoretical_loss": 3.39302835191201, + "tokens_seen": 2537211904 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018696088264794384, + "loss": 1.1377, + "theoretical_loss": 3.3930214403078414, + "tokens_seen": 2537277440 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018694483450351054, + "loss": 1.2662, + "theoretical_loss": 3.393014528932177, + "tokens_seen": 2537342976 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018692878635907724, + "loss": 1.2686, + "theoretical_loss": 3.393007617785003, + "tokens_seen": 2537408512 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018691273821464394, + "loss": 1.172, + "theoretical_loss": 3.393000706866306, + "tokens_seen": 2537474048 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018689669007021064, + "loss": 1.1375, + "theoretical_loss": 3.3929937961760728, + "tokens_seen": 2537539584 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018688064192577734, + "loss": 1.2914, + "theoretical_loss": 3.392986885714289, + "tokens_seen": 2537605120 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018686459378134405, + "loss": 1.2415, + "theoretical_loss": 3.3929799754809427, + "tokens_seen": 2537670656 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018684854563691075, + "loss": 1.1967, + "theoretical_loss": 3.392973065476019, + "tokens_seen": 2537736192 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018683249749247745, + "loss": 1.2271, + "theoretical_loss": 3.392966155699505, + "tokens_seen": 2537801728 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018681644934804415, + "loss": 1.1927, + "theoretical_loss": 3.3929592461513876, + "tokens_seen": 2537867264 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018680040120361085, + "loss": 1.3154, + "theoretical_loss": 3.392952336831653, + "tokens_seen": 2537932800 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018678435305917755, + "loss": 1.2857, + "theoretical_loss": 3.3929454277402877, + "tokens_seen": 2537998336 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018676830491474425, + "loss": 1.1663, + "theoretical_loss": 3.392938518877278, + "tokens_seen": 2538063872 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018675225677031095, + "loss": 1.1745, + "theoretical_loss": 3.3929316102426115, + "tokens_seen": 2538129408 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018673620862587765, + "loss": 1.2983, + "theoretical_loss": 3.392924701836274, + "tokens_seen": 2538194944 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018672016048144435, + "loss": 1.1918, + "theoretical_loss": 3.3929177936582517, + "tokens_seen": 2538260480 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018670411233701105, + "loss": 1.2124, + "theoretical_loss": 3.3929108857085315, + "tokens_seen": 2538326016 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018668806419257773, + "loss": 1.2423, + "theoretical_loss": 3.3929039779871, + "tokens_seen": 2538391552 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018667201604814443, + "loss": 1.2215, + "theoretical_loss": 3.3928970704939445, + "tokens_seen": 2538457088 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018665596790371113, + "loss": 1.2735, + "theoretical_loss": 3.3928901632290502, + "tokens_seen": 2538522624 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018663991975927783, + "loss": 1.2345, + "theoretical_loss": 3.3928832561924045, + "tokens_seen": 2538588160 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018662387161484453, + "loss": 1.2151, + "theoretical_loss": 3.392876349383994, + "tokens_seen": 2538653696 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018660782347041123, + "loss": 1.2536, + "theoretical_loss": 3.392869442803805, + "tokens_seen": 2538719232 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018659177532597793, + "loss": 1.2032, + "theoretical_loss": 3.392862536451824, + "tokens_seen": 2538784768 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018657572718154463, + "loss": 1.2865, + "theoretical_loss": 3.3928556303280377, + "tokens_seen": 2538850304 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018655967903711133, + "loss": 1.237, + "theoretical_loss": 3.3928487244324326, + "tokens_seen": 2538915840 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018654363089267803, + "loss": 1.0268, + "theoretical_loss": 3.392841818764995, + "tokens_seen": 2538981376 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018652758274824473, + "loss": 1.2765, + "theoretical_loss": 3.3928349133257125, + "tokens_seen": 2539046912 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018651153460381143, + "loss": 1.2708, + "theoretical_loss": 3.392828008114571, + "tokens_seen": 2539112448 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018649548645937813, + "loss": 1.2666, + "theoretical_loss": 3.3928211031315567, + "tokens_seen": 2539177984 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018647943831494486, + "loss": 1.173, + "theoretical_loss": 3.3928141983766564, + "tokens_seen": 2539243520 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018646339017051156, + "loss": 1.2938, + "theoretical_loss": 3.3928072938498572, + "tokens_seen": 2539309056 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018644734202607826, + "loss": 1.2162, + "theoretical_loss": 3.392800389551145, + "tokens_seen": 2539374592 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018643129388164496, + "loss": 1.2558, + "theoretical_loss": 3.3927934854805066, + "tokens_seen": 2539440128 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018641524573721166, + "loss": 1.2004, + "theoretical_loss": 3.392786581637929, + "tokens_seen": 2539505664 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018639919759277836, + "loss": 1.126, + "theoretical_loss": 3.3927796780233983, + "tokens_seen": 2539571200 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018638314944834506, + "loss": 1.2285, + "theoretical_loss": 3.392772774636901, + "tokens_seen": 2539636736 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018636710130391174, + "loss": 1.2518, + "theoretical_loss": 3.3927658714784243, + "tokens_seen": 2539702272 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018635105315947844, + "loss": 1.2355, + "theoretical_loss": 3.3927589685479544, + "tokens_seen": 2539767808 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018633500501504514, + "loss": 1.2372, + "theoretical_loss": 3.3927520658454773, + "tokens_seen": 2539833344 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018631895687061184, + "loss": 1.1445, + "theoretical_loss": 3.3927451633709804, + "tokens_seen": 2539898880 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018630290872617854, + "loss": 1.223, + "theoretical_loss": 3.3927382611244505, + "tokens_seen": 2539964416 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018628686058174524, + "loss": 1.2795, + "theoretical_loss": 3.3927313591058734, + "tokens_seen": 2540029952 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018627081243731194, + "loss": 1.1221, + "theoretical_loss": 3.3927244573152358, + "tokens_seen": 2540095488 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018625476429287864, + "loss": 1.255, + "theoretical_loss": 3.3927175557525246, + "tokens_seen": 2540161024 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018623871614844534, + "loss": 1.1977, + "theoretical_loss": 3.3927106544177263, + "tokens_seen": 2540226560 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018622266800401204, + "loss": 1.1528, + "theoretical_loss": 3.392703753310828, + "tokens_seen": 2540292096 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018620661985957874, + "loss": 1.2234, + "theoretical_loss": 3.3926968524318153, + "tokens_seen": 2540357632 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018619057171514544, + "loss": 1.2504, + "theoretical_loss": 3.3926899517806755, + "tokens_seen": 2540423168 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018617452357071215, + "loss": 1.3251, + "theoretical_loss": 3.3926830513573947, + "tokens_seen": 2540488704 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018615847542627885, + "loss": 1.1185, + "theoretical_loss": 3.39267615116196, + "tokens_seen": 2540554240 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018614242728184555, + "loss": 1.3375, + "theoretical_loss": 3.3926692511943575, + "tokens_seen": 2540619776 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018612637913741225, + "loss": 1.1454, + "theoretical_loss": 3.3926623514545744, + "tokens_seen": 2540685312 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018611033099297895, + "loss": 1.2337, + "theoretical_loss": 3.392655451942597, + "tokens_seen": 2540750848 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018609428284854565, + "loss": 1.2649, + "theoretical_loss": 3.392648552658412, + "tokens_seen": 2540816384 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018607823470411235, + "loss": 1.1191, + "theoretical_loss": 3.3926416536020056, + "tokens_seen": 2540881920 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018606218655967905, + "loss": 1.1985, + "theoretical_loss": 3.392634754773365, + "tokens_seen": 2540947456 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018604613841524575, + "loss": 1.1728, + "theoretical_loss": 3.392627856172476, + "tokens_seen": 2541012992 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018603009027081245, + "loss": 1.2345, + "theoretical_loss": 3.392620957799326, + "tokens_seen": 2541078528 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018601404212637915, + "loss": 1.15, + "theoretical_loss": 3.3926140596539014, + "tokens_seen": 2541144064 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018599799398194585, + "loss": 1.2469, + "theoretical_loss": 3.392607161736189, + "tokens_seen": 2541209600 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018598194583751253, + "loss": 1.218, + "theoretical_loss": 3.392600264046175, + "tokens_seen": 2541275136 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018596589769307923, + "loss": 1.2844, + "theoretical_loss": 3.3925933665838457, + "tokens_seen": 2541340672 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018594984954864593, + "loss": 1.2628, + "theoretical_loss": 3.3925864693491885, + "tokens_seen": 2541406208 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018593380140421263, + "loss": 1.1835, + "theoretical_loss": 3.3925795723421897, + "tokens_seen": 2541471744 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018591775325977933, + "loss": 1.2661, + "theoretical_loss": 3.392572675562836, + "tokens_seen": 2541537280 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018590170511534603, + "loss": 1.1425, + "theoretical_loss": 3.3925657790111137, + "tokens_seen": 2541602816 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018588565697091276, + "loss": 1.2121, + "theoretical_loss": 3.3925588826870094, + "tokens_seen": 2541668352 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018586960882647946, + "loss": 1.1686, + "theoretical_loss": 3.3925519865905107, + "tokens_seen": 2541733888 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018585356068204616, + "loss": 1.3281, + "theoretical_loss": 3.392545090721603, + "tokens_seen": 2541799424 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018583751253761286, + "loss": 1.224, + "theoretical_loss": 3.3925381950802733, + "tokens_seen": 2541864960 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018582146439317956, + "loss": 1.1853, + "theoretical_loss": 3.3925312996665085, + "tokens_seen": 2541930496 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018580541624874626, + "loss": 1.2034, + "theoretical_loss": 3.392524404480295, + "tokens_seen": 2541996032 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018578936810431296, + "loss": 1.1387, + "theoretical_loss": 3.3925175095216193, + "tokens_seen": 2542061568 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018577331995987966, + "loss": 1.1388, + "theoretical_loss": 3.392510614790468, + "tokens_seen": 2542127104 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018575727181544636, + "loss": 1.1822, + "theoretical_loss": 3.3925037202868285, + "tokens_seen": 2542192640 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018574122367101306, + "loss": 1.2131, + "theoretical_loss": 3.3924968260106865, + "tokens_seen": 2542258176 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018572517552657976, + "loss": 1.2571, + "theoretical_loss": 3.392489931962029, + "tokens_seen": 2542323712 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018570912738214646, + "loss": 1.2219, + "theoretical_loss": 3.392483038140843, + "tokens_seen": 2542389248 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018569307923771316, + "loss": 1.302, + "theoretical_loss": 3.392476144547114, + "tokens_seen": 2542454784 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018567703109327987, + "loss": 1.2599, + "theoretical_loss": 3.39246925118083, + "tokens_seen": 2542520320 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018566098294884654, + "loss": 1.2071, + "theoretical_loss": 3.3924623580419766, + "tokens_seen": 2542585856 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018564493480441324, + "loss": 1.2535, + "theoretical_loss": 3.392455465130541, + "tokens_seen": 2542651392 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018562888665997994, + "loss": 1.2716, + "theoretical_loss": 3.3924485724465097, + "tokens_seen": 2542716928 + }, + { + "epoch": 1.15, + "objective/train/docs_used": 938851, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2138210535049438, + "objective/train/theoretical_loss": 3.392441679989869, + "objective/train/tokens_used": 2563242464, + "theoretical_loss": 3.392441679989869, + "tokens_seen": 2542782464 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018561283851554664, + "loss": 1.2138, + "theoretical_loss": 3.392441679989869, + "tokens_seen": 2542782464 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018559679037111334, + "loss": 1.3097, + "theoretical_loss": 3.3924347877606067, + "tokens_seen": 2542848000 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018558074222668004, + "loss": 1.2534, + "theoretical_loss": 3.3924278957587077, + "tokens_seen": 2542913536 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018556469408224674, + "loss": 1.192, + "theoretical_loss": 3.39242100398416, + "tokens_seen": 2542979072 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018554864593781344, + "loss": 1.1615, + "theoretical_loss": 3.392414112436949, + "tokens_seen": 2543044608 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018553259779338014, + "loss": 1.2565, + "theoretical_loss": 3.392407221117063, + "tokens_seen": 2543110144 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018551654964894684, + "loss": 1.1628, + "theoretical_loss": 3.3924003300244876, + "tokens_seen": 2543175680 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018550050150451355, + "loss": 1.1413, + "theoretical_loss": 3.392393439159209, + "tokens_seen": 2543241216 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018548445336008025, + "loss": 1.2011, + "theoretical_loss": 3.3923865485212152, + "tokens_seen": 2543306752 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018546840521564695, + "loss": 1.1515, + "theoretical_loss": 3.392379658110492, + "tokens_seen": 2543372288 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018545235707121365, + "loss": 1.2582, + "theoretical_loss": 3.392372767927026, + "tokens_seen": 2543437824 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018543630892678035, + "loss": 1.1513, + "theoretical_loss": 3.3923658779708035, + "tokens_seen": 2543503360 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018542026078234705, + "loss": 1.1133, + "theoretical_loss": 3.392358988241812, + "tokens_seen": 2543568896 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018540421263791375, + "loss": 1.2257, + "theoretical_loss": 3.392352098740038, + "tokens_seen": 2543634432 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018538816449348045, + "loss": 1.2187, + "theoretical_loss": 3.392345209465468, + "tokens_seen": 2543699968 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018537211634904715, + "loss": 1.1859, + "theoretical_loss": 3.392338320418088, + "tokens_seen": 2543765504 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018535606820461385, + "loss": 1.2266, + "theoretical_loss": 3.392331431597886, + "tokens_seen": 2543831040 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018534002006018055, + "loss": 1.225, + "theoretical_loss": 3.392324543004847, + "tokens_seen": 2543896576 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018532397191574725, + "loss": 1.2529, + "theoretical_loss": 3.3923176546389593, + "tokens_seen": 2543962112 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018530792377131395, + "loss": 1.1976, + "theoretical_loss": 3.3923107665002084, + "tokens_seen": 2544027648 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018529187562688065, + "loss": 1.2792, + "theoretical_loss": 3.3923038785885815, + "tokens_seen": 2544093184 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018527582748244735, + "loss": 1.2247, + "theoretical_loss": 3.3922969909040654, + "tokens_seen": 2544158720 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018525977933801405, + "loss": 1.2531, + "theoretical_loss": 3.3922901034466464, + "tokens_seen": 2544224256 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018524373119358076, + "loss": 1.121, + "theoretical_loss": 3.392283216216311, + "tokens_seen": 2544289792 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018522768304914746, + "loss": 1.1797, + "theoretical_loss": 3.3922763292130464, + "tokens_seen": 2544355328 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018521163490471416, + "loss": 1.1928, + "theoretical_loss": 3.392269442436839, + "tokens_seen": 2544420864 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018519558676028086, + "loss": 1.1782, + "theoretical_loss": 3.392262555887675, + "tokens_seen": 2544486400 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018517953861584756, + "loss": 1.1209, + "theoretical_loss": 3.392255669565542, + "tokens_seen": 2544551936 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018516349047141426, + "loss": 1.2789, + "theoretical_loss": 3.3922487834704262, + "tokens_seen": 2544617472 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018514744232698096, + "loss": 1.221, + "theoretical_loss": 3.3922418976023145, + "tokens_seen": 2544683008 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018513139418254766, + "loss": 1.172, + "theoretical_loss": 3.392235011961193, + "tokens_seen": 2544748544 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018511534603811436, + "loss": 1.2127, + "theoretical_loss": 3.3922281265470486, + "tokens_seen": 2544814080 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018509929789368106, + "loss": 1.2273, + "theoretical_loss": 3.3922212413598682, + "tokens_seen": 2544879616 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018508324974924776, + "loss": 1.1462, + "theoretical_loss": 3.392214356399638, + "tokens_seen": 2544945152 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018506720160481446, + "loss": 1.1965, + "theoretical_loss": 3.3922074716663455, + "tokens_seen": 2545010688 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018505115346038116, + "loss": 1.1958, + "theoretical_loss": 3.3922005871599765, + "tokens_seen": 2545076224 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018503510531594786, + "loss": 1.2627, + "theoretical_loss": 3.392193702880519, + "tokens_seen": 2545141760 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018501905717151456, + "loss": 1.126, + "theoretical_loss": 3.392186818827958, + "tokens_seen": 2545207296 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018500300902708126, + "loss": 1.1829, + "theoretical_loss": 3.392179935002281, + "tokens_seen": 2545272832 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018498696088264797, + "loss": 1.2339, + "theoretical_loss": 3.3921730514034745, + "tokens_seen": 2545338368 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018497091273821464, + "loss": 1.1252, + "theoretical_loss": 3.3921661680315256, + "tokens_seen": 2545403904 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018495486459378134, + "loss": 1.1115, + "theoretical_loss": 3.3921592848864206, + "tokens_seen": 2545469440 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018493881644934804, + "loss": 1.2031, + "theoretical_loss": 3.392152401968146, + "tokens_seen": 2545534976 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018492276830491474, + "loss": 1.1882, + "theoretical_loss": 3.3921455192766894, + "tokens_seen": 2545600512 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018490672016048144, + "loss": 1.2258, + "theoretical_loss": 3.392138636812036, + "tokens_seen": 2545666048 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018489067201604814, + "loss": 1.2111, + "theoretical_loss": 3.3921317545741743, + "tokens_seen": 2545731584 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018487462387161484, + "loss": 1.302, + "theoretical_loss": 3.3921248725630893, + "tokens_seen": 2545797120 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018485857572718154, + "loss": 1.2001, + "theoretical_loss": 3.3921179907787686, + "tokens_seen": 2545862656 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018484252758274824, + "loss": 1.2549, + "theoretical_loss": 3.3921111092211986, + "tokens_seen": 2545928192 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018482647943831494, + "loss": 1.2089, + "theoretical_loss": 3.3921042278903664, + "tokens_seen": 2545993728 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018481043129388165, + "loss": 1.1956, + "theoretical_loss": 3.392097346786258, + "tokens_seen": 2546059264 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018479438314944835, + "loss": 1.193, + "theoretical_loss": 3.392090465908861, + "tokens_seen": 2546124800 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018477833500501505, + "loss": 1.2541, + "theoretical_loss": 3.3920835852581614, + "tokens_seen": 2546190336 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018476228686058175, + "loss": 1.2099, + "theoretical_loss": 3.3920767048341456, + "tokens_seen": 2546255872 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018474623871614845, + "loss": 1.2444, + "theoretical_loss": 3.3920698246368013, + "tokens_seen": 2546321408 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018473019057171515, + "loss": 1.2215, + "theoretical_loss": 3.3920629446661144, + "tokens_seen": 2546386944 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018471414242728185, + "loss": 1.1478, + "theoretical_loss": 3.3920560649220723, + "tokens_seen": 2546452480 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018469809428284855, + "loss": 1.2325, + "theoretical_loss": 3.3920491854046606, + "tokens_seen": 2546518016 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018468204613841528, + "loss": 1.219, + "theoretical_loss": 3.392042306113867, + "tokens_seen": 2546583552 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018466599799398198, + "loss": 1.1675, + "theoretical_loss": 3.392035427049678, + "tokens_seen": 2546649088 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018464994984954865, + "loss": 1.2124, + "theoretical_loss": 3.39202854821208, + "tokens_seen": 2546714624 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018463390170511535, + "loss": 1.1921, + "theoretical_loss": 3.39202166960106, + "tokens_seen": 2546780160 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018461785356068205, + "loss": 1.2118, + "theoretical_loss": 3.3920147912166048, + "tokens_seen": 2546845696 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018460180541624875, + "loss": 1.2022, + "theoretical_loss": 3.3920079130587, + "tokens_seen": 2546911232 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018458575727181545, + "loss": 1.1862, + "theoretical_loss": 3.3920010351273344, + "tokens_seen": 2546976768 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018456970912738215, + "loss": 1.1053, + "theoretical_loss": 3.3919941574224928, + "tokens_seen": 2547042304 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018455366098294886, + "loss": 1.2129, + "theoretical_loss": 3.391987279944163, + "tokens_seen": 2547107840 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018453761283851556, + "loss": 1.1903, + "theoretical_loss": 3.3919804026923313, + "tokens_seen": 2547173376 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018452156469408226, + "loss": 1.1184, + "theoretical_loss": 3.391973525666984, + "tokens_seen": 2547238912 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018450551654964896, + "loss": 1.2284, + "theoretical_loss": 3.3919666488681086, + "tokens_seen": 2547304448 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018448946840521566, + "loss": 1.2783, + "theoretical_loss": 3.3919597722956913, + "tokens_seen": 2547369984 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018447342026078236, + "loss": 1.2556, + "theoretical_loss": 3.3919528959497196, + "tokens_seen": 2547435520 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018445737211634906, + "loss": 1.2119, + "theoretical_loss": 3.391946019830179, + "tokens_seen": 2547501056 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018444132397191576, + "loss": 1.3343, + "theoretical_loss": 3.3919391439370568, + "tokens_seen": 2547566592 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018442527582748246, + "loss": 1.237, + "theoretical_loss": 3.39193226827034, + "tokens_seen": 2547632128 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018440922768304916, + "loss": 1.1574, + "theoretical_loss": 3.3919253928300153, + "tokens_seen": 2547697664 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018439317953861586, + "loss": 1.2857, + "theoretical_loss": 3.391918517616069, + "tokens_seen": 2547763200 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018437713139418256, + "loss": 1.2027, + "theoretical_loss": 3.391911642628488, + "tokens_seen": 2547828736 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018436108324974926, + "loss": 1.1059, + "theoretical_loss": 3.391904767867259, + "tokens_seen": 2547894272 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018434503510531596, + "loss": 1.2316, + "theoretical_loss": 3.391897893332369, + "tokens_seen": 2547959808 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018432898696088266, + "loss": 1.2054, + "theoretical_loss": 3.391891019023804, + "tokens_seen": 2548025344 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018431293881644936, + "loss": 1.2075, + "theoretical_loss": 3.3918841449415518, + "tokens_seen": 2548090880 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018429689067201607, + "loss": 1.2665, + "theoretical_loss": 3.3918772710855984, + "tokens_seen": 2548156416 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018428084252758277, + "loss": 1.1424, + "theoretical_loss": 3.3918703974559303, + "tokens_seen": 2548221952 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018426479438314944, + "loss": 1.1888, + "theoretical_loss": 3.3918635240525354, + "tokens_seen": 2548287488 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018424874623871614, + "loss": 1.2266, + "theoretical_loss": 3.391856650875399, + "tokens_seen": 2548353024 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018423269809428284, + "loss": 1.2429, + "theoretical_loss": 3.391849777924509, + "tokens_seen": 2548418560 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018421664994984954, + "loss": 1.247, + "theoretical_loss": 3.391842905199851, + "tokens_seen": 2548484096 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018420060180541624, + "loss": 1.0909, + "theoretical_loss": 3.3918360327014128, + "tokens_seen": 2548549632 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018418455366098294, + "loss": 1.1297, + "theoretical_loss": 3.3918291604291806, + "tokens_seen": 2548615168 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018416850551654964, + "loss": 1.2006, + "theoretical_loss": 3.3918222883831413, + "tokens_seen": 2548680704 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018415245737211634, + "loss": 1.1655, + "theoretical_loss": 3.3918154165632814, + "tokens_seen": 2548746240 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018413640922768304, + "loss": 1.1632, + "theoretical_loss": 3.391808544969588, + "tokens_seen": 2548811776 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018412036108324975, + "loss": 1.2852, + "theoretical_loss": 3.3918016736020475, + "tokens_seen": 2548877312 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018410431293881645, + "loss": 1.2459, + "theoretical_loss": 3.391794802460647, + "tokens_seen": 2548942848 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018408826479438315, + "loss": 1.2364, + "theoretical_loss": 3.3917879315453727, + "tokens_seen": 2549008384 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018407221664994987, + "loss": 1.217, + "theoretical_loss": 3.391781060856212, + "tokens_seen": 2549073920 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018405616850551658, + "loss": 1.1295, + "theoretical_loss": 3.3917741903931513, + "tokens_seen": 2549139456 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018404012036108328, + "loss": 1.2733, + "theoretical_loss": 3.391767320156177, + "tokens_seen": 2549204992 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018402407221664998, + "loss": 1.0434, + "theoretical_loss": 3.3917604501452767, + "tokens_seen": 2549270528 + }, + { + "epoch": 1.15, + "objective/train/docs_used": 941629, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1619378328323364, + "objective/train/theoretical_loss": 3.3917535803604366, + "objective/train/tokens_used": 2569796064, + "theoretical_loss": 3.3917535803604366, + "tokens_seen": 2549336064 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018400802407221668, + "loss": 1.1619, + "theoretical_loss": 3.3917535803604366, + "tokens_seen": 2549336064 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018399197592778338, + "loss": 1.1306, + "theoretical_loss": 3.3917467108016432, + "tokens_seen": 2549401600 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018397592778335008, + "loss": 1.2527, + "theoretical_loss": 3.3917398414688837, + "tokens_seen": 2549467136 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018395987963891678, + "loss": 1.1646, + "theoretical_loss": 3.391732972362145, + "tokens_seen": 2549532672 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018394383149448345, + "loss": 1.2409, + "theoretical_loss": 3.3917261034814135, + "tokens_seen": 2549598208 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018392778335005015, + "loss": 1.2879, + "theoretical_loss": 3.391719234826676, + "tokens_seen": 2549663744 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018391173520561685, + "loss": 1.2351, + "theoretical_loss": 3.391712366397919, + "tokens_seen": 2549729280 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018389568706118355, + "loss": 1.2012, + "theoretical_loss": 3.39170549819513, + "tokens_seen": 2549794816 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018387963891675025, + "loss": 1.1935, + "theoretical_loss": 3.391698630218295, + "tokens_seen": 2549860352 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018386359077231696, + "loss": 1.1112, + "theoretical_loss": 3.3916917624674014, + "tokens_seen": 2549925888 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018384754262788366, + "loss": 1.1587, + "theoretical_loss": 3.3916848949424354, + "tokens_seen": 2549991424 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018383149448345036, + "loss": 1.1711, + "theoretical_loss": 3.391678027643384, + "tokens_seen": 2550056960 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018381544633901706, + "loss": 1.1475, + "theoretical_loss": 3.391671160570234, + "tokens_seen": 2550122496 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018379939819458376, + "loss": 1.21, + "theoretical_loss": 3.391664293722972, + "tokens_seen": 2550188032 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018378335005015046, + "loss": 1.2351, + "theoretical_loss": 3.391657427101585, + "tokens_seen": 2550253568 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018376730190571716, + "loss": 1.1714, + "theoretical_loss": 3.3916505607060596, + "tokens_seen": 2550319104 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018375125376128386, + "loss": 1.2803, + "theoretical_loss": 3.391643694536383, + "tokens_seen": 2550384640 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018373520561685056, + "loss": 1.2036, + "theoretical_loss": 3.3916368285925413, + "tokens_seen": 2550450176 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018371915747241726, + "loss": 1.2006, + "theoretical_loss": 3.3916299628745215, + "tokens_seen": 2550515712 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018370310932798396, + "loss": 1.2251, + "theoretical_loss": 3.3916230973823103, + "tokens_seen": 2550581248 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018368706118355066, + "loss": 1.1145, + "theoretical_loss": 3.3916162321158945, + "tokens_seen": 2550646784 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018367101303911736, + "loss": 1.224, + "theoretical_loss": 3.3916093670752616, + "tokens_seen": 2550712320 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018365496489468406, + "loss": 1.1343, + "theoretical_loss": 3.391602502260397, + "tokens_seen": 2550777856 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018363891675025076, + "loss": 1.2266, + "theoretical_loss": 3.391595637671289, + "tokens_seen": 2550843392 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018362286860581747, + "loss": 1.1083, + "theoretical_loss": 3.391588773307923, + "tokens_seen": 2550908928 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018360682046138417, + "loss": 1.2214, + "theoretical_loss": 3.391581909170286, + "tokens_seen": 2550974464 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018359077231695087, + "loss": 1.2614, + "theoretical_loss": 3.391575045258366, + "tokens_seen": 2551040000 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018357472417251757, + "loss": 1.1864, + "theoretical_loss": 3.391568181572149, + "tokens_seen": 2551105536 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018355867602808424, + "loss": 1.2397, + "theoretical_loss": 3.391561318111621, + "tokens_seen": 2551171072 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018354262788365094, + "loss": 1.1761, + "theoretical_loss": 3.3915544548767698, + "tokens_seen": 2551236608 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018352657973921764, + "loss": 1.1754, + "theoretical_loss": 3.3915475918675817, + "tokens_seen": 2551302144 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018351053159478434, + "loss": 1.1096, + "theoretical_loss": 3.391540729084044, + "tokens_seen": 2551367680 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018349448345035104, + "loss": 1.1897, + "theoretical_loss": 3.3915338665261427, + "tokens_seen": 2551433216 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018347843530591777, + "loss": 1.2109, + "theoretical_loss": 3.3915270041938657, + "tokens_seen": 2551498752 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018346238716148447, + "loss": 1.2049, + "theoretical_loss": 3.3915201420871983, + "tokens_seen": 2551564288 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018344633901705117, + "loss": 1.2241, + "theoretical_loss": 3.3915132802061283, + "tokens_seen": 2551629824 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018343029087261787, + "loss": 1.2235, + "theoretical_loss": 3.3915064185506427, + "tokens_seen": 2551695360 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018341424272818457, + "loss": 1.2426, + "theoretical_loss": 3.391499557120728, + "tokens_seen": 2551760896 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018339819458375127, + "loss": 1.2736, + "theoretical_loss": 3.3914926959163703, + "tokens_seen": 2551826432 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018338214643931797, + "loss": 1.1663, + "theoretical_loss": 3.391485834937557, + "tokens_seen": 2551891968 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018336609829488468, + "loss": 1.2303, + "theoretical_loss": 3.3914789741842752, + "tokens_seen": 2551957504 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018335005015045138, + "loss": 1.3021, + "theoretical_loss": 3.391472113656511, + "tokens_seen": 2552023040 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018333400200601808, + "loss": 1.2369, + "theoretical_loss": 3.391465253354252, + "tokens_seen": 2552088576 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018331795386158478, + "loss": 1.2917, + "theoretical_loss": 3.3914583932774844, + "tokens_seen": 2552154112 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018330190571715148, + "loss": 1.2558, + "theoretical_loss": 3.391451533426195, + "tokens_seen": 2552219648 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018328585757271818, + "loss": 1.216, + "theoretical_loss": 3.391444673800371, + "tokens_seen": 2552285184 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018326980942828488, + "loss": 1.1685, + "theoretical_loss": 3.391437814399999, + "tokens_seen": 2552350720 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018325376128385155, + "loss": 1.3043, + "theoretical_loss": 3.3914309552250654, + "tokens_seen": 2552416256 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018323771313941825, + "loss": 1.1334, + "theoretical_loss": 3.391424096275557, + "tokens_seen": 2552481792 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018322166499498495, + "loss": 1.1816, + "theoretical_loss": 3.3914172375514617, + "tokens_seen": 2552547328 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018320561685055165, + "loss": 1.1524, + "theoretical_loss": 3.3914103790527657, + "tokens_seen": 2552612864 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018318956870611836, + "loss": 1.3365, + "theoretical_loss": 3.3914035207794546, + "tokens_seen": 2552678400 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018317352056168506, + "loss": 1.1144, + "theoretical_loss": 3.391396662731517, + "tokens_seen": 2552743936 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018315747241725176, + "loss": 1.3457, + "theoretical_loss": 3.3913898049089393, + "tokens_seen": 2552809472 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018314142427281846, + "loss": 1.2478, + "theoretical_loss": 3.3913829473117074, + "tokens_seen": 2552875008 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018312537612838516, + "loss": 1.2696, + "theoretical_loss": 3.391376089939809, + "tokens_seen": 2552940544 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018310932798395186, + "loss": 1.2348, + "theoretical_loss": 3.3913692327932305, + "tokens_seen": 2553006080 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018309327983951856, + "loss": 1.2718, + "theoretical_loss": 3.391362375871959, + "tokens_seen": 2553071616 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018307723169508526, + "loss": 1.191, + "theoretical_loss": 3.3913555191759808, + "tokens_seen": 2553137152 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018306118355065196, + "loss": 1.1799, + "theoretical_loss": 3.3913486627052833, + "tokens_seen": 2553202688 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018304513540621866, + "loss": 1.2226, + "theoretical_loss": 3.391341806459853, + "tokens_seen": 2553268224 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018302908726178536, + "loss": 1.1349, + "theoretical_loss": 3.391334950439677, + "tokens_seen": 2553333760 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018301303911735206, + "loss": 1.1729, + "theoretical_loss": 3.3913280946447415, + "tokens_seen": 2553399296 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018299699097291876, + "loss": 1.2197, + "theoretical_loss": 3.391321239075034, + "tokens_seen": 2553464832 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018298094282848546, + "loss": 1.1398, + "theoretical_loss": 3.391314383730541, + "tokens_seen": 2553530368 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018296489468405216, + "loss": 1.1889, + "theoretical_loss": 3.3913075286112493, + "tokens_seen": 2553595904 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018294884653961886, + "loss": 1.2795, + "theoretical_loss": 3.391300673717146, + "tokens_seen": 2553661440 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018293279839518557, + "loss": 1.2932, + "theoretical_loss": 3.3912938190482174, + "tokens_seen": 2553726976 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018291675025075227, + "loss": 1.2703, + "theoretical_loss": 3.3912869646044506, + "tokens_seen": 2553792512 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018290070210631897, + "loss": 1.1821, + "theoretical_loss": 3.391280110385833, + "tokens_seen": 2553858048 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018288465396188567, + "loss": 1.2408, + "theoretical_loss": 3.39127325639235, + "tokens_seen": 2553923584 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018286860581745237, + "loss": 1.1913, + "theoretical_loss": 3.39126640262399, + "tokens_seen": 2553989120 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018285255767301907, + "loss": 1.3273, + "theoretical_loss": 3.3912595490807393, + "tokens_seen": 2554054656 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018283650952858577, + "loss": 1.3286, + "theoretical_loss": 3.391252695762584, + "tokens_seen": 2554120192 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018282046138415247, + "loss": 1.1993, + "theoretical_loss": 3.391245842669512, + "tokens_seen": 2554185728 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018280441323971917, + "loss": 1.1911, + "theoretical_loss": 3.3912389898015096, + "tokens_seen": 2554251264 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018278836509528587, + "loss": 1.1957, + "theoretical_loss": 3.391232137158563, + "tokens_seen": 2554316800 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018277231695085257, + "loss": 1.139, + "theoretical_loss": 3.3912252847406608, + "tokens_seen": 2554382336 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018275626880641927, + "loss": 1.2192, + "theoretical_loss": 3.391218432547788, + "tokens_seen": 2554447872 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018274022066198597, + "loss": 1.2074, + "theoretical_loss": 3.3912115805799323, + "tokens_seen": 2554513408 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018272417251755267, + "loss": 1.255, + "theoretical_loss": 3.3912047288370806, + "tokens_seen": 2554578944 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018270812437311937, + "loss": 1.18, + "theoretical_loss": 3.3911978773192195, + "tokens_seen": 2554644480 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018269207622868607, + "loss": 1.1374, + "theoretical_loss": 3.391191026026336, + "tokens_seen": 2554710016 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018267602808425278, + "loss": 1.2629, + "theoretical_loss": 3.3911841749584166, + "tokens_seen": 2554775552 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018265997993981948, + "loss": 1.2804, + "theoretical_loss": 3.391177324115448, + "tokens_seen": 2554841088 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018264393179538618, + "loss": 1.2151, + "theoretical_loss": 3.3911704734974184, + "tokens_seen": 2554906624 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018262788365095288, + "loss": 1.2566, + "theoretical_loss": 3.391163623104313, + "tokens_seen": 2554972160 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018261183550651958, + "loss": 1.208, + "theoretical_loss": 3.3911567729361196, + "tokens_seen": 2555037696 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018259578736208628, + "loss": 1.2317, + "theoretical_loss": 3.391149922992825, + "tokens_seen": 2555103232 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018257973921765298, + "loss": 1.2029, + "theoretical_loss": 3.3911430732744154, + "tokens_seen": 2555168768 + }, + { + "epoch": 1.15, + "learning_rate": 0.00018256369107321968, + "loss": 1.1694, + "theoretical_loss": 3.391136223780878, + "tokens_seen": 2555234304 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018254764292878635, + "loss": 1.1175, + "theoretical_loss": 3.3911293745122, + "tokens_seen": 2555299840 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018253159478435305, + "loss": 1.2077, + "theoretical_loss": 3.391122525468368, + "tokens_seen": 2555365376 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018251554663991975, + "loss": 1.1494, + "theoretical_loss": 3.3911156766493686, + "tokens_seen": 2555430912 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018249949849548646, + "loss": 1.1765, + "theoretical_loss": 3.391108828055189, + "tokens_seen": 2555496448 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018248345035105316, + "loss": 1.1155, + "theoretical_loss": 3.391101979685816, + "tokens_seen": 2555561984 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018246740220661986, + "loss": 1.2447, + "theoretical_loss": 3.391095131541236, + "tokens_seen": 2555627520 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018245135406218656, + "loss": 1.246, + "theoretical_loss": 3.391088283621437, + "tokens_seen": 2555693056 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018243530591775326, + "loss": 1.2423, + "theoretical_loss": 3.3910814359264045, + "tokens_seen": 2555758592 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018241925777331996, + "loss": 1.2753, + "theoretical_loss": 3.391074588456126, + "tokens_seen": 2555824128 + }, + { + "epoch": 1.16, + "objective/train/docs_used": 944043, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2063616514205933, + "objective/train/theoretical_loss": 3.3910677412105885, + "objective/train/tokens_used": 2576349664, + "theoretical_loss": 3.3910677412105885, + "tokens_seen": 2555889664 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018240320962888666, + "loss": 1.2064, + "theoretical_loss": 3.3910677412105885, + "tokens_seen": 2555889664 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018238716148445336, + "loss": 1.2675, + "theoretical_loss": 3.3910608941897786, + "tokens_seen": 2555955200 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018237111334002006, + "loss": 1.2545, + "theoretical_loss": 3.391054047393683, + "tokens_seen": 2556020736 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018235506519558676, + "loss": 1.2051, + "theoretical_loss": 3.3910472008222894, + "tokens_seen": 2556086272 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018233901705115346, + "loss": 1.2873, + "theoretical_loss": 3.3910403544755834, + "tokens_seen": 2556151808 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018232296890672016, + "loss": 1.2303, + "theoretical_loss": 3.391033508353553, + "tokens_seen": 2556217344 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018230692076228686, + "loss": 1.2579, + "theoretical_loss": 3.3910266624561847, + "tokens_seen": 2556282880 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018229087261785356, + "loss": 1.1753, + "theoretical_loss": 3.391019816783465, + "tokens_seen": 2556348416 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001822748244734203, + "loss": 1.3149, + "theoretical_loss": 3.391012971335381, + "tokens_seen": 2556413952 + }, + { + "epoch": 1.16, + "learning_rate": 0.000182258776328987, + "loss": 1.2766, + "theoretical_loss": 3.39100612611192, + "tokens_seen": 2556479488 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001822427281845537, + "loss": 1.1963, + "theoretical_loss": 3.390999281113068, + "tokens_seen": 2556545024 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018222668004012037, + "loss": 1.3239, + "theoretical_loss": 3.3909924363388124, + "tokens_seen": 2556610560 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018221063189568707, + "loss": 1.2125, + "theoretical_loss": 3.39098559178914, + "tokens_seen": 2556676096 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018219458375125377, + "loss": 1.1714, + "theoretical_loss": 3.3909787474640383, + "tokens_seen": 2556741632 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018217853560682047, + "loss": 1.1587, + "theoretical_loss": 3.390971903363493, + "tokens_seen": 2556807168 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018216248746238717, + "loss": 1.2357, + "theoretical_loss": 3.390965059487492, + "tokens_seen": 2556872704 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018214643931795387, + "loss": 1.2559, + "theoretical_loss": 3.3909582158360214, + "tokens_seen": 2556938240 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018213039117352057, + "loss": 1.1486, + "theoretical_loss": 3.3909513724090683, + "tokens_seen": 2557003776 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018211434302908727, + "loss": 1.2408, + "theoretical_loss": 3.39094452920662, + "tokens_seen": 2557069312 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018209829488465397, + "loss": 1.1899, + "theoretical_loss": 3.390937686228663, + "tokens_seen": 2557134848 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018208224674022067, + "loss": 1.2512, + "theoretical_loss": 3.390930843475184, + "tokens_seen": 2557200384 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018206619859578737, + "loss": 1.2918, + "theoretical_loss": 3.3909240009461703, + "tokens_seen": 2557265920 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018205015045135407, + "loss": 1.2349, + "theoretical_loss": 3.3909171586416087, + "tokens_seen": 2557331456 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018203410230692077, + "loss": 1.0706, + "theoretical_loss": 3.3909103165614862, + "tokens_seen": 2557396992 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018201805416248747, + "loss": 1.2644, + "theoretical_loss": 3.390903474705789, + "tokens_seen": 2557462528 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018200200601805417, + "loss": 1.195, + "theoretical_loss": 3.390896633074505, + "tokens_seen": 2557528064 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018198595787362088, + "loss": 1.1906, + "theoretical_loss": 3.3908897916676204, + "tokens_seen": 2557593600 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018196990972918758, + "loss": 1.2405, + "theoretical_loss": 3.390882950485122, + "tokens_seen": 2557659136 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018195386158475428, + "loss": 1.269, + "theoretical_loss": 3.3908761095269977, + "tokens_seen": 2557724672 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018193781344032098, + "loss": 1.2105, + "theoretical_loss": 3.390869268793233, + "tokens_seen": 2557790208 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018192176529588768, + "loss": 1.31, + "theoretical_loss": 3.3908624282838153, + "tokens_seen": 2557855744 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018190571715145438, + "loss": 1.1505, + "theoretical_loss": 3.390855587998732, + "tokens_seen": 2557921280 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018188966900702108, + "loss": 1.2172, + "theoretical_loss": 3.3908487479379694, + "tokens_seen": 2557986816 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018187362086258778, + "loss": 1.1343, + "theoretical_loss": 3.390841908101515, + "tokens_seen": 2558052352 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018185757271815448, + "loss": 1.3137, + "theoretical_loss": 3.390835068489355, + "tokens_seen": 2558117888 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018184152457372115, + "loss": 1.1937, + "theoretical_loss": 3.390828229101477, + "tokens_seen": 2558183424 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018182547642928785, + "loss": 1.408, + "theoretical_loss": 3.390821389937867, + "tokens_seen": 2558248960 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018180942828485456, + "loss": 1.3373, + "theoretical_loss": 3.3908145509985133, + "tokens_seen": 2558314496 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018179338014042126, + "loss": 1.2736, + "theoretical_loss": 3.390807712283401, + "tokens_seen": 2558380032 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018177733199598796, + "loss": 1.3084, + "theoretical_loss": 3.3908008737925184, + "tokens_seen": 2558445568 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018176128385155466, + "loss": 1.1598, + "theoretical_loss": 3.3907940355258517, + "tokens_seen": 2558511104 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018174523570712136, + "loss": 1.3083, + "theoretical_loss": 3.3907871974833883, + "tokens_seen": 2558576640 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018172918756268806, + "loss": 1.2847, + "theoretical_loss": 3.3907803596651145, + "tokens_seen": 2558642176 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018171313941825476, + "loss": 1.2195, + "theoretical_loss": 3.390773522071018, + "tokens_seen": 2558707712 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018169709127382146, + "loss": 1.1802, + "theoretical_loss": 3.390766684701085, + "tokens_seen": 2558773248 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001816810431293882, + "loss": 1.2094, + "theoretical_loss": 3.3907598475553025, + "tokens_seen": 2558838784 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001816649949849549, + "loss": 1.179, + "theoretical_loss": 3.390753010633658, + "tokens_seen": 2558904320 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001816489468405216, + "loss": 1.1668, + "theoretical_loss": 3.3907461739361375, + "tokens_seen": 2558969856 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001816328986960883, + "loss": 1.1204, + "theoretical_loss": 3.390739337462729, + "tokens_seen": 2559035392 + }, + { + "epoch": 1.16, + "learning_rate": 0.000181616850551655, + "loss": 1.2029, + "theoretical_loss": 3.3907325012134186, + "tokens_seen": 2559100928 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001816008024072217, + "loss": 1.2193, + "theoretical_loss": 3.390725665188193, + "tokens_seen": 2559166464 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001815847542627884, + "loss": 1.3485, + "theoretical_loss": 3.3907188293870405, + "tokens_seen": 2559232000 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001815687061183551, + "loss": 1.2006, + "theoretical_loss": 3.3907119938099464, + "tokens_seen": 2559297536 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001815526579739218, + "loss": 1.2594, + "theoretical_loss": 3.3907051584568983, + "tokens_seen": 2559363072 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001815366098294885, + "loss": 1.1363, + "theoretical_loss": 3.3906983233278836, + "tokens_seen": 2559428608 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018152056168505517, + "loss": 1.1684, + "theoretical_loss": 3.3906914884228883, + "tokens_seen": 2559494144 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018150451354062187, + "loss": 1.2834, + "theoretical_loss": 3.3906846537419, + "tokens_seen": 2559559680 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018148846539618857, + "loss": 1.2066, + "theoretical_loss": 3.390677819284905, + "tokens_seen": 2559625216 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018147241725175527, + "loss": 1.1868, + "theoretical_loss": 3.390670985051891, + "tokens_seen": 2559690752 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018145636910732197, + "loss": 1.2843, + "theoretical_loss": 3.390664151042844, + "tokens_seen": 2559756288 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018144032096288867, + "loss": 1.2268, + "theoretical_loss": 3.390657317257752, + "tokens_seen": 2559821824 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018142427281845537, + "loss": 1.1108, + "theoretical_loss": 3.3906504836966014, + "tokens_seen": 2559887360 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018140822467402207, + "loss": 1.2788, + "theoretical_loss": 3.3906436503593786, + "tokens_seen": 2559952896 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018139217652958877, + "loss": 1.3052, + "theoretical_loss": 3.3906368172460715, + "tokens_seen": 2560018432 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018137612838515547, + "loss": 1.2499, + "theoretical_loss": 3.3906299843566665, + "tokens_seen": 2560083968 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018136008024072217, + "loss": 1.1823, + "theoretical_loss": 3.390623151691151, + "tokens_seen": 2560149504 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018134403209628887, + "loss": 1.2399, + "theoretical_loss": 3.390616319249511, + "tokens_seen": 2560215040 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018132798395185557, + "loss": 1.2865, + "theoretical_loss": 3.390609487031734, + "tokens_seen": 2560280576 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018131193580742228, + "loss": 1.2572, + "theoretical_loss": 3.390602655037807, + "tokens_seen": 2560346112 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018129588766298898, + "loss": 1.2639, + "theoretical_loss": 3.390595823267717, + "tokens_seen": 2560411648 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018127983951855568, + "loss": 1.2442, + "theoretical_loss": 3.3905889917214505, + "tokens_seen": 2560477184 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018126379137412238, + "loss": 1.2415, + "theoretical_loss": 3.3905821603989947, + "tokens_seen": 2560542720 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018124774322968908, + "loss": 1.2579, + "theoretical_loss": 3.390575329300337, + "tokens_seen": 2560608256 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018123169508525578, + "loss": 1.2222, + "theoretical_loss": 3.3905684984254636, + "tokens_seen": 2560673792 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018121564694082248, + "loss": 1.1789, + "theoretical_loss": 3.3905616677743615, + "tokens_seen": 2560739328 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018119959879638918, + "loss": 1.2195, + "theoretical_loss": 3.3905548373470182, + "tokens_seen": 2560804864 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018118355065195588, + "loss": 1.2462, + "theoretical_loss": 3.3905480071434204, + "tokens_seen": 2560870400 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018116750250752258, + "loss": 1.2673, + "theoretical_loss": 3.3905411771635547, + "tokens_seen": 2560935936 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018115145436308928, + "loss": 1.1731, + "theoretical_loss": 3.3905343474074088, + "tokens_seen": 2561001472 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018113540621865595, + "loss": 1.2284, + "theoretical_loss": 3.3905275178749688, + "tokens_seen": 2561067008 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018111935807422266, + "loss": 1.2324, + "theoretical_loss": 3.390520688566222, + "tokens_seen": 2561132544 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018110330992978936, + "loss": 1.2871, + "theoretical_loss": 3.3905138594811555, + "tokens_seen": 2561198080 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018108726178535606, + "loss": 1.2114, + "theoretical_loss": 3.390507030619756, + "tokens_seen": 2561263616 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018107121364092278, + "loss": 1.0945, + "theoretical_loss": 3.3905002019820105, + "tokens_seen": 2561329152 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018105516549648949, + "loss": 1.2435, + "theoretical_loss": 3.390493373567906, + "tokens_seen": 2561394688 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018103911735205619, + "loss": 1.2938, + "theoretical_loss": 3.39048654537743, + "tokens_seen": 2561460224 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001810230692076229, + "loss": 1.255, + "theoretical_loss": 3.3904797174105683, + "tokens_seen": 2561525760 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001810070210631896, + "loss": 1.1538, + "theoretical_loss": 3.390472889667309, + "tokens_seen": 2561591296 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001809909729187563, + "loss": 1.1898, + "theoretical_loss": 3.390466062147638, + "tokens_seen": 2561656832 + }, + { + "epoch": 1.16, + "learning_rate": 0.000180974924774323, + "loss": 1.2578, + "theoretical_loss": 3.3904592348515434, + "tokens_seen": 2561722368 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001809588766298897, + "loss": 1.2315, + "theoretical_loss": 3.3904524077790112, + "tokens_seen": 2561787904 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001809428284854564, + "loss": 1.2968, + "theoretical_loss": 3.390445580930029, + "tokens_seen": 2561853440 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001809267803410231, + "loss": 1.1861, + "theoretical_loss": 3.3904387543045833, + "tokens_seen": 2561918976 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001809107321965898, + "loss": 1.1677, + "theoretical_loss": 3.390431927902661, + "tokens_seen": 2561984512 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001808946840521565, + "loss": 1.1568, + "theoretical_loss": 3.3904251017242495, + "tokens_seen": 2562050048 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001808786359077232, + "loss": 1.2153, + "theoretical_loss": 3.3904182757693357, + "tokens_seen": 2562115584 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001808625877632899, + "loss": 1.1931, + "theoretical_loss": 3.3904114500379063, + "tokens_seen": 2562181120 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001808465396188566, + "loss": 1.2108, + "theoretical_loss": 3.3904046245299484, + "tokens_seen": 2562246656 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018083049147442327, + "loss": 1.2506, + "theoretical_loss": 3.390397799245449, + "tokens_seen": 2562312192 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018081444332998997, + "loss": 1.2422, + "theoretical_loss": 3.3903909741843954, + "tokens_seen": 2562377728 + }, + { + "epoch": 1.16, + "objective/train/docs_used": 946300, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.250570297241211, + "objective/train/theoretical_loss": 3.3903841493467737, + "objective/train/tokens_used": 2582903264, + "theoretical_loss": 3.3903841493467737, + "tokens_seen": 2562443264 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018079839518555667, + "loss": 1.2506, + "theoretical_loss": 3.3903841493467737, + "tokens_seen": 2562443264 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018078234704112337, + "loss": 1.2007, + "theoretical_loss": 3.3903773247325715, + "tokens_seen": 2562508800 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018076629889669007, + "loss": 1.2751, + "theoretical_loss": 3.390370500341776, + "tokens_seen": 2562574336 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018075025075225677, + "loss": 1.0572, + "theoretical_loss": 3.390363676174373, + "tokens_seen": 2562639872 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018073420260782347, + "loss": 1.2956, + "theoretical_loss": 3.3903568522303513, + "tokens_seen": 2562705408 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018071815446339017, + "loss": 1.1405, + "theoretical_loss": 3.3903500285096966, + "tokens_seen": 2562770944 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018070210631895687, + "loss": 1.2861, + "theoretical_loss": 3.390343205012396, + "tokens_seen": 2562836480 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018068605817452357, + "loss": 1.2365, + "theoretical_loss": 3.3903363817384364, + "tokens_seen": 2562902016 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018067001003009027, + "loss": 1.2484, + "theoretical_loss": 3.3903295586878057, + "tokens_seen": 2562967552 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018065396188565697, + "loss": 1.1789, + "theoretical_loss": 3.3903227358604893, + "tokens_seen": 2563033088 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018063791374122367, + "loss": 1.2155, + "theoretical_loss": 3.390315913256476, + "tokens_seen": 2563098624 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018062186559679038, + "loss": 1.2158, + "theoretical_loss": 3.390309090875751, + "tokens_seen": 2563164160 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018060581745235708, + "loss": 1.2641, + "theoretical_loss": 3.390302268718303, + "tokens_seen": 2563229696 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018058976930792378, + "loss": 1.214, + "theoretical_loss": 3.3902954467841173, + "tokens_seen": 2563295232 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018057372116349048, + "loss": 1.1491, + "theoretical_loss": 3.390288625073182, + "tokens_seen": 2563360768 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018055767301905718, + "loss": 1.1692, + "theoretical_loss": 3.3902818035854843, + "tokens_seen": 2563426304 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018054162487462388, + "loss": 1.3277, + "theoretical_loss": 3.39027498232101, + "tokens_seen": 2563491840 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018052557673019058, + "loss": 1.1688, + "theoretical_loss": 3.390268161279747, + "tokens_seen": 2563557376 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018050952858575728, + "loss": 1.2209, + "theoretical_loss": 3.3902613404616826, + "tokens_seen": 2563622912 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018049348044132398, + "loss": 1.1218, + "theoretical_loss": 3.3902545198668026, + "tokens_seen": 2563688448 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018047743229689068, + "loss": 1.1812, + "theoretical_loss": 3.3902476994950947, + "tokens_seen": 2563753984 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018046138415245738, + "loss": 1.2303, + "theoretical_loss": 3.390240879346546, + "tokens_seen": 2563819520 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018044533600802408, + "loss": 1.338, + "theoretical_loss": 3.3902340594211435, + "tokens_seen": 2563885056 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018042928786359078, + "loss": 1.3011, + "theoretical_loss": 3.3902272397188735, + "tokens_seen": 2563950592 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018041323971915748, + "loss": 1.1693, + "theoretical_loss": 3.390220420239724, + "tokens_seen": 2564016128 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018039719157472418, + "loss": 1.2794, + "theoretical_loss": 3.3902136009836816, + "tokens_seen": 2564081664 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018038114343029088, + "loss": 1.2329, + "theoretical_loss": 3.390206781950733, + "tokens_seen": 2564147200 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018036509528585759, + "loss": 1.1828, + "theoretical_loss": 3.3901999631408657, + "tokens_seen": 2564212736 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018034904714142429, + "loss": 1.3166, + "theoretical_loss": 3.390193144554066, + "tokens_seen": 2564278272 + }, + { + "epoch": 1.16, + "learning_rate": 0.000180332998996991, + "loss": 1.1993, + "theoretical_loss": 3.390186326190322, + "tokens_seen": 2564343808 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001803169508525577, + "loss": 1.2988, + "theoretical_loss": 3.390179508049619, + "tokens_seen": 2564409344 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001803009027081244, + "loss": 1.2524, + "theoretical_loss": 3.3901726901319456, + "tokens_seen": 2564474880 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001802848545636911, + "loss": 1.1891, + "theoretical_loss": 3.3901658724372883, + "tokens_seen": 2564540416 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001802688064192578, + "loss": 1.2113, + "theoretical_loss": 3.390159054965634, + "tokens_seen": 2564605952 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001802527582748245, + "loss": 1.1947, + "theoretical_loss": 3.3901522377169697, + "tokens_seen": 2564671488 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001802367101303912, + "loss": 1.1657, + "theoretical_loss": 3.3901454206912827, + "tokens_seen": 2564737024 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001802206619859579, + "loss": 1.1468, + "theoretical_loss": 3.3901386038885595, + "tokens_seen": 2564802560 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001802046138415246, + "loss": 1.2678, + "theoretical_loss": 3.3901317873087873, + "tokens_seen": 2564868096 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001801885656970913, + "loss": 1.2437, + "theoretical_loss": 3.390124970951953, + "tokens_seen": 2564933632 + }, + { + "epoch": 1.16, + "learning_rate": 0.000180172517552658, + "loss": 1.1974, + "theoretical_loss": 3.390118154818045, + "tokens_seen": 2564999168 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001801564694082247, + "loss": 1.2706, + "theoretical_loss": 3.390111338907048, + "tokens_seen": 2565064704 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001801404212637914, + "loss": 1.292, + "theoretical_loss": 3.3901045232189504, + "tokens_seen": 2565130240 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018012437311935807, + "loss": 1.2541, + "theoretical_loss": 3.390097707753739, + "tokens_seen": 2565195776 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018010832497492477, + "loss": 1.2322, + "theoretical_loss": 3.3900908925114006, + "tokens_seen": 2565261312 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018009227683049147, + "loss": 1.1783, + "theoretical_loss": 3.3900840774919225, + "tokens_seen": 2565326848 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018007622868605817, + "loss": 1.2671, + "theoretical_loss": 3.3900772626952915, + "tokens_seen": 2565392384 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018006018054162487, + "loss": 1.2092, + "theoretical_loss": 3.390070448121495, + "tokens_seen": 2565457920 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018004413239719157, + "loss": 1.1608, + "theoretical_loss": 3.3900636337705197, + "tokens_seen": 2565523456 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018002808425275827, + "loss": 1.1749, + "theoretical_loss": 3.3900568196423526, + "tokens_seen": 2565588992 + }, + { + "epoch": 1.16, + "learning_rate": 0.00018001203610832497, + "loss": 1.1814, + "theoretical_loss": 3.3900500057369807, + "tokens_seen": 2565654528 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017999598796389167, + "loss": 1.2163, + "theoretical_loss": 3.390043192054391, + "tokens_seen": 2565720064 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017997993981945837, + "loss": 1.1678, + "theoretical_loss": 3.390036378594571, + "tokens_seen": 2565785600 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017996389167502507, + "loss": 1.3126, + "theoretical_loss": 3.390029565357507, + "tokens_seen": 2565851136 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017994784353059177, + "loss": 1.1822, + "theoretical_loss": 3.3900227523431865, + "tokens_seen": 2565916672 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017993179538615848, + "loss": 1.1731, + "theoretical_loss": 3.3900159395515965, + "tokens_seen": 2565982208 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017991574724172518, + "loss": 1.2436, + "theoretical_loss": 3.390009126982724, + "tokens_seen": 2566047744 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017989969909729188, + "loss": 1.2782, + "theoretical_loss": 3.3900023146365563, + "tokens_seen": 2566113280 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017988365095285858, + "loss": 1.199, + "theoretical_loss": 3.38999550251308, + "tokens_seen": 2566178816 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001798676028084253, + "loss": 1.2486, + "theoretical_loss": 3.389988690612282, + "tokens_seen": 2566244352 + }, + { + "epoch": 1.16, + "learning_rate": 0.000179851554663992, + "loss": 1.3191, + "theoretical_loss": 3.389981878934149, + "tokens_seen": 2566309888 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001798355065195587, + "loss": 1.2238, + "theoretical_loss": 3.3899750674786695, + "tokens_seen": 2566375424 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001798194583751254, + "loss": 1.1894, + "theoretical_loss": 3.3899682562458295, + "tokens_seen": 2566440960 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017980341023069208, + "loss": 1.2654, + "theoretical_loss": 3.3899614452356164, + "tokens_seen": 2566506496 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017978736208625878, + "loss": 1.2092, + "theoretical_loss": 3.3899546344480167, + "tokens_seen": 2566572032 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017977131394182548, + "loss": 1.1732, + "theoretical_loss": 3.389947823883018, + "tokens_seen": 2566637568 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017975526579739218, + "loss": 1.2119, + "theoretical_loss": 3.3899410135406067, + "tokens_seen": 2566703104 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017973921765295888, + "loss": 1.1489, + "theoretical_loss": 3.3899342034207707, + "tokens_seen": 2566768640 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017972316950852558, + "loss": 1.2211, + "theoretical_loss": 3.3899273935234966, + "tokens_seen": 2566834176 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017970712136409228, + "loss": 1.2554, + "theoretical_loss": 3.389920583848771, + "tokens_seen": 2566899712 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017969107321965898, + "loss": 1.2329, + "theoretical_loss": 3.389913774396582, + "tokens_seen": 2566965248 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017967502507522569, + "loss": 1.1761, + "theoretical_loss": 3.3899069651669156, + "tokens_seen": 2567030784 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017965897693079239, + "loss": 1.1669, + "theoretical_loss": 3.3899001561597593, + "tokens_seen": 2567096320 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001796429287863591, + "loss": 1.265, + "theoretical_loss": 3.3898933473751005, + "tokens_seen": 2567161856 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001796268806419258, + "loss": 1.2792, + "theoretical_loss": 3.3898865388129256, + "tokens_seen": 2567227392 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001796108324974925, + "loss": 1.3447, + "theoretical_loss": 3.389879730473222, + "tokens_seen": 2567292928 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001795947843530592, + "loss": 1.1945, + "theoretical_loss": 3.389872922355977, + "tokens_seen": 2567358464 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001795787362086259, + "loss": 1.1828, + "theoretical_loss": 3.3898661144611766, + "tokens_seen": 2567424000 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001795626880641926, + "loss": 1.1434, + "theoretical_loss": 3.389859306788809, + "tokens_seen": 2567489536 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001795466399197593, + "loss": 1.3061, + "theoretical_loss": 3.3898524993388612, + "tokens_seen": 2567555072 + }, + { + "epoch": 1.16, + "learning_rate": 0.000179530591775326, + "loss": 1.1876, + "theoretical_loss": 3.3898456921113196, + "tokens_seen": 2567620608 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001795145436308927, + "loss": 1.2504, + "theoretical_loss": 3.3898388851061716, + "tokens_seen": 2567686144 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001794984954864594, + "loss": 1.2762, + "theoretical_loss": 3.389832078323404, + "tokens_seen": 2567751680 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001794824473420261, + "loss": 1.2423, + "theoretical_loss": 3.389825271763004, + "tokens_seen": 2567817216 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001794663991975928, + "loss": 1.1592, + "theoretical_loss": 3.3898184654249595, + "tokens_seen": 2567882752 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001794503510531595, + "loss": 1.1761, + "theoretical_loss": 3.3898116593092564, + "tokens_seen": 2567948288 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001794343029087262, + "loss": 1.2309, + "theoretical_loss": 3.389804853415882, + "tokens_seen": 2568013824 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017941825476429287, + "loss": 1.2883, + "theoretical_loss": 3.3897980477448235, + "tokens_seen": 2568079360 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017940220661985957, + "loss": 1.2083, + "theoretical_loss": 3.3897912422960683, + "tokens_seen": 2568144896 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017938615847542627, + "loss": 1.1685, + "theoretical_loss": 3.389784437069603, + "tokens_seen": 2568210432 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017937011033099297, + "loss": 1.3019, + "theoretical_loss": 3.3897776320654147, + "tokens_seen": 2568275968 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017935406218655967, + "loss": 1.261, + "theoretical_loss": 3.389770827283491, + "tokens_seen": 2568341504 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017933801404212637, + "loss": 1.0566, + "theoretical_loss": 3.389764022723818, + "tokens_seen": 2568407040 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017932196589769307, + "loss": 1.2816, + "theoretical_loss": 3.389757218386384, + "tokens_seen": 2568472576 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017930591775325977, + "loss": 1.2221, + "theoretical_loss": 3.389750414271175, + "tokens_seen": 2568538112 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017928986960882647, + "loss": 1.2275, + "theoretical_loss": 3.3897436103781784, + "tokens_seen": 2568603648 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001792738214643932, + "loss": 1.2077, + "theoretical_loss": 3.3897368067073814, + "tokens_seen": 2568669184 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001792577733199599, + "loss": 1.2595, + "theoretical_loss": 3.389730003258771, + "tokens_seen": 2568734720 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001792417251755266, + "loss": 1.1128, + "theoretical_loss": 3.3897232000323347, + "tokens_seen": 2568800256 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001792256770310933, + "loss": 1.2581, + "theoretical_loss": 3.389716397028059, + "tokens_seen": 2568865792 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017920962888666, + "loss": 1.3185, + "theoretical_loss": 3.389709594245931, + "tokens_seen": 2568931328 + }, + { + "epoch": 1.16, + "objective/train/docs_used": 948560, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.149373173713684, + "objective/train/theoretical_loss": 3.389702791685938, + "objective/train/tokens_used": 2589456864, + "theoretical_loss": 3.389702791685938, + "tokens_seen": 2568996864 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001791935807422267, + "loss": 1.1494, + "theoretical_loss": 3.389702791685938, + "tokens_seen": 2568996864 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001791775325977934, + "loss": 1.1976, + "theoretical_loss": 3.389695989348067, + "tokens_seen": 2569062400 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001791614844533601, + "loss": 1.2038, + "theoretical_loss": 3.389689187232305, + "tokens_seen": 2569127936 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001791454363089268, + "loss": 1.2849, + "theoretical_loss": 3.3896823853386393, + "tokens_seen": 2569193472 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001791293881644935, + "loss": 1.2709, + "theoretical_loss": 3.389675583667057, + "tokens_seen": 2569259008 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017911334002006018, + "loss": 1.2497, + "theoretical_loss": 3.389668782217545, + "tokens_seen": 2569324544 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017909729187562688, + "loss": 1.2469, + "theoretical_loss": 3.3896619809900903, + "tokens_seen": 2569390080 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017908124373119358, + "loss": 1.2214, + "theoretical_loss": 3.3896551799846804, + "tokens_seen": 2569455616 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017906519558676028, + "loss": 1.1877, + "theoretical_loss": 3.3896483792013017, + "tokens_seen": 2569521152 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017904914744232698, + "loss": 1.2797, + "theoretical_loss": 3.389641578639942, + "tokens_seen": 2569586688 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017903309929789368, + "loss": 1.2731, + "theoretical_loss": 3.389634778300588, + "tokens_seen": 2569652224 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017901705115346038, + "loss": 1.2008, + "theoretical_loss": 3.389627978183227, + "tokens_seen": 2569717760 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017900100300902709, + "loss": 1.292, + "theoretical_loss": 3.389621178287846, + "tokens_seen": 2569783296 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017898495486459379, + "loss": 1.2853, + "theoretical_loss": 3.3896143786144317, + "tokens_seen": 2569848832 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001789689067201605, + "loss": 1.0676, + "theoretical_loss": 3.3896075791629716, + "tokens_seen": 2569914368 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001789528585757272, + "loss": 1.2124, + "theoretical_loss": 3.389600779933453, + "tokens_seen": 2569979904 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001789368104312939, + "loss": 1.259, + "theoretical_loss": 3.389593980925863, + "tokens_seen": 2570045440 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001789207622868606, + "loss": 1.1756, + "theoretical_loss": 3.389587182140188, + "tokens_seen": 2570110976 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001789047141424273, + "loss": 1.2054, + "theoretical_loss": 3.3895803835764156, + "tokens_seen": 2570176512 + }, + { + "epoch": 1.16, + "learning_rate": 0.000178888665997994, + "loss": 1.339, + "theoretical_loss": 3.3895735852345332, + "tokens_seen": 2570242048 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001788726178535607, + "loss": 1.1023, + "theoretical_loss": 3.389566787114527, + "tokens_seen": 2570307584 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001788565697091274, + "loss": 1.1937, + "theoretical_loss": 3.389559989216385, + "tokens_seen": 2570373120 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001788405215646941, + "loss": 1.1588, + "theoretical_loss": 3.389553191540094, + "tokens_seen": 2570438656 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001788244734202608, + "loss": 1.2349, + "theoretical_loss": 3.3895463940856407, + "tokens_seen": 2570504192 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001788084252758275, + "loss": 1.264, + "theoretical_loss": 3.389539596853013, + "tokens_seen": 2570569728 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001787923771313942, + "loss": 1.1922, + "theoretical_loss": 3.3895327998421974, + "tokens_seen": 2570635264 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001787763289869609, + "loss": 1.1862, + "theoretical_loss": 3.389526003053181, + "tokens_seen": 2570700800 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001787602808425276, + "loss": 1.219, + "theoretical_loss": 3.3895192064859514, + "tokens_seen": 2570766336 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001787442326980943, + "loss": 1.2586, + "theoretical_loss": 3.389512410140495, + "tokens_seen": 2570831872 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017872818455366097, + "loss": 1.2381, + "theoretical_loss": 3.3895056140168, + "tokens_seen": 2570897408 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017871213640922767, + "loss": 1.2847, + "theoretical_loss": 3.389498818114852, + "tokens_seen": 2570962944 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017869608826479437, + "loss": 1.1902, + "theoretical_loss": 3.389492022434639, + "tokens_seen": 2571028480 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017868004012036107, + "loss": 1.2918, + "theoretical_loss": 3.3894852269761486, + "tokens_seen": 2571094016 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001786639919759278, + "loss": 1.1032, + "theoretical_loss": 3.389478431739367, + "tokens_seen": 2571159552 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001786479438314945, + "loss": 1.2743, + "theoretical_loss": 3.3894716367242816, + "tokens_seen": 2571225088 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001786318956870612, + "loss": 1.3503, + "theoretical_loss": 3.38946484193088, + "tokens_seen": 2571290624 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001786158475426279, + "loss": 1.2659, + "theoretical_loss": 3.3894580473591485, + "tokens_seen": 2571356160 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001785997993981946, + "loss": 1.1938, + "theoretical_loss": 3.3894512530090743, + "tokens_seen": 2571421696 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001785837512537613, + "loss": 1.1901, + "theoretical_loss": 3.3894444588806456, + "tokens_seen": 2571487232 + }, + { + "epoch": 1.16, + "learning_rate": 0.000178567703109328, + "loss": 1.1207, + "theoretical_loss": 3.3894376649738485, + "tokens_seen": 2571552768 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001785516549648947, + "loss": 1.2013, + "theoretical_loss": 3.38943087128867, + "tokens_seen": 2571618304 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001785356068204614, + "loss": 1.1792, + "theoretical_loss": 3.3894240778250984, + "tokens_seen": 2571683840 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001785195586760281, + "loss": 1.3441, + "theoretical_loss": 3.389417284583119, + "tokens_seen": 2571749376 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001785035105315948, + "loss": 1.1272, + "theoretical_loss": 3.389410491562721, + "tokens_seen": 2571814912 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001784874623871615, + "loss": 1.259, + "theoretical_loss": 3.38940369876389, + "tokens_seen": 2571880448 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001784714142427282, + "loss": 1.2025, + "theoretical_loss": 3.3893969061866134, + "tokens_seen": 2571945984 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001784553660982949, + "loss": 1.273, + "theoretical_loss": 3.3893901138308786, + "tokens_seen": 2572011520 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001784393179538616, + "loss": 1.1861, + "theoretical_loss": 3.389383321696673, + "tokens_seen": 2572077056 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001784232698094283, + "loss": 1.1832, + "theoretical_loss": 3.3893765297839833, + "tokens_seen": 2572142592 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017840722166499498, + "loss": 1.2162, + "theoretical_loss": 3.3893697380927965, + "tokens_seen": 2572208128 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017839117352056168, + "loss": 1.3886, + "theoretical_loss": 3.3893629466231, + "tokens_seen": 2572273664 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017837512537612838, + "loss": 1.2324, + "theoretical_loss": 3.389356155374881, + "tokens_seen": 2572339200 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017835907723169508, + "loss": 1.153, + "theoretical_loss": 3.3893493643481265, + "tokens_seen": 2572404736 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017834302908726178, + "loss": 1.1819, + "theoretical_loss": 3.389342573542824, + "tokens_seen": 2572470272 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017832698094282848, + "loss": 1.2438, + "theoretical_loss": 3.38933578295896, + "tokens_seen": 2572535808 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017831093279839519, + "loss": 1.2489, + "theoretical_loss": 3.3893289925965218, + "tokens_seen": 2572601344 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017829488465396189, + "loss": 1.2707, + "theoretical_loss": 3.3893222024554968, + "tokens_seen": 2572666880 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001782788365095286, + "loss": 1.1542, + "theoretical_loss": 3.389315412535872, + "tokens_seen": 2572732416 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001782627883650953, + "loss": 1.135, + "theoretical_loss": 3.3893086228376346, + "tokens_seen": 2572797952 + }, + { + "epoch": 1.16, + "learning_rate": 0.000178246740220662, + "loss": 1.3316, + "theoretical_loss": 3.389301833360772, + "tokens_seen": 2572863488 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001782306920762287, + "loss": 1.1597, + "theoretical_loss": 3.3892950441052707, + "tokens_seen": 2572929024 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001782146439317954, + "loss": 1.3166, + "theoretical_loss": 3.3892882550711185, + "tokens_seen": 2572994560 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001781985957873621, + "loss": 1.1909, + "theoretical_loss": 3.3892814662583017, + "tokens_seen": 2573060096 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001781825476429288, + "loss": 1.1882, + "theoretical_loss": 3.3892746776668083, + "tokens_seen": 2573125632 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001781664994984955, + "loss": 1.1173, + "theoretical_loss": 3.3892678892966255, + "tokens_seen": 2573191168 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001781504513540622, + "loss": 1.2148, + "theoretical_loss": 3.3892611011477394, + "tokens_seen": 2573256704 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001781344032096289, + "loss": 1.1598, + "theoretical_loss": 3.3892543132201385, + "tokens_seen": 2573322240 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001781183550651956, + "loss": 1.2284, + "theoretical_loss": 3.389247525513809, + "tokens_seen": 2573387776 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001781023069207623, + "loss": 1.2449, + "theoretical_loss": 3.389240738028738, + "tokens_seen": 2573453312 + }, + { + "epoch": 1.16, + "learning_rate": 0.000178086258776329, + "loss": 1.3157, + "theoretical_loss": 3.389233950764914, + "tokens_seen": 2573518848 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001780702106318957, + "loss": 1.1598, + "theoretical_loss": 3.389227163722322, + "tokens_seen": 2573584384 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001780541624874624, + "loss": 1.2362, + "theoretical_loss": 3.389220376900951, + "tokens_seen": 2573649920 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001780381143430291, + "loss": 1.2559, + "theoretical_loss": 3.389213590300787, + "tokens_seen": 2573715456 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001780220661985958, + "loss": 1.1396, + "theoretical_loss": 3.3892068039218177, + "tokens_seen": 2573780992 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001780060180541625, + "loss": 1.2409, + "theoretical_loss": 3.3892000177640305, + "tokens_seen": 2573846528 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001779899699097292, + "loss": 1.3117, + "theoretical_loss": 3.389193231827412, + "tokens_seen": 2573912064 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001779739217652959, + "loss": 1.184, + "theoretical_loss": 3.3891864461119496, + "tokens_seen": 2573977600 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001779578736208626, + "loss": 1.2317, + "theoretical_loss": 3.3891796606176308, + "tokens_seen": 2574043136 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001779418254764293, + "loss": 1.1646, + "theoretical_loss": 3.389172875344442, + "tokens_seen": 2574108672 + }, + { + "epoch": 1.16, + "learning_rate": 0.000177925777331996, + "loss": 1.2377, + "theoretical_loss": 3.389166090292371, + "tokens_seen": 2574174208 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001779097291875627, + "loss": 1.2267, + "theoretical_loss": 3.389159305461405, + "tokens_seen": 2574239744 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001778936810431294, + "loss": 1.2603, + "theoretical_loss": 3.38915252085153, + "tokens_seen": 2574305280 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001778776328986961, + "loss": 1.2116, + "theoretical_loss": 3.389145736462735, + "tokens_seen": 2574370816 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001778615847542628, + "loss": 1.2139, + "theoretical_loss": 3.389138952295006, + "tokens_seen": 2574436352 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001778455366098295, + "loss": 1.2207, + "theoretical_loss": 3.38913216834833, + "tokens_seen": 2574501888 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001778294884653962, + "loss": 1.1468, + "theoretical_loss": 3.3891253846226954, + "tokens_seen": 2574567424 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001778134403209629, + "loss": 1.1741, + "theoretical_loss": 3.389118601118088, + "tokens_seen": 2574632960 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001777973921765296, + "loss": 1.158, + "theoretical_loss": 3.3891118178344954, + "tokens_seen": 2574698496 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001777813440320963, + "loss": 1.1798, + "theoretical_loss": 3.389105034771905, + "tokens_seen": 2574764032 + }, + { + "epoch": 1.16, + "learning_rate": 0.000177765295887663, + "loss": 1.1237, + "theoretical_loss": 3.3890982519303043, + "tokens_seen": 2574829568 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001777492477432297, + "loss": 1.2104, + "theoretical_loss": 3.3890914693096796, + "tokens_seen": 2574895104 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001777331995987964, + "loss": 1.2295, + "theoretical_loss": 3.389084686910019, + "tokens_seen": 2574960640 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001777171514543631, + "loss": 1.3218, + "theoretical_loss": 3.389077904731309, + "tokens_seen": 2575026176 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017770110330992978, + "loss": 1.2411, + "theoretical_loss": 3.389071122773537, + "tokens_seen": 2575091712 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017768505516549648, + "loss": 1.2335, + "theoretical_loss": 3.38906434103669, + "tokens_seen": 2575157248 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017766900702106318, + "loss": 1.284, + "theoretical_loss": 3.3890575595207553, + "tokens_seen": 2575222784 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017765295887662988, + "loss": 1.1905, + "theoretical_loss": 3.3890507782257204, + "tokens_seen": 2575288320 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017763691073219658, + "loss": 1.2113, + "theoretical_loss": 3.389043997151572, + "tokens_seen": 2575353856 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017762086258776329, + "loss": 1.1702, + "theoretical_loss": 3.389037216298298, + "tokens_seen": 2575419392 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017760481444332999, + "loss": 1.235, + "theoretical_loss": 3.3890304356658847, + "tokens_seen": 2575484928 + }, + { + "epoch": 1.16, + "objective/train/docs_used": 950738, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1043345928192139, + "objective/train/theoretical_loss": 3.3890236552543196, + "objective/train/tokens_used": 2596010464, + "theoretical_loss": 3.3890236552543196, + "tokens_seen": 2575550464 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001775887662988967, + "loss": 1.1043, + "theoretical_loss": 3.3890236552543196, + "tokens_seen": 2575550464 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001775727181544634, + "loss": 1.1824, + "theoretical_loss": 3.3890168750635903, + "tokens_seen": 2575616000 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001775566700100301, + "loss": 1.1578, + "theoretical_loss": 3.389010095093683, + "tokens_seen": 2575681536 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001775406218655968, + "loss": 1.2334, + "theoretical_loss": 3.389003315344586, + "tokens_seen": 2575747072 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001775245737211635, + "loss": 1.208, + "theoretical_loss": 3.388996535816286, + "tokens_seen": 2575812608 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001775085255767302, + "loss": 1.2466, + "theoretical_loss": 3.3889897565087703, + "tokens_seen": 2575878144 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001774924774322969, + "loss": 1.2199, + "theoretical_loss": 3.3889829774220264, + "tokens_seen": 2575943680 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017747642928786362, + "loss": 1.1235, + "theoretical_loss": 3.3889761985560405, + "tokens_seen": 2576009216 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017746038114343032, + "loss": 1.2753, + "theoretical_loss": 3.3889694199108007, + "tokens_seen": 2576074752 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017744433299899702, + "loss": 1.2658, + "theoretical_loss": 3.3889626414862937, + "tokens_seen": 2576140288 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017742828485456372, + "loss": 1.1266, + "theoretical_loss": 3.388955863282507, + "tokens_seen": 2576205824 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017741223671013042, + "loss": 1.171, + "theoretical_loss": 3.3889490852994273, + "tokens_seen": 2576271360 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001773961885656971, + "loss": 1.1446, + "theoretical_loss": 3.388942307537043, + "tokens_seen": 2576336896 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001773801404212638, + "loss": 1.1974, + "theoretical_loss": 3.38893552999534, + "tokens_seen": 2576402432 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001773640922768305, + "loss": 1.1889, + "theoretical_loss": 3.388928752674306, + "tokens_seen": 2576467968 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001773480441323972, + "loss": 1.2927, + "theoretical_loss": 3.388921975573928, + "tokens_seen": 2576533504 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001773319959879639, + "loss": 1.2174, + "theoretical_loss": 3.388915198694194, + "tokens_seen": 2576599040 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001773159478435306, + "loss": 1.0474, + "theoretical_loss": 3.3889084220350902, + "tokens_seen": 2576664576 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001772998996990973, + "loss": 1.2403, + "theoretical_loss": 3.3889016455966043, + "tokens_seen": 2576730112 + }, + { + "epoch": 1.16, + "learning_rate": 0.000177283851554664, + "loss": 1.2286, + "theoretical_loss": 3.3888948693787233, + "tokens_seen": 2576795648 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001772678034102307, + "loss": 1.3149, + "theoretical_loss": 3.3888880933814347, + "tokens_seen": 2576861184 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001772517552657974, + "loss": 1.1003, + "theoretical_loss": 3.3888813176047257, + "tokens_seen": 2576926720 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001772357071213641, + "loss": 1.2653, + "theoretical_loss": 3.3888745420485833, + "tokens_seen": 2576992256 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001772196589769308, + "loss": 1.2399, + "theoretical_loss": 3.3888677667129947, + "tokens_seen": 2577057792 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001772036108324975, + "loss": 1.2415, + "theoretical_loss": 3.388860991597947, + "tokens_seen": 2577123328 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001771875626880642, + "loss": 1.2191, + "theoretical_loss": 3.388854216703428, + "tokens_seen": 2577188864 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001771715145436309, + "loss": 1.269, + "theoretical_loss": 3.388847442029424, + "tokens_seen": 2577254400 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001771554663991976, + "loss": 1.2742, + "theoretical_loss": 3.3888406675759226, + "tokens_seen": 2577319936 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001771394182547643, + "loss": 1.2089, + "theoretical_loss": 3.3888338933429116, + "tokens_seen": 2577385472 + }, + { + "epoch": 1.16, + "learning_rate": 0.000177123370110331, + "loss": 1.247, + "theoretical_loss": 3.3888271193303776, + "tokens_seen": 2577451008 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001771073219658977, + "loss": 1.2098, + "theoretical_loss": 3.388820345538308, + "tokens_seen": 2577516544 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001770912738214644, + "loss": 1.1889, + "theoretical_loss": 3.38881357196669, + "tokens_seen": 2577582080 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001770752256770311, + "loss": 1.2533, + "theoretical_loss": 3.3888067986155104, + "tokens_seen": 2577647616 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001770591775325978, + "loss": 1.1983, + "theoretical_loss": 3.3888000254847572, + "tokens_seen": 2577713152 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001770431293881645, + "loss": 1.3011, + "theoretical_loss": 3.3887932525744175, + "tokens_seen": 2577778688 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001770270812437312, + "loss": 1.1373, + "theoretical_loss": 3.388786479884478, + "tokens_seen": 2577844224 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017701103309929788, + "loss": 1.169, + "theoretical_loss": 3.388779707414926, + "tokens_seen": 2577909760 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017699498495486458, + "loss": 1.1784, + "theoretical_loss": 3.3887729351657496, + "tokens_seen": 2577975296 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017697893681043128, + "loss": 1.2522, + "theoretical_loss": 3.3887661631369346, + "tokens_seen": 2578040832 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017696288866599798, + "loss": 1.1794, + "theoretical_loss": 3.3887593913284695, + "tokens_seen": 2578106368 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017694684052156469, + "loss": 1.1721, + "theoretical_loss": 3.3887526197403406, + "tokens_seen": 2578171904 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017693079237713139, + "loss": 1.1631, + "theoretical_loss": 3.388745848372536, + "tokens_seen": 2578237440 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017691474423269809, + "loss": 1.309, + "theoretical_loss": 3.388739077225042, + "tokens_seen": 2578302976 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001768986960882648, + "loss": 1.2385, + "theoretical_loss": 3.3887323062978467, + "tokens_seen": 2578368512 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001768826479438315, + "loss": 1.1879, + "theoretical_loss": 3.388725535590937, + "tokens_seen": 2578434048 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017686659979939822, + "loss": 1.1843, + "theoretical_loss": 3.3887187651043, + "tokens_seen": 2578499584 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017685055165496492, + "loss": 1.273, + "theoretical_loss": 3.3887119948379225, + "tokens_seen": 2578565120 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017683450351053162, + "loss": 1.1547, + "theoretical_loss": 3.3887052247917926, + "tokens_seen": 2578630656 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017681845536609832, + "loss": 1.2427, + "theoretical_loss": 3.388698454965897, + "tokens_seen": 2578696192 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017680240722166502, + "loss": 1.155, + "theoretical_loss": 3.3886916853602234, + "tokens_seen": 2578761728 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017678635907723172, + "loss": 1.227, + "theoretical_loss": 3.3886849159747587, + "tokens_seen": 2578827264 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017677031093279842, + "loss": 1.2267, + "theoretical_loss": 3.38867814680949, + "tokens_seen": 2578892800 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017675426278836512, + "loss": 1.274, + "theoretical_loss": 3.388671377864405, + "tokens_seen": 2578958336 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017673821464393182, + "loss": 1.2173, + "theoretical_loss": 3.3886646091394907, + "tokens_seen": 2579023872 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017672216649949852, + "loss": 1.1838, + "theoretical_loss": 3.3886578406347345, + "tokens_seen": 2579089408 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017670611835506522, + "loss": 1.1292, + "theoretical_loss": 3.388651072350123, + "tokens_seen": 2579154944 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001766900702106319, + "loss": 1.204, + "theoretical_loss": 3.388644304285644, + "tokens_seen": 2579220480 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001766740220661986, + "loss": 1.2563, + "theoretical_loss": 3.388637536441285, + "tokens_seen": 2579286016 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001766579739217653, + "loss": 1.1433, + "theoretical_loss": 3.3886307688170327, + "tokens_seen": 2579351552 + }, + { + "epoch": 1.16, + "learning_rate": 0.000176641925777332, + "loss": 1.1913, + "theoretical_loss": 3.3886240014128743, + "tokens_seen": 2579417088 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001766258776328987, + "loss": 1.245, + "theoretical_loss": 3.388617234228798, + "tokens_seen": 2579482624 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001766098294884654, + "loss": 1.1278, + "theoretical_loss": 3.38861046726479, + "tokens_seen": 2579548160 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001765937813440321, + "loss": 1.2103, + "theoretical_loss": 3.388603700520838, + "tokens_seen": 2579613696 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001765777331995988, + "loss": 1.138, + "theoretical_loss": 3.3885969339969293, + "tokens_seen": 2579679232 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001765616850551655, + "loss": 1.2585, + "theoretical_loss": 3.3885901676930503, + "tokens_seen": 2579744768 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001765456369107322, + "loss": 1.135, + "theoretical_loss": 3.3885834016091896, + "tokens_seen": 2579810304 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001765295887662989, + "loss": 1.1629, + "theoretical_loss": 3.3885766357453337, + "tokens_seen": 2579875840 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001765135406218656, + "loss": 1.1576, + "theoretical_loss": 3.3885698701014704, + "tokens_seen": 2579941376 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001764974924774323, + "loss": 1.2326, + "theoretical_loss": 3.388563104677586, + "tokens_seen": 2580006912 + }, + { + "epoch": 1.16, + "learning_rate": 0.000176481444332999, + "loss": 1.1181, + "theoretical_loss": 3.3885563394736686, + "tokens_seen": 2580072448 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001764653961885657, + "loss": 1.2132, + "theoretical_loss": 3.388549574489705, + "tokens_seen": 2580137984 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001764493480441324, + "loss": 1.2611, + "theoretical_loss": 3.388542809725683, + "tokens_seen": 2580203520 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001764332998996991, + "loss": 1.2759, + "theoretical_loss": 3.3885360451815894, + "tokens_seen": 2580269056 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001764172517552658, + "loss": 1.1962, + "theoretical_loss": 3.3885292808574112, + "tokens_seen": 2580334592 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001764012036108325, + "loss": 1.2611, + "theoretical_loss": 3.3885225167531363, + "tokens_seen": 2580400128 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001763851554663992, + "loss": 1.2473, + "theoretical_loss": 3.3885157528687517, + "tokens_seen": 2580465664 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001763691073219659, + "loss": 1.1882, + "theoretical_loss": 3.388508989204245, + "tokens_seen": 2580531200 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001763530591775326, + "loss": 1.2217, + "theoretical_loss": 3.388502225759603, + "tokens_seen": 2580596736 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001763370110330993, + "loss": 1.1946, + "theoretical_loss": 3.388495462534813, + "tokens_seen": 2580662272 + }, + { + "epoch": 1.16, + "learning_rate": 0.000176320962888666, + "loss": 1.2247, + "theoretical_loss": 3.388488699529862, + "tokens_seen": 2580727808 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017630491474423268, + "loss": 1.3672, + "theoretical_loss": 3.388481936744738, + "tokens_seen": 2580793344 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017628886659979938, + "loss": 1.2518, + "theoretical_loss": 3.388475174179428, + "tokens_seen": 2580858880 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001762728184553661, + "loss": 1.305, + "theoretical_loss": 3.388468411833919, + "tokens_seen": 2580924416 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001762567703109328, + "loss": 1.2392, + "theoretical_loss": 3.3884616497081987, + "tokens_seen": 2580989952 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001762407221664995, + "loss": 1.2692, + "theoretical_loss": 3.388454887802254, + "tokens_seen": 2581055488 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017622467402206621, + "loss": 1.1717, + "theoretical_loss": 3.388448126116072, + "tokens_seen": 2581121024 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017620862587763291, + "loss": 1.0989, + "theoretical_loss": 3.388441364649641, + "tokens_seen": 2581186560 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017619257773319961, + "loss": 1.1663, + "theoretical_loss": 3.388434603402947, + "tokens_seen": 2581252096 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017617652958876632, + "loss": 1.1909, + "theoretical_loss": 3.388427842375978, + "tokens_seen": 2581317632 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017616048144433302, + "loss": 1.2616, + "theoretical_loss": 3.3884210815687212, + "tokens_seen": 2581383168 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017614443329989972, + "loss": 1.1623, + "theoretical_loss": 3.388414320981164, + "tokens_seen": 2581448704 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017612838515546642, + "loss": 1.2322, + "theoretical_loss": 3.3884075606132935, + "tokens_seen": 2581514240 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017611233701103312, + "loss": 1.246, + "theoretical_loss": 3.3884008004650967, + "tokens_seen": 2581579776 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017609628886659982, + "loss": 1.2962, + "theoretical_loss": 3.3883940405365616, + "tokens_seen": 2581645312 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017608024072216652, + "loss": 1.312, + "theoretical_loss": 3.3883872808276747, + "tokens_seen": 2581710848 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017606419257773322, + "loss": 1.19, + "theoretical_loss": 3.3883805213384237, + "tokens_seen": 2581776384 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017604814443329992, + "loss": 1.2326, + "theoretical_loss": 3.388373762068796, + "tokens_seen": 2581841920 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017603209628886662, + "loss": 1.2149, + "theoretical_loss": 3.3883670030187787, + "tokens_seen": 2581907456 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017601604814443332, + "loss": 1.2551, + "theoretical_loss": 3.388360244188359, + "tokens_seen": 2581972992 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017600000000000002, + "loss": 1.1511, + "theoretical_loss": 3.3883534855775244, + "tokens_seen": 2582038528 + }, + { + "epoch": 1.16, + "objective/train/docs_used": 953437, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2303626537322998, + "objective/train/theoretical_loss": 3.388346727186262, + "objective/train/tokens_used": 2602564064, + "theoretical_loss": 3.388346727186262, + "tokens_seen": 2582104064 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001759839518555667, + "loss": 1.2304, + "theoretical_loss": 3.388346727186262, + "tokens_seen": 2582104064 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001759679037111334, + "loss": 1.1613, + "theoretical_loss": 3.3883399690145595, + "tokens_seen": 2582169600 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001759518555667001, + "loss": 1.2458, + "theoretical_loss": 3.3883332110624034, + "tokens_seen": 2582235136 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001759358074222668, + "loss": 1.2043, + "theoretical_loss": 3.388326453329782, + "tokens_seen": 2582300672 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001759197592778335, + "loss": 1.2049, + "theoretical_loss": 3.3883196958166817, + "tokens_seen": 2582366208 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001759037111334002, + "loss": 1.3225, + "theoretical_loss": 3.3883129385230903, + "tokens_seen": 2582431744 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001758876629889669, + "loss": 1.1862, + "theoretical_loss": 3.3883061814489954, + "tokens_seen": 2582497280 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001758716148445336, + "loss": 1.1541, + "theoretical_loss": 3.388299424594383, + "tokens_seen": 2582562816 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001758555667001003, + "loss": 1.1136, + "theoretical_loss": 3.388292667959242, + "tokens_seen": 2582628352 + }, + { + "epoch": 1.16, + "learning_rate": 0.000175839518555667, + "loss": 1.119, + "theoretical_loss": 3.388285911543559, + "tokens_seen": 2582693888 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001758234704112337, + "loss": 1.2555, + "theoretical_loss": 3.388279155347321, + "tokens_seen": 2582759424 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001758074222668004, + "loss": 1.3688, + "theoretical_loss": 3.388272399370516, + "tokens_seen": 2582824960 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001757913741223671, + "loss": 1.2634, + "theoretical_loss": 3.3882656436131304, + "tokens_seen": 2582890496 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001757753259779338, + "loss": 1.2077, + "theoretical_loss": 3.3882588880751525, + "tokens_seen": 2582956032 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001757592778335005, + "loss": 1.2804, + "theoretical_loss": 3.388252132756569, + "tokens_seen": 2583021568 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001757432296890672, + "loss": 1.1935, + "theoretical_loss": 3.3882453776573667, + "tokens_seen": 2583087104 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001757271815446339, + "loss": 1.2591, + "theoretical_loss": 3.3882386227775343, + "tokens_seen": 2583152640 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001757111334002006, + "loss": 1.331, + "theoretical_loss": 3.388231868117058, + "tokens_seen": 2583218176 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001756950852557673, + "loss": 1.2, + "theoretical_loss": 3.3882251136759254, + "tokens_seen": 2583283712 + }, + { + "epoch": 1.16, + "learning_rate": 0.000175679037111334, + "loss": 1.2039, + "theoretical_loss": 3.388218359454124, + "tokens_seen": 2583349248 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001756629889669007, + "loss": 1.2781, + "theoretical_loss": 3.388211605451641, + "tokens_seen": 2583414784 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001756469408224674, + "loss": 1.1304, + "theoretical_loss": 3.3882048516684637, + "tokens_seen": 2583480320 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001756308926780341, + "loss": 1.2184, + "theoretical_loss": 3.3881980981045796, + "tokens_seen": 2583545856 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001756148445336008, + "loss": 1.2655, + "theoretical_loss": 3.388191344759975, + "tokens_seen": 2583611392 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001755987963891675, + "loss": 1.2348, + "theoretical_loss": 3.3881845916346394, + "tokens_seen": 2583676928 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001755827482447342, + "loss": 1.1938, + "theoretical_loss": 3.388177838728558, + "tokens_seen": 2583742464 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001755667001003009, + "loss": 1.2184, + "theoretical_loss": 3.388171086041719, + "tokens_seen": 2583808000 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001755506519558676, + "loss": 1.234, + "theoretical_loss": 3.3881643335741094, + "tokens_seen": 2583873536 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017553460381143431, + "loss": 1.1717, + "theoretical_loss": 3.388157581325717, + "tokens_seen": 2583939072 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017551855566700101, + "loss": 1.0969, + "theoretical_loss": 3.3881508292965288, + "tokens_seen": 2584004608 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017550250752256772, + "loss": 1.1507, + "theoretical_loss": 3.3881440774865323, + "tokens_seen": 2584070144 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017548645937813442, + "loss": 1.3194, + "theoretical_loss": 3.3881373258957144, + "tokens_seen": 2584135680 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017547041123370112, + "loss": 1.2, + "theoretical_loss": 3.388130574524063, + "tokens_seen": 2584201216 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017545436308926782, + "loss": 1.2726, + "theoretical_loss": 3.388123823371565, + "tokens_seen": 2584266752 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017543831494483452, + "loss": 1.1746, + "theoretical_loss": 3.3881170724382077, + "tokens_seen": 2584332288 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017542226680040122, + "loss": 1.2855, + "theoretical_loss": 3.388110321723979, + "tokens_seen": 2584397824 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017540621865596792, + "loss": 1.1897, + "theoretical_loss": 3.388103571228866, + "tokens_seen": 2584463360 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017539017051153462, + "loss": 1.4237, + "theoretical_loss": 3.3880968209528555, + "tokens_seen": 2584528896 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017537412236710132, + "loss": 1.1977, + "theoretical_loss": 3.3880900708959354, + "tokens_seen": 2584594432 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017535807422266802, + "loss": 1.3524, + "theoretical_loss": 3.388083321058093, + "tokens_seen": 2584659968 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017534202607823472, + "loss": 1.3111, + "theoretical_loss": 3.3880765714393153, + "tokens_seen": 2584725504 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017532597793380142, + "loss": 1.1726, + "theoretical_loss": 3.3880698220395895, + "tokens_seen": 2584791040 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017530992978936812, + "loss": 1.1994, + "theoretical_loss": 3.3880630728589036, + "tokens_seen": 2584856576 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017529388164493482, + "loss": 1.2716, + "theoretical_loss": 3.3880563238972448, + "tokens_seen": 2584922112 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001752778335005015, + "loss": 1.1422, + "theoretical_loss": 3.3880495751546, + "tokens_seen": 2584987648 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001752617853560682, + "loss": 1.282, + "theoretical_loss": 3.388042826630957, + "tokens_seen": 2585053184 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001752457372116349, + "loss": 1.2179, + "theoretical_loss": 3.3880360783263024, + "tokens_seen": 2585118720 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001752296890672016, + "loss": 1.2494, + "theoretical_loss": 3.3880293302406246, + "tokens_seen": 2585184256 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001752136409227683, + "loss": 1.1948, + "theoretical_loss": 3.3880225823739103, + "tokens_seen": 2585249792 + }, + { + "epoch": 1.16, + "learning_rate": 0.000175197592778335, + "loss": 1.2187, + "theoretical_loss": 3.3880158347261466, + "tokens_seen": 2585315328 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001751815446339017, + "loss": 1.168, + "theoretical_loss": 3.3880090872973216, + "tokens_seen": 2585380864 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001751654964894684, + "loss": 1.2153, + "theoretical_loss": 3.388002340087422, + "tokens_seen": 2585446400 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001751494483450351, + "loss": 1.243, + "theoretical_loss": 3.3879955930964356, + "tokens_seen": 2585511936 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001751334002006018, + "loss": 1.2663, + "theoretical_loss": 3.3879888463243493, + "tokens_seen": 2585577472 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001751173520561685, + "loss": 1.1802, + "theoretical_loss": 3.387982099771151, + "tokens_seen": 2585643008 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001751013039117352, + "loss": 1.2322, + "theoretical_loss": 3.3879753534368273, + "tokens_seen": 2585708544 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001750852557673019, + "loss": 1.2133, + "theoretical_loss": 3.3879686073213664, + "tokens_seen": 2585774080 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017506920762286863, + "loss": 1.2161, + "theoretical_loss": 3.3879618614247553, + "tokens_seen": 2585839616 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017505315947843533, + "loss": 1.2487, + "theoretical_loss": 3.3879551157469807, + "tokens_seen": 2585905152 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017503711133400203, + "loss": 1.2267, + "theoretical_loss": 3.387948370288031, + "tokens_seen": 2585970688 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017502106318956873, + "loss": 1.2815, + "theoretical_loss": 3.387941625047893, + "tokens_seen": 2586036224 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017500501504513543, + "loss": 1.2256, + "theoretical_loss": 3.3879348800265543, + "tokens_seen": 2586101760 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017498896690070214, + "loss": 1.1945, + "theoretical_loss": 3.3879281352240023, + "tokens_seen": 2586167296 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001749729187562688, + "loss": 1.3377, + "theoretical_loss": 3.387921390640224, + "tokens_seen": 2586232832 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001749568706118355, + "loss": 1.2671, + "theoretical_loss": 3.387914646275207, + "tokens_seen": 2586298368 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001749408224674022, + "loss": 1.3867, + "theoretical_loss": 3.3879079021289384, + "tokens_seen": 2586363904 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001749247743229689, + "loss": 1.2552, + "theoretical_loss": 3.387901158201406, + "tokens_seen": 2586429440 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001749087261785356, + "loss": 1.2646, + "theoretical_loss": 3.387894414492597, + "tokens_seen": 2586494976 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001748926780341023, + "loss": 1.2461, + "theoretical_loss": 3.387887671002498, + "tokens_seen": 2586560512 + }, + { + "epoch": 1.16, + "learning_rate": 0.000174876629889669, + "loss": 1.2345, + "theoretical_loss": 3.3878809277310977, + "tokens_seen": 2586626048 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001748605817452357, + "loss": 1.3483, + "theoretical_loss": 3.387874184678383, + "tokens_seen": 2586691584 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017484453360080241, + "loss": 1.1491, + "theoretical_loss": 3.3878674418443406, + "tokens_seen": 2586757120 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017482848545636911, + "loss": 1.2748, + "theoretical_loss": 3.3878606992289586, + "tokens_seen": 2586822656 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017481243731193582, + "loss": 1.2487, + "theoretical_loss": 3.387853956832224, + "tokens_seen": 2586888192 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017479638916750252, + "loss": 1.2389, + "theoretical_loss": 3.3878472146541245, + "tokens_seen": 2586953728 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017478034102306922, + "loss": 1.3271, + "theoretical_loss": 3.387840472694647, + "tokens_seen": 2587019264 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017476429287863592, + "loss": 1.2126, + "theoretical_loss": 3.3878337309537794, + "tokens_seen": 2587084800 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017474824473420262, + "loss": 1.274, + "theoretical_loss": 3.3878269894315087, + "tokens_seen": 2587150336 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017473219658976932, + "loss": 1.205, + "theoretical_loss": 3.387820248127823, + "tokens_seen": 2587215872 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017471614844533602, + "loss": 1.1171, + "theoretical_loss": 3.3878135070427082, + "tokens_seen": 2587281408 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017470010030090272, + "loss": 1.2883, + "theoretical_loss": 3.387806766176153, + "tokens_seen": 2587346944 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017468405215646942, + "loss": 1.1676, + "theoretical_loss": 3.3878000255281444, + "tokens_seen": 2587412480 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017466800401203612, + "loss": 1.3479, + "theoretical_loss": 3.3877932850986694, + "tokens_seen": 2587478016 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017465195586760282, + "loss": 1.1182, + "theoretical_loss": 3.387786544887716, + "tokens_seen": 2587543552 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017463590772316952, + "loss": 1.202, + "theoretical_loss": 3.387779804895271, + "tokens_seen": 2587609088 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017461985957873622, + "loss": 1.2599, + "theoretical_loss": 3.3877730651213223, + "tokens_seen": 2587674624 + }, + { + "epoch": 1.16, + "learning_rate": 0.00017460381143430292, + "loss": 1.2146, + "theoretical_loss": 3.387766325565857, + "tokens_seen": 2587740160 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001745877632898696, + "loss": 1.2577, + "theoretical_loss": 3.3877595862288623, + "tokens_seen": 2587805696 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001745717151454363, + "loss": 1.2429, + "theoretical_loss": 3.3877528471103258, + "tokens_seen": 2587871232 + }, + { + "epoch": 1.16, + "learning_rate": 0.000174555667001003, + "loss": 1.1859, + "theoretical_loss": 3.3877461082102354, + "tokens_seen": 2587936768 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001745396188565697, + "loss": 1.1363, + "theoretical_loss": 3.3877393695285773, + "tokens_seen": 2588002304 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001745235707121364, + "loss": 1.2847, + "theoretical_loss": 3.38773263106534, + "tokens_seen": 2588067840 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001745075225677031, + "loss": 1.3115, + "theoretical_loss": 3.3877258928205105, + "tokens_seen": 2588133376 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001744914744232698, + "loss": 1.2377, + "theoretical_loss": 3.387719154794076, + "tokens_seen": 2588198912 + }, + { + "epoch": 1.16, + "learning_rate": 0.0001744754262788365, + "loss": 1.2076, + "theoretical_loss": 3.387712416986024, + "tokens_seen": 2588264448 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017445937813440323, + "loss": 1.2123, + "theoretical_loss": 3.3877056793963423, + "tokens_seen": 2588329984 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017444332998996993, + "loss": 1.1432, + "theoretical_loss": 3.3876989420250174, + "tokens_seen": 2588395520 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017442728184553663, + "loss": 1.1529, + "theoretical_loss": 3.3876922048720375, + "tokens_seen": 2588461056 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017441123370110333, + "loss": 1.2171, + "theoretical_loss": 3.3876854679373896, + "tokens_seen": 2588526592 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017439518555667003, + "loss": 1.2329, + "theoretical_loss": 3.3876787312210617, + "tokens_seen": 2588592128 + }, + { + "epoch": 1.17, + "objective/train/docs_used": 955843, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1700377464294434, + "objective/train/theoretical_loss": 3.38767199472304, + "objective/train/tokens_used": 2609117664, + "theoretical_loss": 3.38767199472304, + "tokens_seen": 2588657664 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017437913741223673, + "loss": 1.17, + "theoretical_loss": 3.38767199472304, + "tokens_seen": 2588657664 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017436308926780343, + "loss": 1.3166, + "theoretical_loss": 3.387665258443313, + "tokens_seen": 2588723200 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017434704112337013, + "loss": 1.1575, + "theoretical_loss": 3.3876585223818676, + "tokens_seen": 2588788736 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017433099297893683, + "loss": 1.2042, + "theoretical_loss": 3.3876517865386915, + "tokens_seen": 2588854272 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017431494483450353, + "loss": 1.2111, + "theoretical_loss": 3.387645050913772, + "tokens_seen": 2588919808 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017429889669007024, + "loss": 1.3094, + "theoretical_loss": 3.387638315507096, + "tokens_seen": 2588985344 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017428284854563694, + "loss": 1.2472, + "theoretical_loss": 3.3876315803186516, + "tokens_seen": 2589050880 + }, + { + "epoch": 1.17, + "learning_rate": 0.0001742668004012036, + "loss": 1.1795, + "theoretical_loss": 3.3876248453484257, + "tokens_seen": 2589116416 + }, + { + "epoch": 1.17, + "learning_rate": 0.0001742507522567703, + "loss": 1.3042, + "theoretical_loss": 3.387618110596406, + "tokens_seen": 2589181952 + }, + { + "epoch": 1.17, + "learning_rate": 0.000174234704112337, + "loss": 1.2694, + "theoretical_loss": 3.3876113760625803, + "tokens_seen": 2589247488 + }, + { + "epoch": 1.17, + "learning_rate": 0.0001742186559679037, + "loss": 1.1646, + "theoretical_loss": 3.387604641746935, + "tokens_seen": 2589313024 + }, + { + "epoch": 1.17, + "learning_rate": 0.0001742026078234704, + "loss": 1.2418, + "theoretical_loss": 3.3875979076494582, + "tokens_seen": 2589378560 + }, + { + "epoch": 1.17, + "learning_rate": 0.0001741865596790371, + "loss": 1.1886, + "theoretical_loss": 3.3875911737701374, + "tokens_seen": 2589444096 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017417051153460381, + "loss": 1.2577, + "theoretical_loss": 3.38758444010896, + "tokens_seen": 2589509632 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017415446339017051, + "loss": 1.2545, + "theoretical_loss": 3.3875777066659127, + "tokens_seen": 2589575168 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017413841524573721, + "loss": 1.195, + "theoretical_loss": 3.387570973440983, + "tokens_seen": 2589640704 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017412236710130392, + "loss": 1.1953, + "theoretical_loss": 3.38756424043416, + "tokens_seen": 2589706240 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017410631895687062, + "loss": 1.222, + "theoretical_loss": 3.387557507645429, + "tokens_seen": 2589771776 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017409027081243732, + "loss": 1.1382, + "theoretical_loss": 3.387550775074778, + "tokens_seen": 2589837312 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017407422266800402, + "loss": 1.1933, + "theoretical_loss": 3.3875440427221952, + "tokens_seen": 2589902848 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017405817452357072, + "loss": 1.233, + "theoretical_loss": 3.3875373105876676, + "tokens_seen": 2589968384 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017404212637913742, + "loss": 1.257, + "theoretical_loss": 3.3875305786711825, + "tokens_seen": 2590033920 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017402607823470412, + "loss": 1.2804, + "theoretical_loss": 3.387523846972727, + "tokens_seen": 2590099456 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017401003009027082, + "loss": 1.1993, + "theoretical_loss": 3.387517115492289, + "tokens_seen": 2590164992 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017399398194583752, + "loss": 1.2148, + "theoretical_loss": 3.3875103842298557, + "tokens_seen": 2590230528 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017397793380140422, + "loss": 1.1762, + "theoretical_loss": 3.387503653185415, + "tokens_seen": 2590296064 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017396188565697092, + "loss": 1.1676, + "theoretical_loss": 3.3874969223589533, + "tokens_seen": 2590361600 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017394583751253762, + "loss": 1.2293, + "theoretical_loss": 3.3874901917504587, + "tokens_seen": 2590427136 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017392978936810432, + "loss": 1.2578, + "theoretical_loss": 3.387483461359919, + "tokens_seen": 2590492672 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017391374122367102, + "loss": 1.3496, + "theoretical_loss": 3.3874767311873213, + "tokens_seen": 2590558208 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017389769307923772, + "loss": 1.2435, + "theoretical_loss": 3.3874700012326526, + "tokens_seen": 2590623744 + }, + { + "epoch": 1.17, + "learning_rate": 0.0001738816449348044, + "loss": 1.1839, + "theoretical_loss": 3.3874632714959008, + "tokens_seen": 2590689280 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017386559679037113, + "loss": 1.2224, + "theoretical_loss": 3.3874565419770533, + "tokens_seen": 2590754816 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017384954864593783, + "loss": 1.2304, + "theoretical_loss": 3.3874498126760977, + "tokens_seen": 2590820352 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017383350050150453, + "loss": 1.1684, + "theoretical_loss": 3.3874430835930207, + "tokens_seen": 2590885888 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017381745235707123, + "loss": 1.1041, + "theoretical_loss": 3.3874363547278104, + "tokens_seen": 2590951424 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017380140421263793, + "loss": 1.2407, + "theoretical_loss": 3.387429626080454, + "tokens_seen": 2591016960 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017378535606820463, + "loss": 1.1887, + "theoretical_loss": 3.387422897650939, + "tokens_seen": 2591082496 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017376930792377133, + "loss": 1.1934, + "theoretical_loss": 3.387416169439253, + "tokens_seen": 2591148032 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017375325977933803, + "loss": 1.2786, + "theoretical_loss": 3.387409441445383, + "tokens_seen": 2591213568 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017373721163490473, + "loss": 1.3162, + "theoretical_loss": 3.3874027136693163, + "tokens_seen": 2591279104 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017372116349047143, + "loss": 1.13, + "theoretical_loss": 3.3873959861110414, + "tokens_seen": 2591344640 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017370511534603813, + "loss": 1.0969, + "theoretical_loss": 3.387389258770545, + "tokens_seen": 2591410176 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017368906720160483, + "loss": 1.2338, + "theoretical_loss": 3.3873825316478143, + "tokens_seen": 2591475712 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017367301905717153, + "loss": 1.3081, + "theoretical_loss": 3.3873758047428373, + "tokens_seen": 2591541248 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017365697091273823, + "loss": 1.1796, + "theoretical_loss": 3.387369078055601, + "tokens_seen": 2591606784 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017364092276830493, + "loss": 1.1187, + "theoretical_loss": 3.3873623515860936, + "tokens_seen": 2591672320 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017362487462387164, + "loss": 1.2508, + "theoretical_loss": 3.3873556253343016, + "tokens_seen": 2591737856 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017360882647943834, + "loss": 1.1168, + "theoretical_loss": 3.3873488993002128, + "tokens_seen": 2591803392 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017359277833500504, + "loss": 1.1917, + "theoretical_loss": 3.387342173483815, + "tokens_seen": 2591868928 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017357673019057174, + "loss": 1.2098, + "theoretical_loss": 3.387335447885095, + "tokens_seen": 2591934464 + }, + { + "epoch": 1.17, + "learning_rate": 0.0001735606820461384, + "loss": 1.363, + "theoretical_loss": 3.3873287225040407, + "tokens_seen": 2592000000 + }, + { + "epoch": 1.17, + "learning_rate": 0.0001735446339017051, + "loss": 1.1587, + "theoretical_loss": 3.3873219973406394, + "tokens_seen": 2592065536 + }, + { + "epoch": 1.17, + "learning_rate": 0.0001735285857572718, + "loss": 1.1186, + "theoretical_loss": 3.387315272394879, + "tokens_seen": 2592131072 + }, + { + "epoch": 1.17, + "learning_rate": 0.0001735125376128385, + "loss": 1.1753, + "theoretical_loss": 3.387308547666746, + "tokens_seen": 2592196608 + }, + { + "epoch": 1.17, + "learning_rate": 0.0001734964894684052, + "loss": 1.2661, + "theoretical_loss": 3.387301823156229, + "tokens_seen": 2592262144 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017348044132397191, + "loss": 1.1313, + "theoretical_loss": 3.387295098863315, + "tokens_seen": 2592327680 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017346439317953861, + "loss": 1.277, + "theoretical_loss": 3.3872883747879903, + "tokens_seen": 2592393216 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017344834503510532, + "loss": 1.2672, + "theoretical_loss": 3.387281650930244, + "tokens_seen": 2592458752 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017343229689067202, + "loss": 1.2234, + "theoretical_loss": 3.387274927290063, + "tokens_seen": 2592524288 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017341624874623872, + "loss": 1.1685, + "theoretical_loss": 3.3872682038674347, + "tokens_seen": 2592589824 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017340020060180542, + "loss": 1.1413, + "theoretical_loss": 3.387261480662347, + "tokens_seen": 2592655360 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017338415245737212, + "loss": 1.2293, + "theoretical_loss": 3.3872547576747865, + "tokens_seen": 2592720896 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017336810431293882, + "loss": 1.2597, + "theoretical_loss": 3.387248034904741, + "tokens_seen": 2592786432 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017335205616850552, + "loss": 1.3209, + "theoretical_loss": 3.387241312352198, + "tokens_seen": 2592851968 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017333600802407222, + "loss": 1.2558, + "theoretical_loss": 3.387234590017145, + "tokens_seen": 2592917504 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017331995987963892, + "loss": 1.2985, + "theoretical_loss": 3.3872278678995698, + "tokens_seen": 2592983040 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017330391173520562, + "loss": 1.194, + "theoretical_loss": 3.3872211459994594, + "tokens_seen": 2593048576 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017328786359077232, + "loss": 1.2392, + "theoretical_loss": 3.3872144243168014, + "tokens_seen": 2593114112 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017327181544633905, + "loss": 1.3443, + "theoretical_loss": 3.3872077028515837, + "tokens_seen": 2593179648 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017325576730190572, + "loss": 1.1924, + "theoretical_loss": 3.387200981603793, + "tokens_seen": 2593245184 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017323971915747242, + "loss": 1.2944, + "theoretical_loss": 3.387194260573417, + "tokens_seen": 2593310720 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017322367101303912, + "loss": 1.2667, + "theoretical_loss": 3.387187539760444, + "tokens_seen": 2593376256 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017320762286860582, + "loss": 1.1736, + "theoretical_loss": 3.3871808191648602, + "tokens_seen": 2593441792 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017319157472417253, + "loss": 1.2772, + "theoretical_loss": 3.387174098786654, + "tokens_seen": 2593507328 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017317552657973923, + "loss": 1.2262, + "theoretical_loss": 3.3871673786258123, + "tokens_seen": 2593572864 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017315947843530593, + "loss": 1.2542, + "theoretical_loss": 3.3871606586823226, + "tokens_seen": 2593638400 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017314343029087263, + "loss": 1.2214, + "theoretical_loss": 3.387153938956173, + "tokens_seen": 2593703936 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017312738214643933, + "loss": 1.1927, + "theoretical_loss": 3.3871472194473506, + "tokens_seen": 2593769472 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017311133400200603, + "loss": 1.2071, + "theoretical_loss": 3.3871405001558426, + "tokens_seen": 2593835008 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017309528585757273, + "loss": 1.2671, + "theoretical_loss": 3.387133781081637, + "tokens_seen": 2593900544 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017307923771313943, + "loss": 1.2397, + "theoretical_loss": 3.387127062224721, + "tokens_seen": 2593966080 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017306318956870613, + "loss": 1.2688, + "theoretical_loss": 3.387120343585082, + "tokens_seen": 2594031616 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017304714142427283, + "loss": 1.2338, + "theoretical_loss": 3.3871136251627076, + "tokens_seen": 2594097152 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017303109327983953, + "loss": 1.2859, + "theoretical_loss": 3.387106906957585, + "tokens_seen": 2594162688 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017301504513540623, + "loss": 1.1858, + "theoretical_loss": 3.3871001889697023, + "tokens_seen": 2594228224 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017299899699097293, + "loss": 1.1423, + "theoretical_loss": 3.387093471199047, + "tokens_seen": 2594293760 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017298294884653963, + "loss": 1.2077, + "theoretical_loss": 3.3870867536456055, + "tokens_seen": 2594359296 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017296690070210633, + "loss": 1.1478, + "theoretical_loss": 3.3870800363093667, + "tokens_seen": 2594424832 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017295085255767303, + "loss": 1.2735, + "theoretical_loss": 3.3870733191903173, + "tokens_seen": 2594490368 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017293480441323974, + "loss": 1.1609, + "theoretical_loss": 3.387066602288445, + "tokens_seen": 2594555904 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017291875626880644, + "loss": 1.1282, + "theoretical_loss": 3.387059885603737, + "tokens_seen": 2594621440 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017290270812437314, + "loss": 1.3059, + "theoretical_loss": 3.3870531691361805, + "tokens_seen": 2594686976 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017288665997993984, + "loss": 1.2485, + "theoretical_loss": 3.3870464528857642, + "tokens_seen": 2594752512 + }, + { + "epoch": 1.17, + "learning_rate": 0.0001728706118355065, + "loss": 1.2039, + "theoretical_loss": 3.3870397368524743, + "tokens_seen": 2594818048 + }, + { + "epoch": 1.17, + "learning_rate": 0.0001728545636910732, + "loss": 1.2158, + "theoretical_loss": 3.3870330210362996, + "tokens_seen": 2594883584 + }, + { + "epoch": 1.17, + "learning_rate": 0.0001728385155466399, + "loss": 1.1855, + "theoretical_loss": 3.3870263054372263, + "tokens_seen": 2594949120 + }, + { + "epoch": 1.17, + "learning_rate": 0.0001728224674022066, + "loss": 1.1718, + "theoretical_loss": 3.387019590055243, + "tokens_seen": 2595014656 + }, + { + "epoch": 1.17, + "learning_rate": 0.0001728064192577733, + "loss": 1.2675, + "theoretical_loss": 3.387012874890336, + "tokens_seen": 2595080192 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017279037111334001, + "loss": 1.2388, + "theoretical_loss": 3.387006159942494, + "tokens_seen": 2595145728 + }, + { + "epoch": 1.17, + "objective/train/docs_used": 958162, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2600939273834229, + "objective/train/theoretical_loss": 3.386999445211704, + "objective/train/tokens_used": 2615671264, + "theoretical_loss": 3.386999445211704, + "tokens_seen": 2595211264 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017277432296890671, + "loss": 1.2601, + "theoretical_loss": 3.386999445211704, + "tokens_seen": 2595211264 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017275827482447342, + "loss": 1.1829, + "theoretical_loss": 3.386992730697953, + "tokens_seen": 2595276800 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017274222668004012, + "loss": 1.2412, + "theoretical_loss": 3.3869860164012295, + "tokens_seen": 2595342336 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017272617853560682, + "loss": 1.1974, + "theoretical_loss": 3.386979302321521, + "tokens_seen": 2595407872 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017271013039117352, + "loss": 1.1535, + "theoretical_loss": 3.3869725884588133, + "tokens_seen": 2595473408 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017269408224674022, + "loss": 1.2709, + "theoretical_loss": 3.3869658748130957, + "tokens_seen": 2595538944 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017267803410230692, + "loss": 1.2672, + "theoretical_loss": 3.3869591613843553, + "tokens_seen": 2595604480 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017266198595787365, + "loss": 1.2555, + "theoretical_loss": 3.3869524481725795, + "tokens_seen": 2595670016 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017264593781344035, + "loss": 1.3085, + "theoretical_loss": 3.3869457351777554, + "tokens_seen": 2595735552 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017262988966900705, + "loss": 1.1888, + "theoretical_loss": 3.386939022399871, + "tokens_seen": 2595801088 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017261384152457375, + "loss": 1.1934, + "theoretical_loss": 3.3869323098389135, + "tokens_seen": 2595866624 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017259779338014045, + "loss": 1.1885, + "theoretical_loss": 3.3869255974948707, + "tokens_seen": 2595932160 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017258174523570715, + "loss": 1.2064, + "theoretical_loss": 3.38691888536773, + "tokens_seen": 2595997696 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017256569709127385, + "loss": 1.246, + "theoretical_loss": 3.386912173457479, + "tokens_seen": 2596063232 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017254964894684052, + "loss": 1.219, + "theoretical_loss": 3.3869054617641052, + "tokens_seen": 2596128768 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017253360080240722, + "loss": 1.2372, + "theoretical_loss": 3.386898750287596, + "tokens_seen": 2596194304 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017251755265797392, + "loss": 1.2944, + "theoretical_loss": 3.3868920390279387, + "tokens_seen": 2596259840 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017250150451354063, + "loss": 1.1603, + "theoretical_loss": 3.386885327985121, + "tokens_seen": 2596325376 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017248545636910733, + "loss": 1.3457, + "theoretical_loss": 3.386878617159131, + "tokens_seen": 2596390912 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017246940822467403, + "loss": 1.1654, + "theoretical_loss": 3.3868719065499557, + "tokens_seen": 2596456448 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017245336008024073, + "loss": 1.2186, + "theoretical_loss": 3.3868651961575824, + "tokens_seen": 2596521984 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017243731193580743, + "loss": 1.1437, + "theoretical_loss": 3.386858485981999, + "tokens_seen": 2596587520 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017242126379137413, + "loss": 1.2842, + "theoretical_loss": 3.386851776023193, + "tokens_seen": 2596653056 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017240521564694083, + "loss": 1.1487, + "theoretical_loss": 3.3868450662811513, + "tokens_seen": 2596718592 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017238916750250753, + "loss": 1.2149, + "theoretical_loss": 3.3868383567558626, + "tokens_seen": 2596784128 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017237311935807423, + "loss": 1.2, + "theoretical_loss": 3.3868316474473135, + "tokens_seen": 2596849664 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017235707121364093, + "loss": 1.2015, + "theoretical_loss": 3.386824938355492, + "tokens_seen": 2596915200 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017234102306920763, + "loss": 1.2355, + "theoretical_loss": 3.386818229480385, + "tokens_seen": 2596980736 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017232497492477433, + "loss": 1.2097, + "theoretical_loss": 3.386811520821981, + "tokens_seen": 2597046272 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017230892678034103, + "loss": 1.142, + "theoretical_loss": 3.3868048123802663, + "tokens_seen": 2597111808 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017229287863590773, + "loss": 1.3213, + "theoretical_loss": 3.38679810415523, + "tokens_seen": 2597177344 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017227683049147443, + "loss": 1.1497, + "theoretical_loss": 3.386791396146858, + "tokens_seen": 2597242880 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017226078234704113, + "loss": 1.2109, + "theoretical_loss": 3.3867846883551387, + "tokens_seen": 2597308416 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017224473420260784, + "loss": 1.2123, + "theoretical_loss": 3.38677798078006, + "tokens_seen": 2597373952 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017222868605817454, + "loss": 1.184, + "theoretical_loss": 3.386771273421609, + "tokens_seen": 2597439488 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017221263791374124, + "loss": 1.3114, + "theoretical_loss": 3.3867645662797727, + "tokens_seen": 2597505024 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017219658976930794, + "loss": 1.259, + "theoretical_loss": 3.3867578593545398, + "tokens_seen": 2597570560 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017218054162487464, + "loss": 1.2472, + "theoretical_loss": 3.3867511526458967, + "tokens_seen": 2597636096 + }, + { + "epoch": 1.17, + "learning_rate": 0.0001721644934804413, + "loss": 1.1938, + "theoretical_loss": 3.3867444461538314, + "tokens_seen": 2597701632 + }, + { + "epoch": 1.17, + "learning_rate": 0.000172148445336008, + "loss": 1.2529, + "theoretical_loss": 3.3867377398783316, + "tokens_seen": 2597767168 + }, + { + "epoch": 1.17, + "learning_rate": 0.0001721323971915747, + "loss": 1.2882, + "theoretical_loss": 3.386731033819385, + "tokens_seen": 2597832704 + }, + { + "epoch": 1.17, + "learning_rate": 0.0001721163490471414, + "loss": 1.2478, + "theoretical_loss": 3.3867243279769785, + "tokens_seen": 2597898240 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017210030090270811, + "loss": 1.222, + "theoretical_loss": 3.3867176223511004, + "tokens_seen": 2597963776 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017208425275827481, + "loss": 1.2294, + "theoretical_loss": 3.3867109169417375, + "tokens_seen": 2598029312 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017206820461384154, + "loss": 1.2728, + "theoretical_loss": 3.386704211748878, + "tokens_seen": 2598094848 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017205215646940824, + "loss": 1.1902, + "theoretical_loss": 3.3866975067725087, + "tokens_seen": 2598160384 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017203610832497494, + "loss": 1.2732, + "theoretical_loss": 3.386690802012618, + "tokens_seen": 2598225920 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017202006018054164, + "loss": 1.184, + "theoretical_loss": 3.386684097469193, + "tokens_seen": 2598291456 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017200401203610835, + "loss": 1.1831, + "theoretical_loss": 3.3866773931422216, + "tokens_seen": 2598356992 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017198796389167505, + "loss": 1.2174, + "theoretical_loss": 3.3866706890316904, + "tokens_seen": 2598422528 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017197191574724175, + "loss": 1.2453, + "theoretical_loss": 3.386663985137588, + "tokens_seen": 2598488064 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017195586760280845, + "loss": 1.2239, + "theoretical_loss": 3.3866572814599016, + "tokens_seen": 2598553600 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017193981945837515, + "loss": 1.2081, + "theoretical_loss": 3.3866505779986187, + "tokens_seen": 2598619136 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017192377131394185, + "loss": 1.3134, + "theoretical_loss": 3.3866438747537266, + "tokens_seen": 2598684672 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017190772316950855, + "loss": 1.2588, + "theoretical_loss": 3.3866371717252135, + "tokens_seen": 2598750208 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017189167502507525, + "loss": 1.2006, + "theoretical_loss": 3.386630468913067, + "tokens_seen": 2598815744 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017187562688064195, + "loss": 1.2889, + "theoretical_loss": 3.3866237663172734, + "tokens_seen": 2598881280 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017185957873620865, + "loss": 1.2163, + "theoretical_loss": 3.3866170639378215, + "tokens_seen": 2598946816 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017184353059177532, + "loss": 1.1039, + "theoretical_loss": 3.3866103617746983, + "tokens_seen": 2599012352 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017182748244734202, + "loss": 1.2245, + "theoretical_loss": 3.386603659827892, + "tokens_seen": 2599077888 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017181143430290873, + "loss": 1.1833, + "theoretical_loss": 3.386596958097389, + "tokens_seen": 2599143424 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017179538615847543, + "loss": 1.1245, + "theoretical_loss": 3.3865902565831782, + "tokens_seen": 2599208960 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017177933801404213, + "loss": 1.2788, + "theoretical_loss": 3.3865835552852466, + "tokens_seen": 2599274496 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017176328986960883, + "loss": 1.1982, + "theoretical_loss": 3.3865768542035815, + "tokens_seen": 2599340032 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017174724172517553, + "loss": 1.155, + "theoretical_loss": 3.3865701533381705, + "tokens_seen": 2599405568 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017173119358074223, + "loss": 1.2232, + "theoretical_loss": 3.386563452689001, + "tokens_seen": 2599471104 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017171514543630893, + "loss": 1.2162, + "theoretical_loss": 3.3865567522560616, + "tokens_seen": 2599536640 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017169909729187563, + "loss": 1.2229, + "theoretical_loss": 3.3865500520393392, + "tokens_seen": 2599602176 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017168304914744233, + "loss": 1.2163, + "theoretical_loss": 3.386543352038821, + "tokens_seen": 2599667712 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017166700100300903, + "loss": 1.215, + "theoretical_loss": 3.3865366522544953, + "tokens_seen": 2599733248 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017165095285857573, + "loss": 1.1652, + "theoretical_loss": 3.386529952686349, + "tokens_seen": 2599798784 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017163490471414243, + "loss": 1.3837, + "theoretical_loss": 3.38652325333437, + "tokens_seen": 2599864320 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017161885656970913, + "loss": 1.2816, + "theoretical_loss": 3.386516554198546, + "tokens_seen": 2599929856 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017160280842527583, + "loss": 1.2256, + "theoretical_loss": 3.3865098552788644, + "tokens_seen": 2599995392 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017158676028084253, + "loss": 1.1454, + "theoretical_loss": 3.386503156575313, + "tokens_seen": 2600060928 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017157071213640924, + "loss": 1.2299, + "theoretical_loss": 3.386496458087879, + "tokens_seen": 2600126464 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017155466399197594, + "loss": 1.146, + "theoretical_loss": 3.3864897598165498, + "tokens_seen": 2600192000 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017155466399197594, + "loss": 1.3089, + "theoretical_loss": 3.3864830617613135, + "tokens_seen": 2600257536 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017153861584754264, + "loss": 1.0647, + "theoretical_loss": 3.3864763639221582, + "tokens_seen": 2600323072 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017152256770310934, + "loss": 1.2474, + "theoretical_loss": 3.38646966629907, + "tokens_seen": 2600388608 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017150651955867604, + "loss": 1.2261, + "theoretical_loss": 3.386462968892038, + "tokens_seen": 2600454144 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017149047141424274, + "loss": 1.2939, + "theoretical_loss": 3.3864562717010487, + "tokens_seen": 2600519680 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017149047141424274, + "loss": 1.2528, + "theoretical_loss": 3.38644957472609, + "tokens_seen": 2600585216 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017147442326980944, + "loss": 1.1806, + "theoretical_loss": 3.38644287796715, + "tokens_seen": 2600650752 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017145837512537614, + "loss": 1.2045, + "theoretical_loss": 3.3864361814242154, + "tokens_seen": 2600716288 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017144232698094284, + "loss": 1.1717, + "theoretical_loss": 3.3864294850972745, + "tokens_seen": 2600781824 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017142627883650954, + "loss": 1.2168, + "theoretical_loss": 3.3864227889863145, + "tokens_seen": 2600847360 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017141023069207624, + "loss": 1.1905, + "theoretical_loss": 3.386416093091323, + "tokens_seen": 2600912896 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017139418254764294, + "loss": 1.2439, + "theoretical_loss": 3.3864093974122884, + "tokens_seen": 2600978432 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017137813440320964, + "loss": 1.2733, + "theoretical_loss": 3.386402701949197, + "tokens_seen": 2601043968 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017136208625877634, + "loss": 1.2608, + "theoretical_loss": 3.386396006702037, + "tokens_seen": 2601109504 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017134603811434304, + "loss": 1.2579, + "theoretical_loss": 3.386389311670796, + "tokens_seen": 2601175040 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017132998996990974, + "loss": 1.2106, + "theoretical_loss": 3.386382616855462, + "tokens_seen": 2601240576 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017131394182547645, + "loss": 1.2207, + "theoretical_loss": 3.386375922256022, + "tokens_seen": 2601306112 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017129789368104315, + "loss": 1.202, + "theoretical_loss": 3.3863692278724637, + "tokens_seen": 2601371648 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017128184553660985, + "loss": 1.2008, + "theoretical_loss": 3.386362533704775, + "tokens_seen": 2601437184 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017126579739217655, + "loss": 1.2448, + "theoretical_loss": 3.386355839752943, + "tokens_seen": 2601502720 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017124974924774325, + "loss": 1.263, + "theoretical_loss": 3.386349146016956, + "tokens_seen": 2601568256 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017123370110330995, + "loss": 1.2187, + "theoretical_loss": 3.386342452496801, + "tokens_seen": 2601633792 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017121765295887665, + "loss": 1.1607, + "theoretical_loss": 3.3863357591924657, + "tokens_seen": 2601699328 + }, + { + "epoch": 1.17, + "objective/train/docs_used": 960707, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1961839199066162, + "objective/train/theoretical_loss": 3.3863290661039382, + "objective/train/tokens_used": 2622224864, + "theoretical_loss": 3.3863290661039382, + "tokens_seen": 2601764864 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017120160481444335, + "loss": 1.1962, + "theoretical_loss": 3.3863290661039382, + "tokens_seen": 2601764864 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017118555667001005, + "loss": 1.1779, + "theoretical_loss": 3.3863223732312053, + "tokens_seen": 2601830400 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017116950852557675, + "loss": 1.3081, + "theoretical_loss": 3.3863156805742554, + "tokens_seen": 2601895936 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017115346038114342, + "loss": 1.2206, + "theoretical_loss": 3.386308988133075, + "tokens_seen": 2601961472 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017113741223671013, + "loss": 1.1936, + "theoretical_loss": 3.386302295907653, + "tokens_seen": 2602027008 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017112136409227683, + "loss": 1.2337, + "theoretical_loss": 3.3862956038979766, + "tokens_seen": 2602092544 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017110531594784353, + "loss": 1.2623, + "theoretical_loss": 3.386288912104033, + "tokens_seen": 2602158080 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017108926780341023, + "loss": 1.1761, + "theoretical_loss": 3.38628222052581, + "tokens_seen": 2602223616 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017107321965897693, + "loss": 1.1928, + "theoretical_loss": 3.3862755291632958, + "tokens_seen": 2602289152 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017105717151454363, + "loss": 1.1816, + "theoretical_loss": 3.386268838016477, + "tokens_seen": 2602354688 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017104112337011033, + "loss": 1.235, + "theoretical_loss": 3.386262147085342, + "tokens_seen": 2602420224 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017102507522567703, + "loss": 1.2112, + "theoretical_loss": 3.386255456369878, + "tokens_seen": 2602485760 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017100902708124373, + "loss": 1.1938, + "theoretical_loss": 3.386248765870073, + "tokens_seen": 2602551296 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017099297893681043, + "loss": 1.1856, + "theoretical_loss": 3.3862420755859137, + "tokens_seen": 2602616832 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017097693079237713, + "loss": 1.1373, + "theoretical_loss": 3.3862353855173892, + "tokens_seen": 2602682368 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017096088264794383, + "loss": 1.348, + "theoretical_loss": 3.386228695664486, + "tokens_seen": 2602747904 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017094483450351053, + "loss": 1.2224, + "theoretical_loss": 3.386222006027192, + "tokens_seen": 2602813440 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017092878635907723, + "loss": 1.2269, + "theoretical_loss": 3.3862153166054947, + "tokens_seen": 2602878976 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017091273821464393, + "loss": 1.2283, + "theoretical_loss": 3.386208627399382, + "tokens_seen": 2602944512 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017089669007021063, + "loss": 1.1928, + "theoretical_loss": 3.3862019384088415, + "tokens_seen": 2603010048 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017088064192577734, + "loss": 1.2597, + "theoretical_loss": 3.386195249633861, + "tokens_seen": 2603075584 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017086459378134406, + "loss": 1.2441, + "theoretical_loss": 3.3861885610744276, + "tokens_seen": 2603141120 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017084854563691076, + "loss": 1.1897, + "theoretical_loss": 3.386181872730529, + "tokens_seen": 2603206656 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017083249749247744, + "loss": 1.1693, + "theoretical_loss": 3.3861751846021533, + "tokens_seen": 2603272192 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017081644934804414, + "loss": 1.1934, + "theoretical_loss": 3.3861684966892875, + "tokens_seen": 2603337728 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017080040120361084, + "loss": 1.2851, + "theoretical_loss": 3.38616180899192, + "tokens_seen": 2603403264 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017078435305917754, + "loss": 1.2204, + "theoretical_loss": 3.386155121510038, + "tokens_seen": 2603468800 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017076830491474424, + "loss": 1.1594, + "theoretical_loss": 3.386148434243629, + "tokens_seen": 2603534336 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017075225677031094, + "loss": 1.2745, + "theoretical_loss": 3.3861417471926805, + "tokens_seen": 2603599872 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017073620862587764, + "loss": 1.1953, + "theoretical_loss": 3.3861350603571805, + "tokens_seen": 2603665408 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017072016048144434, + "loss": 1.1795, + "theoretical_loss": 3.386128373737117, + "tokens_seen": 2603730944 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017070411233701104, + "loss": 1.2609, + "theoretical_loss": 3.3861216873324764, + "tokens_seen": 2603796480 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017068806419257774, + "loss": 1.2537, + "theoretical_loss": 3.386115001143248, + "tokens_seen": 2603862016 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017067201604814444, + "loss": 1.2406, + "theoretical_loss": 3.386108315169418, + "tokens_seen": 2603927552 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017065596790371114, + "loss": 1.164, + "theoretical_loss": 3.386101629410975, + "tokens_seen": 2603993088 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017063991975927784, + "loss": 1.2997, + "theoretical_loss": 3.3860949438679055, + "tokens_seen": 2604058624 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017062387161484455, + "loss": 1.2378, + "theoretical_loss": 3.3860882585401986, + "tokens_seen": 2604124160 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017060782347041125, + "loss": 1.1737, + "theoretical_loss": 3.386081573427841, + "tokens_seen": 2604189696 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017059177532597795, + "loss": 1.1851, + "theoretical_loss": 3.3860748885308203, + "tokens_seen": 2604255232 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017057572718154465, + "loss": 1.229, + "theoretical_loss": 3.386068203849125, + "tokens_seen": 2604320768 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017055967903711135, + "loss": 1.2305, + "theoretical_loss": 3.386061519382742, + "tokens_seen": 2604386304 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017054363089267805, + "loss": 1.2092, + "theoretical_loss": 3.3860548351316586, + "tokens_seen": 2604451840 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017052758274824475, + "loss": 1.1222, + "theoretical_loss": 3.3860481510958635, + "tokens_seen": 2604517376 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017051153460381145, + "loss": 1.2501, + "theoretical_loss": 3.3860414672753434, + "tokens_seen": 2604582912 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017049548645937815, + "loss": 1.1873, + "theoretical_loss": 3.3860347836700866, + "tokens_seen": 2604648448 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017047943831494485, + "loss": 1.2569, + "theoretical_loss": 3.3860281002800807, + "tokens_seen": 2604713984 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017046339017051155, + "loss": 1.151, + "theoretical_loss": 3.386021417105313, + "tokens_seen": 2604779520 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017044734202607823, + "loss": 1.1226, + "theoretical_loss": 3.386014734145771, + "tokens_seen": 2604845056 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017043129388164493, + "loss": 1.199, + "theoretical_loss": 3.386008051401443, + "tokens_seen": 2604910592 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017041524573721163, + "loss": 1.2857, + "theoretical_loss": 3.386001368872316, + "tokens_seen": 2604976128 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017039919759277833, + "loss": 1.2242, + "theoretical_loss": 3.3859946865583783, + "tokens_seen": 2605041664 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017038314944834503, + "loss": 1.1876, + "theoretical_loss": 3.3859880044596173, + "tokens_seen": 2605107200 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017036710130391173, + "loss": 1.2254, + "theoretical_loss": 3.3859813225760202, + "tokens_seen": 2605172736 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017035105315947843, + "loss": 1.3176, + "theoretical_loss": 3.3859746409075755, + "tokens_seen": 2605238272 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017033500501504513, + "loss": 1.261, + "theoretical_loss": 3.38596795945427, + "tokens_seen": 2605303808 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017031895687061183, + "loss": 1.1971, + "theoretical_loss": 3.385961278216092, + "tokens_seen": 2605369344 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017030290872617853, + "loss": 1.1919, + "theoretical_loss": 3.385954597193029, + "tokens_seen": 2605434880 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017028686058174523, + "loss": 1.2433, + "theoretical_loss": 3.3859479163850685, + "tokens_seen": 2605500416 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017027081243731193, + "loss": 1.1655, + "theoretical_loss": 3.3859412357921985, + "tokens_seen": 2605565952 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017025476429287866, + "loss": 1.2037, + "theoretical_loss": 3.385934555414406, + "tokens_seen": 2605631488 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017023871614844536, + "loss": 1.2094, + "theoretical_loss": 3.3859278752516793, + "tokens_seen": 2605697024 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017022266800401206, + "loss": 1.2277, + "theoretical_loss": 3.3859211953040056, + "tokens_seen": 2605762560 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017020661985957876, + "loss": 1.2055, + "theoretical_loss": 3.3859145155713732, + "tokens_seen": 2605828096 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017019057171514546, + "loss": 1.151, + "theoretical_loss": 3.3859078360537693, + "tokens_seen": 2605893632 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017017452357071216, + "loss": 1.2943, + "theoretical_loss": 3.385901156751182, + "tokens_seen": 2605959168 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017015847542627886, + "loss": 1.1127, + "theoretical_loss": 3.385894477663598, + "tokens_seen": 2606024704 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017014242728184556, + "loss": 1.1788, + "theoretical_loss": 3.385887798791006, + "tokens_seen": 2606090240 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017012637913741224, + "loss": 1.2448, + "theoretical_loss": 3.3858811201333934, + "tokens_seen": 2606155776 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017011033099297894, + "loss": 1.2337, + "theoretical_loss": 3.3858744416907474, + "tokens_seen": 2606221312 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017009428284854564, + "loss": 1.157, + "theoretical_loss": 3.385867763463056, + "tokens_seen": 2606286848 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017007823470411234, + "loss": 1.2605, + "theoretical_loss": 3.3858610854503075, + "tokens_seen": 2606352384 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017006218655967904, + "loss": 1.2821, + "theoretical_loss": 3.3858544076524884, + "tokens_seen": 2606417920 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017004613841524574, + "loss": 1.2758, + "theoretical_loss": 3.385847730069587, + "tokens_seen": 2606483456 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017003009027081244, + "loss": 1.2071, + "theoretical_loss": 3.385841052701591, + "tokens_seen": 2606548992 + }, + { + "epoch": 1.17, + "learning_rate": 0.00017001404212637914, + "loss": 1.2931, + "theoretical_loss": 3.3858343755484883, + "tokens_seen": 2606614528 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016999799398194584, + "loss": 1.1842, + "theoretical_loss": 3.3858276986102664, + "tokens_seen": 2606680064 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016998194583751254, + "loss": 1.1546, + "theoretical_loss": 3.3858210218869127, + "tokens_seen": 2606745600 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016996589769307924, + "loss": 1.0724, + "theoretical_loss": 3.3858143453784146, + "tokens_seen": 2606811136 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016994984954864594, + "loss": 1.1896, + "theoretical_loss": 3.3858076690847607, + "tokens_seen": 2606876672 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016993380140421265, + "loss": 1.3133, + "theoretical_loss": 3.3858009930059385, + "tokens_seen": 2606942208 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016991775325977935, + "loss": 1.1738, + "theoretical_loss": 3.385794317141935, + "tokens_seen": 2607007744 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016990170511534605, + "loss": 1.2886, + "theoretical_loss": 3.3857876414927386, + "tokens_seen": 2607073280 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016988565697091275, + "loss": 1.3111, + "theoretical_loss": 3.385780966058337, + "tokens_seen": 2607138816 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016986960882647945, + "loss": 1.2871, + "theoretical_loss": 3.3857742908387167, + "tokens_seen": 2607204352 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016985356068204615, + "loss": 1.2396, + "theoretical_loss": 3.385767615833867, + "tokens_seen": 2607269888 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016983751253761285, + "loss": 1.2108, + "theoretical_loss": 3.385760941043775, + "tokens_seen": 2607335424 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016982146439317955, + "loss": 1.192, + "theoretical_loss": 3.3857542664684277, + "tokens_seen": 2607400960 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016980541624874625, + "loss": 1.2474, + "theoretical_loss": 3.3857475921078137, + "tokens_seen": 2607466496 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016978936810431295, + "loss": 1.2321, + "theoretical_loss": 3.38574091796192, + "tokens_seen": 2607532032 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016977331995987965, + "loss": 1.1813, + "theoretical_loss": 3.3857342440307354, + "tokens_seen": 2607597568 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016975727181544635, + "loss": 1.1222, + "theoretical_loss": 3.385727570314246, + "tokens_seen": 2607663104 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016974122367101303, + "loss": 1.1537, + "theoretical_loss": 3.3857208968124413, + "tokens_seen": 2607728640 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016972517552657973, + "loss": 1.2583, + "theoretical_loss": 3.3857142235253073, + "tokens_seen": 2607794176 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016970912738214643, + "loss": 1.1884, + "theoretical_loss": 3.3857075504528327, + "tokens_seen": 2607859712 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016969307923771313, + "loss": 1.2495, + "theoretical_loss": 3.385700877595005, + "tokens_seen": 2607925248 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016967703109327983, + "loss": 1.2728, + "theoretical_loss": 3.385694204951812, + "tokens_seen": 2607990784 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016966098294884656, + "loss": 1.1743, + "theoretical_loss": 3.385687532523241, + "tokens_seen": 2608056320 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016964493480441326, + "loss": 1.2671, + "theoretical_loss": 3.3856808603092796, + "tokens_seen": 2608121856 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016962888665997996, + "loss": 1.2429, + "theoretical_loss": 3.385674188309917, + "tokens_seen": 2608187392 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016961283851554666, + "loss": 1.153, + "theoretical_loss": 3.385667516525139, + "tokens_seen": 2608252928 + }, + { + "epoch": 1.17, + "objective/train/docs_used": 962986, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1198704242706299, + "objective/train/theoretical_loss": 3.3856608449549337, + "objective/train/tokens_used": 2628778464, + "theoretical_loss": 3.3856608449549337, + "tokens_seen": 2608318464 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016959679037111336, + "loss": 1.1199, + "theoretical_loss": 3.3856608449549337, + "tokens_seen": 2608318464 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016958074222668006, + "loss": 1.2465, + "theoretical_loss": 3.3856541735992898, + "tokens_seen": 2608384000 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016956469408224676, + "loss": 1.2575, + "theoretical_loss": 3.3856475024581942, + "tokens_seen": 2608449536 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016954864593781346, + "loss": 1.2418, + "theoretical_loss": 3.385640831531635, + "tokens_seen": 2608515072 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016953259779338016, + "loss": 1.2864, + "theoretical_loss": 3.3856341608195994, + "tokens_seen": 2608580608 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016951654964894686, + "loss": 1.1997, + "theoretical_loss": 3.3856274903220758, + "tokens_seen": 2608646144 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016950050150451356, + "loss": 1.1928, + "theoretical_loss": 3.385620820039051, + "tokens_seen": 2608711680 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016948445336008026, + "loss": 1.3696, + "theoretical_loss": 3.385614149970514, + "tokens_seen": 2608777216 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016946840521564696, + "loss": 1.2636, + "theoretical_loss": 3.3856074801164517, + "tokens_seen": 2608842752 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016945235707121366, + "loss": 1.2538, + "theoretical_loss": 3.385600810476851, + "tokens_seen": 2608908288 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016943630892678034, + "loss": 1.2079, + "theoretical_loss": 3.385594141051701, + "tokens_seen": 2608973824 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016942026078234704, + "loss": 1.0777, + "theoretical_loss": 3.3855874718409895, + "tokens_seen": 2609039360 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016940421263791374, + "loss": 1.1801, + "theoretical_loss": 3.385580802844703, + "tokens_seen": 2609104896 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016938816449348044, + "loss": 1.1379, + "theoretical_loss": 3.3855741340628303, + "tokens_seen": 2609170432 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016937211634904714, + "loss": 1.1434, + "theoretical_loss": 3.3855674654953583, + "tokens_seen": 2609235968 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016935606820461384, + "loss": 1.2946, + "theoretical_loss": 3.385560797142275, + "tokens_seen": 2609301504 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016934002006018054, + "loss": 1.2133, + "theoretical_loss": 3.3855541290035687, + "tokens_seen": 2609367040 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016932397191574724, + "loss": 1.1865, + "theoretical_loss": 3.385547461079226, + "tokens_seen": 2609432576 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016930792377131394, + "loss": 1.1555, + "theoretical_loss": 3.3855407933692363, + "tokens_seen": 2609498112 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016929187562688064, + "loss": 1.2957, + "theoretical_loss": 3.3855341258735856, + "tokens_seen": 2609563648 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016927582748244734, + "loss": 1.2109, + "theoretical_loss": 3.3855274585922626, + "tokens_seen": 2609629184 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016925977933801405, + "loss": 1.1652, + "theoretical_loss": 3.3855207915252548, + "tokens_seen": 2609694720 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016924373119358075, + "loss": 1.2116, + "theoretical_loss": 3.3855141246725493, + "tokens_seen": 2609760256 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016922768304914745, + "loss": 1.2872, + "theoretical_loss": 3.385507458034135, + "tokens_seen": 2609825792 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016921163490471415, + "loss": 1.1593, + "theoretical_loss": 3.385500791609999, + "tokens_seen": 2609891328 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016919558676028085, + "loss": 1.1929, + "theoretical_loss": 3.385494125400129, + "tokens_seen": 2609956864 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016917953861584755, + "loss": 1.1921, + "theoretical_loss": 3.3854874594045126, + "tokens_seen": 2610022400 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016916349047141425, + "loss": 1.221, + "theoretical_loss": 3.385480793623138, + "tokens_seen": 2610087936 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016914744232698095, + "loss": 1.1524, + "theoretical_loss": 3.385474128055993, + "tokens_seen": 2610153472 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016913139418254765, + "loss": 1.2192, + "theoretical_loss": 3.3854674627030645, + "tokens_seen": 2610219008 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016911534603811435, + "loss": 1.276, + "theoretical_loss": 3.385460797564341, + "tokens_seen": 2610284544 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016909929789368105, + "loss": 1.3053, + "theoretical_loss": 3.3854541326398095, + "tokens_seen": 2610350080 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016908324974924775, + "loss": 1.29, + "theoretical_loss": 3.385447467929459, + "tokens_seen": 2610415616 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016906720160481445, + "loss": 1.2001, + "theoretical_loss": 3.385440803433276, + "tokens_seen": 2610481152 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016905115346038115, + "loss": 1.1762, + "theoretical_loss": 3.385434139151249, + "tokens_seen": 2610546688 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016903510531594785, + "loss": 1.1823, + "theoretical_loss": 3.3854274750833655, + "tokens_seen": 2610612224 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016901905717151455, + "loss": 1.1935, + "theoretical_loss": 3.3854208112296127, + "tokens_seen": 2610677760 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016900300902708126, + "loss": 1.2293, + "theoretical_loss": 3.3854141475899793, + "tokens_seen": 2610743296 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016898696088264796, + "loss": 1.2605, + "theoretical_loss": 3.3854074841644524, + "tokens_seen": 2610808832 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016897091273821466, + "loss": 1.214, + "theoretical_loss": 3.38540082095302, + "tokens_seen": 2610874368 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016895486459378136, + "loss": 1.1866, + "theoretical_loss": 3.38539415795567, + "tokens_seen": 2610939904 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016893881644934806, + "loss": 1.205, + "theoretical_loss": 3.38538749517239, + "tokens_seen": 2611005440 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016892276830491476, + "loss": 1.1668, + "theoretical_loss": 3.3853808326031674, + "tokens_seen": 2611070976 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016890672016048146, + "loss": 1.2214, + "theoretical_loss": 3.38537417024799, + "tokens_seen": 2611136512 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016889067201604816, + "loss": 1.2485, + "theoretical_loss": 3.385367508106846, + "tokens_seen": 2611202048 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016887462387161486, + "loss": 1.1966, + "theoretical_loss": 3.385360846179723, + "tokens_seen": 2611267584 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016885857572718156, + "loss": 1.2685, + "theoretical_loss": 3.385354184466609, + "tokens_seen": 2611333120 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016884252758274826, + "loss": 1.2521, + "theoretical_loss": 3.385347522967491, + "tokens_seen": 2611398656 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016882647943831496, + "loss": 1.1742, + "theoretical_loss": 3.3853408616823577, + "tokens_seen": 2611464192 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016881043129388166, + "loss": 1.2678, + "theoretical_loss": 3.385334200611196, + "tokens_seen": 2611529728 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016879438314944836, + "loss": 1.104, + "theoretical_loss": 3.385327539753994, + "tokens_seen": 2611595264 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016877833500501506, + "loss": 1.3696, + "theoretical_loss": 3.38532087911074, + "tokens_seen": 2611660800 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016876228686058176, + "loss": 1.262, + "theoretical_loss": 3.3853142186814207, + "tokens_seen": 2611726336 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016874623871614847, + "loss": 1.1972, + "theoretical_loss": 3.385307558466024, + "tokens_seen": 2611791872 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016873019057171514, + "loss": 1.2537, + "theoretical_loss": 3.3853008984645387, + "tokens_seen": 2611857408 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016871414242728184, + "loss": 1.1724, + "theoretical_loss": 3.385294238676952, + "tokens_seen": 2611922944 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016869809428284854, + "loss": 1.3025, + "theoretical_loss": 3.3852875791032515, + "tokens_seen": 2611988480 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016868204613841524, + "loss": 1.2502, + "theoretical_loss": 3.385280919743425, + "tokens_seen": 2612054016 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016866599799398194, + "loss": 1.1794, + "theoretical_loss": 3.38527426059746, + "tokens_seen": 2612119552 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016864994984954864, + "loss": 1.2246, + "theoretical_loss": 3.385267601665345, + "tokens_seen": 2612185088 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016863390170511534, + "loss": 1.2439, + "theoretical_loss": 3.385260942947067, + "tokens_seen": 2612250624 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016861785356068204, + "loss": 1.2171, + "theoretical_loss": 3.385254284442614, + "tokens_seen": 2612316160 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016860180541624874, + "loss": 1.2589, + "theoretical_loss": 3.3852476261519744, + "tokens_seen": 2612381696 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016858575727181544, + "loss": 1.2437, + "theoretical_loss": 3.3852409680751356, + "tokens_seen": 2612447232 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016856970912738215, + "loss": 1.1616, + "theoretical_loss": 3.385234310212085, + "tokens_seen": 2612512768 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016855366098294885, + "loss": 1.1599, + "theoretical_loss": 3.3852276525628104, + "tokens_seen": 2612578304 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016853761283851555, + "loss": 1.1999, + "theoretical_loss": 3.3852209951273, + "tokens_seen": 2612643840 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016852156469408225, + "loss": 1.1862, + "theoretical_loss": 3.385214337905541, + "tokens_seen": 2612709376 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016850551654964895, + "loss": 1.0941, + "theoretical_loss": 3.3852076808975218, + "tokens_seen": 2612774912 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016848946840521565, + "loss": 1.115, + "theoretical_loss": 3.38520102410323, + "tokens_seen": 2612840448 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016847342026078235, + "loss": 1.2289, + "theoretical_loss": 3.385194367522653, + "tokens_seen": 2612905984 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016845737211634908, + "loss": 1.2949, + "theoretical_loss": 3.385187711155779, + "tokens_seen": 2612971520 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016844132397191578, + "loss": 1.3069, + "theoretical_loss": 3.3851810550025956, + "tokens_seen": 2613037056 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016842527582748248, + "loss": 1.2532, + "theoretical_loss": 3.3851743990630907, + "tokens_seen": 2613102592 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016840922768304915, + "loss": 1.3009, + "theoretical_loss": 3.3851677433372522, + "tokens_seen": 2613168128 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016839317953861585, + "loss": 1.2127, + "theoretical_loss": 3.3851610878250673, + "tokens_seen": 2613233664 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016837713139418255, + "loss": 1.2245, + "theoretical_loss": 3.3851544325265244, + "tokens_seen": 2613299200 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016836108324974925, + "loss": 1.2004, + "theoretical_loss": 3.385147777441611, + "tokens_seen": 2613364736 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016834503510531595, + "loss": 1.2635, + "theoretical_loss": 3.3851411225703147, + "tokens_seen": 2613430272 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016832898696088265, + "loss": 1.1198, + "theoretical_loss": 3.385134467912624, + "tokens_seen": 2613495808 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016831293881644936, + "loss": 1.2387, + "theoretical_loss": 3.385127813468526, + "tokens_seen": 2613561344 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016829689067201606, + "loss": 1.2425, + "theoretical_loss": 3.385121159238009, + "tokens_seen": 2613626880 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016828084252758276, + "loss": 1.2069, + "theoretical_loss": 3.38511450522106, + "tokens_seen": 2613692416 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016826479438314946, + "loss": 1.14, + "theoretical_loss": 3.3851078514176676, + "tokens_seen": 2613757952 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016824874623871616, + "loss": 1.1675, + "theoretical_loss": 3.3851011978278187, + "tokens_seen": 2613823488 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016823269809428286, + "loss": 1.2985, + "theoretical_loss": 3.385094544451502, + "tokens_seen": 2613889024 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016821664994984956, + "loss": 1.2064, + "theoretical_loss": 3.3850878912887055, + "tokens_seen": 2613954560 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016820060180541626, + "loss": 1.265, + "theoretical_loss": 3.3850812383394158, + "tokens_seen": 2614020096 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016818455366098296, + "loss": 1.1815, + "theoretical_loss": 3.3850745856036215, + "tokens_seen": 2614085632 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016816850551654966, + "loss": 1.2056, + "theoretical_loss": 3.3850679330813103, + "tokens_seen": 2614151168 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016815245737211636, + "loss": 1.176, + "theoretical_loss": 3.3850612807724705, + "tokens_seen": 2614216704 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016813640922768306, + "loss": 1.2838, + "theoretical_loss": 3.3850546286770884, + "tokens_seen": 2614282240 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016812036108324976, + "loss": 1.2312, + "theoretical_loss": 3.3850479767951533, + "tokens_seen": 2614347776 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016810431293881646, + "loss": 1.2476, + "theoretical_loss": 3.3850413251266525, + "tokens_seen": 2614413312 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016808826479438316, + "loss": 1.195, + "theoretical_loss": 3.3850346736715733, + "tokens_seen": 2614478848 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016807221664994986, + "loss": 1.164, + "theoretical_loss": 3.3850280224299043, + "tokens_seen": 2614544384 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016805616850551657, + "loss": 1.0851, + "theoretical_loss": 3.385021371401633, + "tokens_seen": 2614609920 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016804012036108327, + "loss": 1.2975, + "theoretical_loss": 3.3850147205867467, + "tokens_seen": 2614675456 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016802407221664994, + "loss": 1.2682, + "theoretical_loss": 3.385008069985234, + "tokens_seen": 2614740992 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016800802407221664, + "loss": 1.1754, + "theoretical_loss": 3.3850014195970823, + "tokens_seen": 2614806528 + }, + { + "epoch": 1.17, + "objective/train/docs_used": 965905, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3353825807571411, + "objective/train/theoretical_loss": 3.38499476942228, + "objective/train/tokens_used": 2635332064, + "theoretical_loss": 3.38499476942228, + "tokens_seen": 2614872064 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016799197592778334, + "loss": 1.3354, + "theoretical_loss": 3.38499476942228, + "tokens_seen": 2614872064 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016797592778335004, + "loss": 1.1731, + "theoretical_loss": 3.3849881194608136, + "tokens_seen": 2614937600 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016795987963891674, + "loss": 1.2494, + "theoretical_loss": 3.384981469712672, + "tokens_seen": 2615003136 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016794383149448344, + "loss": 1.1981, + "theoretical_loss": 3.3849748201778427, + "tokens_seen": 2615068672 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016792778335005014, + "loss": 1.2227, + "theoretical_loss": 3.3849681708563137, + "tokens_seen": 2615134208 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016791173520561684, + "loss": 1.141, + "theoretical_loss": 3.384961521748073, + "tokens_seen": 2615199744 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016789568706118354, + "loss": 1.2529, + "theoretical_loss": 3.384954872853107, + "tokens_seen": 2615265280 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016787963891675025, + "loss": 1.1851, + "theoretical_loss": 3.3849482241714055, + "tokens_seen": 2615330816 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016786359077231697, + "loss": 1.2137, + "theoretical_loss": 3.384941575702955, + "tokens_seen": 2615396352 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016784754262788367, + "loss": 1.278, + "theoretical_loss": 3.3849349274477434, + "tokens_seen": 2615461888 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016783149448345037, + "loss": 1.1949, + "theoretical_loss": 3.3849282794057594, + "tokens_seen": 2615527424 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016781544633901708, + "loss": 1.287, + "theoretical_loss": 3.3849216315769897, + "tokens_seen": 2615592960 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016779939819458378, + "loss": 1.2242, + "theoretical_loss": 3.3849149839614228, + "tokens_seen": 2615658496 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016778335005015048, + "loss": 1.1467, + "theoretical_loss": 3.3849083365590467, + "tokens_seen": 2615724032 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016776730190571718, + "loss": 1.2564, + "theoretical_loss": 3.3849016893698485, + "tokens_seen": 2615789568 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016775125376128388, + "loss": 1.2541, + "theoretical_loss": 3.3848950423938167, + "tokens_seen": 2615855104 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016773520561685058, + "loss": 1.198, + "theoretical_loss": 3.384888395630939, + "tokens_seen": 2615920640 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016771915747241728, + "loss": 1.1067, + "theoretical_loss": 3.3848817490812024, + "tokens_seen": 2615986176 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016770310932798395, + "loss": 1.2013, + "theoretical_loss": 3.3848751027445956, + "tokens_seen": 2616051712 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016768706118355065, + "loss": 1.1906, + "theoretical_loss": 3.3848684566211062, + "tokens_seen": 2616117248 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016767101303911735, + "loss": 1.3591, + "theoretical_loss": 3.3848618107107225, + "tokens_seen": 2616182784 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016765496489468405, + "loss": 1.2529, + "theoretical_loss": 3.3848551650134318, + "tokens_seen": 2616248320 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016763891675025075, + "loss": 1.1866, + "theoretical_loss": 3.3848485195292213, + "tokens_seen": 2616313856 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016762286860581746, + "loss": 1.1055, + "theoretical_loss": 3.3848418742580804, + "tokens_seen": 2616379392 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016760682046138416, + "loss": 1.1592, + "theoretical_loss": 3.3848352291999957, + "tokens_seen": 2616444928 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016759077231695086, + "loss": 1.2197, + "theoretical_loss": 3.3848285843549553, + "tokens_seen": 2616510464 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016757472417251756, + "loss": 1.3036, + "theoretical_loss": 3.384821939722947, + "tokens_seen": 2616576000 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016755867602808426, + "loss": 1.2686, + "theoretical_loss": 3.384815295303959, + "tokens_seen": 2616641536 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016754262788365096, + "loss": 1.264, + "theoretical_loss": 3.384808651097979, + "tokens_seen": 2616707072 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016752657973921766, + "loss": 1.1528, + "theoretical_loss": 3.384802007104995, + "tokens_seen": 2616772608 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016751053159478436, + "loss": 1.3016, + "theoretical_loss": 3.384795363324994, + "tokens_seen": 2616838144 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016749448345035106, + "loss": 1.1915, + "theoretical_loss": 3.3847887197579647, + "tokens_seen": 2616903680 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016747843530591776, + "loss": 1.2905, + "theoretical_loss": 3.3847820764038947, + "tokens_seen": 2616969216 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016746238716148446, + "loss": 1.2237, + "theoretical_loss": 3.3847754332627717, + "tokens_seen": 2617034752 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016744633901705116, + "loss": 1.1574, + "theoretical_loss": 3.3847687903345838, + "tokens_seen": 2617100288 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016743029087261786, + "loss": 1.3101, + "theoretical_loss": 3.3847621476193184, + "tokens_seen": 2617165824 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016741424272818456, + "loss": 1.2996, + "theoretical_loss": 3.3847555051169635, + "tokens_seen": 2617231360 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016739819458375126, + "loss": 1.1477, + "theoretical_loss": 3.3847488628275073, + "tokens_seen": 2617296896 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016738214643931797, + "loss": 1.1858, + "theoretical_loss": 3.3847422207509377, + "tokens_seen": 2617362432 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016736609829488467, + "loss": 1.307, + "theoretical_loss": 3.3847355788872417, + "tokens_seen": 2617427968 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016735005015045137, + "loss": 1.2552, + "theoretical_loss": 3.3847289372364084, + "tokens_seen": 2617493504 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016733400200601807, + "loss": 1.1782, + "theoretical_loss": 3.3847222957984244, + "tokens_seen": 2617559040 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016731795386158474, + "loss": 1.2424, + "theoretical_loss": 3.384715654573278, + "tokens_seen": 2617624576 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016730190571715144, + "loss": 1.1205, + "theoretical_loss": 3.3847090135609577, + "tokens_seen": 2617690112 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016728585757271814, + "loss": 1.236, + "theoretical_loss": 3.3847023727614505, + "tokens_seen": 2617755648 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016726980942828484, + "loss": 1.2783, + "theoretical_loss": 3.3846957321747446, + "tokens_seen": 2617821184 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016725376128385157, + "loss": 1.2771, + "theoretical_loss": 3.384689091800828, + "tokens_seen": 2617886720 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016723771313941827, + "loss": 1.1101, + "theoretical_loss": 3.384682451639688, + "tokens_seen": 2617952256 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016722166499498497, + "loss": 1.2628, + "theoretical_loss": 3.384675811691313, + "tokens_seen": 2618017792 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016720561685055167, + "loss": 1.1015, + "theoretical_loss": 3.3846691719556907, + "tokens_seen": 2618083328 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016718956870611837, + "loss": 1.1783, + "theoretical_loss": 3.384662532432809, + "tokens_seen": 2618148864 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016717352056168507, + "loss": 1.1786, + "theoretical_loss": 3.384655893122656, + "tokens_seen": 2618214400 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016715747241725177, + "loss": 1.1626, + "theoretical_loss": 3.3846492540252187, + "tokens_seen": 2618279936 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016714142427281847, + "loss": 1.2342, + "theoretical_loss": 3.3846426151404856, + "tokens_seen": 2618345472 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016712537612838518, + "loss": 1.2207, + "theoretical_loss": 3.384635976468445, + "tokens_seen": 2618411008 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016710932798395188, + "loss": 1.221, + "theoretical_loss": 3.3846293380090837, + "tokens_seen": 2618476544 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016709327983951858, + "loss": 1.2719, + "theoretical_loss": 3.3846226997623905, + "tokens_seen": 2618542080 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016707723169508528, + "loss": 1.2681, + "theoretical_loss": 3.3846160617283525, + "tokens_seen": 2618607616 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016706118355065198, + "loss": 1.139, + "theoretical_loss": 3.384609423906958, + "tokens_seen": 2618673152 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016704513540621868, + "loss": 1.2608, + "theoretical_loss": 3.384602786298195, + "tokens_seen": 2618738688 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016702908726178538, + "loss": 1.1791, + "theoretical_loss": 3.3845961489020513, + "tokens_seen": 2618804224 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016701303911735205, + "loss": 1.1683, + "theoretical_loss": 3.3845895117185143, + "tokens_seen": 2618869760 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016699699097291875, + "loss": 1.2251, + "theoretical_loss": 3.384582874747572, + "tokens_seen": 2618935296 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016698094282848545, + "loss": 1.1766, + "theoretical_loss": 3.3845762379892133, + "tokens_seen": 2619000832 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016696489468405215, + "loss": 1.2122, + "theoretical_loss": 3.384569601443425, + "tokens_seen": 2619066368 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016694884653961886, + "loss": 1.2616, + "theoretical_loss": 3.3845629651101947, + "tokens_seen": 2619131904 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016693279839518556, + "loss": 1.1797, + "theoretical_loss": 3.384556328989511, + "tokens_seen": 2619197440 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016691675025075226, + "loss": 1.2005, + "theoretical_loss": 3.384549693081362, + "tokens_seen": 2619262976 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016690070210631896, + "loss": 1.268, + "theoretical_loss": 3.3845430573857347, + "tokens_seen": 2619328512 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016688465396188566, + "loss": 1.2522, + "theoretical_loss": 3.3845364219026175, + "tokens_seen": 2619394048 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016686860581745236, + "loss": 1.1585, + "theoretical_loss": 3.3845297866319983, + "tokens_seen": 2619459584 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016685255767301906, + "loss": 1.1684, + "theoretical_loss": 3.384523151573865, + "tokens_seen": 2619525120 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016683650952858576, + "loss": 1.1938, + "theoretical_loss": 3.384516516728205, + "tokens_seen": 2619590656 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016682046138415246, + "loss": 1.105, + "theoretical_loss": 3.3845098820950072, + "tokens_seen": 2619656192 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016680441323971916, + "loss": 1.2484, + "theoretical_loss": 3.3845032476742585, + "tokens_seen": 2619721728 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016678836509528586, + "loss": 1.1558, + "theoretical_loss": 3.384496613465947, + "tokens_seen": 2619787264 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016677231695085256, + "loss": 1.1836, + "theoretical_loss": 3.3844899794700605, + "tokens_seen": 2619852800 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016675626880641926, + "loss": 1.1948, + "theoretical_loss": 3.3844833456865873, + "tokens_seen": 2619918336 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016674022066198596, + "loss": 1.1014, + "theoretical_loss": 3.3844767121155153, + "tokens_seen": 2619983872 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016672417251755266, + "loss": 1.2119, + "theoretical_loss": 3.3844700787568316, + "tokens_seen": 2620049408 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016670812437311936, + "loss": 1.2087, + "theoretical_loss": 3.384463445610525, + "tokens_seen": 2620114944 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016669207622868607, + "loss": 1.2371, + "theoretical_loss": 3.384456812676583, + "tokens_seen": 2620180480 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016667602808425277, + "loss": 1.032, + "theoretical_loss": 3.3844501799549938, + "tokens_seen": 2620246016 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016665997993981947, + "loss": 1.2069, + "theoretical_loss": 3.3844435474457444, + "tokens_seen": 2620311552 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016664393179538617, + "loss": 1.1761, + "theoretical_loss": 3.384436915148824, + "tokens_seen": 2620377088 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016662788365095287, + "loss": 1.2072, + "theoretical_loss": 3.3844302830642192, + "tokens_seen": 2620442624 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016661183550651957, + "loss": 1.2036, + "theoretical_loss": 3.3844236511919186, + "tokens_seen": 2620508160 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016659578736208627, + "loss": 1.1862, + "theoretical_loss": 3.3844170195319103, + "tokens_seen": 2620573696 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016657973921765297, + "loss": 1.1578, + "theoretical_loss": 3.3844103880841816, + "tokens_seen": 2620639232 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016656369107321967, + "loss": 1.2764, + "theoretical_loss": 3.3844037568487204, + "tokens_seen": 2620704768 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016654764292878637, + "loss": 1.2296, + "theoretical_loss": 3.384397125825515, + "tokens_seen": 2620770304 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016653159478435307, + "loss": 1.2326, + "theoretical_loss": 3.384390495014554, + "tokens_seen": 2620835840 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016651554663991977, + "loss": 1.2257, + "theoretical_loss": 3.384383864415824, + "tokens_seen": 2620901376 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016649949849548647, + "loss": 1.1454, + "theoretical_loss": 3.384377234029313, + "tokens_seen": 2620966912 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016648345035105317, + "loss": 1.1672, + "theoretical_loss": 3.384370603855009, + "tokens_seen": 2621032448 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016646740220661987, + "loss": 1.1522, + "theoretical_loss": 3.384363973892901, + "tokens_seen": 2621097984 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016645135406218657, + "loss": 1.2173, + "theoretical_loss": 3.384357344142976, + "tokens_seen": 2621163520 + }, + { + "epoch": 1.17, + "learning_rate": 0.00016643530591775328, + "loss": 1.1282, + "theoretical_loss": 3.3843507146052216, + "tokens_seen": 2621229056 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016641925777331998, + "loss": 1.1438, + "theoretical_loss": 3.384344085279626, + "tokens_seen": 2621294592 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016640320962888668, + "loss": 1.1938, + "theoretical_loss": 3.384337456166177, + "tokens_seen": 2621360128 + }, + { + "debugging/Compilability": 0.96875, + "debugging/distinct-1-grams": 0.7223527597750008, + "debugging/entropy-1-grams": 5.542038399192005, + "debugging/length": 471.3125, + "debugging/num_segments": 32, + "debugging/score": 0.0036147918259581175, + "debugging/score_std": 0.002343087994424332, + "epoch": 1.18, + "objective/train/docs_used": 968228, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1880791187286377, + "objective/train/theoretical_loss": 3.3843308272648636, + "objective/train/tokens_used": 2641885664, + "theoretical_loss": 3.3843308272648636, + "tokens_seen": 2621425664 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016638716148445338, + "loss": 1.1881, + "theoretical_loss": 3.3843308272648636, + "tokens_seen": 2621425664 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016637111334002008, + "loss": 1.2257, + "theoretical_loss": 3.3843241985756722, + "tokens_seen": 2621491200 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016635506519558678, + "loss": 1.2737, + "theoretical_loss": 3.3843175700985912, + "tokens_seen": 2621556736 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016633901705115348, + "loss": 1.1744, + "theoretical_loss": 3.384310941833609, + "tokens_seen": 2621622272 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016632296890672018, + "loss": 1.2804, + "theoretical_loss": 3.384304313780713, + "tokens_seen": 2621687808 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016630692076228685, + "loss": 1.1669, + "theoretical_loss": 3.384297685939891, + "tokens_seen": 2621753344 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016629087261785355, + "loss": 1.1476, + "theoretical_loss": 3.3842910583111316, + "tokens_seen": 2621818880 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016627482447342025, + "loss": 1.1962, + "theoretical_loss": 3.384284430894422, + "tokens_seen": 2621884416 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016625877632898696, + "loss": 1.1634, + "theoretical_loss": 3.3842778036897503, + "tokens_seen": 2621949952 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016624272818455366, + "loss": 1.2402, + "theoretical_loss": 3.384271176697105, + "tokens_seen": 2622015488 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016622668004012036, + "loss": 1.2914, + "theoretical_loss": 3.384264549916473, + "tokens_seen": 2622081024 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016621063189568706, + "loss": 1.2383, + "theoretical_loss": 3.384257923347843, + "tokens_seen": 2622146560 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016619458375125376, + "loss": 1.1642, + "theoretical_loss": 3.3842512969912026, + "tokens_seen": 2622212096 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016617853560682046, + "loss": 1.2288, + "theoretical_loss": 3.38424467084654, + "tokens_seen": 2622277632 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016616248746238716, + "loss": 1.1766, + "theoretical_loss": 3.3842380449138423, + "tokens_seen": 2622343168 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016614643931795386, + "loss": 1.1684, + "theoretical_loss": 3.3842314191930987, + "tokens_seen": 2622408704 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016613039117352056, + "loss": 1.1897, + "theoretical_loss": 3.384224793684296, + "tokens_seen": 2622474240 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016611434302908726, + "loss": 1.2827, + "theoretical_loss": 3.384218168387423, + "tokens_seen": 2622539776 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016609829488465396, + "loss": 1.2531, + "theoretical_loss": 3.384211543302467, + "tokens_seen": 2622605312 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016608224674022066, + "loss": 1.1345, + "theoretical_loss": 3.384204918429416, + "tokens_seen": 2622670848 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016606619859578736, + "loss": 1.2023, + "theoretical_loss": 3.384198293768258, + "tokens_seen": 2622736384 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001660501504513541, + "loss": 1.2505, + "theoretical_loss": 3.3841916693189815, + "tokens_seen": 2622801920 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001660341023069208, + "loss": 1.1257, + "theoretical_loss": 3.3841850450815736, + "tokens_seen": 2622867456 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001660180541624875, + "loss": 1.1934, + "theoretical_loss": 3.3841784210560224, + "tokens_seen": 2622932992 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001660020060180542, + "loss": 1.2373, + "theoretical_loss": 3.384171797242316, + "tokens_seen": 2622998528 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016598595787362087, + "loss": 1.3191, + "theoretical_loss": 3.3841651736404423, + "tokens_seen": 2623064064 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016596990972918757, + "loss": 1.1993, + "theoretical_loss": 3.3841585502503895, + "tokens_seen": 2623129600 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016595386158475427, + "loss": 1.3093, + "theoretical_loss": 3.384151927072145, + "tokens_seen": 2623195136 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016593781344032097, + "loss": 1.1585, + "theoretical_loss": 3.3841453041056972, + "tokens_seen": 2623260672 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016592176529588767, + "loss": 1.2126, + "theoretical_loss": 3.384138681351034, + "tokens_seen": 2623326208 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016590571715145437, + "loss": 1.1898, + "theoretical_loss": 3.3841320588081425, + "tokens_seen": 2623391744 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016588966900702107, + "loss": 1.2065, + "theoretical_loss": 3.384125436477012, + "tokens_seen": 2623457280 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016587362086258777, + "loss": 1.2556, + "theoretical_loss": 3.3841188143576293, + "tokens_seen": 2623522816 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016585757271815447, + "loss": 1.1929, + "theoretical_loss": 3.384112192449983, + "tokens_seen": 2623588352 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016584152457372117, + "loss": 1.2682, + "theoretical_loss": 3.3841055707540613, + "tokens_seen": 2623653888 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016582547642928787, + "loss": 1.1896, + "theoretical_loss": 3.384098949269851, + "tokens_seen": 2623719424 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016580942828485457, + "loss": 1.2294, + "theoretical_loss": 3.384092327997341, + "tokens_seen": 2623784960 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016579338014042127, + "loss": 1.2847, + "theoretical_loss": 3.384085706936519, + "tokens_seen": 2623850496 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016577733199598797, + "loss": 1.219, + "theoretical_loss": 3.384079086087373, + "tokens_seen": 2623916032 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016576128385155468, + "loss": 1.2723, + "theoretical_loss": 3.384072465449891, + "tokens_seen": 2623981568 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016574523570712138, + "loss": 1.2593, + "theoretical_loss": 3.38406584502406, + "tokens_seen": 2624047104 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016572918756268808, + "loss": 1.2771, + "theoretical_loss": 3.3840592248098695, + "tokens_seen": 2624112640 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016571313941825478, + "loss": 1.2209, + "theoretical_loss": 3.3840526048073065, + "tokens_seen": 2624178176 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016569709127382148, + "loss": 1.1973, + "theoretical_loss": 3.384045985016359, + "tokens_seen": 2624243712 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016568104312938818, + "loss": 1.2966, + "theoretical_loss": 3.3840393654370153, + "tokens_seen": 2624309248 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016566499498495488, + "loss": 1.1802, + "theoretical_loss": 3.384032746069263, + "tokens_seen": 2624374784 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016564894684052158, + "loss": 1.1965, + "theoretical_loss": 3.3840261269130907, + "tokens_seen": 2624440320 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016563289869608828, + "loss": 1.2324, + "theoretical_loss": 3.3840195079684854, + "tokens_seen": 2624505856 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016561685055165498, + "loss": 1.1992, + "theoretical_loss": 3.3840128892354358, + "tokens_seen": 2624571392 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016560080240722165, + "loss": 1.2536, + "theoretical_loss": 3.3840062707139293, + "tokens_seen": 2624636928 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016558475426278835, + "loss": 1.2409, + "theoretical_loss": 3.383999652403954, + "tokens_seen": 2624702464 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016556870611835506, + "loss": 1.1673, + "theoretical_loss": 3.3839930343054987, + "tokens_seen": 2624768000 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016555265797392176, + "loss": 1.2993, + "theoretical_loss": 3.38398641641855, + "tokens_seen": 2624833536 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016553660982948846, + "loss": 1.2979, + "theoretical_loss": 3.3839797987430966, + "tokens_seen": 2624899072 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016552056168505516, + "loss": 1.1491, + "theoretical_loss": 3.383973181279127, + "tokens_seen": 2624964608 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016550451354062186, + "loss": 1.2543, + "theoretical_loss": 3.3839665640266277, + "tokens_seen": 2625030144 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016548846539618856, + "loss": 1.2726, + "theoretical_loss": 3.3839599469855877, + "tokens_seen": 2625095680 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016547241725175526, + "loss": 1.2213, + "theoretical_loss": 3.383953330155995, + "tokens_seen": 2625161216 + }, + { + "epoch": 1.18, + "learning_rate": 0.000165456369107322, + "loss": 1.167, + "theoretical_loss": 3.383946713537837, + "tokens_seen": 2625226752 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001654403209628887, + "loss": 1.2872, + "theoretical_loss": 3.383940097131102, + "tokens_seen": 2625292288 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001654242728184554, + "loss": 1.3252, + "theoretical_loss": 3.3839334809357786, + "tokens_seen": 2625357824 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001654082246740221, + "loss": 1.192, + "theoretical_loss": 3.383926864951854, + "tokens_seen": 2625423360 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001653921765295888, + "loss": 1.2447, + "theoretical_loss": 3.3839202491793157, + "tokens_seen": 2625488896 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001653761283851555, + "loss": 1.2325, + "theoretical_loss": 3.3839136336181523, + "tokens_seen": 2625554432 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001653600802407222, + "loss": 1.1936, + "theoretical_loss": 3.383907018268352, + "tokens_seen": 2625619968 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001653440320962889, + "loss": 1.2154, + "theoretical_loss": 3.3839004031299025, + "tokens_seen": 2625685504 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001653279839518556, + "loss": 1.1386, + "theoretical_loss": 3.383893788202792, + "tokens_seen": 2625751040 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001653119358074223, + "loss": 1.2013, + "theoretical_loss": 3.383887173487008, + "tokens_seen": 2625816576 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016529588766298897, + "loss": 1.1993, + "theoretical_loss": 3.3838805589825385, + "tokens_seen": 2625882112 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016527983951855567, + "loss": 1.1831, + "theoretical_loss": 3.383873944689372, + "tokens_seen": 2625947648 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016526379137412237, + "loss": 1.2255, + "theoretical_loss": 3.3838673306074964, + "tokens_seen": 2626013184 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016524774322968907, + "loss": 1.2862, + "theoretical_loss": 3.383860716736899, + "tokens_seen": 2626078720 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016523169508525577, + "loss": 1.1675, + "theoretical_loss": 3.383854103077568, + "tokens_seen": 2626144256 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016521564694082247, + "loss": 1.1459, + "theoretical_loss": 3.3838474896294923, + "tokens_seen": 2626209792 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016519959879638917, + "loss": 1.152, + "theoretical_loss": 3.383840876392659, + "tokens_seen": 2626275328 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016518355065195587, + "loss": 1.2533, + "theoretical_loss": 3.3838342633670564, + "tokens_seen": 2626340864 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016516750250752257, + "loss": 1.0901, + "theoretical_loss": 3.383827650552672, + "tokens_seen": 2626406400 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016515145436308927, + "loss": 1.2321, + "theoretical_loss": 3.3838210379494944, + "tokens_seen": 2626471936 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016513540621865597, + "loss": 1.209, + "theoretical_loss": 3.383814425557511, + "tokens_seen": 2626537472 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016511935807422267, + "loss": 1.289, + "theoretical_loss": 3.3838078133767104, + "tokens_seen": 2626603008 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016510330992978937, + "loss": 1.1804, + "theoretical_loss": 3.3838012014070804, + "tokens_seen": 2626668544 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016508726178535607, + "loss": 1.2903, + "theoretical_loss": 3.3837945896486086, + "tokens_seen": 2626734080 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016507121364092278, + "loss": 1.1896, + "theoretical_loss": 3.3837879781012834, + "tokens_seen": 2626799616 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016505516549648948, + "loss": 1.2486, + "theoretical_loss": 3.383781366765093, + "tokens_seen": 2626865152 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016503911735205618, + "loss": 1.182, + "theoretical_loss": 3.383774755640024, + "tokens_seen": 2626930688 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016502306920762288, + "loss": 1.1819, + "theoretical_loss": 3.3837681447260666, + "tokens_seen": 2626996224 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016500702106318958, + "loss": 1.1579, + "theoretical_loss": 3.3837615340232072, + "tokens_seen": 2627061760 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016499097291875628, + "loss": 1.3285, + "theoretical_loss": 3.383754923531434, + "tokens_seen": 2627127296 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016497492477432298, + "loss": 1.198, + "theoretical_loss": 3.3837483132507358, + "tokens_seen": 2627192832 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016495887662988968, + "loss": 1.1859, + "theoretical_loss": 3.3837417031810997, + "tokens_seen": 2627258368 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016494282848545638, + "loss": 1.2898, + "theoretical_loss": 3.383735093322514, + "tokens_seen": 2627323904 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016492678034102308, + "loss": 1.2372, + "theoretical_loss": 3.383728483674967, + "tokens_seen": 2627389440 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016491073219658975, + "loss": 1.19, + "theoretical_loss": 3.3837218742384456, + "tokens_seen": 2627454976 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016489468405215646, + "loss": 1.196, + "theoretical_loss": 3.3837152650129396, + "tokens_seen": 2627520512 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016487863590772316, + "loss": 1.1546, + "theoretical_loss": 3.3837086559984355, + "tokens_seen": 2627586048 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016486258776328988, + "loss": 1.2652, + "theoretical_loss": 3.3837020471949217, + "tokens_seen": 2627651584 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016484653961885658, + "loss": 1.3183, + "theoretical_loss": 3.3836954386023868, + "tokens_seen": 2627717120 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016483049147442328, + "loss": 1.2272, + "theoretical_loss": 3.3836888302208177, + "tokens_seen": 2627782656 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016481444332998999, + "loss": 1.0728, + "theoretical_loss": 3.3836822220502034, + "tokens_seen": 2627848192 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016479839518555669, + "loss": 1.2002, + "theoretical_loss": 3.3836756140905315, + "tokens_seen": 2627913728 + }, + { + "epoch": 1.18, + "objective/train/docs_used": 970711, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2031419277191162, + "objective/train/theoretical_loss": 3.38366900634179, + "objective/train/tokens_used": 2648439264, + "theoretical_loss": 3.38366900634179, + "tokens_seen": 2627979264 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001647823470411234, + "loss": 1.2031, + "theoretical_loss": 3.38366900634179, + "tokens_seen": 2627979264 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001647662988966901, + "loss": 1.154, + "theoretical_loss": 3.383662398803967, + "tokens_seen": 2628044800 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001647502507522568, + "loss": 1.2274, + "theoretical_loss": 3.38365579147705, + "tokens_seen": 2628110336 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001647342026078235, + "loss": 1.1877, + "theoretical_loss": 3.3836491843610284, + "tokens_seen": 2628175872 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001647181544633902, + "loss": 1.279, + "theoretical_loss": 3.3836425774558885, + "tokens_seen": 2628241408 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001647021063189569, + "loss": 1.2354, + "theoretical_loss": 3.3836359707616195, + "tokens_seen": 2628306944 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001646860581745236, + "loss": 1.2298, + "theoretical_loss": 3.3836293642782085, + "tokens_seen": 2628372480 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001646700100300903, + "loss": 1.2392, + "theoretical_loss": 3.3836227580056444, + "tokens_seen": 2628438016 + }, + { + "epoch": 1.18, + "learning_rate": 0.000164653961885657, + "loss": 1.2424, + "theoretical_loss": 3.383616151943915, + "tokens_seen": 2628503552 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001646379137412237, + "loss": 1.3078, + "theoretical_loss": 3.383609546093008, + "tokens_seen": 2628569088 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001646218655967904, + "loss": 1.2143, + "theoretical_loss": 3.383602940452911, + "tokens_seen": 2628634624 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001646058174523571, + "loss": 1.1126, + "theoretical_loss": 3.383596335023613, + "tokens_seen": 2628700160 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016458976930792377, + "loss": 1.2921, + "theoretical_loss": 3.383589729805102, + "tokens_seen": 2628765696 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016457372116349047, + "loss": 1.2429, + "theoretical_loss": 3.383583124797365, + "tokens_seen": 2628831232 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016455767301905717, + "loss": 1.2327, + "theoretical_loss": 3.383576520000391, + "tokens_seen": 2628896768 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016454162487462387, + "loss": 1.1617, + "theoretical_loss": 3.383569915414167, + "tokens_seen": 2628962304 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016452557673019057, + "loss": 1.2584, + "theoretical_loss": 3.3835633110386825, + "tokens_seen": 2629027840 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016450952858575727, + "loss": 1.135, + "theoretical_loss": 3.3835567068739243, + "tokens_seen": 2629093376 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016449348044132397, + "loss": 1.2386, + "theoretical_loss": 3.3835501029198807, + "tokens_seen": 2629158912 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016447743229689067, + "loss": 1.2256, + "theoretical_loss": 3.3835434991765396, + "tokens_seen": 2629224448 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016446138415245737, + "loss": 1.2862, + "theoretical_loss": 3.38353689564389, + "tokens_seen": 2629289984 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016444533600802407, + "loss": 1.2617, + "theoretical_loss": 3.3835302923219186, + "tokens_seen": 2629355520 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016442928786359077, + "loss": 1.2535, + "theoretical_loss": 3.383523689210614, + "tokens_seen": 2629421056 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016441323971915747, + "loss": 1.2154, + "theoretical_loss": 3.383517086309965, + "tokens_seen": 2629486592 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016439719157472417, + "loss": 1.2095, + "theoretical_loss": 3.383510483619958, + "tokens_seen": 2629552128 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016438114343029088, + "loss": 1.232, + "theoretical_loss": 3.383503881140582, + "tokens_seen": 2629617664 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016436509528585758, + "loss": 1.3033, + "theoretical_loss": 3.3834972788718254, + "tokens_seen": 2629683200 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016434904714142428, + "loss": 1.1369, + "theoretical_loss": 3.3834906768136754, + "tokens_seen": 2629748736 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016433299899699098, + "loss": 1.1716, + "theoretical_loss": 3.3834840749661206, + "tokens_seen": 2629814272 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016431695085255768, + "loss": 1.3206, + "theoretical_loss": 3.383477473329149, + "tokens_seen": 2629879808 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016430090270812438, + "loss": 1.3305, + "theoretical_loss": 3.383470871902748, + "tokens_seen": 2629945344 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016428485456369108, + "loss": 1.1959, + "theoretical_loss": 3.3834642706869067, + "tokens_seen": 2630010880 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016426880641925778, + "loss": 1.2626, + "theoretical_loss": 3.3834576696816123, + "tokens_seen": 2630076416 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016425275827482448, + "loss": 1.2745, + "theoretical_loss": 3.3834510688868527, + "tokens_seen": 2630141952 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016423671013039118, + "loss": 1.1216, + "theoretical_loss": 3.383444468302617, + "tokens_seen": 2630207488 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016422066198595788, + "loss": 1.2413, + "theoretical_loss": 3.383437867928892, + "tokens_seen": 2630273024 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016420461384152458, + "loss": 1.242, + "theoretical_loss": 3.3834312677656664, + "tokens_seen": 2630338560 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016418856569709128, + "loss": 1.2148, + "theoretical_loss": 3.3834246678129283, + "tokens_seen": 2630404096 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016417251755265798, + "loss": 1.1897, + "theoretical_loss": 3.3834180680706654, + "tokens_seen": 2630469632 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016415646940822468, + "loss": 1.2755, + "theoretical_loss": 3.383411468538866, + "tokens_seen": 2630535168 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016414042126379138, + "loss": 1.1893, + "theoretical_loss": 3.3834048692175185, + "tokens_seen": 2630600704 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016412437311935809, + "loss": 1.2498, + "theoretical_loss": 3.38339827010661, + "tokens_seen": 2630666240 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016410832497492479, + "loss": 1.1415, + "theoretical_loss": 3.3833916712061294, + "tokens_seen": 2630731776 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001640922768304915, + "loss": 1.2647, + "theoretical_loss": 3.3833850725160644, + "tokens_seen": 2630797312 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001640762286860582, + "loss": 1.2593, + "theoretical_loss": 3.383378474036403, + "tokens_seen": 2630862848 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001640601805416249, + "loss": 1.1737, + "theoretical_loss": 3.3833718757671334, + "tokens_seen": 2630928384 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001640441323971916, + "loss": 1.314, + "theoretical_loss": 3.3833652777082435, + "tokens_seen": 2630993920 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001640280842527583, + "loss": 1.2087, + "theoretical_loss": 3.3833586798597213, + "tokens_seen": 2631059456 + }, + { + "epoch": 1.18, + "learning_rate": 0.000164012036108325, + "loss": 1.1181, + "theoretical_loss": 3.3833520822215553, + "tokens_seen": 2631124992 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001639959879638917, + "loss": 1.1265, + "theoretical_loss": 3.3833454847937325, + "tokens_seen": 2631190528 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001639799398194584, + "loss": 1.1972, + "theoretical_loss": 3.3833388875762425, + "tokens_seen": 2631256064 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001639638916750251, + "loss": 1.1627, + "theoretical_loss": 3.383332290569072, + "tokens_seen": 2631321600 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001639478435305918, + "loss": 1.2018, + "theoretical_loss": 3.38332569377221, + "tokens_seen": 2631387136 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001639317953861585, + "loss": 1.1334, + "theoretical_loss": 3.383319097185644, + "tokens_seen": 2631452672 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001639157472417252, + "loss": 1.2964, + "theoretical_loss": 3.383312500809362, + "tokens_seen": 2631518208 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001638996990972919, + "loss": 1.2264, + "theoretical_loss": 3.3833059046433527, + "tokens_seen": 2631583744 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016388365095285857, + "loss": 1.1789, + "theoretical_loss": 3.3832993086876035, + "tokens_seen": 2631649280 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016386760280842527, + "loss": 1.2692, + "theoretical_loss": 3.3832927129421027, + "tokens_seen": 2631714816 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016385155466399197, + "loss": 1.2887, + "theoretical_loss": 3.3832861174068385, + "tokens_seen": 2631780352 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016383550651955867, + "loss": 1.228, + "theoretical_loss": 3.383279522081799, + "tokens_seen": 2631845888 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016381945837512537, + "loss": 1.2399, + "theoretical_loss": 3.3832729269669715, + "tokens_seen": 2631911424 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016380341023069207, + "loss": 1.1583, + "theoretical_loss": 3.383266332062345, + "tokens_seen": 2631976960 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016378736208625877, + "loss": 1.1633, + "theoretical_loss": 3.3832597373679074, + "tokens_seen": 2632042496 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016377131394182547, + "loss": 1.2093, + "theoretical_loss": 3.383253142883646, + "tokens_seen": 2632108032 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016375526579739217, + "loss": 1.1976, + "theoretical_loss": 3.3832465486095504, + "tokens_seen": 2632173568 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016373921765295887, + "loss": 1.1768, + "theoretical_loss": 3.383239954545607, + "tokens_seen": 2632239104 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016372316950852557, + "loss": 1.2769, + "theoretical_loss": 3.383233360691805, + "tokens_seen": 2632304640 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016370712136409227, + "loss": 1.2672, + "theoretical_loss": 3.3832267670481317, + "tokens_seen": 2632370176 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016369107321965898, + "loss": 1.2583, + "theoretical_loss": 3.3832201736145757, + "tokens_seen": 2632435712 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016367502507522568, + "loss": 1.1784, + "theoretical_loss": 3.383213580391125, + "tokens_seen": 2632501248 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001636589769307924, + "loss": 1.2463, + "theoretical_loss": 3.3832069873777675, + "tokens_seen": 2632566784 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001636429287863591, + "loss": 1.1616, + "theoretical_loss": 3.3832003945744913, + "tokens_seen": 2632632320 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001636268806419258, + "loss": 1.3823, + "theoretical_loss": 3.383193801981285, + "tokens_seen": 2632697856 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001636108324974925, + "loss": 1.1693, + "theoretical_loss": 3.3831872095981357, + "tokens_seen": 2632763392 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001635947843530592, + "loss": 1.1543, + "theoretical_loss": 3.3831806174250323, + "tokens_seen": 2632828928 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016357873620862588, + "loss": 1.231, + "theoretical_loss": 3.383174025461962, + "tokens_seen": 2632894464 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016356268806419258, + "loss": 1.2281, + "theoretical_loss": 3.383167433708914, + "tokens_seen": 2632960000 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016354663991975928, + "loss": 1.2138, + "theoretical_loss": 3.383160842165876, + "tokens_seen": 2633025536 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016353059177532598, + "loss": 1.2425, + "theoretical_loss": 3.3831542508328356, + "tokens_seen": 2633091072 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016351454363089268, + "loss": 1.2767, + "theoretical_loss": 3.383147659709781, + "tokens_seen": 2633156608 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016349849548645938, + "loss": 1.2658, + "theoretical_loss": 3.383141068796701, + "tokens_seen": 2633222144 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016348244734202608, + "loss": 1.3488, + "theoretical_loss": 3.383134478093583, + "tokens_seen": 2633287680 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016346639919759278, + "loss": 1.2118, + "theoretical_loss": 3.3831278876004154, + "tokens_seen": 2633353216 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016345035105315949, + "loss": 1.22, + "theoretical_loss": 3.3831212973171856, + "tokens_seen": 2633418752 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016343430290872619, + "loss": 1.1797, + "theoretical_loss": 3.383114707243883, + "tokens_seen": 2633484288 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016341825476429289, + "loss": 1.2043, + "theoretical_loss": 3.383108117380494, + "tokens_seen": 2633549824 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001634022066198596, + "loss": 1.2403, + "theoretical_loss": 3.3831015277270087, + "tokens_seen": 2633615360 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001633861584754263, + "loss": 1.2472, + "theoretical_loss": 3.3830949382834135, + "tokens_seen": 2633680896 + }, + { + "epoch": 1.18, + "learning_rate": 0.000163370110330993, + "loss": 1.2637, + "theoretical_loss": 3.383088349049697, + "tokens_seen": 2633746432 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001633540621865597, + "loss": 1.224, + "theoretical_loss": 3.3830817600258474, + "tokens_seen": 2633811968 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001633380140421264, + "loss": 1.2834, + "theoretical_loss": 3.3830751712118534, + "tokens_seen": 2633877504 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001633219658976931, + "loss": 1.1402, + "theoretical_loss": 3.383068582607702, + "tokens_seen": 2633943040 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001633059177532598, + "loss": 1.2285, + "theoretical_loss": 3.3830619942133815, + "tokens_seen": 2634008576 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001632898696088265, + "loss": 1.3339, + "theoretical_loss": 3.3830554060288804, + "tokens_seen": 2634074112 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001632738214643932, + "loss": 1.1723, + "theoretical_loss": 3.383048818054187, + "tokens_seen": 2634139648 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001632577733199599, + "loss": 1.2615, + "theoretical_loss": 3.383042230289289, + "tokens_seen": 2634205184 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001632417251755266, + "loss": 1.2933, + "theoretical_loss": 3.383035642734175, + "tokens_seen": 2634270720 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001632256770310933, + "loss": 1.2156, + "theoretical_loss": 3.383029055388832, + "tokens_seen": 2634336256 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016320962888666, + "loss": 1.2666, + "theoretical_loss": 3.3830224682532486, + "tokens_seen": 2634401792 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016319358074222667, + "loss": 1.1795, + "theoretical_loss": 3.3830158813274136, + "tokens_seen": 2634467328 + }, + { + "epoch": 1.18, + "objective/train/docs_used": 973046, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1985998153686523, + "objective/train/theoretical_loss": 3.3830092946113144, + "objective/train/tokens_used": 2654992864, + "theoretical_loss": 3.3830092946113144, + "tokens_seen": 2634532864 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016317753259779337, + "loss": 1.1986, + "theoretical_loss": 3.3830092946113144, + "tokens_seen": 2634532864 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016316148445336007, + "loss": 1.1661, + "theoretical_loss": 3.3830027081049394, + "tokens_seen": 2634598400 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016314543630892677, + "loss": 1.1226, + "theoretical_loss": 3.3829961218082762, + "tokens_seen": 2634663936 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016312938816449347, + "loss": 1.1659, + "theoretical_loss": 3.382989535721314, + "tokens_seen": 2634729472 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016311334002006017, + "loss": 1.1737, + "theoretical_loss": 3.3829829498440396, + "tokens_seen": 2634795008 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016309729187562687, + "loss": 1.285, + "theoretical_loss": 3.382976364176442, + "tokens_seen": 2634860544 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016308124373119357, + "loss": 1.2623, + "theoretical_loss": 3.382969778718509, + "tokens_seen": 2634926080 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016306519558676027, + "loss": 1.1833, + "theoretical_loss": 3.3829631934702284, + "tokens_seen": 2634991616 + }, + { + "epoch": 1.18, + "learning_rate": 0.000163049147442327, + "loss": 1.2918, + "theoretical_loss": 3.3829566084315887, + "tokens_seen": 2635057152 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001630330992978937, + "loss": 1.256, + "theoretical_loss": 3.3829500236025787, + "tokens_seen": 2635122688 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001630170511534604, + "loss": 1.1775, + "theoretical_loss": 3.382943438983185, + "tokens_seen": 2635188224 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001630010030090271, + "loss": 1.1629, + "theoretical_loss": 3.382936854573397, + "tokens_seen": 2635253760 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001629849548645938, + "loss": 1.2941, + "theoretical_loss": 3.3829302703732016, + "tokens_seen": 2635319296 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001629689067201605, + "loss": 1.267, + "theoretical_loss": 3.382923686382588, + "tokens_seen": 2635384832 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001629528585757272, + "loss": 1.2195, + "theoretical_loss": 3.382917102601544, + "tokens_seen": 2635450368 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001629368104312939, + "loss": 1.2385, + "theoretical_loss": 3.382910519030058, + "tokens_seen": 2635515904 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001629207622868606, + "loss": 1.1978, + "theoretical_loss": 3.3829039356681174, + "tokens_seen": 2635581440 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001629047141424273, + "loss": 1.108, + "theoretical_loss": 3.3828973525157107, + "tokens_seen": 2635646976 + }, + { + "epoch": 1.18, + "learning_rate": 0.000162888665997994, + "loss": 1.1619, + "theoretical_loss": 3.3828907695728256, + "tokens_seen": 2635712512 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016287261785356068, + "loss": 1.133, + "theoretical_loss": 3.3828841868394512, + "tokens_seen": 2635778048 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016285656970912738, + "loss": 1.3919, + "theoretical_loss": 3.382877604315575, + "tokens_seen": 2635843584 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016284052156469408, + "loss": 1.2497, + "theoretical_loss": 3.382871022001185, + "tokens_seen": 2635909120 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016282447342026078, + "loss": 1.1896, + "theoretical_loss": 3.3828644398962697, + "tokens_seen": 2635974656 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016280842527582748, + "loss": 1.2407, + "theoretical_loss": 3.382857858000817, + "tokens_seen": 2636040192 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016279237713139418, + "loss": 1.2087, + "theoretical_loss": 3.382851276314815, + "tokens_seen": 2636105728 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016277632898696088, + "loss": 1.2208, + "theoretical_loss": 3.382844694838252, + "tokens_seen": 2636171264 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016276028084252759, + "loss": 1.2966, + "theoretical_loss": 3.3828381135711156, + "tokens_seen": 2636236800 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016274423269809429, + "loss": 1.1476, + "theoretical_loss": 3.3828315325133946, + "tokens_seen": 2636302336 + }, + { + "epoch": 1.18, + "learning_rate": 0.000162728184553661, + "loss": 1.2773, + "theoretical_loss": 3.3828249516650772, + "tokens_seen": 2636367872 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001627121364092277, + "loss": 1.182, + "theoretical_loss": 3.382818371026151, + "tokens_seen": 2636433408 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001626960882647944, + "loss": 1.1726, + "theoretical_loss": 3.3828117905966044, + "tokens_seen": 2636498944 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001626800401203611, + "loss": 1.2505, + "theoretical_loss": 3.382805210376425, + "tokens_seen": 2636564480 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001626639919759278, + "loss": 1.2428, + "theoretical_loss": 3.3827986303656017, + "tokens_seen": 2636630016 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001626479438314945, + "loss": 1.2294, + "theoretical_loss": 3.382792050564123, + "tokens_seen": 2636695552 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001626318956870612, + "loss": 1.2222, + "theoretical_loss": 3.3827854709719754, + "tokens_seen": 2636761088 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001626158475426279, + "loss": 1.2789, + "theoretical_loss": 3.3827788915891484, + "tokens_seen": 2636826624 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001625997993981946, + "loss": 1.1941, + "theoretical_loss": 3.38277231241563, + "tokens_seen": 2636892160 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001625837512537613, + "loss": 1.2198, + "theoretical_loss": 3.3827657334514076, + "tokens_seen": 2636957696 + }, + { + "epoch": 1.18, + "learning_rate": 0.000162567703109328, + "loss": 1.1634, + "theoretical_loss": 3.38275915469647, + "tokens_seen": 2637023232 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001625516549648947, + "loss": 1.213, + "theoretical_loss": 3.3827525761508053, + "tokens_seen": 2637088768 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001625356068204614, + "loss": 1.2889, + "theoretical_loss": 3.3827459978144017, + "tokens_seen": 2637154304 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001625195586760281, + "loss": 1.308, + "theoretical_loss": 3.382739419687247, + "tokens_seen": 2637219840 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001625035105315948, + "loss": 1.1917, + "theoretical_loss": 3.3827328417693296, + "tokens_seen": 2637285376 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016248746238716147, + "loss": 1.2326, + "theoretical_loss": 3.3827262640606373, + "tokens_seen": 2637350912 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016247141424272817, + "loss": 1.2836, + "theoretical_loss": 3.382719686561159, + "tokens_seen": 2637416448 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001624553660982949, + "loss": 1.1423, + "theoretical_loss": 3.382713109270882, + "tokens_seen": 2637481984 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001624393179538616, + "loss": 1.2439, + "theoretical_loss": 3.3827065321897947, + "tokens_seen": 2637547520 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001624232698094283, + "loss": 1.2521, + "theoretical_loss": 3.3826999553178854, + "tokens_seen": 2637613056 + }, + { + "epoch": 1.18, + "learning_rate": 0.000162407221664995, + "loss": 1.165, + "theoretical_loss": 3.382693378655142, + "tokens_seen": 2637678592 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001623911735205617, + "loss": 1.255, + "theoretical_loss": 3.3826868022015533, + "tokens_seen": 2637744128 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001623751253761284, + "loss": 1.1027, + "theoretical_loss": 3.382680225957107, + "tokens_seen": 2637809664 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001623590772316951, + "loss": 1.2494, + "theoretical_loss": 3.3826736499217906, + "tokens_seen": 2637875200 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001623430290872618, + "loss": 1.182, + "theoretical_loss": 3.3826670740955938, + "tokens_seen": 2637940736 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001623269809428285, + "loss": 1.1793, + "theoretical_loss": 3.3826604984785034, + "tokens_seen": 2638006272 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001623109327983952, + "loss": 1.1869, + "theoretical_loss": 3.382653923070508, + "tokens_seen": 2638071808 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001622948846539619, + "loss": 1.222, + "theoretical_loss": 3.382647347871596, + "tokens_seen": 2638137344 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001622788365095286, + "loss": 1.1943, + "theoretical_loss": 3.382640772881755, + "tokens_seen": 2638202880 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001622627883650953, + "loss": 1.2238, + "theoretical_loss": 3.3826341981009733, + "tokens_seen": 2638268416 + }, + { + "epoch": 1.18, + "learning_rate": 0.000162246740220662, + "loss": 1.1979, + "theoretical_loss": 3.38262762352924, + "tokens_seen": 2638333952 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001622306920762287, + "loss": 1.1712, + "theoretical_loss": 3.382621049166542, + "tokens_seen": 2638399488 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001622146439317954, + "loss": 1.2423, + "theoretical_loss": 3.382614475012868, + "tokens_seen": 2638465024 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001621985957873621, + "loss": 1.2351, + "theoretical_loss": 3.3826079010682064, + "tokens_seen": 2638530560 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001621825476429288, + "loss": 1.2828, + "theoretical_loss": 3.382601327332545, + "tokens_seen": 2638596096 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016216649949849548, + "loss": 1.2748, + "theoretical_loss": 3.3825947538058716, + "tokens_seen": 2638661632 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016215045135406218, + "loss": 1.176, + "theoretical_loss": 3.3825881804881757, + "tokens_seen": 2638727168 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016213440320962888, + "loss": 1.1609, + "theoretical_loss": 3.3825816073794437, + "tokens_seen": 2638792704 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016211835506519558, + "loss": 1.1624, + "theoretical_loss": 3.3825750344796655, + "tokens_seen": 2638858240 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016210230692076228, + "loss": 1.1964, + "theoretical_loss": 3.3825684617888276, + "tokens_seen": 2638923776 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016208625877632898, + "loss": 1.2127, + "theoretical_loss": 3.3825618893069196, + "tokens_seen": 2638989312 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016207021063189569, + "loss": 1.0538, + "theoretical_loss": 3.382555317033929, + "tokens_seen": 2639054848 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016205416248746239, + "loss": 1.2275, + "theoretical_loss": 3.382548744969844, + "tokens_seen": 2639120384 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001620381143430291, + "loss": 1.2643, + "theoretical_loss": 3.3825421731146523, + "tokens_seen": 2639185920 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001620220661985958, + "loss": 1.2454, + "theoretical_loss": 3.3825356014683434, + "tokens_seen": 2639251456 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001620060180541625, + "loss": 1.2337, + "theoretical_loss": 3.3825290300309043, + "tokens_seen": 2639316992 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001619899699097292, + "loss": 1.2249, + "theoretical_loss": 3.3825224588023235, + "tokens_seen": 2639382528 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001619739217652959, + "loss": 1.2001, + "theoretical_loss": 3.3825158877825894, + "tokens_seen": 2639448064 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001619578736208626, + "loss": 1.1768, + "theoretical_loss": 3.3825093169716896, + "tokens_seen": 2639513600 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001619418254764293, + "loss": 1.2343, + "theoretical_loss": 3.382502746369613, + "tokens_seen": 2639579136 + }, + { + "epoch": 1.18, + "learning_rate": 0.000161925777331996, + "loss": 1.1769, + "theoretical_loss": 3.3824961759763474, + "tokens_seen": 2639644672 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001619097291875627, + "loss": 1.1797, + "theoretical_loss": 3.382489605791881, + "tokens_seen": 2639710208 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001618936810431294, + "loss": 1.272, + "theoretical_loss": 3.382483035816202, + "tokens_seen": 2639775744 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001618776328986961, + "loss": 1.2081, + "theoretical_loss": 3.3824764660492987, + "tokens_seen": 2639841280 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001618615847542628, + "loss": 1.2157, + "theoretical_loss": 3.382469896491159, + "tokens_seen": 2639906816 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001618455366098295, + "loss": 1.3783, + "theoretical_loss": 3.3824633271417714, + "tokens_seen": 2639972352 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001618294884653962, + "loss": 1.2215, + "theoretical_loss": 3.382456758001124, + "tokens_seen": 2640037888 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001618134403209629, + "loss": 1.2745, + "theoretical_loss": 3.382450189069205, + "tokens_seen": 2640103424 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001617973921765296, + "loss": 1.2477, + "theoretical_loss": 3.3824436203460024, + "tokens_seen": 2640168960 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001617813440320963, + "loss": 1.1615, + "theoretical_loss": 3.3824370518315043, + "tokens_seen": 2640234496 + }, + { + "epoch": 1.18, + "learning_rate": 0.000161765295887663, + "loss": 1.2707, + "theoretical_loss": 3.3824304835256993, + "tokens_seen": 2640300032 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001617492477432297, + "loss": 1.2722, + "theoretical_loss": 3.3824239154285753, + "tokens_seen": 2640365568 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001617331995987964, + "loss": 1.1896, + "theoretical_loss": 3.382417347540121, + "tokens_seen": 2640431104 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001617171514543631, + "loss": 1.1822, + "theoretical_loss": 3.3824107798603236, + "tokens_seen": 2640496640 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001617011033099298, + "loss": 1.1771, + "theoretical_loss": 3.382404212389172, + "tokens_seen": 2640562176 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001616850551654965, + "loss": 1.3092, + "theoretical_loss": 3.3823976451266544, + "tokens_seen": 2640627712 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001616690070210632, + "loss": 1.1635, + "theoretical_loss": 3.3823910780727586, + "tokens_seen": 2640693248 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001616529588766299, + "loss": 1.2177, + "theoretical_loss": 3.3823845112274733, + "tokens_seen": 2640758784 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001616369107321966, + "loss": 1.2628, + "theoretical_loss": 3.382377944590786, + "tokens_seen": 2640824320 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001616208625877633, + "loss": 1.1217, + "theoretical_loss": 3.382371378162686, + "tokens_seen": 2640889856 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016160481444333, + "loss": 1.1916, + "theoretical_loss": 3.3823648119431606, + "tokens_seen": 2640955392 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001615887662988967, + "loss": 1.2015, + "theoretical_loss": 3.382358245932198, + "tokens_seen": 2641020928 + }, + { + "epoch": 1.18, + "objective/train/docs_used": 975405, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2501837015151978, + "objective/train/theoretical_loss": 3.3823516801297866, + "objective/train/tokens_used": 2661546464, + "theoretical_loss": 3.3823516801297866, + "tokens_seen": 2641086464 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001615727181544634, + "loss": 1.2502, + "theoretical_loss": 3.3823516801297866, + "tokens_seen": 2641086464 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001615566700100301, + "loss": 1.2699, + "theoretical_loss": 3.382345114535915, + "tokens_seen": 2641152000 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001615406218655968, + "loss": 1.3219, + "theoretical_loss": 3.3823385491505706, + "tokens_seen": 2641217536 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001615245737211635, + "loss": 1.2117, + "theoretical_loss": 3.3823319839737422, + "tokens_seen": 2641283072 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001615085255767302, + "loss": 1.2572, + "theoretical_loss": 3.382325419005418, + "tokens_seen": 2641348608 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001614924774322969, + "loss": 1.351, + "theoretical_loss": 3.382318854245586, + "tokens_seen": 2641414144 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016147642928786358, + "loss": 1.2175, + "theoretical_loss": 3.3823122896942346, + "tokens_seen": 2641479680 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016146038114343028, + "loss": 1.202, + "theoretical_loss": 3.3823057253513515, + "tokens_seen": 2641545216 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016144433299899698, + "loss": 1.3158, + "theoretical_loss": 3.3822991612169258, + "tokens_seen": 2641610752 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016142828485456368, + "loss": 1.2169, + "theoretical_loss": 3.382292597290945, + "tokens_seen": 2641676288 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016141223671013038, + "loss": 1.1776, + "theoretical_loss": 3.382286033573397, + "tokens_seen": 2641741824 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016139618856569708, + "loss": 1.1766, + "theoretical_loss": 3.3822794700642707, + "tokens_seen": 2641807360 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016138014042126379, + "loss": 1.2223, + "theoretical_loss": 3.3822729067635544, + "tokens_seen": 2641872896 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016136409227683049, + "loss": 1.215, + "theoretical_loss": 3.382266343671236, + "tokens_seen": 2641938432 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001613480441323972, + "loss": 1.217, + "theoretical_loss": 3.3822597807873036, + "tokens_seen": 2642003968 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001613319959879639, + "loss": 1.3193, + "theoretical_loss": 3.3822532181117455, + "tokens_seen": 2642069504 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001613159478435306, + "loss": 1.2127, + "theoretical_loss": 3.38224665564455, + "tokens_seen": 2642135040 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001612998996990973, + "loss": 1.3031, + "theoretical_loss": 3.3822400933857053, + "tokens_seen": 2642200576 + }, + { + "epoch": 1.18, + "learning_rate": 0.000161283851554664, + "loss": 1.2567, + "theoretical_loss": 3.3822335313351997, + "tokens_seen": 2642266112 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001612678034102307, + "loss": 1.232, + "theoretical_loss": 3.382226969493021, + "tokens_seen": 2642331648 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016125175526579742, + "loss": 1.2279, + "theoretical_loss": 3.3822204078591582, + "tokens_seen": 2642397184 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016123570712136412, + "loss": 1.3445, + "theoretical_loss": 3.382213846433599, + "tokens_seen": 2642462720 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016121965897693082, + "loss": 1.2729, + "theoretical_loss": 3.382207285216331, + "tokens_seen": 2642528256 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016120361083249752, + "loss": 1.1858, + "theoretical_loss": 3.382200724207344, + "tokens_seen": 2642593792 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016118756268806422, + "loss": 1.2684, + "theoretical_loss": 3.382194163406625, + "tokens_seen": 2642659328 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016117151454363092, + "loss": 1.2132, + "theoretical_loss": 3.382187602814162, + "tokens_seen": 2642724864 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001611554663991976, + "loss": 1.2284, + "theoretical_loss": 3.3821810424299446, + "tokens_seen": 2642790400 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001611394182547643, + "loss": 1.2375, + "theoretical_loss": 3.38217448225396, + "tokens_seen": 2642855936 + }, + { + "epoch": 1.18, + "learning_rate": 0.000161123370110331, + "loss": 1.114, + "theoretical_loss": 3.3821679222861967, + "tokens_seen": 2642921472 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001611073219658977, + "loss": 1.2318, + "theoretical_loss": 3.3821613625266425, + "tokens_seen": 2642987008 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001610912738214644, + "loss": 1.216, + "theoretical_loss": 3.382154802975286, + "tokens_seen": 2643052544 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001610752256770311, + "loss": 1.241, + "theoretical_loss": 3.3821482436321153, + "tokens_seen": 2643118080 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001610591775325978, + "loss": 1.2799, + "theoretical_loss": 3.3821416844971193, + "tokens_seen": 2643183616 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001610431293881645, + "loss": 1.14, + "theoretical_loss": 3.3821351255702856, + "tokens_seen": 2643249152 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001610270812437312, + "loss": 1.2257, + "theoretical_loss": 3.382128566851602, + "tokens_seen": 2643314688 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001610110330992979, + "loss": 1.1862, + "theoretical_loss": 3.3821220083410575, + "tokens_seen": 2643380224 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001609949849548646, + "loss": 1.1564, + "theoretical_loss": 3.3821154500386403, + "tokens_seen": 2643445760 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001609789368104313, + "loss": 1.1758, + "theoretical_loss": 3.3821088919443385, + "tokens_seen": 2643511296 + }, + { + "epoch": 1.18, + "learning_rate": 0.000160962888665998, + "loss": 1.162, + "theoretical_loss": 3.38210233405814, + "tokens_seen": 2643576832 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001609468405215647, + "loss": 1.2369, + "theoretical_loss": 3.3820957763800332, + "tokens_seen": 2643642368 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001609307923771314, + "loss": 1.2544, + "theoretical_loss": 3.3820892189100062, + "tokens_seen": 2643707904 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001609147442326981, + "loss": 1.1422, + "theoretical_loss": 3.382082661648048, + "tokens_seen": 2643773440 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001608986960882648, + "loss": 1.1856, + "theoretical_loss": 3.3820761045941463, + "tokens_seen": 2643838976 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001608826479438315, + "loss": 1.3066, + "theoretical_loss": 3.382069547748289, + "tokens_seen": 2643904512 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001608665997993982, + "loss": 1.2822, + "theoretical_loss": 3.382062991110465, + "tokens_seen": 2643970048 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001608505516549649, + "loss": 1.2527, + "theoretical_loss": 3.382056434680662, + "tokens_seen": 2644035584 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001608345035105316, + "loss": 1.2053, + "theoretical_loss": 3.3820498784588686, + "tokens_seen": 2644101120 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001608184553660983, + "loss": 1.1896, + "theoretical_loss": 3.3820433224450728, + "tokens_seen": 2644166656 + }, + { + "epoch": 1.18, + "learning_rate": 0.000160802407221665, + "loss": 1.131, + "theoretical_loss": 3.382036766639263, + "tokens_seen": 2644232192 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001607863590772317, + "loss": 1.3084, + "theoretical_loss": 3.3820302110414278, + "tokens_seen": 2644297728 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016077031093279838, + "loss": 1.1926, + "theoretical_loss": 3.3820236556515546, + "tokens_seen": 2644363264 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016075426278836508, + "loss": 1.1866, + "theoretical_loss": 3.3820171004696324, + "tokens_seen": 2644428800 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016073821464393178, + "loss": 1.2475, + "theoretical_loss": 3.382010545495649, + "tokens_seen": 2644494336 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016072216649949848, + "loss": 1.2252, + "theoretical_loss": 3.3820039907295927, + "tokens_seen": 2644559872 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016070611835506519, + "loss": 1.2232, + "theoretical_loss": 3.381997436171452, + "tokens_seen": 2644625408 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016069007021063189, + "loss": 1.1566, + "theoretical_loss": 3.381990881821215, + "tokens_seen": 2644690944 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016067402206619859, + "loss": 1.2255, + "theoretical_loss": 3.38198432767887, + "tokens_seen": 2644756480 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016065797392176531, + "loss": 1.1328, + "theoretical_loss": 3.3819777737444054, + "tokens_seen": 2644822016 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016064192577733201, + "loss": 1.2156, + "theoretical_loss": 3.3819712200178094, + "tokens_seen": 2644887552 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016062587763289872, + "loss": 1.2202, + "theoretical_loss": 3.3819646664990697, + "tokens_seen": 2644953088 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016060982948846542, + "loss": 1.2565, + "theoretical_loss": 3.3819581131881753, + "tokens_seen": 2645018624 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016059378134403212, + "loss": 1.1939, + "theoretical_loss": 3.381951560085114, + "tokens_seen": 2645084160 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016057773319959882, + "loss": 1.2139, + "theoretical_loss": 3.3819450071898745, + "tokens_seen": 2645149696 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016056168505516552, + "loss": 1.1563, + "theoretical_loss": 3.3819384545024445, + "tokens_seen": 2645215232 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016054563691073222, + "loss": 1.2475, + "theoretical_loss": 3.3819319020228127, + "tokens_seen": 2645280768 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016052958876629892, + "loss": 1.1489, + "theoretical_loss": 3.3819253497509667, + "tokens_seen": 2645346304 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016051354062186562, + "loss": 1.1603, + "theoretical_loss": 3.381918797686896, + "tokens_seen": 2645411840 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016049749247743232, + "loss": 1.2862, + "theoretical_loss": 3.3819122458305877, + "tokens_seen": 2645477376 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016048144433299902, + "loss": 1.235, + "theoretical_loss": 3.3819056941820307, + "tokens_seen": 2645542912 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016046539618856572, + "loss": 1.1783, + "theoretical_loss": 3.381899142741213, + "tokens_seen": 2645608448 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001604493480441324, + "loss": 1.2396, + "theoretical_loss": 3.381892591508123, + "tokens_seen": 2645673984 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001604332998996991, + "loss": 1.2104, + "theoretical_loss": 3.3818860404827484, + "tokens_seen": 2645739520 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001604172517552658, + "loss": 1.2343, + "theoretical_loss": 3.3818794896650783, + "tokens_seen": 2645805056 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001604012036108325, + "loss": 1.2588, + "theoretical_loss": 3.381872939055101, + "tokens_seen": 2645870592 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001603851554663992, + "loss": 1.1631, + "theoretical_loss": 3.381866388652804, + "tokens_seen": 2645936128 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001603691073219659, + "loss": 1.1294, + "theoretical_loss": 3.381859838458176, + "tokens_seen": 2646001664 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001603530591775326, + "loss": 1.2823, + "theoretical_loss": 3.3818532884712056, + "tokens_seen": 2646067200 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001603370110330993, + "loss": 1.1517, + "theoretical_loss": 3.3818467386918805, + "tokens_seen": 2646132736 + }, + { + "epoch": 1.18, + "learning_rate": 0.000160320962888666, + "loss": 1.2093, + "theoretical_loss": 3.3818401891201892, + "tokens_seen": 2646198272 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001603049147442327, + "loss": 1.1997, + "theoretical_loss": 3.38183363975612, + "tokens_seen": 2646263808 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001602888665997994, + "loss": 1.1906, + "theoretical_loss": 3.381827090599661, + "tokens_seen": 2646329344 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001602728184553661, + "loss": 1.2064, + "theoretical_loss": 3.381820541650801, + "tokens_seen": 2646394880 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001602567703109328, + "loss": 1.2859, + "theoretical_loss": 3.3818139929095272, + "tokens_seen": 2646460416 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001602407221664995, + "loss": 1.2384, + "theoretical_loss": 3.3818074443758293, + "tokens_seen": 2646525952 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001602246740220662, + "loss": 1.2584, + "theoretical_loss": 3.3818008960496946, + "tokens_seen": 2646591488 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001602086258776329, + "loss": 1.2254, + "theoretical_loss": 3.3817943479311117, + "tokens_seen": 2646657024 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001601925777331996, + "loss": 1.2604, + "theoretical_loss": 3.381787800020069, + "tokens_seen": 2646722560 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001601765295887663, + "loss": 1.166, + "theoretical_loss": 3.3817812523165545, + "tokens_seen": 2646788096 + }, + { + "epoch": 1.18, + "learning_rate": 0.000160160481444333, + "loss": 1.1835, + "theoretical_loss": 3.381774704820556, + "tokens_seen": 2646853632 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001601444332998997, + "loss": 1.1786, + "theoretical_loss": 3.381768157532063, + "tokens_seen": 2646919168 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001601283851554664, + "loss": 1.1668, + "theoretical_loss": 3.3817616104510635, + "tokens_seen": 2646984704 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001601123370110331, + "loss": 1.29, + "theoretical_loss": 3.381755063577545, + "tokens_seen": 2647050240 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001600962888665998, + "loss": 1.1857, + "theoretical_loss": 3.3817485169114962, + "tokens_seen": 2647115776 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001600802407221665, + "loss": 1.2333, + "theoretical_loss": 3.3817419704529055, + "tokens_seen": 2647181312 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016006419257773318, + "loss": 1.185, + "theoretical_loss": 3.381735424201761, + "tokens_seen": 2647246848 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001600481444332999, + "loss": 1.2268, + "theoretical_loss": 3.381728878158052, + "tokens_seen": 2647312384 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001600320962888666, + "loss": 1.1851, + "theoretical_loss": 3.3817223323217647, + "tokens_seen": 2647377920 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001600160481444333, + "loss": 1.1136, + "theoretical_loss": 3.3817157866928893, + "tokens_seen": 2647443456 + }, + { + "epoch": 1.18, + "learning_rate": 0.00016, + "loss": 1.1718, + "theoretical_loss": 3.3817092412714134, + "tokens_seen": 2647508992 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015998395185556671, + "loss": 1.1592, + "theoretical_loss": 3.381702696057325, + "tokens_seen": 2647574528 + }, + { + "epoch": 1.18, + "objective/train/docs_used": 978151, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.213378667831421, + "objective/train/theoretical_loss": 3.3816961510506127, + "objective/train/tokens_used": 2668100064, + "theoretical_loss": 3.3816961510506127, + "tokens_seen": 2647640064 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015996790371113341, + "loss": 1.2134, + "theoretical_loss": 3.3816961510506127, + "tokens_seen": 2647640064 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015995185556670011, + "loss": 1.167, + "theoretical_loss": 3.381689606251265, + "tokens_seen": 2647705600 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015993580742226682, + "loss": 1.3064, + "theoretical_loss": 3.38168306165927, + "tokens_seen": 2647771136 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015991975927783352, + "loss": 1.1508, + "theoretical_loss": 3.3816765172746157, + "tokens_seen": 2647836672 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015990371113340022, + "loss": 1.2009, + "theoretical_loss": 3.381669973097291, + "tokens_seen": 2647902208 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015988766298896692, + "loss": 1.2418, + "theoretical_loss": 3.3816634291272836, + "tokens_seen": 2647967744 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015987161484453362, + "loss": 1.2009, + "theoretical_loss": 3.381656885364582, + "tokens_seen": 2648033280 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015985556670010032, + "loss": 1.1228, + "theoretical_loss": 3.3816503418091752, + "tokens_seen": 2648098816 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015983951855566702, + "loss": 1.0753, + "theoretical_loss": 3.3816437984610506, + "tokens_seen": 2648164352 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015982347041123372, + "loss": 1.2235, + "theoretical_loss": 3.3816372553201965, + "tokens_seen": 2648229888 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015980742226680042, + "loss": 1.302, + "theoretical_loss": 3.3816307123866016, + "tokens_seen": 2648295424 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015979137412236712, + "loss": 1.1481, + "theoretical_loss": 3.3816241696602543, + "tokens_seen": 2648360960 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015977532597793382, + "loss": 1.1495, + "theoretical_loss": 3.381617627141143, + "tokens_seen": 2648426496 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015975927783350052, + "loss": 1.2084, + "theoretical_loss": 3.381611084829255, + "tokens_seen": 2648492032 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001597432296890672, + "loss": 1.283, + "theoretical_loss": 3.3816045427245798, + "tokens_seen": 2648557568 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001597271815446339, + "loss": 1.2107, + "theoretical_loss": 3.381598000827105, + "tokens_seen": 2648623104 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001597111334002006, + "loss": 1.1023, + "theoretical_loss": 3.3815914591368195, + "tokens_seen": 2648688640 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001596950852557673, + "loss": 1.1319, + "theoretical_loss": 3.3815849176537105, + "tokens_seen": 2648754176 + }, + { + "epoch": 1.18, + "learning_rate": 0.000159679037111334, + "loss": 1.1647, + "theoretical_loss": 3.381578376377768, + "tokens_seen": 2648819712 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001596629889669007, + "loss": 1.2027, + "theoretical_loss": 3.381571835308979, + "tokens_seen": 2648885248 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001596469408224674, + "loss": 1.1313, + "theoretical_loss": 3.381565294447332, + "tokens_seen": 2648950784 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001596308926780341, + "loss": 1.1557, + "theoretical_loss": 3.381558753792816, + "tokens_seen": 2649016320 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001596148445336008, + "loss": 1.2196, + "theoretical_loss": 3.3815522133454183, + "tokens_seen": 2649081856 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001595987963891675, + "loss": 1.1775, + "theoretical_loss": 3.381545673105128, + "tokens_seen": 2649147392 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001595827482447342, + "loss": 1.1749, + "theoretical_loss": 3.381539133071933, + "tokens_seen": 2649212928 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001595667001003009, + "loss": 1.1992, + "theoretical_loss": 3.3815325932458222, + "tokens_seen": 2649278464 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001595506519558676, + "loss": 1.1835, + "theoretical_loss": 3.381526053626783, + "tokens_seen": 2649344000 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001595346038114343, + "loss": 1.1627, + "theoretical_loss": 3.381519514214805, + "tokens_seen": 2649409536 + }, + { + "epoch": 1.18, + "learning_rate": 0.000159518555667001, + "loss": 1.1648, + "theoretical_loss": 3.3815129750098754, + "tokens_seen": 2649475072 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001595025075225677, + "loss": 1.2731, + "theoretical_loss": 3.3815064360119824, + "tokens_seen": 2649540608 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001594864593781344, + "loss": 1.2337, + "theoretical_loss": 3.3814998972211154, + "tokens_seen": 2649606144 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001594704112337011, + "loss": 1.2063, + "theoretical_loss": 3.381493358637262, + "tokens_seen": 2649671680 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015945436308926783, + "loss": 1.2517, + "theoretical_loss": 3.3814868202604105, + "tokens_seen": 2649737216 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001594383149448345, + "loss": 1.2694, + "theoretical_loss": 3.3814802820905494, + "tokens_seen": 2649802752 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001594222668004012, + "loss": 1.2063, + "theoretical_loss": 3.3814737441276668, + "tokens_seen": 2649868288 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001594062186559679, + "loss": 1.2478, + "theoretical_loss": 3.3814672063717515, + "tokens_seen": 2649933824 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001593901705115346, + "loss": 1.1846, + "theoretical_loss": 3.3814606688227915, + "tokens_seen": 2649999360 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001593741223671013, + "loss": 1.2568, + "theoretical_loss": 3.3814541314807753, + "tokens_seen": 2650064896 + }, + { + "epoch": 1.18, + "learning_rate": 0.000159358074222668, + "loss": 1.2993, + "theoretical_loss": 3.3814475943456914, + "tokens_seen": 2650130432 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001593420260782347, + "loss": 1.2162, + "theoretical_loss": 3.3814410574175273, + "tokens_seen": 2650195968 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001593259779338014, + "loss": 1.1599, + "theoretical_loss": 3.3814345206962724, + "tokens_seen": 2650261504 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001593099297893681, + "loss": 1.1967, + "theoretical_loss": 3.381427984181914, + "tokens_seen": 2650327040 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015929388164493481, + "loss": 1.2672, + "theoretical_loss": 3.3814214478744415, + "tokens_seen": 2650392576 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015927783350050151, + "loss": 1.2286, + "theoretical_loss": 3.3814149117738426, + "tokens_seen": 2650458112 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015926178535606822, + "loss": 1.1898, + "theoretical_loss": 3.3814083758801052, + "tokens_seen": 2650523648 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015924573721163492, + "loss": 1.1636, + "theoretical_loss": 3.381401840193219, + "tokens_seen": 2650589184 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015922968906720162, + "loss": 1.1835, + "theoretical_loss": 3.381395304713171, + "tokens_seen": 2650654720 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015921364092276832, + "loss": 1.2986, + "theoretical_loss": 3.3813887694399503, + "tokens_seen": 2650720256 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015919759277833502, + "loss": 1.1781, + "theoretical_loss": 3.381382234373545, + "tokens_seen": 2650785792 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015918154463390172, + "loss": 1.2075, + "theoretical_loss": 3.381375699513943, + "tokens_seen": 2650851328 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015916549648946842, + "loss": 1.1949, + "theoretical_loss": 3.3813691648611335, + "tokens_seen": 2650916864 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015914944834503512, + "loss": 1.1665, + "theoretical_loss": 3.3813626304151043, + "tokens_seen": 2650982400 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015913340020060182, + "loss": 1.1386, + "theoretical_loss": 3.381356096175844, + "tokens_seen": 2651047936 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015911735205616852, + "loss": 1.2442, + "theoretical_loss": 3.381349562143341, + "tokens_seen": 2651113472 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015910130391173522, + "loss": 1.1957, + "theoretical_loss": 3.381343028317583, + "tokens_seen": 2651179008 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015908525576730192, + "loss": 1.2878, + "theoretical_loss": 3.381336494698559, + "tokens_seen": 2651244544 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015906920762286862, + "loss": 1.2211, + "theoretical_loss": 3.381329961286257, + "tokens_seen": 2651310080 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001590531594784353, + "loss": 1.2437, + "theoretical_loss": 3.381323428080666, + "tokens_seen": 2651375616 + }, + { + "epoch": 1.18, + "learning_rate": 0.000159037111334002, + "loss": 1.1457, + "theoretical_loss": 3.3813168950817736, + "tokens_seen": 2651441152 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001590210631895687, + "loss": 1.2875, + "theoretical_loss": 3.381310362289568, + "tokens_seen": 2651506688 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001590050150451354, + "loss": 1.2175, + "theoretical_loss": 3.3813038297040388, + "tokens_seen": 2651572224 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001589889669007021, + "loss": 1.1862, + "theoretical_loss": 3.381297297325173, + "tokens_seen": 2651637760 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001589729187562688, + "loss": 1.113, + "theoretical_loss": 3.3812907651529596, + "tokens_seen": 2651703296 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001589568706118355, + "loss": 1.2669, + "theoretical_loss": 3.3812842331873867, + "tokens_seen": 2651768832 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001589408224674022, + "loss": 1.239, + "theoretical_loss": 3.381277701428443, + "tokens_seen": 2651834368 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001589247743229689, + "loss": 1.2037, + "theoretical_loss": 3.3812711698761166, + "tokens_seen": 2651899904 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001589087261785356, + "loss": 1.3002, + "theoretical_loss": 3.381264638530396, + "tokens_seen": 2651965440 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001588926780341023, + "loss": 1.2208, + "theoretical_loss": 3.381258107391269, + "tokens_seen": 2652030976 + }, + { + "epoch": 1.18, + "learning_rate": 0.000158876629889669, + "loss": 1.1792, + "theoretical_loss": 3.381251576458725, + "tokens_seen": 2652096512 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001588605817452357, + "loss": 1.2327, + "theoretical_loss": 3.3812450457327516, + "tokens_seen": 2652162048 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015884453360080243, + "loss": 1.1816, + "theoretical_loss": 3.381238515213337, + "tokens_seen": 2652227584 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015882848545636913, + "loss": 1.2601, + "theoretical_loss": 3.38123198490047, + "tokens_seen": 2652293120 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015881243731193583, + "loss": 1.1848, + "theoretical_loss": 3.3812254547941394, + "tokens_seen": 2652358656 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015879638916750253, + "loss": 1.1932, + "theoretical_loss": 3.381218924894333, + "tokens_seen": 2652424192 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015878034102306923, + "loss": 1.2528, + "theoretical_loss": 3.3812123952010387, + "tokens_seen": 2652489728 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015876429287863593, + "loss": 1.2247, + "theoretical_loss": 3.3812058657142456, + "tokens_seen": 2652555264 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015874824473420264, + "loss": 1.2273, + "theoretical_loss": 3.381199336433942, + "tokens_seen": 2652620800 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001587321965897693, + "loss": 1.3129, + "theoretical_loss": 3.381192807360116, + "tokens_seen": 2652686336 + }, + { + "epoch": 1.18, + "learning_rate": 0.000158716148445336, + "loss": 1.1492, + "theoretical_loss": 3.381186278492756, + "tokens_seen": 2652751872 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001587001003009027, + "loss": 1.404, + "theoretical_loss": 3.3811797498318503, + "tokens_seen": 2652817408 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001586840521564694, + "loss": 1.1538, + "theoretical_loss": 3.381173221377388, + "tokens_seen": 2652882944 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001586680040120361, + "loss": 1.2735, + "theoretical_loss": 3.3811666931293565, + "tokens_seen": 2652948480 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001586519558676028, + "loss": 1.2176, + "theoretical_loss": 3.3811601650877443, + "tokens_seen": 2653014016 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001586359077231695, + "loss": 1.1033, + "theoretical_loss": 3.3811536372525404, + "tokens_seen": 2653079552 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001586198595787362, + "loss": 1.2168, + "theoretical_loss": 3.3811471096237327, + "tokens_seen": 2653145088 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015860381143430291, + "loss": 1.2016, + "theoretical_loss": 3.3811405822013096, + "tokens_seen": 2653210624 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015858776328986961, + "loss": 1.2849, + "theoretical_loss": 3.38113405498526, + "tokens_seen": 2653276160 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015857171514543632, + "loss": 1.28, + "theoretical_loss": 3.3811275279755715, + "tokens_seen": 2653341696 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015855566700100302, + "loss": 1.2294, + "theoretical_loss": 3.381121001172233, + "tokens_seen": 2653407232 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015853961885656972, + "loss": 1.2403, + "theoretical_loss": 3.3811144745752326, + "tokens_seen": 2653472768 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015852357071213642, + "loss": 1.2123, + "theoretical_loss": 3.3811079481845585, + "tokens_seen": 2653538304 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015850752256770312, + "loss": 1.2316, + "theoretical_loss": 3.3811014220002, + "tokens_seen": 2653603840 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015849147442326982, + "loss": 1.2373, + "theoretical_loss": 3.3810948960221445, + "tokens_seen": 2653669376 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015847542627883652, + "loss": 1.2454, + "theoretical_loss": 3.3810883702503807, + "tokens_seen": 2653734912 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015845937813440322, + "loss": 1.2126, + "theoretical_loss": 3.381081844684897, + "tokens_seen": 2653800448 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015844332998996992, + "loss": 1.2149, + "theoretical_loss": 3.3810753193256815, + "tokens_seen": 2653865984 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015842728184553662, + "loss": 1.2949, + "theoretical_loss": 3.3810687941727235, + "tokens_seen": 2653931520 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015841123370110332, + "loss": 1.1568, + "theoretical_loss": 3.3810622692260104, + "tokens_seen": 2653997056 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015839518555667002, + "loss": 1.2948, + "theoretical_loss": 3.3810557444855314, + "tokens_seen": 2654062592 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015837913741223672, + "loss": 1.1784, + "theoretical_loss": 3.381049219951274, + "tokens_seen": 2654128128 + }, + { + "epoch": 1.18, + "objective/train/docs_used": 980437, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2051327228546143, + "objective/train/theoretical_loss": 3.3810426956232273, + "objective/train/tokens_used": 2674653664, + "theoretical_loss": 3.3810426956232273, + "tokens_seen": 2654193664 + }, + { + "epoch": 1.18, + "learning_rate": 0.00015836308926780342, + "loss": 1.2051, + "theoretical_loss": 3.3810426956232273, + "tokens_seen": 2654193664 + }, + { + "epoch": 1.18, + "learning_rate": 0.0001583470411233701, + "loss": 1.1942, + "theoretical_loss": 3.3810361715013793, + "tokens_seen": 2654259200 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001583309929789368, + "loss": 1.2006, + "theoretical_loss": 3.3810296475857187, + "tokens_seen": 2654324736 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001583149448345035, + "loss": 1.2245, + "theoretical_loss": 3.3810231238762336, + "tokens_seen": 2654390272 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001582988966900702, + "loss": 1.1493, + "theoretical_loss": 3.3810166003729125, + "tokens_seen": 2654455808 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001582828485456369, + "loss": 1.2702, + "theoretical_loss": 3.381010077075744, + "tokens_seen": 2654521344 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001582668004012036, + "loss": 1.191, + "theoretical_loss": 3.381003553984716, + "tokens_seen": 2654586880 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015825075225677033, + "loss": 1.2894, + "theoretical_loss": 3.380997031099817, + "tokens_seen": 2654652416 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015823470411233703, + "loss": 1.2169, + "theoretical_loss": 3.3809905084210365, + "tokens_seen": 2654717952 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015821865596790373, + "loss": 1.1988, + "theoretical_loss": 3.3809839859483612, + "tokens_seen": 2654783488 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015820260782347043, + "loss": 1.2079, + "theoretical_loss": 3.3809774636817806, + "tokens_seen": 2654849024 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015818655967903713, + "loss": 1.2466, + "theoretical_loss": 3.3809709416212828, + "tokens_seen": 2654914560 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015817051153460383, + "loss": 1.1627, + "theoretical_loss": 3.3809644197668565, + "tokens_seen": 2654980096 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015815446339017053, + "loss": 1.2228, + "theoretical_loss": 3.3809578981184893, + "tokens_seen": 2655045632 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015813841524573723, + "loss": 1.3419, + "theoretical_loss": 3.3809513766761703, + "tokens_seen": 2655111168 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015812236710130393, + "loss": 1.2141, + "theoretical_loss": 3.3809448554398878, + "tokens_seen": 2655176704 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015810631895687063, + "loss": 1.1834, + "theoretical_loss": 3.38093833440963, + "tokens_seen": 2655242240 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015809027081243733, + "loss": 1.2589, + "theoretical_loss": 3.3809318135853856, + "tokens_seen": 2655307776 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015807422266800404, + "loss": 1.2636, + "theoretical_loss": 3.3809252929671425, + "tokens_seen": 2655373312 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015805817452357074, + "loss": 1.1953, + "theoretical_loss": 3.3809187725548897, + "tokens_seen": 2655438848 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015804212637913744, + "loss": 1.2105, + "theoretical_loss": 3.3809122523486153, + "tokens_seen": 2655504384 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001580260782347041, + "loss": 1.2109, + "theoretical_loss": 3.3809057323483076, + "tokens_seen": 2655569920 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001580100300902708, + "loss": 1.2013, + "theoretical_loss": 3.3808992125539556, + "tokens_seen": 2655635456 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001579939819458375, + "loss": 1.2298, + "theoretical_loss": 3.380892692965547, + "tokens_seen": 2655700992 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001579779338014042, + "loss": 1.1172, + "theoretical_loss": 3.3808861735830704, + "tokens_seen": 2655766528 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001579618856569709, + "loss": 1.1954, + "theoretical_loss": 3.3808796544065145, + "tokens_seen": 2655832064 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001579458375125376, + "loss": 1.2096, + "theoretical_loss": 3.3808731354358676, + "tokens_seen": 2655897600 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015792978936810431, + "loss": 1.1963, + "theoretical_loss": 3.380866616671118, + "tokens_seen": 2655963136 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015791374122367101, + "loss": 1.2544, + "theoretical_loss": 3.380860098112254, + "tokens_seen": 2656028672 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015789769307923771, + "loss": 1.1732, + "theoretical_loss": 3.3808535797592643, + "tokens_seen": 2656094208 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015788164493480442, + "loss": 1.2074, + "theoretical_loss": 3.380847061612137, + "tokens_seen": 2656159744 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015786559679037112, + "loss": 1.2583, + "theoretical_loss": 3.380840543670861, + "tokens_seen": 2656225280 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015784954864593782, + "loss": 1.1258, + "theoretical_loss": 3.380834025935424, + "tokens_seen": 2656290816 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015783350050150452, + "loss": 1.1893, + "theoretical_loss": 3.3808275084058153, + "tokens_seen": 2656356352 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015781745235707122, + "loss": 1.2465, + "theoretical_loss": 3.3808209910820226, + "tokens_seen": 2656421888 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015780140421263792, + "loss": 1.1351, + "theoretical_loss": 3.380814473964035, + "tokens_seen": 2656487424 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015778535606820462, + "loss": 1.1955, + "theoretical_loss": 3.3808079570518403, + "tokens_seen": 2656552960 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015776930792377132, + "loss": 1.1322, + "theoretical_loss": 3.380801440345427, + "tokens_seen": 2656618496 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015775325977933802, + "loss": 1.2057, + "theoretical_loss": 3.3807949238447836, + "tokens_seen": 2656684032 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015773721163490472, + "loss": 1.1734, + "theoretical_loss": 3.380788407549899, + "tokens_seen": 2656749568 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015772116349047142, + "loss": 1.2227, + "theoretical_loss": 3.380781891460761, + "tokens_seen": 2656815104 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015770511534603812, + "loss": 1.134, + "theoretical_loss": 3.3807753755773584, + "tokens_seen": 2656880640 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015768906720160482, + "loss": 1.2184, + "theoretical_loss": 3.3807688598996792, + "tokens_seen": 2656946176 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015767301905717152, + "loss": 1.2444, + "theoretical_loss": 3.3807623444277124, + "tokens_seen": 2657011712 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015765697091273822, + "loss": 1.1167, + "theoretical_loss": 3.380755829161446, + "tokens_seen": 2657077248 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015764092276830493, + "loss": 1.1962, + "theoretical_loss": 3.3807493141008687, + "tokens_seen": 2657142784 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015762487462387163, + "loss": 1.2268, + "theoretical_loss": 3.3807427992459687, + "tokens_seen": 2657208320 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015760882647943833, + "loss": 1.1845, + "theoretical_loss": 3.3807362845967344, + "tokens_seen": 2657273856 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015759277833500503, + "loss": 1.1113, + "theoretical_loss": 3.3807297701531547, + "tokens_seen": 2657339392 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015757673019057173, + "loss": 1.2143, + "theoretical_loss": 3.3807232559152176, + "tokens_seen": 2657404928 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015756068204613843, + "loss": 1.2366, + "theoretical_loss": 3.3807167418829116, + "tokens_seen": 2657470464 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015754463390170513, + "loss": 1.2419, + "theoretical_loss": 3.380710228056225, + "tokens_seen": 2657536000 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015752858575727183, + "loss": 1.2296, + "theoretical_loss": 3.380703714435147, + "tokens_seen": 2657601536 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015751253761283853, + "loss": 1.1574, + "theoretical_loss": 3.3806972010196654, + "tokens_seen": 2657667072 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015749648946840523, + "loss": 1.2016, + "theoretical_loss": 3.3806906878097682, + "tokens_seen": 2657732608 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015748044132397193, + "loss": 1.307, + "theoretical_loss": 3.380684174805445, + "tokens_seen": 2657798144 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015746439317953863, + "loss": 1.1953, + "theoretical_loss": 3.3806776620066827, + "tokens_seen": 2657863680 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015744834503510533, + "loss": 1.2394, + "theoretical_loss": 3.3806711494134714, + "tokens_seen": 2657929216 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015743229689067203, + "loss": 1.2547, + "theoretical_loss": 3.3806646370257987, + "tokens_seen": 2657994752 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015741624874623873, + "loss": 1.3257, + "theoretical_loss": 3.380658124843653, + "tokens_seen": 2658060288 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015740020060180543, + "loss": 1.1647, + "theoretical_loss": 3.3806516128670228, + "tokens_seen": 2658125824 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015738415245737214, + "loss": 1.2506, + "theoretical_loss": 3.380645101095897, + "tokens_seen": 2658191360 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015736810431293884, + "loss": 1.233, + "theoretical_loss": 3.380638589530263, + "tokens_seen": 2658256896 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015735205616850554, + "loss": 1.1276, + "theoretical_loss": 3.3806320781701107, + "tokens_seen": 2658322432 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001573360080240722, + "loss": 1.1521, + "theoretical_loss": 3.3806255670154273, + "tokens_seen": 2658387968 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001573199598796389, + "loss": 1.2197, + "theoretical_loss": 3.3806190560662017, + "tokens_seen": 2658453504 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001573039117352056, + "loss": 1.222, + "theoretical_loss": 3.380612545322423, + "tokens_seen": 2658519040 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001572878635907723, + "loss": 1.2628, + "theoretical_loss": 3.3806060347840785, + "tokens_seen": 2658584576 + }, + { + "epoch": 1.19, + "learning_rate": 0.000157271815446339, + "loss": 1.2326, + "theoretical_loss": 3.3805995244511573, + "tokens_seen": 2658650112 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001572557673019057, + "loss": 1.2181, + "theoretical_loss": 3.380593014323648, + "tokens_seen": 2658715648 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015723971915747241, + "loss": 1.2418, + "theoretical_loss": 3.3805865044015384, + "tokens_seen": 2658781184 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015722367101303911, + "loss": 1.2192, + "theoretical_loss": 3.380579994684817, + "tokens_seen": 2658846720 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015720762286860582, + "loss": 1.2161, + "theoretical_loss": 3.3805734851734734, + "tokens_seen": 2658912256 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015719157472417252, + "loss": 1.2511, + "theoretical_loss": 3.380566975867495, + "tokens_seen": 2658977792 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015717552657973922, + "loss": 1.2942, + "theoretical_loss": 3.3805604667668705, + "tokens_seen": 2659043328 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015715947843530592, + "loss": 1.1442, + "theoretical_loss": 3.380553957871588, + "tokens_seen": 2659108864 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015714343029087262, + "loss": 1.2843, + "theoretical_loss": 3.380547449181637, + "tokens_seen": 2659174400 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015712738214643932, + "loss": 1.1226, + "theoretical_loss": 3.380540940697005, + "tokens_seen": 2659239936 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015711133400200602, + "loss": 1.1354, + "theoretical_loss": 3.380534432417681, + "tokens_seen": 2659305472 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015709528585757272, + "loss": 1.1703, + "theoretical_loss": 3.380527924343653, + "tokens_seen": 2659371008 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015707923771313942, + "loss": 1.2124, + "theoretical_loss": 3.38052141647491, + "tokens_seen": 2659436544 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015706318956870612, + "loss": 1.1983, + "theoretical_loss": 3.38051490881144, + "tokens_seen": 2659502080 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015704714142427285, + "loss": 1.2448, + "theoretical_loss": 3.3805084013532314, + "tokens_seen": 2659567616 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015703109327983955, + "loss": 1.3283, + "theoretical_loss": 3.3805018941002736, + "tokens_seen": 2659633152 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015701504513540622, + "loss": 1.2602, + "theoretical_loss": 3.380495387052554, + "tokens_seen": 2659698688 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015699899699097292, + "loss": 1.2153, + "theoretical_loss": 3.380488880210061, + "tokens_seen": 2659764224 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015698294884653962, + "loss": 1.195, + "theoretical_loss": 3.380482373572784, + "tokens_seen": 2659829760 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015696690070210632, + "loss": 1.2788, + "theoretical_loss": 3.380475867140711, + "tokens_seen": 2659895296 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015695085255767303, + "loss": 1.1621, + "theoretical_loss": 3.38046936091383, + "tokens_seen": 2659960832 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015693480441323973, + "loss": 1.1952, + "theoretical_loss": 3.3804628548921305, + "tokens_seen": 2660026368 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015691875626880643, + "loss": 1.2209, + "theoretical_loss": 3.3804563490756, + "tokens_seen": 2660091904 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015690270812437313, + "loss": 1.3066, + "theoretical_loss": 3.380449843464228, + "tokens_seen": 2660157440 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015688665997993983, + "loss": 1.2075, + "theoretical_loss": 3.3804433380580017, + "tokens_seen": 2660222976 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015687061183550653, + "loss": 1.1433, + "theoretical_loss": 3.3804368328569105, + "tokens_seen": 2660288512 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015685456369107323, + "loss": 1.1777, + "theoretical_loss": 3.3804303278609424, + "tokens_seen": 2660354048 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015683851554663993, + "loss": 1.2524, + "theoretical_loss": 3.3804238230700867, + "tokens_seen": 2660419584 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015682246740220663, + "loss": 1.2951, + "theoretical_loss": 3.380417318484331, + "tokens_seen": 2660485120 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015680641925777333, + "loss": 1.2285, + "theoretical_loss": 3.380410814103664, + "tokens_seen": 2660550656 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015679037111334003, + "loss": 1.2661, + "theoretical_loss": 3.380404309928074, + "tokens_seen": 2660616192 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015677432296890673, + "loss": 1.2477, + "theoretical_loss": 3.38039780595755, + "tokens_seen": 2660681728 + }, + { + "epoch": 1.19, + "objective/train/docs_used": 982605, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2494752407073975, + "objective/train/theoretical_loss": 3.38039130219208, + "objective/train/tokens_used": 2681207264, + "theoretical_loss": 3.38039130219208, + "tokens_seen": 2660747264 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015675827482447343, + "loss": 1.2495, + "theoretical_loss": 3.38039130219208, + "tokens_seen": 2660747264 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015674222668004013, + "loss": 1.1756, + "theoretical_loss": 3.380384798631653, + "tokens_seen": 2660812800 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015672617853560683, + "loss": 1.2101, + "theoretical_loss": 3.380378295276257, + "tokens_seen": 2660878336 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015671013039117353, + "loss": 1.0956, + "theoretical_loss": 3.3803717921258807, + "tokens_seen": 2660943872 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015669408224674024, + "loss": 1.1738, + "theoretical_loss": 3.3803652891805127, + "tokens_seen": 2661009408 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015667803410230694, + "loss": 1.2644, + "theoretical_loss": 3.380358786440141, + "tokens_seen": 2661074944 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015666198595787364, + "loss": 1.1757, + "theoretical_loss": 3.3803522839047546, + "tokens_seen": 2661140480 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015664593781344034, + "loss": 1.1884, + "theoretical_loss": 3.380345781574342, + "tokens_seen": 2661206016 + }, + { + "epoch": 1.19, + "learning_rate": 0.000156629889669007, + "loss": 1.1889, + "theoretical_loss": 3.3803392794488913, + "tokens_seen": 2661271552 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001566138415245737, + "loss": 1.1971, + "theoretical_loss": 3.380332777528391, + "tokens_seen": 2661337088 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001565977933801404, + "loss": 1.229, + "theoretical_loss": 3.3803262758128305, + "tokens_seen": 2661402624 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001565817452357071, + "loss": 1.154, + "theoretical_loss": 3.380319774302197, + "tokens_seen": 2661468160 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001565656970912738, + "loss": 1.2352, + "theoretical_loss": 3.3803132729964798, + "tokens_seen": 2661533696 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015654964894684051, + "loss": 1.2578, + "theoretical_loss": 3.3803067718956674, + "tokens_seen": 2661599232 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015653360080240721, + "loss": 1.1845, + "theoretical_loss": 3.3803002709997476, + "tokens_seen": 2661664768 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015651755265797392, + "loss": 1.1632, + "theoretical_loss": 3.3802937703087097, + "tokens_seen": 2661730304 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015650150451354062, + "loss": 1.2888, + "theoretical_loss": 3.3802872698225417, + "tokens_seen": 2661795840 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015648545636910732, + "loss": 1.2022, + "theoretical_loss": 3.3802807695412325, + "tokens_seen": 2661861376 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015646940822467402, + "loss": 1.0493, + "theoretical_loss": 3.38027426946477, + "tokens_seen": 2661926912 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015645336008024074, + "loss": 1.158, + "theoretical_loss": 3.3802677695931433, + "tokens_seen": 2661992448 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015643731193580745, + "loss": 1.1732, + "theoretical_loss": 3.380261269926341, + "tokens_seen": 2662057984 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015642126379137415, + "loss": 1.0538, + "theoretical_loss": 3.380254770464351, + "tokens_seen": 2662123520 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015640521564694085, + "loss": 1.1614, + "theoretical_loss": 3.380248271207162, + "tokens_seen": 2662189056 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015638916750250755, + "loss": 1.1507, + "theoretical_loss": 3.3802417721547626, + "tokens_seen": 2662254592 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015637311935807425, + "loss": 1.2083, + "theoretical_loss": 3.3802352733071412, + "tokens_seen": 2662320128 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015635707121364095, + "loss": 1.2305, + "theoretical_loss": 3.3802287746642867, + "tokens_seen": 2662385664 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015634102306920765, + "loss": 1.2565, + "theoretical_loss": 3.380222276226187, + "tokens_seen": 2662451200 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015632497492477435, + "loss": 1.1357, + "theoretical_loss": 3.3802157779928312, + "tokens_seen": 2662516736 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015630892678034102, + "loss": 1.1794, + "theoretical_loss": 3.3802092799642076, + "tokens_seen": 2662582272 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015629287863590772, + "loss": 1.2459, + "theoretical_loss": 3.3802027821403042, + "tokens_seen": 2662647808 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015627683049147442, + "loss": 1.0889, + "theoretical_loss": 3.3801962845211104, + "tokens_seen": 2662713344 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015626078234704113, + "loss": 1.3336, + "theoretical_loss": 3.380189787106614, + "tokens_seen": 2662778880 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015624473420260783, + "loss": 1.1696, + "theoretical_loss": 3.380183289896804, + "tokens_seen": 2662844416 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015622868605817453, + "loss": 1.1649, + "theoretical_loss": 3.380176792891669, + "tokens_seen": 2662909952 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015621263791374123, + "loss": 1.1388, + "theoretical_loss": 3.3801702960911966, + "tokens_seen": 2662975488 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015619658976930793, + "loss": 1.2327, + "theoretical_loss": 3.380163799495376, + "tokens_seen": 2663041024 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015618054162487463, + "loss": 1.2273, + "theoretical_loss": 3.380157303104196, + "tokens_seen": 2663106560 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015616449348044133, + "loss": 1.1641, + "theoretical_loss": 3.3801508069176447, + "tokens_seen": 2663172096 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015614844533600803, + "loss": 1.0944, + "theoretical_loss": 3.3801443109357105, + "tokens_seen": 2663237632 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015613239719157473, + "loss": 1.3299, + "theoretical_loss": 3.380137815158382, + "tokens_seen": 2663303168 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015611634904714143, + "loss": 1.1123, + "theoretical_loss": 3.380131319585648, + "tokens_seen": 2663368704 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015610030090270813, + "loss": 1.2655, + "theoretical_loss": 3.380124824217497, + "tokens_seen": 2663434240 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015608425275827483, + "loss": 1.3306, + "theoretical_loss": 3.380118329053917, + "tokens_seen": 2663499776 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015606820461384153, + "loss": 1.1014, + "theoretical_loss": 3.3801118340948975, + "tokens_seen": 2663565312 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015605215646940823, + "loss": 1.2027, + "theoretical_loss": 3.380105339340426, + "tokens_seen": 2663630848 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015603610832497493, + "loss": 1.2648, + "theoretical_loss": 3.380098844790491, + "tokens_seen": 2663696384 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015602006018054163, + "loss": 1.1408, + "theoretical_loss": 3.380092350445082, + "tokens_seen": 2663761920 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015600401203610834, + "loss": 1.1068, + "theoretical_loss": 3.380085856304187, + "tokens_seen": 2663827456 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015598796389167504, + "loss": 1.1903, + "theoretical_loss": 3.3800793623677947, + "tokens_seen": 2663892992 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015597191574724174, + "loss": 1.282, + "theoretical_loss": 3.380072868635893, + "tokens_seen": 2663958528 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015595586760280844, + "loss": 1.2046, + "theoretical_loss": 3.380066375108471, + "tokens_seen": 2664024064 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015593981945837514, + "loss": 1.2679, + "theoretical_loss": 3.3800598817855176, + "tokens_seen": 2664089600 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001559237713139418, + "loss": 1.0932, + "theoretical_loss": 3.38005338866702, + "tokens_seen": 2664155136 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001559077231695085, + "loss": 1.2569, + "theoretical_loss": 3.3800468957529684, + "tokens_seen": 2664220672 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001558916750250752, + "loss": 1.2388, + "theoretical_loss": 3.38004040304335, + "tokens_seen": 2664286208 + }, + { + "epoch": 1.19, + "learning_rate": 0.0001558756268806419, + "loss": 1.177, + "theoretical_loss": 3.3800339105381543, + "tokens_seen": 2664351744 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015585957873620861, + "loss": 1.1951, + "theoretical_loss": 3.380027418237369, + "tokens_seen": 2664417280 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015584353059177534, + "loss": 1.218, + "theoretical_loss": 3.380020926140983, + "tokens_seen": 2664482816 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015582748244734204, + "loss": 1.2029, + "theoretical_loss": 3.380014434248985, + "tokens_seen": 2664548352 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015581143430290874, + "loss": 1.218, + "theoretical_loss": 3.380007942561363, + "tokens_seen": 2664613888 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015579538615847544, + "loss": 1.2065, + "theoretical_loss": 3.3800014510781065, + "tokens_seen": 2664679424 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015577933801404214, + "loss": 1.1732, + "theoretical_loss": 3.3799949597992036, + "tokens_seen": 2664744960 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015576328986960885, + "loss": 1.1997, + "theoretical_loss": 3.379988468724642, + "tokens_seen": 2664810496 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015574724172517555, + "loss": 1.1265, + "theoretical_loss": 3.3799819778544116, + "tokens_seen": 2664876032 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015573119358074225, + "loss": 1.1803, + "theoretical_loss": 3.3799754871885, + "tokens_seen": 2664941568 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015571514543630895, + "loss": 1.1616, + "theoretical_loss": 3.3799689967268964, + "tokens_seen": 2665007104 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015569909729187565, + "loss": 1.2241, + "theoretical_loss": 3.379962506469589, + "tokens_seen": 2665072640 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015568304914744235, + "loss": 1.1342, + "theoretical_loss": 3.379956016416566, + "tokens_seen": 2665138176 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015566700100300905, + "loss": 1.1623, + "theoretical_loss": 3.3799495265678163, + "tokens_seen": 2665203712 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015565095285857575, + "loss": 1.219, + "theoretical_loss": 3.3799430369233283, + "tokens_seen": 2665269248 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015563490471414245, + "loss": 1.2178, + "theoretical_loss": 3.3799365474830907, + "tokens_seen": 2665334784 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015561885656970912, + "loss": 1.2252, + "theoretical_loss": 3.3799300582470924, + "tokens_seen": 2665400320 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015560280842527582, + "loss": 1.2378, + "theoretical_loss": 3.3799235692153213, + "tokens_seen": 2665465856 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015558676028084252, + "loss": 1.3035, + "theoretical_loss": 3.3799170803877665, + "tokens_seen": 2665531392 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015557071213640923, + "loss": 1.1744, + "theoretical_loss": 3.3799105917644163, + "tokens_seen": 2665596928 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015555466399197593, + "loss": 1.2548, + "theoretical_loss": 3.379904103345259, + "tokens_seen": 2665662464 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015553861584754263, + "loss": 1.1545, + "theoretical_loss": 3.3798976151302833, + "tokens_seen": 2665728000 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015552256770310933, + "loss": 1.128, + "theoretical_loss": 3.3798911271194783, + "tokens_seen": 2665793536 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015550651955867603, + "loss": 1.1697, + "theoretical_loss": 3.3798846393128317, + "tokens_seen": 2665859072 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015549047141424273, + "loss": 1.2226, + "theoretical_loss": 3.379878151710333, + "tokens_seen": 2665924608 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015547442326980943, + "loss": 1.278, + "theoretical_loss": 3.3798716643119695, + "tokens_seen": 2665990144 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015545837512537613, + "loss": 1.2199, + "theoretical_loss": 3.379865177117731, + "tokens_seen": 2666055680 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015544232698094283, + "loss": 1.2004, + "theoretical_loss": 3.379858690127605, + "tokens_seen": 2666121216 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015542627883650953, + "loss": 1.2339, + "theoretical_loss": 3.379852203341581, + "tokens_seen": 2666186752 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015541023069207623, + "loss": 1.2863, + "theoretical_loss": 3.379845716759647, + "tokens_seen": 2666252288 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015539418254764293, + "loss": 1.1852, + "theoretical_loss": 3.379839230381792, + "tokens_seen": 2666317824 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015537813440320963, + "loss": 1.2379, + "theoretical_loss": 3.379832744208004, + "tokens_seen": 2666383360 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015536208625877633, + "loss": 1.1313, + "theoretical_loss": 3.379826258238272, + "tokens_seen": 2666448896 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015534603811434303, + "loss": 1.2122, + "theoretical_loss": 3.3798197724725845, + "tokens_seen": 2666514432 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015532998996990974, + "loss": 1.1634, + "theoretical_loss": 3.37981328691093, + "tokens_seen": 2666579968 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015531394182547644, + "loss": 1.1292, + "theoretical_loss": 3.379806801553297, + "tokens_seen": 2666645504 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015529789368104314, + "loss": 1.1621, + "theoretical_loss": 3.379800316399674, + "tokens_seen": 2666711040 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015528184553660984, + "loss": 1.1643, + "theoretical_loss": 3.3797938314500495, + "tokens_seen": 2666776576 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015526579739217654, + "loss": 1.127, + "theoretical_loss": 3.3797873467044126, + "tokens_seen": 2666842112 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015524974924774324, + "loss": 1.1175, + "theoretical_loss": 3.3797808621627516, + "tokens_seen": 2666907648 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015523370110330994, + "loss": 1.0904, + "theoretical_loss": 3.379774377825055, + "tokens_seen": 2666973184 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015521765295887664, + "loss": 1.22, + "theoretical_loss": 3.379767893691311, + "tokens_seen": 2667038720 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015520160481444334, + "loss": 1.2155, + "theoretical_loss": 3.379761409761509, + "tokens_seen": 2667104256 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015518555667001004, + "loss": 1.0873, + "theoretical_loss": 3.3797549260356368, + "tokens_seen": 2667169792 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015516950852557674, + "loss": 1.1819, + "theoretical_loss": 3.3797484425136832, + "tokens_seen": 2667235328 + }, + { + "epoch": 1.19, + "objective/train/docs_used": 985090, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1423561573028564, + "objective/train/theoretical_loss": 3.3797419591956372, + "objective/train/tokens_used": 2687760864, + "theoretical_loss": 3.3797419591956372, + "tokens_seen": 2667300864 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015515346038114344, + "loss": 1.1424, + "theoretical_loss": 3.3797419591956372, + "tokens_seen": 2667300864 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015513741223671014, + "loss": 1.2677, + "theoretical_loss": 3.379735476081487, + "tokens_seen": 2667366400 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015512136409227684, + "loss": 1.2571, + "theoretical_loss": 3.379728993171221, + "tokens_seen": 2667431936 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015510531594784354, + "loss": 1.2319, + "theoretical_loss": 3.3797225104648283, + "tokens_seen": 2667497472 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015508926780341024, + "loss": 1.2008, + "theoretical_loss": 3.379716027962297, + "tokens_seen": 2667563008 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015507321965897695, + "loss": 1.1075, + "theoretical_loss": 3.3797095456636157, + "tokens_seen": 2667628544 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015505717151454365, + "loss": 1.1495, + "theoretical_loss": 3.3797030635687735, + "tokens_seen": 2667694080 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015504112337011035, + "loss": 1.1608, + "theoretical_loss": 3.3796965816777584, + "tokens_seen": 2667759616 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015502507522567705, + "loss": 1.1501, + "theoretical_loss": 3.3796900999905595, + "tokens_seen": 2667825152 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015500902708124375, + "loss": 1.1954, + "theoretical_loss": 3.379683618507165, + "tokens_seen": 2667890688 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015499297893681045, + "loss": 1.2131, + "theoretical_loss": 3.3796771372275636, + "tokens_seen": 2667956224 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015497693079237715, + "loss": 1.1931, + "theoretical_loss": 3.3796706561517436, + "tokens_seen": 2668021760 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015496088264794385, + "loss": 1.1973, + "theoretical_loss": 3.3796641752796943, + "tokens_seen": 2668087296 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015494483450351055, + "loss": 1.2533, + "theoretical_loss": 3.3796576946114034, + "tokens_seen": 2668152832 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015492878635907725, + "loss": 1.2268, + "theoretical_loss": 3.37965121414686, + "tokens_seen": 2668218368 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015491273821464392, + "loss": 1.2054, + "theoretical_loss": 3.379644733886053, + "tokens_seen": 2668283904 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015489669007021063, + "loss": 1.2484, + "theoretical_loss": 3.37963825382897, + "tokens_seen": 2668349440 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015488064192577733, + "loss": 1.0945, + "theoretical_loss": 3.379631773975601, + "tokens_seen": 2668414976 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015486459378134403, + "loss": 1.179, + "theoretical_loss": 3.3796252943259333, + "tokens_seen": 2668480512 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015484854563691073, + "loss": 1.2839, + "theoretical_loss": 3.379618814879956, + "tokens_seen": 2668546048 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015483249749247743, + "loss": 1.2097, + "theoretical_loss": 3.3796123356376584, + "tokens_seen": 2668611584 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015481644934804413, + "loss": 1.0911, + "theoretical_loss": 3.379605856599028, + "tokens_seen": 2668677120 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015480040120361083, + "loss": 1.1372, + "theoretical_loss": 3.379599377764053, + "tokens_seen": 2668742656 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015478435305917753, + "loss": 1.2454, + "theoretical_loss": 3.3795928991327235, + "tokens_seen": 2668808192 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015476830491474423, + "loss": 1.2773, + "theoretical_loss": 3.3795864207050275, + "tokens_seen": 2668873728 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015475225677031093, + "loss": 1.1427, + "theoretical_loss": 3.3795799424809534, + "tokens_seen": 2668939264 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015473620862587763, + "loss": 1.1537, + "theoretical_loss": 3.37957346446049, + "tokens_seen": 2669004800 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015472016048144433, + "loss": 1.2583, + "theoretical_loss": 3.379566986643625, + "tokens_seen": 2669070336 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015470411233701103, + "loss": 1.137, + "theoretical_loss": 3.3795605090303487, + "tokens_seen": 2669135872 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015468806419257773, + "loss": 1.1211, + "theoretical_loss": 3.379554031620648, + "tokens_seen": 2669201408 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015467201604814443, + "loss": 1.2516, + "theoretical_loss": 3.379547554414513, + "tokens_seen": 2669266944 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015465596790371113, + "loss": 1.194, + "theoretical_loss": 3.379541077411931, + "tokens_seen": 2669332480 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015463991975927786, + "loss": 1.1779, + "theoretical_loss": 3.379534600612892, + "tokens_seen": 2669398016 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015462387161484456, + "loss": 1.2159, + "theoretical_loss": 3.379528124017383, + "tokens_seen": 2669463552 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015460782347041126, + "loss": 1.1715, + "theoretical_loss": 3.3795216476253938, + "tokens_seen": 2669529088 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015459177532597794, + "loss": 1.2817, + "theoretical_loss": 3.3795151714369127, + "tokens_seen": 2669594624 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015457572718154464, + "loss": 1.1393, + "theoretical_loss": 3.379508695451928, + "tokens_seen": 2669660160 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015455967903711134, + "loss": 1.2293, + "theoretical_loss": 3.379502219670429, + "tokens_seen": 2669725696 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015454363089267804, + "loss": 1.2258, + "theoretical_loss": 3.379495744092403, + "tokens_seen": 2669791232 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015452758274824474, + "loss": 1.2564, + "theoretical_loss": 3.3794892687178404, + "tokens_seen": 2669856768 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015451153460381144, + "loss": 1.2323, + "theoretical_loss": 3.379482793546728, + "tokens_seen": 2669922304 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015449548645937814, + "loss": 1.279, + "theoretical_loss": 3.379476318579056, + "tokens_seen": 2669987840 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015447943831494484, + "loss": 1.2858, + "theoretical_loss": 3.379469843814812, + "tokens_seen": 2670053376 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015446339017051154, + "loss": 1.1549, + "theoretical_loss": 3.379463369253985, + "tokens_seen": 2670118912 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015444734202607824, + "loss": 1.2604, + "theoretical_loss": 3.3794568948965638, + "tokens_seen": 2670184448 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015443129388164494, + "loss": 1.2726, + "theoretical_loss": 3.3794504207425367, + "tokens_seen": 2670249984 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015441524573721164, + "loss": 1.1508, + "theoretical_loss": 3.379443946791892, + "tokens_seen": 2670315520 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015439919759277834, + "loss": 1.343, + "theoretical_loss": 3.379437473044619, + "tokens_seen": 2670381056 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015438314944834505, + "loss": 1.2176, + "theoretical_loss": 3.379430999500706, + "tokens_seen": 2670446592 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015436710130391175, + "loss": 1.3652, + "theoretical_loss": 3.3794245261601414, + "tokens_seen": 2670512128 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015435105315947845, + "loss": 1.3006, + "theoretical_loss": 3.3794180530229143, + "tokens_seen": 2670577664 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015433500501504515, + "loss": 1.1725, + "theoretical_loss": 3.379411580089013, + "tokens_seen": 2670643200 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015431895687061185, + "loss": 1.1047, + "theoretical_loss": 3.3794051073584264, + "tokens_seen": 2670708736 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015430290872617855, + "loss": 1.1702, + "theoretical_loss": 3.379398634831143, + "tokens_seen": 2670774272 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015428686058174525, + "loss": 1.1907, + "theoretical_loss": 3.379392162507151, + "tokens_seen": 2670839808 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015427081243731195, + "loss": 1.1067, + "theoretical_loss": 3.3793856903864397, + "tokens_seen": 2670905344 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015425476429287865, + "loss": 1.1237, + "theoretical_loss": 3.379379218468997, + "tokens_seen": 2670970880 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015423871614844535, + "loss": 1.2099, + "theoretical_loss": 3.3793727467548123, + "tokens_seen": 2671036416 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015422266800401205, + "loss": 1.1741, + "theoretical_loss": 3.379366275243874, + "tokens_seen": 2671101952 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015420661985957873, + "loss": 1.2297, + "theoretical_loss": 3.37935980393617, + "tokens_seen": 2671167488 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015419057171514543, + "loss": 1.2137, + "theoretical_loss": 3.3793533328316903, + "tokens_seen": 2671233024 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015417452357071213, + "loss": 1.1759, + "theoretical_loss": 3.379346861930422, + "tokens_seen": 2671298560 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015415847542627883, + "loss": 1.1707, + "theoretical_loss": 3.379340391232355, + "tokens_seen": 2671364096 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015414242728184553, + "loss": 1.1712, + "theoretical_loss": 3.3793339207374773, + "tokens_seen": 2671429632 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015412637913741223, + "loss": 1.1363, + "theoretical_loss": 3.379327450445778, + "tokens_seen": 2671495168 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015411033099297893, + "loss": 1.151, + "theoretical_loss": 3.379320980357245, + "tokens_seen": 2671560704 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015409428284854563, + "loss": 1.1613, + "theoretical_loss": 3.379314510471868, + "tokens_seen": 2671626240 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015407823470411233, + "loss": 1.2192, + "theoretical_loss": 3.379308040789634, + "tokens_seen": 2671691776 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015406218655967903, + "loss": 1.1512, + "theoretical_loss": 3.379301571310533, + "tokens_seen": 2671757312 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015404613841524576, + "loss": 1.259, + "theoretical_loss": 3.3792951020345536, + "tokens_seen": 2671822848 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015403009027081246, + "loss": 1.2594, + "theoretical_loss": 3.379288632961684, + "tokens_seen": 2671888384 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015401404212637916, + "loss": 1.2504, + "theoretical_loss": 3.3792821640919124, + "tokens_seen": 2671953920 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015399799398194586, + "loss": 1.2448, + "theoretical_loss": 3.3792756954252288, + "tokens_seen": 2672019456 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015398194583751256, + "loss": 1.2183, + "theoretical_loss": 3.3792692269616205, + "tokens_seen": 2672084992 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015396589769307926, + "loss": 1.1081, + "theoretical_loss": 3.379262758701077, + "tokens_seen": 2672150528 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015394984954864596, + "loss": 1.19, + "theoretical_loss": 3.3792562906435863, + "tokens_seen": 2672216064 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015393380140421266, + "loss": 1.2246, + "theoretical_loss": 3.3792498227891374, + "tokens_seen": 2672281600 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015391775325977936, + "loss": 1.1452, + "theoretical_loss": 3.379243355137719, + "tokens_seen": 2672347136 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015390170511534606, + "loss": 1.2506, + "theoretical_loss": 3.37923688768932, + "tokens_seen": 2672412672 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015388565697091274, + "loss": 1.2483, + "theoretical_loss": 3.3792304204439283, + "tokens_seen": 2672478208 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015386960882647944, + "loss": 1.1942, + "theoretical_loss": 3.379223953401533, + "tokens_seen": 2672543744 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015385356068204614, + "loss": 1.1045, + "theoretical_loss": 3.3792174865621227, + "tokens_seen": 2672609280 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015383751253761284, + "loss": 1.2307, + "theoretical_loss": 3.379211019925686, + "tokens_seen": 2672674816 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015382146439317954, + "loss": 1.273, + "theoretical_loss": 3.3792045534922117, + "tokens_seen": 2672740352 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015380541624874624, + "loss": 1.1438, + "theoretical_loss": 3.3791980872616887, + "tokens_seen": 2672805888 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015378936810431294, + "loss": 1.1727, + "theoretical_loss": 3.379191621234105, + "tokens_seen": 2672871424 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015377331995987964, + "loss": 1.2766, + "theoretical_loss": 3.3791851554094494, + "tokens_seen": 2672936960 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015375727181544634, + "loss": 1.1374, + "theoretical_loss": 3.379178689787711, + "tokens_seen": 2673002496 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015374122367101304, + "loss": 1.198, + "theoretical_loss": 3.379172224368878, + "tokens_seen": 2673068032 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015372517552657974, + "loss": 1.2557, + "theoretical_loss": 3.3791657591529396, + "tokens_seen": 2673133568 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015370912738214644, + "loss": 1.1338, + "theoretical_loss": 3.379159294139884, + "tokens_seen": 2673199104 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015369307923771315, + "loss": 1.2393, + "theoretical_loss": 3.3791528293296995, + "tokens_seen": 2673264640 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015367703109327985, + "loss": 1.1769, + "theoretical_loss": 3.379146364722376, + "tokens_seen": 2673330176 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015366098294884655, + "loss": 1.2064, + "theoretical_loss": 3.3791399003179006, + "tokens_seen": 2673395712 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015364493480441325, + "loss": 1.16, + "theoretical_loss": 3.3791334361162635, + "tokens_seen": 2673461248 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015362888665997995, + "loss": 1.2086, + "theoretical_loss": 3.379126972117452, + "tokens_seen": 2673526784 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015361283851554665, + "loss": 1.1937, + "theoretical_loss": 3.3791205083214555, + "tokens_seen": 2673592320 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015359679037111335, + "loss": 1.3207, + "theoretical_loss": 3.379114044728263, + "tokens_seen": 2673657856 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015358074222668005, + "loss": 1.2128, + "theoretical_loss": 3.379107581337862, + "tokens_seen": 2673723392 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015356469408224675, + "loss": 1.2616, + "theoretical_loss": 3.379101118150243, + "tokens_seen": 2673788928 + }, + { + "epoch": 1.19, + "objective/train/docs_used": 987333, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1841596364974976, + "objective/train/theoretical_loss": 3.3790946551653924, + "objective/train/tokens_used": 2694314464, + "theoretical_loss": 3.3790946551653924, + "tokens_seen": 2673854464 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015354864593781345, + "loss": 1.1842, + "theoretical_loss": 3.3790946551653924, + "tokens_seen": 2673854464 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015353259779338015, + "loss": 1.1575, + "theoretical_loss": 3.3790881923833003, + "tokens_seen": 2673920000 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015351654964894685, + "loss": 1.2274, + "theoretical_loss": 3.3790817298039553, + "tokens_seen": 2673985536 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015350050150451353, + "loss": 1.1944, + "theoretical_loss": 3.379075267427346, + "tokens_seen": 2674051072 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015348445336008023, + "loss": 1.2192, + "theoretical_loss": 3.3790688052534605, + "tokens_seen": 2674116608 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015346840521564693, + "loss": 1.1011, + "theoretical_loss": 3.379062343282288, + "tokens_seen": 2674182144 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015345235707121366, + "loss": 1.2277, + "theoretical_loss": 3.379055881513817, + "tokens_seen": 2674247680 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015343630892678036, + "loss": 1.2527, + "theoretical_loss": 3.3790494199480365, + "tokens_seen": 2674313216 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015342026078234706, + "loss": 1.1794, + "theoretical_loss": 3.379042958584935, + "tokens_seen": 2674378752 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015340421263791376, + "loss": 1.2612, + "theoretical_loss": 3.379036497424501, + "tokens_seen": 2674444288 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015338816449348046, + "loss": 1.226, + "theoretical_loss": 3.379030036466723, + "tokens_seen": 2674509824 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015337211634904716, + "loss": 1.1481, + "theoretical_loss": 3.3790235757115905, + "tokens_seen": 2674575360 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015335606820461386, + "loss": 1.2308, + "theoretical_loss": 3.3790171151590913, + "tokens_seen": 2674640896 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015334002006018056, + "loss": 1.1922, + "theoretical_loss": 3.379010654809214, + "tokens_seen": 2674706432 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015332397191574726, + "loss": 1.1363, + "theoretical_loss": 3.3790041946619485, + "tokens_seen": 2674771968 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015330792377131396, + "loss": 1.1906, + "theoretical_loss": 3.378997734717282, + "tokens_seen": 2674837504 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015329187562688066, + "loss": 1.2291, + "theoretical_loss": 3.3789912749752045, + "tokens_seen": 2674903040 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015327582748244736, + "loss": 1.1739, + "theoretical_loss": 3.3789848154357034, + "tokens_seen": 2674968576 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015325977933801406, + "loss": 1.1767, + "theoretical_loss": 3.3789783560987683, + "tokens_seen": 2675034112 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015324373119358076, + "loss": 1.1714, + "theoretical_loss": 3.378971896964388, + "tokens_seen": 2675099648 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015322768304914746, + "loss": 1.1657, + "theoretical_loss": 3.3789654380325502, + "tokens_seen": 2675165184 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015321163490471416, + "loss": 1.2616, + "theoretical_loss": 3.378958979303244, + "tokens_seen": 2675230720 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015319558676028084, + "loss": 1.1551, + "theoretical_loss": 3.3789525207764592, + "tokens_seen": 2675296256 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015317953861584754, + "loss": 1.3164, + "theoretical_loss": 3.378946062452183, + "tokens_seen": 2675361792 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015316349047141424, + "loss": 1.2621, + "theoretical_loss": 3.3789396043304047, + "tokens_seen": 2675427328 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015314744232698094, + "loss": 1.1817, + "theoretical_loss": 3.3789331464111125, + "tokens_seen": 2675492864 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015313139418254764, + "loss": 1.2849, + "theoretical_loss": 3.378926688694296, + "tokens_seen": 2675558400 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015311534603811434, + "loss": 1.2727, + "theoretical_loss": 3.3789202311799436, + "tokens_seen": 2675623936 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015309929789368104, + "loss": 1.198, + "theoretical_loss": 3.3789137738680437, + "tokens_seen": 2675689472 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015308324974924774, + "loss": 1.1716, + "theoretical_loss": 3.378907316758585, + "tokens_seen": 2675755008 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015306720160481444, + "loss": 1.2521, + "theoretical_loss": 3.3789008598515564, + "tokens_seen": 2675820544 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015305115346038114, + "loss": 1.2743, + "theoretical_loss": 3.3788944031469463, + "tokens_seen": 2675886080 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015303510531594784, + "loss": 1.2146, + "theoretical_loss": 3.3788879466447437, + "tokens_seen": 2675951616 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015301905717151455, + "loss": 1.1647, + "theoretical_loss": 3.3788814903449373, + "tokens_seen": 2676017152 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015300300902708125, + "loss": 1.0713, + "theoretical_loss": 3.378875034247516, + "tokens_seen": 2676082688 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015298696088264795, + "loss": 1.1475, + "theoretical_loss": 3.3788685783524675, + "tokens_seen": 2676148224 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015297091273821465, + "loss": 1.3014, + "theoretical_loss": 3.3788621226597817, + "tokens_seen": 2676213760 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015295486459378135, + "loss": 1.1641, + "theoretical_loss": 3.3788556671694465, + "tokens_seen": 2676279296 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015293881644934805, + "loss": 1.2732, + "theoretical_loss": 3.378849211881451, + "tokens_seen": 2676344832 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015292276830491475, + "loss": 1.3385, + "theoretical_loss": 3.378842756795784, + "tokens_seen": 2676410368 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015290672016048145, + "loss": 1.2223, + "theoretical_loss": 3.3788363019124334, + "tokens_seen": 2676475904 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015289067201604815, + "loss": 1.2742, + "theoretical_loss": 3.3788298472313896, + "tokens_seen": 2676541440 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015287462387161485, + "loss": 1.1882, + "theoretical_loss": 3.3788233927526394, + "tokens_seen": 2676606976 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015285857572718155, + "loss": 1.3456, + "theoretical_loss": 3.3788169384761724, + "tokens_seen": 2676672512 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015284252758274825, + "loss": 1.3291, + "theoretical_loss": 3.3788104844019773, + "tokens_seen": 2676738048 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015282647943831495, + "loss": 1.2014, + "theoretical_loss": 3.378804030530043, + "tokens_seen": 2676803584 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015281043129388165, + "loss": 1.1073, + "theoretical_loss": 3.3787975768603578, + "tokens_seen": 2676869120 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015279438314944835, + "loss": 1.2673, + "theoretical_loss": 3.3787911233929107, + "tokens_seen": 2676934656 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015277833500501505, + "loss": 1.1821, + "theoretical_loss": 3.37878467012769, + "tokens_seen": 2677000192 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015276228686058176, + "loss": 1.2277, + "theoretical_loss": 3.378778217064685, + "tokens_seen": 2677065728 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015274623871614846, + "loss": 1.1289, + "theoretical_loss": 3.3787717642038837, + "tokens_seen": 2677131264 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015273019057171516, + "loss": 1.1303, + "theoretical_loss": 3.3787653115452754, + "tokens_seen": 2677196800 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015271414242728186, + "loss": 1.2861, + "theoretical_loss": 3.3787588590888484, + "tokens_seen": 2677262336 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015269809428284856, + "loss": 1.2252, + "theoretical_loss": 3.378752406834592, + "tokens_seen": 2677327872 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015268204613841526, + "loss": 1.2211, + "theoretical_loss": 3.3787459547824947, + "tokens_seen": 2677393408 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015266599799398196, + "loss": 1.1605, + "theoretical_loss": 3.378739502932545, + "tokens_seen": 2677458944 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015264994984954866, + "loss": 1.2664, + "theoretical_loss": 3.3787330512847316, + "tokens_seen": 2677524480 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015263390170511536, + "loss": 1.1702, + "theoretical_loss": 3.3787265998390428, + "tokens_seen": 2677590016 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015261785356068206, + "loss": 1.2184, + "theoretical_loss": 3.3787201485954688, + "tokens_seen": 2677655552 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015260180541624876, + "loss": 1.1523, + "theoretical_loss": 3.3787136975539966, + "tokens_seen": 2677721088 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015258575727181546, + "loss": 1.2575, + "theoretical_loss": 3.378707246714616, + "tokens_seen": 2677786624 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015256970912738216, + "loss": 1.1438, + "theoretical_loss": 3.3787007960773154, + "tokens_seen": 2677852160 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015255366098294886, + "loss": 1.2365, + "theoretical_loss": 3.3786943456420837, + "tokens_seen": 2677917696 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015253761283851556, + "loss": 1.2821, + "theoretical_loss": 3.378687895408909, + "tokens_seen": 2677983232 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015252156469408226, + "loss": 1.2313, + "theoretical_loss": 3.378681445377781, + "tokens_seen": 2678048768 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015250551654964897, + "loss": 1.2852, + "theoretical_loss": 3.3786749955486877, + "tokens_seen": 2678114304 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015248946840521564, + "loss": 1.1628, + "theoretical_loss": 3.378668545921618, + "tokens_seen": 2678179840 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015247342026078234, + "loss": 1.2387, + "theoretical_loss": 3.378662096496561, + "tokens_seen": 2678245376 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015245737211634904, + "loss": 1.2439, + "theoretical_loss": 3.3786556472735048, + "tokens_seen": 2678310912 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015244132397191574, + "loss": 1.2869, + "theoretical_loss": 3.3786491982524383, + "tokens_seen": 2678376448 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015242527582748244, + "loss": 1.2287, + "theoretical_loss": 3.37864274943335, + "tokens_seen": 2678441984 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015240922768304914, + "loss": 1.1906, + "theoretical_loss": 3.37863630081623, + "tokens_seen": 2678507520 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015239317953861584, + "loss": 1.1735, + "theoretical_loss": 3.3786298524010654, + "tokens_seen": 2678573056 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015237713139418254, + "loss": 1.2033, + "theoretical_loss": 3.378623404187846, + "tokens_seen": 2678638592 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015236108324974924, + "loss": 1.2746, + "theoretical_loss": 3.3786169561765593, + "tokens_seen": 2678704128 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015234503510531594, + "loss": 1.1962, + "theoretical_loss": 3.3786105083671956, + "tokens_seen": 2678769664 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015232898696088265, + "loss": 1.0897, + "theoretical_loss": 3.3786040607597427, + "tokens_seen": 2678835200 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015231293881644935, + "loss": 1.2047, + "theoretical_loss": 3.3785976133541893, + "tokens_seen": 2678900736 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015229689067201605, + "loss": 1.1876, + "theoretical_loss": 3.3785911661505246, + "tokens_seen": 2678966272 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015228084252758275, + "loss": 1.233, + "theoretical_loss": 3.378584719148737, + "tokens_seen": 2679031808 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015226479438314945, + "loss": 1.2106, + "theoretical_loss": 3.378578272348815, + "tokens_seen": 2679097344 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015224874623871618, + "loss": 1.251, + "theoretical_loss": 3.3785718257507478, + "tokens_seen": 2679162880 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015223269809428288, + "loss": 1.2904, + "theoretical_loss": 3.3785653793545243, + "tokens_seen": 2679228416 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015221664994984958, + "loss": 1.2126, + "theoretical_loss": 3.378558933160133, + "tokens_seen": 2679293952 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015220060180541628, + "loss": 1.2804, + "theoretical_loss": 3.3785524871675623, + "tokens_seen": 2679359488 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015218455366098298, + "loss": 1.1025, + "theoretical_loss": 3.3785460413768016, + "tokens_seen": 2679425024 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015216850551654965, + "loss": 1.1881, + "theoretical_loss": 3.3785395957878386, + "tokens_seen": 2679490560 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015215245737211635, + "loss": 1.1693, + "theoretical_loss": 3.3785331504006635, + "tokens_seen": 2679556096 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015213640922768305, + "loss": 1.441, + "theoretical_loss": 3.378526705215264, + "tokens_seen": 2679621632 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015212036108324975, + "loss": 1.2179, + "theoretical_loss": 3.378520260231629, + "tokens_seen": 2679687168 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015210431293881645, + "loss": 1.2138, + "theoretical_loss": 3.3785138154497476, + "tokens_seen": 2679752704 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015208826479438315, + "loss": 1.1292, + "theoretical_loss": 3.378507370869608, + "tokens_seen": 2679818240 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015207221664994986, + "loss": 1.3027, + "theoretical_loss": 3.3785009264911996, + "tokens_seen": 2679883776 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015205616850551656, + "loss": 1.2118, + "theoretical_loss": 3.378494482314511, + "tokens_seen": 2679949312 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015204012036108326, + "loss": 1.2518, + "theoretical_loss": 3.37848803833953, + "tokens_seen": 2680014848 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015202407221664996, + "loss": 1.1442, + "theoretical_loss": 3.3784815945662467, + "tokens_seen": 2680080384 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015200802407221666, + "loss": 1.2375, + "theoretical_loss": 3.3784751509946496, + "tokens_seen": 2680145920 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015199197592778336, + "loss": 1.2094, + "theoretical_loss": 3.3784687076247266, + "tokens_seen": 2680211456 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015197592778335006, + "loss": 1.2822, + "theoretical_loss": 3.3784622644564672, + "tokens_seen": 2680276992 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015195987963891676, + "loss": 1.3232, + "theoretical_loss": 3.37845582148986, + "tokens_seen": 2680342528 + }, + { + "epoch": 1.19, + "objective/train/docs_used": 990114, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.071455955505371, + "objective/train/theoretical_loss": 3.378449378724894, + "objective/train/tokens_used": 2700868064, + "theoretical_loss": 3.378449378724894, + "tokens_seen": 2680408064 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015194383149448346, + "loss": 1.0715, + "theoretical_loss": 3.378449378724894, + "tokens_seen": 2680408064 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015192778335005016, + "loss": 1.0468, + "theoretical_loss": 3.3784429361615573, + "tokens_seen": 2680473600 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015191173520561686, + "loss": 1.2447, + "theoretical_loss": 3.378436493799839, + "tokens_seen": 2680539136 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015189568706118356, + "loss": 1.2762, + "theoretical_loss": 3.3784300516397283, + "tokens_seen": 2680604672 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015187963891675026, + "loss": 1.1348, + "theoretical_loss": 3.3784236096812132, + "tokens_seen": 2680670208 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015186359077231696, + "loss": 1.1346, + "theoretical_loss": 3.378417167924283, + "tokens_seen": 2680735744 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015184754262788366, + "loss": 1.1904, + "theoretical_loss": 3.3784107263689265, + "tokens_seen": 2680801280 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015183149448345037, + "loss": 1.1258, + "theoretical_loss": 3.3784042850151317, + "tokens_seen": 2680866816 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015181544633901707, + "loss": 1.2114, + "theoretical_loss": 3.3783978438628885, + "tokens_seen": 2680932352 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015179939819458377, + "loss": 1.2056, + "theoretical_loss": 3.3783914029121846, + "tokens_seen": 2680997888 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015178335005015044, + "loss": 1.2171, + "theoretical_loss": 3.3783849621630098, + "tokens_seen": 2681063424 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015176730190571714, + "loss": 1.277, + "theoretical_loss": 3.378378521615352, + "tokens_seen": 2681128960 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015175125376128384, + "loss": 1.155, + "theoretical_loss": 3.3783720812692004, + "tokens_seen": 2681194496 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015173520561685054, + "loss": 1.2063, + "theoretical_loss": 3.3783656411245433, + "tokens_seen": 2681260032 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015171915747241724, + "loss": 1.1493, + "theoretical_loss": 3.3783592011813703, + "tokens_seen": 2681325568 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015170310932798394, + "loss": 1.2553, + "theoretical_loss": 3.3783527614396696, + "tokens_seen": 2681391104 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015168706118355064, + "loss": 1.1039, + "theoretical_loss": 3.3783463218994303, + "tokens_seen": 2681456640 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015167101303911734, + "loss": 1.165, + "theoretical_loss": 3.3783398825606406, + "tokens_seen": 2681522176 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015165496489468404, + "loss": 1.244, + "theoretical_loss": 3.3783334434232897, + "tokens_seen": 2681587712 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015163891675025077, + "loss": 1.3172, + "theoretical_loss": 3.3783270044873666, + "tokens_seen": 2681653248 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015162286860581747, + "loss": 1.165, + "theoretical_loss": 3.378320565752859, + "tokens_seen": 2681718784 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015160682046138417, + "loss": 1.1552, + "theoretical_loss": 3.3783141272197574, + "tokens_seen": 2681784320 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015159077231695087, + "loss": 1.1831, + "theoretical_loss": 3.378307688888049, + "tokens_seen": 2681849856 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015157472417251758, + "loss": 1.1906, + "theoretical_loss": 3.378301250757724, + "tokens_seen": 2681915392 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015155867602808428, + "loss": 1.1625, + "theoretical_loss": 3.37829481282877, + "tokens_seen": 2681980928 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015154262788365098, + "loss": 1.3083, + "theoretical_loss": 3.378288375101176, + "tokens_seen": 2682046464 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015152657973921768, + "loss": 1.2417, + "theoretical_loss": 3.378281937574931, + "tokens_seen": 2682112000 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015151053159478438, + "loss": 1.2133, + "theoretical_loss": 3.378275500250024, + "tokens_seen": 2682177536 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015149448345035108, + "loss": 1.2482, + "theoretical_loss": 3.378269063126443, + "tokens_seen": 2682243072 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015147843530591775, + "loss": 1.1676, + "theoretical_loss": 3.378262626204178, + "tokens_seen": 2682308608 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015146238716148445, + "loss": 1.2014, + "theoretical_loss": 3.3782561894832166, + "tokens_seen": 2682374144 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015144633901705115, + "loss": 1.31, + "theoretical_loss": 3.3782497529635487, + "tokens_seen": 2682439680 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015143029087261785, + "loss": 1.1813, + "theoretical_loss": 3.378243316645162, + "tokens_seen": 2682505216 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015141424272818455, + "loss": 1.174, + "theoretical_loss": 3.3782368805280454, + "tokens_seen": 2682570752 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015139819458375126, + "loss": 1.1828, + "theoretical_loss": 3.3782304446121887, + "tokens_seen": 2682636288 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015138214643931796, + "loss": 1.1614, + "theoretical_loss": 3.37822400889758, + "tokens_seen": 2682701824 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015136609829488466, + "loss": 1.1984, + "theoretical_loss": 3.378217573384208, + "tokens_seen": 2682767360 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015135005015045136, + "loss": 1.2663, + "theoretical_loss": 3.378211138072061, + "tokens_seen": 2682832896 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015133400200601806, + "loss": 1.2083, + "theoretical_loss": 3.3782047029611295, + "tokens_seen": 2682898432 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015131795386158476, + "loss": 1.2218, + "theoretical_loss": 3.378198268051401, + "tokens_seen": 2682963968 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015130190571715146, + "loss": 1.211, + "theoretical_loss": 3.3781918333428638, + "tokens_seen": 2683029504 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015128585757271816, + "loss": 1.2768, + "theoretical_loss": 3.378185398835508, + "tokens_seen": 2683095040 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015126980942828486, + "loss": 1.0642, + "theoretical_loss": 3.3781789645293214, + "tokens_seen": 2683160576 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015125376128385156, + "loss": 1.0895, + "theoretical_loss": 3.3781725304242936, + "tokens_seen": 2683226112 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015123771313941826, + "loss": 1.2465, + "theoretical_loss": 3.3781660965204128, + "tokens_seen": 2683291648 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015122166499498496, + "loss": 1.1992, + "theoretical_loss": 3.378159662817668, + "tokens_seen": 2683357184 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015120561685055166, + "loss": 1.1745, + "theoretical_loss": 3.378153229316048, + "tokens_seen": 2683422720 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015118956870611836, + "loss": 1.1548, + "theoretical_loss": 3.3781467960155416, + "tokens_seen": 2683488256 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015117352056168506, + "loss": 1.2105, + "theoretical_loss": 3.3781403629161377, + "tokens_seen": 2683553792 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015115747241725176, + "loss": 1.1835, + "theoretical_loss": 3.378133930017825, + "tokens_seen": 2683619328 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015114142427281847, + "loss": 1.2753, + "theoretical_loss": 3.3781274973205924, + "tokens_seen": 2683684864 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015112537612838517, + "loss": 1.1783, + "theoretical_loss": 3.378121064824428, + "tokens_seen": 2683750400 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015110932798395187, + "loss": 1.2633, + "theoretical_loss": 3.378114632529322, + "tokens_seen": 2683815936 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015109327983951854, + "loss": 1.2827, + "theoretical_loss": 3.3781082004352623, + "tokens_seen": 2683881472 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015107723169508524, + "loss": 1.3858, + "theoretical_loss": 3.3781017685422374, + "tokens_seen": 2683947008 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015106118355065194, + "loss": 1.3045, + "theoretical_loss": 3.3780953368502367, + "tokens_seen": 2684012544 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015104513540621867, + "loss": 1.2511, + "theoretical_loss": 3.378088905359249, + "tokens_seen": 2684078080 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015102908726178537, + "loss": 1.1899, + "theoretical_loss": 3.378082474069263, + "tokens_seen": 2684143616 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015101303911735207, + "loss": 1.1571, + "theoretical_loss": 3.378076042980267, + "tokens_seen": 2684209152 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015099699097291877, + "loss": 1.2381, + "theoretical_loss": 3.3780696120922507, + "tokens_seen": 2684274688 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015098094282848547, + "loss": 1.1824, + "theoretical_loss": 3.3780631814052025, + "tokens_seen": 2684340224 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015096489468405217, + "loss": 1.2171, + "theoretical_loss": 3.378056750919111, + "tokens_seen": 2684405760 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015094884653961887, + "loss": 1.2551, + "theoretical_loss": 3.378050320633965, + "tokens_seen": 2684471296 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015093279839518557, + "loss": 1.3573, + "theoretical_loss": 3.378043890549754, + "tokens_seen": 2684536832 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015091675025075227, + "loss": 1.1565, + "theoretical_loss": 3.3780374606664663, + "tokens_seen": 2684602368 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015090070210631897, + "loss": 1.1733, + "theoretical_loss": 3.378031030984091, + "tokens_seen": 2684667904 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015088465396188568, + "loss": 1.2762, + "theoretical_loss": 3.378024601502616, + "tokens_seen": 2684733440 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015086860581745238, + "loss": 1.2066, + "theoretical_loss": 3.3780181722220313, + "tokens_seen": 2684798976 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015085255767301908, + "loss": 1.3247, + "theoretical_loss": 3.378011743142325, + "tokens_seen": 2684864512 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015083650952858578, + "loss": 1.3662, + "theoretical_loss": 3.3780053142634863, + "tokens_seen": 2684930048 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015082046138415248, + "loss": 1.2677, + "theoretical_loss": 3.3779988855855034, + "tokens_seen": 2684995584 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015080441323971918, + "loss": 1.2478, + "theoretical_loss": 3.377992457108366, + "tokens_seen": 2685061120 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015078836509528588, + "loss": 1.1421, + "theoretical_loss": 3.3779860288320624, + "tokens_seen": 2685126656 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015077231695085255, + "loss": 1.187, + "theoretical_loss": 3.3779796007565817, + "tokens_seen": 2685192192 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015075626880641925, + "loss": 1.358, + "theoretical_loss": 3.3779731728819127, + "tokens_seen": 2685257728 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015074022066198595, + "loss": 1.2504, + "theoretical_loss": 3.3779667452080435, + "tokens_seen": 2685323264 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015072417251755265, + "loss": 1.264, + "theoretical_loss": 3.377960317734964, + "tokens_seen": 2685388800 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015070812437311936, + "loss": 1.1775, + "theoretical_loss": 3.3779538904626625, + "tokens_seen": 2685454336 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015069207622868606, + "loss": 1.2208, + "theoretical_loss": 3.377947463391128, + "tokens_seen": 2685519872 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015067602808425276, + "loss": 1.2579, + "theoretical_loss": 3.3779410365203484, + "tokens_seen": 2685585408 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015065997993981946, + "loss": 1.1228, + "theoretical_loss": 3.377934609850314, + "tokens_seen": 2685650944 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015064393179538616, + "loss": 1.1025, + "theoretical_loss": 3.377928183381013, + "tokens_seen": 2685716480 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015062788365095286, + "loss": 1.1695, + "theoretical_loss": 3.377921757112434, + "tokens_seen": 2685782016 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015061183550651956, + "loss": 1.4228, + "theoretical_loss": 3.377915331044566, + "tokens_seen": 2685847552 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015059578736208626, + "loss": 1.2835, + "theoretical_loss": 3.377908905177398, + "tokens_seen": 2685913088 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015057973921765296, + "loss": 1.2251, + "theoretical_loss": 3.3779024795109187, + "tokens_seen": 2685978624 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015056369107321966, + "loss": 1.2359, + "theoretical_loss": 3.3778960540451166, + "tokens_seen": 2686044160 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015054764292878636, + "loss": 1.2285, + "theoretical_loss": 3.3778896287799816, + "tokens_seen": 2686109696 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015053159478435306, + "loss": 1.2104, + "theoretical_loss": 3.377883203715501, + "tokens_seen": 2686175232 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015051554663991976, + "loss": 1.1901, + "theoretical_loss": 3.3778767788516646, + "tokens_seen": 2686240768 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015049949849548646, + "loss": 1.198, + "theoretical_loss": 3.3778703541884614, + "tokens_seen": 2686306304 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015048345035105316, + "loss": 1.2776, + "theoretical_loss": 3.37786392972588, + "tokens_seen": 2686371840 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015046740220661986, + "loss": 1.1599, + "theoretical_loss": 3.3778575054639086, + "tokens_seen": 2686437376 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015045135406218657, + "loss": 1.2327, + "theoretical_loss": 3.377851081402537, + "tokens_seen": 2686502912 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015043530591775327, + "loss": 1.1879, + "theoretical_loss": 3.377844657541754, + "tokens_seen": 2686568448 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015041925777331997, + "loss": 1.1206, + "theoretical_loss": 3.3778382338815476, + "tokens_seen": 2686633984 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015040320962888667, + "loss": 1.1274, + "theoretical_loss": 3.377831810421907, + "tokens_seen": 2686699520 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015038716148445337, + "loss": 1.2073, + "theoretical_loss": 3.3778253871628214, + "tokens_seen": 2686765056 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015037111334002007, + "loss": 1.176, + "theoretical_loss": 3.3778189641042795, + "tokens_seen": 2686830592 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015035506519558677, + "loss": 1.3159, + "theoretical_loss": 3.37781254124627, + "tokens_seen": 2686896128 + }, + { + "epoch": 1.19, + "objective/train/docs_used": 992300, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1008992195129395, + "objective/train/theoretical_loss": 3.377806118588782, + "objective/train/tokens_used": 2707421664, + "theoretical_loss": 3.377806118588782, + "tokens_seen": 2686961664 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015033901705115347, + "loss": 1.1009, + "theoretical_loss": 3.377806118588782, + "tokens_seen": 2686961664 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015032296890672017, + "loss": 1.2868, + "theoretical_loss": 3.3777996961318038, + "tokens_seen": 2687027200 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015030692076228687, + "loss": 1.3274, + "theoretical_loss": 3.3777932738753247, + "tokens_seen": 2687092736 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015029087261785357, + "loss": 1.1521, + "theoretical_loss": 3.3777868518193337, + "tokens_seen": 2687158272 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015027482447342027, + "loss": 1.2661, + "theoretical_loss": 3.377780429963819, + "tokens_seen": 2687223808 + }, + { + "epoch": 1.19, + "learning_rate": 0.00015025877632898697, + "loss": 1.3037, + "theoretical_loss": 3.3777740083087706, + "tokens_seen": 2687289344 + }, + { + "epoch": 1.2, + "learning_rate": 0.00015024272818455367, + "loss": 1.1444, + "theoretical_loss": 3.3777675868541763, + "tokens_seen": 2687354880 + }, + { + "epoch": 1.2, + "learning_rate": 0.00015022668004012037, + "loss": 1.1672, + "theoretical_loss": 3.3777611656000253, + "tokens_seen": 2687420416 + }, + { + "epoch": 1.2, + "learning_rate": 0.00015021063189568707, + "loss": 1.2196, + "theoretical_loss": 3.3777547445463063, + "tokens_seen": 2687485952 + }, + { + "epoch": 1.2, + "learning_rate": 0.00015019458375125378, + "loss": 1.1564, + "theoretical_loss": 3.3777483236930084, + "tokens_seen": 2687551488 + }, + { + "epoch": 1.2, + "learning_rate": 0.00015017853560682048, + "loss": 1.1815, + "theoretical_loss": 3.3777419030401203, + "tokens_seen": 2687617024 + }, + { + "epoch": 1.2, + "learning_rate": 0.00015016248746238718, + "loss": 1.3307, + "theoretical_loss": 3.3777354825876307, + "tokens_seen": 2687682560 + }, + { + "epoch": 1.2, + "learning_rate": 0.00015014643931795388, + "loss": 1.2194, + "theoretical_loss": 3.377729062335529, + "tokens_seen": 2687748096 + }, + { + "epoch": 1.2, + "learning_rate": 0.00015013039117352058, + "loss": 1.1646, + "theoretical_loss": 3.377722642283804, + "tokens_seen": 2687813632 + }, + { + "epoch": 1.2, + "learning_rate": 0.00015011434302908728, + "loss": 1.2884, + "theoretical_loss": 3.3777162224324435, + "tokens_seen": 2687879168 + }, + { + "epoch": 1.2, + "learning_rate": 0.00015009829488465398, + "loss": 1.2286, + "theoretical_loss": 3.3777098027814376, + "tokens_seen": 2687944704 + }, + { + "epoch": 1.2, + "learning_rate": 0.00015008224674022068, + "loss": 1.2173, + "theoretical_loss": 3.3777033833307746, + "tokens_seen": 2688010240 + }, + { + "epoch": 1.2, + "learning_rate": 0.00015006619859578735, + "loss": 1.1968, + "theoretical_loss": 3.3776969640804437, + "tokens_seen": 2688075776 + }, + { + "epoch": 1.2, + "learning_rate": 0.00015005015045135405, + "loss": 1.3068, + "theoretical_loss": 3.3776905450304335, + "tokens_seen": 2688141312 + }, + { + "epoch": 1.2, + "learning_rate": 0.00015003410230692075, + "loss": 1.2081, + "theoretical_loss": 3.377684126180733, + "tokens_seen": 2688206848 + }, + { + "epoch": 1.2, + "learning_rate": 0.00015001805416248746, + "loss": 1.1577, + "theoretical_loss": 3.3776777075313307, + "tokens_seen": 2688272384 + }, + { + "epoch": 1.2, + "learning_rate": 0.00015000200601805416, + "loss": 1.1528, + "theoretical_loss": 3.3776712890822163, + "tokens_seen": 2688337920 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014998595787362086, + "loss": 1.2713, + "theoretical_loss": 3.377664870833377, + "tokens_seen": 2688403456 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014996990972918756, + "loss": 1.1312, + "theoretical_loss": 3.3776584527848037, + "tokens_seen": 2688468992 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014995386158475426, + "loss": 1.1551, + "theoretical_loss": 3.3776520349364842, + "tokens_seen": 2688534528 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014993781344032096, + "loss": 1.1605, + "theoretical_loss": 3.3776456172884073, + "tokens_seen": 2688600064 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014992176529588766, + "loss": 1.1471, + "theoretical_loss": 3.377639199840562, + "tokens_seen": 2688665600 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014990571715145436, + "loss": 1.1931, + "theoretical_loss": 3.377632782592938, + "tokens_seen": 2688731136 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014988966900702106, + "loss": 1.156, + "theoretical_loss": 3.377626365545523, + "tokens_seen": 2688796672 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014987362086258776, + "loss": 1.1579, + "theoretical_loss": 3.377619948698306, + "tokens_seen": 2688862208 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014985757271815446, + "loss": 1.1199, + "theoretical_loss": 3.3776135320512766, + "tokens_seen": 2688927744 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001498415245737212, + "loss": 1.1421, + "theoretical_loss": 3.377607115604423, + "tokens_seen": 2688993280 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001498254764292879, + "loss": 1.1771, + "theoretical_loss": 3.3776006993577345, + "tokens_seen": 2689058816 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001498094282848546, + "loss": 1.1154, + "theoretical_loss": 3.3775942833111996, + "tokens_seen": 2689124352 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001497933801404213, + "loss": 1.2524, + "theoretical_loss": 3.3775878674648077, + "tokens_seen": 2689189888 + }, + { + "epoch": 1.2, + "learning_rate": 0.000149777331995988, + "loss": 1.1576, + "theoretical_loss": 3.377581451818547, + "tokens_seen": 2689255424 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014976128385155467, + "loss": 1.218, + "theoretical_loss": 3.3775750363724075, + "tokens_seen": 2689320960 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014974523570712137, + "loss": 1.1863, + "theoretical_loss": 3.3775686211263767, + "tokens_seen": 2689386496 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014972918756268807, + "loss": 1.1733, + "theoretical_loss": 3.377562206080444, + "tokens_seen": 2689452032 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014971313941825477, + "loss": 1.1928, + "theoretical_loss": 3.377555791234599, + "tokens_seen": 2689517568 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014969709127382147, + "loss": 1.188, + "theoretical_loss": 3.377549376588829, + "tokens_seen": 2689583104 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014968104312938817, + "loss": 1.1667, + "theoretical_loss": 3.3775429621431248, + "tokens_seen": 2689648640 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014966499498495487, + "loss": 1.2821, + "theoretical_loss": 3.377536547897474, + "tokens_seen": 2689714176 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014964894684052157, + "loss": 1.1912, + "theoretical_loss": 3.377530133851866, + "tokens_seen": 2689779712 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014963289869608827, + "loss": 1.2661, + "theoretical_loss": 3.3775237200062893, + "tokens_seen": 2689845248 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014961685055165497, + "loss": 0.9881, + "theoretical_loss": 3.3775173063607333, + "tokens_seen": 2689910784 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014960080240722167, + "loss": 1.137, + "theoretical_loss": 3.3775108929151862, + "tokens_seen": 2689976320 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014958475426278837, + "loss": 1.2266, + "theoretical_loss": 3.377504479669638, + "tokens_seen": 2690041856 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014956870611835507, + "loss": 1.2261, + "theoretical_loss": 3.377498066624076, + "tokens_seen": 2690107392 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014955265797392177, + "loss": 1.1432, + "theoretical_loss": 3.377491653778491, + "tokens_seen": 2690172928 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014953660982948847, + "loss": 1.2128, + "theoretical_loss": 3.37748524113287, + "tokens_seen": 2690238464 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014952056168505518, + "loss": 1.2784, + "theoretical_loss": 3.3774788286872033, + "tokens_seen": 2690304000 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014950451354062188, + "loss": 1.1671, + "theoretical_loss": 3.3774724164414787, + "tokens_seen": 2690369536 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014948846539618858, + "loss": 1.1845, + "theoretical_loss": 3.377466004395686, + "tokens_seen": 2690435072 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014947241725175528, + "loss": 1.2145, + "theoretical_loss": 3.3774595925498136, + "tokens_seen": 2690500608 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014945636910732198, + "loss": 1.1815, + "theoretical_loss": 3.377453180903851, + "tokens_seen": 2690566144 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014944032096288868, + "loss": 1.3313, + "theoretical_loss": 3.3774467694577863, + "tokens_seen": 2690631680 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014942427281845538, + "loss": 1.2098, + "theoretical_loss": 3.377440358211609, + "tokens_seen": 2690697216 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014940822467402208, + "loss": 1.233, + "theoretical_loss": 3.3774339471653074, + "tokens_seen": 2690762752 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014939217652958878, + "loss": 1.1802, + "theoretical_loss": 3.377427536318871, + "tokens_seen": 2690828288 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014937612838515545, + "loss": 1.1972, + "theoretical_loss": 3.377421125672288, + "tokens_seen": 2690893824 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014936008024072215, + "loss": 1.2162, + "theoretical_loss": 3.377414715225548, + "tokens_seen": 2690959360 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014934403209628885, + "loss": 1.2152, + "theoretical_loss": 3.37740830497864, + "tokens_seen": 2691024896 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014932798395185556, + "loss": 1.2492, + "theoretical_loss": 3.3774018949315523, + "tokens_seen": 2691090432 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014931193580742226, + "loss": 1.2227, + "theoretical_loss": 3.377395485084274, + "tokens_seen": 2691155968 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014929588766298896, + "loss": 1.2101, + "theoretical_loss": 3.377389075436794, + "tokens_seen": 2691221504 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014927983951855566, + "loss": 1.0686, + "theoretical_loss": 3.3773826659891015, + "tokens_seen": 2691287040 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014926379137412236, + "loss": 1.1516, + "theoretical_loss": 3.3773762567411856, + "tokens_seen": 2691352576 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014924774322968909, + "loss": 1.1143, + "theoretical_loss": 3.377369847693034, + "tokens_seen": 2691418112 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001492316950852558, + "loss": 1.1505, + "theoretical_loss": 3.377363438844637, + "tokens_seen": 2691483648 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001492156469408225, + "loss": 1.1989, + "theoretical_loss": 3.3773570301959825, + "tokens_seen": 2691549184 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001491995987963892, + "loss": 1.114, + "theoretical_loss": 3.37735062174706, + "tokens_seen": 2691614720 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001491835506519559, + "loss": 1.1754, + "theoretical_loss": 3.377344213497858, + "tokens_seen": 2691680256 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001491675025075226, + "loss": 1.1916, + "theoretical_loss": 3.3773378054483656, + "tokens_seen": 2691745792 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001491514543630893, + "loss": 1.2999, + "theoretical_loss": 3.377331397598572, + "tokens_seen": 2691811328 + }, + { + "epoch": 1.2, + "learning_rate": 0.000149135406218656, + "loss": 1.2384, + "theoretical_loss": 3.377324989948466, + "tokens_seen": 2691876864 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001491193580742227, + "loss": 1.2029, + "theoretical_loss": 3.3773185824980363, + "tokens_seen": 2691942400 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001491033099297894, + "loss": 1.2255, + "theoretical_loss": 3.377312175247272, + "tokens_seen": 2692007936 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001490872617853561, + "loss": 1.2279, + "theoretical_loss": 3.377305768196162, + "tokens_seen": 2692073472 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001490712136409228, + "loss": 1.151, + "theoretical_loss": 3.377299361344695, + "tokens_seen": 2692139008 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014905516549648947, + "loss": 1.1607, + "theoretical_loss": 3.37729295469286, + "tokens_seen": 2692204544 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014903911735205617, + "loss": 1.1953, + "theoretical_loss": 3.377286548240646, + "tokens_seen": 2692270080 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014902306920762287, + "loss": 1.2182, + "theoretical_loss": 3.377280141988042, + "tokens_seen": 2692335616 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014900702106318957, + "loss": 1.1385, + "theoretical_loss": 3.377273735935037, + "tokens_seen": 2692401152 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014899097291875627, + "loss": 1.174, + "theoretical_loss": 3.3772673300816196, + "tokens_seen": 2692466688 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014897492477432297, + "loss": 1.2049, + "theoretical_loss": 3.3772609244277785, + "tokens_seen": 2692532224 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014895887662988967, + "loss": 1.1089, + "theoretical_loss": 3.3772545189735035, + "tokens_seen": 2692597760 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014894282848545637, + "loss": 1.1899, + "theoretical_loss": 3.3772481137187826, + "tokens_seen": 2692663296 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014892678034102307, + "loss": 1.1419, + "theoretical_loss": 3.3772417086636057, + "tokens_seen": 2692728832 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014891073219658977, + "loss": 1.1197, + "theoretical_loss": 3.3772353038079608, + "tokens_seen": 2692794368 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014889468405215647, + "loss": 1.1989, + "theoretical_loss": 3.377228899151837, + "tokens_seen": 2692859904 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014887863590772317, + "loss": 1.1547, + "theoretical_loss": 3.377222494695224, + "tokens_seen": 2692925440 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014886258776328987, + "loss": 1.1726, + "theoretical_loss": 3.37721609043811, + "tokens_seen": 2692990976 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014884653961885657, + "loss": 1.1804, + "theoretical_loss": 3.377209686380484, + "tokens_seen": 2693056512 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014883049147442328, + "loss": 1.0866, + "theoretical_loss": 3.377203282522335, + "tokens_seen": 2693122048 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014881444332998998, + "loss": 1.1705, + "theoretical_loss": 3.377196878863652, + "tokens_seen": 2693187584 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014879839518555668, + "loss": 1.1102, + "theoretical_loss": 3.377190475404424, + "tokens_seen": 2693253120 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014878234704112338, + "loss": 1.1649, + "theoretical_loss": 3.37718407214464, + "tokens_seen": 2693318656 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014876629889669008, + "loss": 1.1563, + "theoretical_loss": 3.377177669084288, + "tokens_seen": 2693384192 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014875025075225678, + "loss": 1.1939, + "theoretical_loss": 3.3771712662233586, + "tokens_seen": 2693449728 + }, + { + "epoch": 1.2, + "objective/train/docs_used": 994686, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.253936767578125, + "objective/train/theoretical_loss": 3.3771648635618394, + "objective/train/tokens_used": 2713975264, + "theoretical_loss": 3.3771648635618394, + "tokens_seen": 2693515264 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014873420260782348, + "loss": 1.2539, + "theoretical_loss": 3.3771648635618394, + "tokens_seen": 2693515264 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014871815446339018, + "loss": 1.2295, + "theoretical_loss": 3.37715846109972, + "tokens_seen": 2693580800 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014870210631895688, + "loss": 1.1645, + "theoretical_loss": 3.3771520588369888, + "tokens_seen": 2693646336 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014868605817452358, + "loss": 1.1349, + "theoretical_loss": 3.377145656773635, + "tokens_seen": 2693711872 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014867001003009025, + "loss": 1.1436, + "theoretical_loss": 3.377139254909648, + "tokens_seen": 2693777408 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014865396188565696, + "loss": 1.2648, + "theoretical_loss": 3.3771328532450164, + "tokens_seen": 2693842944 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014863791374122368, + "loss": 1.1452, + "theoretical_loss": 3.377126451779729, + "tokens_seen": 2693908480 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014862186559679038, + "loss": 1.2092, + "theoretical_loss": 3.3771200505137746, + "tokens_seen": 2693974016 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014860581745235708, + "loss": 1.1783, + "theoretical_loss": 3.3771136494471423, + "tokens_seen": 2694039552 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014858976930792378, + "loss": 1.2146, + "theoretical_loss": 3.3771072485798213, + "tokens_seen": 2694105088 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014857372116349049, + "loss": 1.2158, + "theoretical_loss": 3.3771008479118003, + "tokens_seen": 2694170624 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014855767301905719, + "loss": 1.126, + "theoretical_loss": 3.3770944474430684, + "tokens_seen": 2694236160 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001485416248746239, + "loss": 1.1771, + "theoretical_loss": 3.3770880471736144, + "tokens_seen": 2694301696 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001485255767301906, + "loss": 1.2332, + "theoretical_loss": 3.3770816471034273, + "tokens_seen": 2694367232 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001485095285857573, + "loss": 1.1154, + "theoretical_loss": 3.3770752472324963, + "tokens_seen": 2694432768 + }, + { + "epoch": 1.2, + "learning_rate": 0.000148493480441324, + "loss": 1.2817, + "theoretical_loss": 3.3770688475608095, + "tokens_seen": 2694498304 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001484774322968907, + "loss": 1.0952, + "theoretical_loss": 3.377062448088357, + "tokens_seen": 2694563840 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001484613841524574, + "loss": 1.1988, + "theoretical_loss": 3.377056048815127, + "tokens_seen": 2694629376 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001484453360080241, + "loss": 1.3198, + "theoretical_loss": 3.3770496497411084, + "tokens_seen": 2694694912 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001484292878635908, + "loss": 1.1981, + "theoretical_loss": 3.377043250866291, + "tokens_seen": 2694760448 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001484132397191575, + "loss": 1.257, + "theoretical_loss": 3.377036852190663, + "tokens_seen": 2694825984 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001483971915747242, + "loss": 1.2421, + "theoretical_loss": 3.3770304537142133, + "tokens_seen": 2694891520 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001483811434302909, + "loss": 1.1803, + "theoretical_loss": 3.377024055436931, + "tokens_seen": 2694957056 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001483650952858576, + "loss": 1.2013, + "theoretical_loss": 3.377017657358805, + "tokens_seen": 2695022592 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014834904714142427, + "loss": 1.1574, + "theoretical_loss": 3.377011259479825, + "tokens_seen": 2695088128 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014833299899699097, + "loss": 1.148, + "theoretical_loss": 3.3770048617999793, + "tokens_seen": 2695153664 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014831695085255767, + "loss": 1.1463, + "theoretical_loss": 3.3769984643192563, + "tokens_seen": 2695219200 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014830090270812437, + "loss": 1.1681, + "theoretical_loss": 3.376992067037646, + "tokens_seen": 2695284736 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014828485456369107, + "loss": 1.2019, + "theoretical_loss": 3.376985669955137, + "tokens_seen": 2695350272 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014826880641925777, + "loss": 1.2531, + "theoretical_loss": 3.376979273071718, + "tokens_seen": 2695415808 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014825275827482447, + "loss": 1.1628, + "theoretical_loss": 3.3769728763873785, + "tokens_seen": 2695481344 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014823671013039117, + "loss": 1.1105, + "theoretical_loss": 3.376966479902107, + "tokens_seen": 2695546880 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014822066198595787, + "loss": 1.0932, + "theoretical_loss": 3.3769600836158924, + "tokens_seen": 2695612416 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014820461384152457, + "loss": 1.2137, + "theoretical_loss": 3.3769536875287236, + "tokens_seen": 2695677952 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014818856569709127, + "loss": 1.2392, + "theoretical_loss": 3.3769472916405903, + "tokens_seen": 2695743488 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014817251755265797, + "loss": 1.2545, + "theoretical_loss": 3.376940895951481, + "tokens_seen": 2695809024 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014815646940822467, + "loss": 1.1523, + "theoretical_loss": 3.3769345004613847, + "tokens_seen": 2695874560 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014814042126379138, + "loss": 1.2315, + "theoretical_loss": 3.3769281051702897, + "tokens_seen": 2695940096 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014812437311935808, + "loss": 1.0799, + "theoretical_loss": 3.3769217100781863, + "tokens_seen": 2696005632 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014810832497492478, + "loss": 1.1637, + "theoretical_loss": 3.3769153151850624, + "tokens_seen": 2696071168 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014809227683049148, + "loss": 1.2509, + "theoretical_loss": 3.3769089204909077, + "tokens_seen": 2696136704 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014807622868605818, + "loss": 1.2851, + "theoretical_loss": 3.3769025259957104, + "tokens_seen": 2696202240 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014806018054162488, + "loss": 1.1169, + "theoretical_loss": 3.37689613169946, + "tokens_seen": 2696267776 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014804413239719158, + "loss": 1.2604, + "theoretical_loss": 3.3768897376021454, + "tokens_seen": 2696333312 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014802808425275828, + "loss": 1.2479, + "theoretical_loss": 3.3768833437037555, + "tokens_seen": 2696398848 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014801203610832498, + "loss": 1.2282, + "theoretical_loss": 3.3768769500042795, + "tokens_seen": 2696464384 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014799598796389168, + "loss": 1.1741, + "theoretical_loss": 3.376870556503706, + "tokens_seen": 2696529920 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014797993981945838, + "loss": 1.2489, + "theoretical_loss": 3.3768641632020246, + "tokens_seen": 2696595456 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014796389167502508, + "loss": 1.15, + "theoretical_loss": 3.3768577700992233, + "tokens_seen": 2696660992 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014794784353059178, + "loss": 1.2201, + "theoretical_loss": 3.376851377195292, + "tokens_seen": 2696726528 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014793179538615848, + "loss": 1.2376, + "theoretical_loss": 3.3768449844902193, + "tokens_seen": 2696792064 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014791574724172518, + "loss": 1.0677, + "theoretical_loss": 3.376838591983994, + "tokens_seen": 2696857600 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014789969909729188, + "loss": 1.2035, + "theoretical_loss": 3.3768321996766053, + "tokens_seen": 2696923136 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014788365095285859, + "loss": 1.0671, + "theoretical_loss": 3.3768258075680424, + "tokens_seen": 2696988672 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014786760280842529, + "loss": 1.1827, + "theoretical_loss": 3.376819415658294, + "tokens_seen": 2697054208 + }, + { + "epoch": 1.2, + "learning_rate": 0.000147851554663992, + "loss": 1.1539, + "theoretical_loss": 3.376813023947349, + "tokens_seen": 2697119744 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001478355065195587, + "loss": 1.2762, + "theoretical_loss": 3.3768066324351964, + "tokens_seen": 2697185280 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001478194583751254, + "loss": 1.1638, + "theoretical_loss": 3.3768002411218254, + "tokens_seen": 2697250816 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001478034102306921, + "loss": 1.1128, + "theoretical_loss": 3.376793850007225, + "tokens_seen": 2697316352 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001477873620862588, + "loss": 1.109, + "theoretical_loss": 3.3767874590913842, + "tokens_seen": 2697381888 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001477713139418255, + "loss": 1.2123, + "theoretical_loss": 3.376781068374292, + "tokens_seen": 2697447424 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001477552657973922, + "loss": 1.206, + "theoretical_loss": 3.376774677855937, + "tokens_seen": 2697512960 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001477392176529589, + "loss": 1.2133, + "theoretical_loss": 3.376768287536308, + "tokens_seen": 2697578496 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001477231695085256, + "loss": 1.176, + "theoretical_loss": 3.376761897415395, + "tokens_seen": 2697644032 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001477071213640923, + "loss": 1.249, + "theoretical_loss": 3.3767555074931863, + "tokens_seen": 2697709568 + }, + { + "epoch": 1.2, + "learning_rate": 0.000147691073219659, + "loss": 1.2315, + "theoretical_loss": 3.376749117769671, + "tokens_seen": 2697775104 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001476750250752257, + "loss": 1.2354, + "theoretical_loss": 3.3767427282448383, + "tokens_seen": 2697840640 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014765897693079237, + "loss": 1.161, + "theoretical_loss": 3.376736338918677, + "tokens_seen": 2697906176 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014764292878635907, + "loss": 1.1682, + "theoretical_loss": 3.376729949791176, + "tokens_seen": 2697971712 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014762688064192577, + "loss": 1.185, + "theoretical_loss": 3.3767235608623247, + "tokens_seen": 2698037248 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014761083249749247, + "loss": 1.1558, + "theoretical_loss": 3.3767171721321114, + "tokens_seen": 2698102784 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014759478435305917, + "loss": 1.1351, + "theoretical_loss": 3.376710783600526, + "tokens_seen": 2698168320 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014757873620862587, + "loss": 1.1682, + "theoretical_loss": 3.3767043952675566, + "tokens_seen": 2698233856 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014756268806419257, + "loss": 1.2846, + "theoretical_loss": 3.376698007133193, + "tokens_seen": 2698299392 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014754663991975927, + "loss": 1.2133, + "theoretical_loss": 3.376691619197423, + "tokens_seen": 2698364928 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014753059177532597, + "loss": 1.0906, + "theoretical_loss": 3.376685231460238, + "tokens_seen": 2698430464 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014751454363089267, + "loss": 1.1014, + "theoretical_loss": 3.376678843921624, + "tokens_seen": 2698496000 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014749849548645937, + "loss": 1.2276, + "theoretical_loss": 3.376672456581572, + "tokens_seen": 2698561536 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014748244734202607, + "loss": 1.2473, + "theoretical_loss": 3.3766660694400703, + "tokens_seen": 2698627072 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014746639919759278, + "loss": 1.1953, + "theoretical_loss": 3.3766596824971082, + "tokens_seen": 2698692608 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014745035105315948, + "loss": 1.0879, + "theoretical_loss": 3.3766532957526745, + "tokens_seen": 2698758144 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001474343029087262, + "loss": 1.2144, + "theoretical_loss": 3.3766469092067584, + "tokens_seen": 2698823680 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001474182547642929, + "loss": 1.1602, + "theoretical_loss": 3.3766405228593483, + "tokens_seen": 2698889216 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001474022066198596, + "loss": 1.2397, + "theoretical_loss": 3.376634136710434, + "tokens_seen": 2698954752 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001473861584754263, + "loss": 1.1277, + "theoretical_loss": 3.3766277507600044, + "tokens_seen": 2699020288 + }, + { + "epoch": 1.2, + "learning_rate": 0.000147370110330993, + "loss": 1.3028, + "theoretical_loss": 3.376621365008048, + "tokens_seen": 2699085824 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001473540621865597, + "loss": 1.159, + "theoretical_loss": 3.3766149794545544, + "tokens_seen": 2699151360 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014733801404212638, + "loss": 1.291, + "theoretical_loss": 3.376608594099512, + "tokens_seen": 2699216896 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014732196589769308, + "loss": 1.2058, + "theoretical_loss": 3.37660220894291, + "tokens_seen": 2699282432 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014730591775325978, + "loss": 1.1976, + "theoretical_loss": 3.3765958239847382, + "tokens_seen": 2699347968 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014728986960882648, + "loss": 1.1973, + "theoretical_loss": 3.376589439224985, + "tokens_seen": 2699413504 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014727382146439318, + "loss": 1.2635, + "theoretical_loss": 3.376583054663639, + "tokens_seen": 2699479040 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014725777331995988, + "loss": 1.1811, + "theoretical_loss": 3.3765766703006896, + "tokens_seen": 2699544576 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014724172517552658, + "loss": 1.1253, + "theoretical_loss": 3.376570286136126, + "tokens_seen": 2699610112 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014722567703109328, + "loss": 1.2744, + "theoretical_loss": 3.376563902169937, + "tokens_seen": 2699675648 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014720962888665999, + "loss": 1.2629, + "theoretical_loss": 3.3765575184021115, + "tokens_seen": 2699741184 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014719358074222669, + "loss": 1.2403, + "theoretical_loss": 3.3765511348326385, + "tokens_seen": 2699806720 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014717753259779339, + "loss": 1.1898, + "theoretical_loss": 3.3765447514615077, + "tokens_seen": 2699872256 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001471614844533601, + "loss": 1.2206, + "theoretical_loss": 3.3765383682887076, + "tokens_seen": 2699937792 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001471454363089268, + "loss": 1.1387, + "theoretical_loss": 3.376531985314227, + "tokens_seen": 2700003328 + }, + { + "epoch": 1.2, + "objective/train/docs_used": 997030, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1506798267364502, + "objective/train/theoretical_loss": 3.3765256025380554, + "objective/train/tokens_used": 2720528864, + "theoretical_loss": 3.3765256025380554, + "tokens_seen": 2700068864 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001471293881644935, + "loss": 1.1507, + "theoretical_loss": 3.3765256025380554, + "tokens_seen": 2700068864 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001471133400200602, + "loss": 1.2809, + "theoretical_loss": 3.3765192199601817, + "tokens_seen": 2700134400 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001470972918756269, + "loss": 1.1721, + "theoretical_loss": 3.3765128375805946, + "tokens_seen": 2700199936 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001470812437311936, + "loss": 1.1927, + "theoretical_loss": 3.3765064553992836, + "tokens_seen": 2700265472 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001470651955867603, + "loss": 1.1428, + "theoretical_loss": 3.376500073416237, + "tokens_seen": 2700331008 + }, + { + "epoch": 1.2, + "learning_rate": 0.000147049147442327, + "loss": 1.1708, + "theoretical_loss": 3.376493691631445, + "tokens_seen": 2700396544 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001470330992978937, + "loss": 1.1204, + "theoretical_loss": 3.3764873100448956, + "tokens_seen": 2700462080 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001470170511534604, + "loss": 1.308, + "theoretical_loss": 3.3764809286565782, + "tokens_seen": 2700527616 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001470010030090271, + "loss": 1.2781, + "theoretical_loss": 3.376474547466482, + "tokens_seen": 2700593152 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001469849548645938, + "loss": 1.1382, + "theoretical_loss": 3.3764681664745955, + "tokens_seen": 2700658688 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001469689067201605, + "loss": 1.1562, + "theoretical_loss": 3.3764617856809083, + "tokens_seen": 2700724224 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014695285857572717, + "loss": 1.1947, + "theoretical_loss": 3.376455405085409, + "tokens_seen": 2700789760 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014693681043129387, + "loss": 1.1597, + "theoretical_loss": 3.376449024688087, + "tokens_seen": 2700855296 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014692076228686057, + "loss": 1.3265, + "theoretical_loss": 3.3764426444889315, + "tokens_seen": 2700920832 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014690471414242727, + "loss": 1.1843, + "theoretical_loss": 3.376436264487931, + "tokens_seen": 2700986368 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014688866599799397, + "loss": 1.1739, + "theoretical_loss": 3.3764298846850744, + "tokens_seen": 2701051904 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014687261785356067, + "loss": 1.2727, + "theoretical_loss": 3.3764235050803517, + "tokens_seen": 2701117440 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014685656970912737, + "loss": 1.2449, + "theoretical_loss": 3.3764171256737514, + "tokens_seen": 2701182976 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001468405215646941, + "loss": 1.1577, + "theoretical_loss": 3.376410746465262, + "tokens_seen": 2701248512 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001468244734202608, + "loss": 1.1253, + "theoretical_loss": 3.376404367454873, + "tokens_seen": 2701314048 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001468084252758275, + "loss": 1.2075, + "theoretical_loss": 3.376397988642574, + "tokens_seen": 2701379584 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001467923771313942, + "loss": 1.1627, + "theoretical_loss": 3.376391610028353, + "tokens_seen": 2701445120 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001467763289869609, + "loss": 1.2294, + "theoretical_loss": 3.3763852316121996, + "tokens_seen": 2701510656 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001467602808425276, + "loss": 1.1753, + "theoretical_loss": 3.3763788533941033, + "tokens_seen": 2701576192 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001467442326980943, + "loss": 1.1624, + "theoretical_loss": 3.376372475374052, + "tokens_seen": 2701641728 + }, + { + "epoch": 1.2, + "learning_rate": 0.000146728184553661, + "loss": 1.1959, + "theoretical_loss": 3.3763660975520358, + "tokens_seen": 2701707264 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001467121364092277, + "loss": 1.2124, + "theoretical_loss": 3.3763597199280433, + "tokens_seen": 2701772800 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001466960882647944, + "loss": 1.2277, + "theoretical_loss": 3.3763533425020635, + "tokens_seen": 2701838336 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001466800401203611, + "loss": 1.132, + "theoretical_loss": 3.3763469652740854, + "tokens_seen": 2701903872 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001466639919759278, + "loss": 1.1648, + "theoretical_loss": 3.3763405882440987, + "tokens_seen": 2701969408 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001466479438314945, + "loss": 1.1743, + "theoretical_loss": 3.3763342114120913, + "tokens_seen": 2702034944 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014663189568706118, + "loss": 1.1684, + "theoretical_loss": 3.3763278347780536, + "tokens_seen": 2702100480 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014661584754262788, + "loss": 1.2426, + "theoretical_loss": 3.376321458341973, + "tokens_seen": 2702166016 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014659979939819458, + "loss": 1.1933, + "theoretical_loss": 3.37631508210384, + "tokens_seen": 2702231552 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014658375125376128, + "loss": 1.215, + "theoretical_loss": 3.3763087060636434, + "tokens_seen": 2702297088 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014656770310932798, + "loss": 1.2343, + "theoretical_loss": 3.376302330221372, + "tokens_seen": 2702362624 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014655165496489468, + "loss": 1.3822, + "theoretical_loss": 3.3762959545770146, + "tokens_seen": 2702428160 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014653560682046138, + "loss": 1.1825, + "theoretical_loss": 3.3762895791305607, + "tokens_seen": 2702493696 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014651955867602809, + "loss": 1.1725, + "theoretical_loss": 3.376283203881999, + "tokens_seen": 2702559232 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014650351053159479, + "loss": 1.136, + "theoretical_loss": 3.376276828831319, + "tokens_seen": 2702624768 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001464874623871615, + "loss": 1.1105, + "theoretical_loss": 3.376270453978509, + "tokens_seen": 2702690304 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001464714142427282, + "loss": 1.1163, + "theoretical_loss": 3.3762640793235588, + "tokens_seen": 2702755840 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001464553660982949, + "loss": 1.1418, + "theoretical_loss": 3.3762577048664575, + "tokens_seen": 2702821376 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001464393179538616, + "loss": 1.2649, + "theoretical_loss": 3.3762513306071935, + "tokens_seen": 2702886912 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001464232698094283, + "loss": 1.2144, + "theoretical_loss": 3.3762449565457566, + "tokens_seen": 2702952448 + }, + { + "epoch": 1.2, + "learning_rate": 0.000146407221664995, + "loss": 1.124, + "theoretical_loss": 3.3762385826821353, + "tokens_seen": 2703017984 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001463911735205617, + "loss": 1.2738, + "theoretical_loss": 3.376232209016319, + "tokens_seen": 2703083520 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001463751253761284, + "loss": 1.089, + "theoretical_loss": 3.376225835548296, + "tokens_seen": 2703149056 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001463590772316951, + "loss": 1.1466, + "theoretical_loss": 3.376219462278057, + "tokens_seen": 2703214592 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001463430290872618, + "loss": 1.247, + "theoretical_loss": 3.3762130892055895, + "tokens_seen": 2703280128 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001463269809428285, + "loss": 1.1412, + "theoretical_loss": 3.3762067163308833, + "tokens_seen": 2703345664 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001463109327983952, + "loss": 1.2691, + "theoretical_loss": 3.3762003436539274, + "tokens_seen": 2703411200 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001462948846539619, + "loss": 1.1775, + "theoretical_loss": 3.3761939711747107, + "tokens_seen": 2703476736 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001462788365095286, + "loss": 1.2068, + "theoretical_loss": 3.3761875988932224, + "tokens_seen": 2703542272 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001462627883650953, + "loss": 1.2526, + "theoretical_loss": 3.376181226809451, + "tokens_seen": 2703607808 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014624674022066197, + "loss": 1.2501, + "theoretical_loss": 3.376174854923387, + "tokens_seen": 2703673344 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001462306920762287, + "loss": 1.1541, + "theoretical_loss": 3.376168483235018, + "tokens_seen": 2703738880 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001462146439317954, + "loss": 1.1948, + "theoretical_loss": 3.3761621117443337, + "tokens_seen": 2703804416 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001461985957873621, + "loss": 1.1728, + "theoretical_loss": 3.376155740451323, + "tokens_seen": 2703869952 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001461825476429288, + "loss": 1.2133, + "theoretical_loss": 3.376149369355976, + "tokens_seen": 2703935488 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001461664994984955, + "loss": 1.2189, + "theoretical_loss": 3.37614299845828, + "tokens_seen": 2704001024 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001461504513540622, + "loss": 1.1919, + "theoretical_loss": 3.3761366277582248, + "tokens_seen": 2704066560 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001461344032096289, + "loss": 1.1377, + "theoretical_loss": 3.3761302572558, + "tokens_seen": 2704132096 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001461183550651956, + "loss": 1.2394, + "theoretical_loss": 3.376123886950994, + "tokens_seen": 2704197632 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001461023069207623, + "loss": 1.2663, + "theoretical_loss": 3.376117516843797, + "tokens_seen": 2704263168 + }, + { + "epoch": 1.2, + "learning_rate": 0.000146086258776329, + "loss": 1.1755, + "theoretical_loss": 3.3761111469341967, + "tokens_seen": 2704328704 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001460702106318957, + "loss": 1.3073, + "theoretical_loss": 3.376104777222183, + "tokens_seen": 2704394240 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001460541624874624, + "loss": 1.1903, + "theoretical_loss": 3.3760984077077443, + "tokens_seen": 2704459776 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001460381143430291, + "loss": 1.2656, + "theoretical_loss": 3.3760920383908704, + "tokens_seen": 2704525312 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001460220661985958, + "loss": 1.1608, + "theoretical_loss": 3.37608566927155, + "tokens_seen": 2704590848 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001460060180541625, + "loss": 1.2072, + "theoretical_loss": 3.3760793003497724, + "tokens_seen": 2704656384 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001459899699097292, + "loss": 1.2839, + "theoretical_loss": 3.3760729316255267, + "tokens_seen": 2704721920 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001459739217652959, + "loss": 1.3388, + "theoretical_loss": 3.3760665630988016, + "tokens_seen": 2704787456 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001459578736208626, + "loss": 1.2021, + "theoretical_loss": 3.3760601947695865, + "tokens_seen": 2704852992 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001459418254764293, + "loss": 1.1836, + "theoretical_loss": 3.376053826637871, + "tokens_seen": 2704918528 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014592577733199598, + "loss": 1.2103, + "theoretical_loss": 3.376047458703643, + "tokens_seen": 2704984064 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014590972918756268, + "loss": 1.1576, + "theoretical_loss": 3.376041090966892, + "tokens_seen": 2705049600 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014589368104312938, + "loss": 1.1502, + "theoretical_loss": 3.376034723427608, + "tokens_seen": 2705115136 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014587763289869608, + "loss": 1.165, + "theoretical_loss": 3.3760283560857793, + "tokens_seen": 2705180672 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014586158475426278, + "loss": 1.183, + "theoretical_loss": 3.3760219889413947, + "tokens_seen": 2705246208 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014584553660982948, + "loss": 1.1413, + "theoretical_loss": 3.376015621994444, + "tokens_seen": 2705311744 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014582948846539619, + "loss": 1.2106, + "theoretical_loss": 3.3760092552449157, + "tokens_seen": 2705377280 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014581344032096289, + "loss": 1.1386, + "theoretical_loss": 3.3760028886927995, + "tokens_seen": 2705442816 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001457973921765296, + "loss": 1.212, + "theoretical_loss": 3.375996522338084, + "tokens_seen": 2705508352 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001457813440320963, + "loss": 1.1845, + "theoretical_loss": 3.375990156180759, + "tokens_seen": 2705573888 + }, + { + "epoch": 1.2, + "learning_rate": 0.000145765295887663, + "loss": 1.2785, + "theoretical_loss": 3.3759837902208125, + "tokens_seen": 2705639424 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001457492477432297, + "loss": 1.2379, + "theoretical_loss": 3.3759774244582346, + "tokens_seen": 2705704960 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001457331995987964, + "loss": 1.1546, + "theoretical_loss": 3.375971058893014, + "tokens_seen": 2705770496 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001457171514543631, + "loss": 1.2413, + "theoretical_loss": 3.3759646935251393, + "tokens_seen": 2705836032 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001457011033099298, + "loss": 1.2681, + "theoretical_loss": 3.3759583283546, + "tokens_seen": 2705901568 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001456850551654965, + "loss": 1.289, + "theoretical_loss": 3.375951963381386, + "tokens_seen": 2705967104 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001456690070210632, + "loss": 1.2206, + "theoretical_loss": 3.3759455986054854, + "tokens_seen": 2706032640 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001456529588766299, + "loss": 1.1597, + "theoretical_loss": 3.3759392340268874, + "tokens_seen": 2706098176 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014563691073219662, + "loss": 1.1898, + "theoretical_loss": 3.3759328696455815, + "tokens_seen": 2706163712 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001456208625877633, + "loss": 1.1663, + "theoretical_loss": 3.375926505461557, + "tokens_seen": 2706229248 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014560481444333, + "loss": 1.226, + "theoretical_loss": 3.375920141474802, + "tokens_seen": 2706294784 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001455887662988967, + "loss": 1.1879, + "theoretical_loss": 3.375913777685306, + "tokens_seen": 2706360320 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001455727181544634, + "loss": 1.2566, + "theoretical_loss": 3.375907414093059, + "tokens_seen": 2706425856 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001455566700100301, + "loss": 1.2043, + "theoretical_loss": 3.375901050698049, + "tokens_seen": 2706491392 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001455406218655968, + "loss": 1.1401, + "theoretical_loss": 3.3758946875002662, + "tokens_seen": 2706556928 + }, + { + "epoch": 1.2, + "objective/train/docs_used": 999762, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.170237421989441, + "objective/train/theoretical_loss": 3.3758883244996984, + "objective/train/tokens_used": 2727082464, + "theoretical_loss": 3.3758883244996984, + "tokens_seen": 2706622464 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001455245737211635, + "loss": 1.1702, + "theoretical_loss": 3.3758883244996984, + "tokens_seen": 2706622464 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001455085255767302, + "loss": 1.1118, + "theoretical_loss": 3.3758819616963356, + "tokens_seen": 2706688000 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001454924774322969, + "loss": 1.353, + "theoretical_loss": 3.375875599090167, + "tokens_seen": 2706753536 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001454764292878636, + "loss": 1.1689, + "theoretical_loss": 3.375869236681181, + "tokens_seen": 2706819072 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001454603811434303, + "loss": 1.156, + "theoretical_loss": 3.3758628744693673, + "tokens_seen": 2706884608 + }, + { + "epoch": 1.2, + "learning_rate": 0.000145444332998997, + "loss": 1.2772, + "theoretical_loss": 3.375856512454715, + "tokens_seen": 2706950144 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001454282848545637, + "loss": 1.2655, + "theoretical_loss": 3.3758501506372127, + "tokens_seen": 2707015680 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001454122367101304, + "loss": 1.1081, + "theoretical_loss": 3.3758437890168502, + "tokens_seen": 2707081216 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001453961885656971, + "loss": 1.1861, + "theoretical_loss": 3.3758374275936163, + "tokens_seen": 2707146752 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001453801404212638, + "loss": 1.1965, + "theoretical_loss": 3.3758310663674997, + "tokens_seen": 2707212288 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001453640922768305, + "loss": 1.2698, + "theoretical_loss": 3.3758247053384904, + "tokens_seen": 2707277824 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001453480441323972, + "loss": 1.3382, + "theoretical_loss": 3.375818344506577, + "tokens_seen": 2707343360 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001453319959879639, + "loss": 1.1206, + "theoretical_loss": 3.375811983871748, + "tokens_seen": 2707408896 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001453159478435306, + "loss": 1.2265, + "theoretical_loss": 3.375805623433994, + "tokens_seen": 2707474432 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001452998996990973, + "loss": 1.15, + "theoretical_loss": 3.3757992631933034, + "tokens_seen": 2707539968 + }, + { + "epoch": 1.2, + "learning_rate": 0.000145283851554664, + "loss": 1.1749, + "theoretical_loss": 3.375792903149665, + "tokens_seen": 2707605504 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001452678034102307, + "loss": 1.1035, + "theoretical_loss": 3.375786543303068, + "tokens_seen": 2707671040 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001452517552657974, + "loss": 1.1158, + "theoretical_loss": 3.375780183653502, + "tokens_seen": 2707736576 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014523570712136408, + "loss": 1.1456, + "theoretical_loss": 3.3757738242009556, + "tokens_seen": 2707802112 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014521965897693078, + "loss": 1.2975, + "theoretical_loss": 3.375767464945418, + "tokens_seen": 2707867648 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014520361083249748, + "loss": 1.171, + "theoretical_loss": 3.3757611058868786, + "tokens_seen": 2707933184 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014518756268806418, + "loss": 1.281, + "theoretical_loss": 3.3757547470253266, + "tokens_seen": 2707998720 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014517151454363088, + "loss": 1.2133, + "theoretical_loss": 3.3757483883607513, + "tokens_seen": 2708064256 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014515546639919759, + "loss": 1.175, + "theoretical_loss": 3.375742029893141, + "tokens_seen": 2708129792 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014513941825476429, + "loss": 1.1347, + "theoretical_loss": 3.3757356716224853, + "tokens_seen": 2708195328 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014512337011033099, + "loss": 1.2223, + "theoretical_loss": 3.3757293135487734, + "tokens_seen": 2708260864 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001451073219658977, + "loss": 1.1641, + "theoretical_loss": 3.3757229556719945, + "tokens_seen": 2708326400 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001450912738214644, + "loss": 1.2733, + "theoretical_loss": 3.3757165979921373, + "tokens_seen": 2708391936 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001450752256770311, + "loss": 1.2114, + "theoretical_loss": 3.375710240509192, + "tokens_seen": 2708457472 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001450591775325978, + "loss": 1.19, + "theoretical_loss": 3.3757038832231463, + "tokens_seen": 2708523008 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014504312938816452, + "loss": 1.3882, + "theoretical_loss": 3.3756975261339903, + "tokens_seen": 2708588544 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014502708124373122, + "loss": 1.1434, + "theoretical_loss": 3.375691169241713, + "tokens_seen": 2708654080 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014501103309929792, + "loss": 1.2097, + "theoretical_loss": 3.375684812546303, + "tokens_seen": 2708719616 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014499498495486462, + "loss": 1.1339, + "theoretical_loss": 3.37567845604775, + "tokens_seen": 2708785152 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014497893681043132, + "loss": 1.2792, + "theoretical_loss": 3.3756720997460428, + "tokens_seen": 2708850688 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014496288866599802, + "loss": 1.2121, + "theoretical_loss": 3.375665743641171, + "tokens_seen": 2708916224 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014494684052156472, + "loss": 1.2146, + "theoretical_loss": 3.3756593877331236, + "tokens_seen": 2708981760 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014493079237713142, + "loss": 1.1483, + "theoretical_loss": 3.3756530320218894, + "tokens_seen": 2709047296 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001449147442326981, + "loss": 1.0648, + "theoretical_loss": 3.3756466765074578, + "tokens_seen": 2709112832 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001448986960882648, + "loss": 1.2121, + "theoretical_loss": 3.375640321189818, + "tokens_seen": 2709178368 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001448826479438315, + "loss": 1.1726, + "theoretical_loss": 3.3756339660689587, + "tokens_seen": 2709243904 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001448665997993982, + "loss": 1.1055, + "theoretical_loss": 3.37562761114487, + "tokens_seen": 2709309440 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001448505516549649, + "loss": 1.1544, + "theoretical_loss": 3.37562125641754, + "tokens_seen": 2709374976 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001448345035105316, + "loss": 1.1448, + "theoretical_loss": 3.3756149018869586, + "tokens_seen": 2709440512 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001448184553660983, + "loss": 1.2626, + "theoretical_loss": 3.375608547553114, + "tokens_seen": 2709506048 + }, + { + "epoch": 1.2, + "learning_rate": 0.000144802407221665, + "loss": 1.1417, + "theoretical_loss": 3.3756021934159968, + "tokens_seen": 2709571584 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001447863590772317, + "loss": 1.2028, + "theoretical_loss": 3.375595839475595, + "tokens_seen": 2709637120 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001447703109327984, + "loss": 1.2917, + "theoretical_loss": 3.375589485731898, + "tokens_seen": 2709702656 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001447542627883651, + "loss": 1.0891, + "theoretical_loss": 3.3755831321848953, + "tokens_seen": 2709768192 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001447382146439318, + "loss": 1.1815, + "theoretical_loss": 3.3755767788345756, + "tokens_seen": 2709833728 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001447221664994985, + "loss": 1.1979, + "theoretical_loss": 3.3755704256809285, + "tokens_seen": 2709899264 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001447061183550652, + "loss": 1.0666, + "theoretical_loss": 3.3755640727239427, + "tokens_seen": 2709964800 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001446900702106319, + "loss": 1.1777, + "theoretical_loss": 3.3755577199636075, + "tokens_seen": 2710030336 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001446740220661986, + "loss": 1.1611, + "theoretical_loss": 3.375551367399912, + "tokens_seen": 2710095872 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001446579739217653, + "loss": 1.204, + "theoretical_loss": 3.375545015032846, + "tokens_seen": 2710161408 + }, + { + "epoch": 1.2, + "learning_rate": 0.000144641925777332, + "loss": 1.3132, + "theoretical_loss": 3.375538662862398, + "tokens_seen": 2710226944 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001446258776328987, + "loss": 1.1225, + "theoretical_loss": 3.3755323108885573, + "tokens_seen": 2710292480 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001446098294884654, + "loss": 1.2071, + "theoretical_loss": 3.3755259591113127, + "tokens_seen": 2710358016 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001445937813440321, + "loss": 1.1262, + "theoretical_loss": 3.3755196075306544, + "tokens_seen": 2710423552 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001445777331995988, + "loss": 1.2085, + "theoretical_loss": 3.3755132561465704, + "tokens_seen": 2710489088 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001445616850551655, + "loss": 1.2788, + "theoretical_loss": 3.3755069049590505, + "tokens_seen": 2710554624 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001445456369107322, + "loss": 1.2512, + "theoretical_loss": 3.3755005539680836, + "tokens_seen": 2710620160 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014452958876629888, + "loss": 1.1441, + "theoretical_loss": 3.375494203173659, + "tokens_seen": 2710685696 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014451354062186558, + "loss": 1.2533, + "theoretical_loss": 3.375487852575766, + "tokens_seen": 2710751232 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014449749247743228, + "loss": 1.2352, + "theoretical_loss": 3.375481502174394, + "tokens_seen": 2710816768 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014448144433299898, + "loss": 1.3086, + "theoretical_loss": 3.375475151969531, + "tokens_seen": 2710882304 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014446539618856569, + "loss": 1.1726, + "theoretical_loss": 3.3754688019611674, + "tokens_seen": 2710947840 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014444934804413239, + "loss": 1.2053, + "theoretical_loss": 3.375462452149292, + "tokens_seen": 2711013376 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014443329989969911, + "loss": 1.132, + "theoretical_loss": 3.3754561025338936, + "tokens_seen": 2711078912 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014441725175526581, + "loss": 1.2625, + "theoretical_loss": 3.375449753114962, + "tokens_seen": 2711144448 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014440120361083251, + "loss": 1.1497, + "theoretical_loss": 3.3754434038924854, + "tokens_seen": 2711209984 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014438515546639922, + "loss": 1.1756, + "theoretical_loss": 3.3754370548664543, + "tokens_seen": 2711275520 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014436910732196592, + "loss": 1.1229, + "theoretical_loss": 3.375430706036857, + "tokens_seen": 2711341056 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014435305917753262, + "loss": 1.2864, + "theoretical_loss": 3.375424357403683, + "tokens_seen": 2711406592 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014433701103309932, + "loss": 1.1275, + "theoretical_loss": 3.3754180089669212, + "tokens_seen": 2711472128 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014432096288866602, + "loss": 1.2392, + "theoretical_loss": 3.3754116607265607, + "tokens_seen": 2711537664 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014430491474423272, + "loss": 1.2055, + "theoretical_loss": 3.375405312682591, + "tokens_seen": 2711603200 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014428886659979942, + "loss": 1.158, + "theoretical_loss": 3.3753989648350013, + "tokens_seen": 2711668736 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014427281845536612, + "loss": 1.2441, + "theoretical_loss": 3.3753926171837807, + "tokens_seen": 2711734272 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014425677031093282, + "loss": 1.2043, + "theoretical_loss": 3.375386269728918, + "tokens_seen": 2711799808 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014424072216649952, + "loss": 1.1546, + "theoretical_loss": 3.3753799224704033, + "tokens_seen": 2711865344 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014422467402206622, + "loss": 1.1689, + "theoretical_loss": 3.375373575408225, + "tokens_seen": 2711930880 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001442086258776329, + "loss": 1.1599, + "theoretical_loss": 3.375367228542372, + "tokens_seen": 2711996416 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001441925777331996, + "loss": 1.1141, + "theoretical_loss": 3.3753608818728345, + "tokens_seen": 2712061952 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001441765295887663, + "loss": 1.1565, + "theoretical_loss": 3.375354535399601, + "tokens_seen": 2712127488 + }, + { + "epoch": 1.2, + "learning_rate": 0.000144160481444333, + "loss": 1.1347, + "theoretical_loss": 3.3753481891226604, + "tokens_seen": 2712193024 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001441444332998997, + "loss": 1.2098, + "theoretical_loss": 3.375341843042003, + "tokens_seen": 2712258560 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001441283851554664, + "loss": 1.1843, + "theoretical_loss": 3.375335497157617, + "tokens_seen": 2712324096 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001441123370110331, + "loss": 1.1497, + "theoretical_loss": 3.375329151469492, + "tokens_seen": 2712389632 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001440962888665998, + "loss": 1.198, + "theoretical_loss": 3.3753228059776164, + "tokens_seen": 2712455168 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001440802407221665, + "loss": 1.2735, + "theoretical_loss": 3.3753164606819808, + "tokens_seen": 2712520704 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001440641925777332, + "loss": 1.1234, + "theoretical_loss": 3.3753101155825735, + "tokens_seen": 2712586240 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001440481444332999, + "loss": 1.2173, + "theoretical_loss": 3.3753037706793836, + "tokens_seen": 2712651776 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001440320962888666, + "loss": 1.2447, + "theoretical_loss": 3.3752974259724007, + "tokens_seen": 2712717312 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001440160481444333, + "loss": 1.1326, + "theoretical_loss": 3.375291081461614, + "tokens_seen": 2712782848 + }, + { + "epoch": 1.2, + "learning_rate": 0.000144, + "loss": 1.2369, + "theoretical_loss": 3.3752847371470125, + "tokens_seen": 2712848384 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001439839518555667, + "loss": 1.1686, + "theoretical_loss": 3.375278393028585, + "tokens_seen": 2712913920 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001439679037111334, + "loss": 1.0901, + "theoretical_loss": 3.3752720491063215, + "tokens_seen": 2712979456 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001439518555667001, + "loss": 1.2075, + "theoretical_loss": 3.375265705380211, + "tokens_seen": 2713044992 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001439358074222668, + "loss": 1.2464, + "theoretical_loss": 3.375259361850242, + "tokens_seen": 2713110528 + }, + { + "epoch": 1.2, + "objective/train/docs_used": 1002147, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.257035493850708, + "objective/train/theoretical_loss": 3.3752530185164042, + "objective/train/tokens_used": 2733636064, + "theoretical_loss": 3.3752530185164042, + "tokens_seen": 2713176064 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001439197592778335, + "loss": 1.257, + "theoretical_loss": 3.3752530185164042, + "tokens_seen": 2713176064 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001439037111334002, + "loss": 1.3244, + "theoretical_loss": 3.375246675378687, + "tokens_seen": 2713241600 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001438876629889669, + "loss": 1.2493, + "theoretical_loss": 3.3752403324370794, + "tokens_seen": 2713307136 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001438716148445336, + "loss": 1.2474, + "theoretical_loss": 3.375233989691571, + "tokens_seen": 2713372672 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001438555667001003, + "loss": 1.207, + "theoretical_loss": 3.3752276471421503, + "tokens_seen": 2713438208 + }, + { + "epoch": 1.2, + "learning_rate": 0.000143839518555667, + "loss": 1.2578, + "theoretical_loss": 3.3752213047888064, + "tokens_seen": 2713503744 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001438234704112337, + "loss": 1.1755, + "theoretical_loss": 3.3752149626315293, + "tokens_seen": 2713569280 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001438074222668004, + "loss": 1.1342, + "theoretical_loss": 3.375208620670308, + "tokens_seen": 2713634816 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001437913741223671, + "loss": 1.0974, + "theoretical_loss": 3.3752022789051312, + "tokens_seen": 2713700352 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001437753259779338, + "loss": 1.3485, + "theoretical_loss": 3.3751959373359885, + "tokens_seen": 2713765888 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001437592778335005, + "loss": 1.2427, + "theoretical_loss": 3.375189595962869, + "tokens_seen": 2713831424 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014374322968906721, + "loss": 1.2371, + "theoretical_loss": 3.375183254785762, + "tokens_seen": 2713896960 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014372718154463391, + "loss": 1.2215, + "theoretical_loss": 3.3751769138046566, + "tokens_seen": 2713962496 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014371113340020062, + "loss": 1.1073, + "theoretical_loss": 3.3751705730195423, + "tokens_seen": 2714028032 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014369508525576732, + "loss": 1.1427, + "theoretical_loss": 3.3751642324304076, + "tokens_seen": 2714093568 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014367903711133402, + "loss": 1.2286, + "theoretical_loss": 3.3751578920372425, + "tokens_seen": 2714159104 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014366298896690072, + "loss": 1.193, + "theoretical_loss": 3.375151551840036, + "tokens_seen": 2714224640 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014364694082246742, + "loss": 1.1983, + "theoretical_loss": 3.375145211838777, + "tokens_seen": 2714290176 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014363089267803412, + "loss": 1.2041, + "theoretical_loss": 3.375138872033455, + "tokens_seen": 2714355712 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014361484453360082, + "loss": 1.1543, + "theoretical_loss": 3.375132532424059, + "tokens_seen": 2714421248 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014359879638916752, + "loss": 1.1643, + "theoretical_loss": 3.375126193010578, + "tokens_seen": 2714486784 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014358274824473422, + "loss": 1.2228, + "theoretical_loss": 3.3751198537930023, + "tokens_seen": 2714552320 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014356670010030092, + "loss": 1.0996, + "theoretical_loss": 3.37511351477132, + "tokens_seen": 2714617856 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014355065195586762, + "loss": 1.1504, + "theoretical_loss": 3.3751071759455207, + "tokens_seen": 2714683392 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014353460381143432, + "loss": 1.1727, + "theoretical_loss": 3.3751008373155935, + "tokens_seen": 2714748928 + }, + { + "epoch": 1.2, + "learning_rate": 0.000143518555667001, + "loss": 1.2552, + "theoretical_loss": 3.375094498881528, + "tokens_seen": 2714814464 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001435025075225677, + "loss": 1.1562, + "theoretical_loss": 3.3750881606433127, + "tokens_seen": 2714880000 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001434864593781344, + "loss": 1.1829, + "theoretical_loss": 3.375081822600938, + "tokens_seen": 2714945536 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001434704112337011, + "loss": 1.2014, + "theoretical_loss": 3.3750754847543916, + "tokens_seen": 2715011072 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001434543630892678, + "loss": 1.1618, + "theoretical_loss": 3.3750691471036642, + "tokens_seen": 2715076608 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001434383149448345, + "loss": 1.1856, + "theoretical_loss": 3.375062809648744, + "tokens_seen": 2715142144 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001434222668004012, + "loss": 1.2094, + "theoretical_loss": 3.37505647238962, + "tokens_seen": 2715207680 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001434062186559679, + "loss": 1.1787, + "theoretical_loss": 3.375050135326283, + "tokens_seen": 2715273216 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001433901705115346, + "loss": 1.2022, + "theoretical_loss": 3.3750437984587207, + "tokens_seen": 2715338752 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001433741223671013, + "loss": 1.3358, + "theoretical_loss": 3.3750374617869228, + "tokens_seen": 2715404288 + }, + { + "epoch": 1.2, + "learning_rate": 0.000143358074222668, + "loss": 1.2302, + "theoretical_loss": 3.375031125310879, + "tokens_seen": 2715469824 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001433420260782347, + "loss": 1.2447, + "theoretical_loss": 3.3750247890305776, + "tokens_seen": 2715535360 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001433259779338014, + "loss": 1.2007, + "theoretical_loss": 3.3750184529460086, + "tokens_seen": 2715600896 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001433099297893681, + "loss": 1.2225, + "theoretical_loss": 3.3750121170571608, + "tokens_seen": 2715666432 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001432938816449348, + "loss": 1.1231, + "theoretical_loss": 3.3750057813640235, + "tokens_seen": 2715731968 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001432778335005015, + "loss": 1.227, + "theoretical_loss": 3.374999445866586, + "tokens_seen": 2715797504 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001432617853560682, + "loss": 1.2491, + "theoretical_loss": 3.374993110564838, + "tokens_seen": 2715863040 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001432457372116349, + "loss": 1.1694, + "theoretical_loss": 3.3749867754587677, + "tokens_seen": 2715928576 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014322968906720163, + "loss": 1.0861, + "theoretical_loss": 3.374980440548365, + "tokens_seen": 2715994112 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014321364092276833, + "loss": 1.3025, + "theoretical_loss": 3.3749741058336196, + "tokens_seen": 2716059648 + }, + { + "epoch": 1.2, + "learning_rate": 0.000143197592778335, + "loss": 1.1332, + "theoretical_loss": 3.3749677713145196, + "tokens_seen": 2716125184 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001431815446339017, + "loss": 1.1475, + "theoretical_loss": 3.374961436991055, + "tokens_seen": 2716190720 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001431654964894684, + "loss": 1.1208, + "theoretical_loss": 3.374955102863215, + "tokens_seen": 2716256256 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001431494483450351, + "loss": 1.1882, + "theoretical_loss": 3.374948768930989, + "tokens_seen": 2716321792 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001431334002006018, + "loss": 1.2194, + "theoretical_loss": 3.3749424351943658, + "tokens_seen": 2716387328 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001431173520561685, + "loss": 1.2465, + "theoretical_loss": 3.374936101653334, + "tokens_seen": 2716452864 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001431013039117352, + "loss": 1.2325, + "theoretical_loss": 3.3749297683078847, + "tokens_seen": 2716518400 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001430852557673019, + "loss": 1.237, + "theoretical_loss": 3.3749234351580055, + "tokens_seen": 2716583936 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001430692076228686, + "loss": 1.175, + "theoretical_loss": 3.3749171022036863, + "tokens_seen": 2716649472 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014305315947843531, + "loss": 1.1226, + "theoretical_loss": 3.374910769444916, + "tokens_seen": 2716715008 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014303711133400201, + "loss": 1.1625, + "theoretical_loss": 3.3749044368816845, + "tokens_seen": 2716780544 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014302106318956872, + "loss": 1.1369, + "theoretical_loss": 3.3748981045139805, + "tokens_seen": 2716846080 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014300501504513542, + "loss": 1.2285, + "theoretical_loss": 3.374891772341793, + "tokens_seen": 2716911616 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014298896690070212, + "loss": 1.1976, + "theoretical_loss": 3.3748854403651123, + "tokens_seen": 2716977152 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014297291875626882, + "loss": 1.2332, + "theoretical_loss": 3.3748791085839267, + "tokens_seen": 2717042688 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014295687061183552, + "loss": 1.1792, + "theoretical_loss": 3.3748727769982256, + "tokens_seen": 2717108224 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014294082246740222, + "loss": 1.1779, + "theoretical_loss": 3.3748664456079984, + "tokens_seen": 2717173760 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014292477432296892, + "loss": 1.176, + "theoretical_loss": 3.374860114413235, + "tokens_seen": 2717239296 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014290872617853562, + "loss": 1.2119, + "theoretical_loss": 3.374853783413923, + "tokens_seen": 2717304832 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014289267803410232, + "loss": 1.1567, + "theoretical_loss": 3.3748474526100534, + "tokens_seen": 2717370368 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014287662988966902, + "loss": 1.1888, + "theoretical_loss": 3.374841122001614, + "tokens_seen": 2717435904 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014286058174523572, + "loss": 1.2173, + "theoretical_loss": 3.3748347915885955, + "tokens_seen": 2717501440 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014284453360080242, + "loss": 1.1595, + "theoretical_loss": 3.3748284613709862, + "tokens_seen": 2717566976 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014282848545636912, + "loss": 1.2309, + "theoretical_loss": 3.3748221313487754, + "tokens_seen": 2717632512 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001428124373119358, + "loss": 1.172, + "theoretical_loss": 3.3748158015219527, + "tokens_seen": 2717698048 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001427963891675025, + "loss": 1.111, + "theoretical_loss": 3.3748094718905066, + "tokens_seen": 2717763584 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001427803410230692, + "loss": 1.1135, + "theoretical_loss": 3.3748031424544274, + "tokens_seen": 2717829120 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001427642928786359, + "loss": 1.1939, + "theoretical_loss": 3.374796813213704, + "tokens_seen": 2717894656 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001427482447342026, + "loss": 1.2321, + "theoretical_loss": 3.3747904841683254, + "tokens_seen": 2717960192 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001427321965897693, + "loss": 1.1229, + "theoretical_loss": 3.374784155318281, + "tokens_seen": 2718025728 + }, + { + "epoch": 1.2, + "learning_rate": 0.000142716148445336, + "loss": 1.2022, + "theoretical_loss": 3.3747778266635597, + "tokens_seen": 2718091264 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001427001003009027, + "loss": 1.2745, + "theoretical_loss": 3.3747714982041517, + "tokens_seen": 2718156800 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001426840521564694, + "loss": 1.2321, + "theoretical_loss": 3.3747651699400456, + "tokens_seen": 2718222336 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001426680040120361, + "loss": 1.3157, + "theoretical_loss": 3.3747588418712304, + "tokens_seen": 2718287872 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001426519558676028, + "loss": 1.2324, + "theoretical_loss": 3.3747525139976964, + "tokens_seen": 2718353408 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014263590772316953, + "loss": 1.1505, + "theoretical_loss": 3.3747461863194315, + "tokens_seen": 2718418944 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014261985957873623, + "loss": 1.2783, + "theoretical_loss": 3.374739858836426, + "tokens_seen": 2718484480 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014260381143430293, + "loss": 1.2235, + "theoretical_loss": 3.3747335315486686, + "tokens_seen": 2718550016 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014258776328986963, + "loss": 1.2094, + "theoretical_loss": 3.3747272044561494, + "tokens_seen": 2718615552 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014257171514543633, + "loss": 1.1921, + "theoretical_loss": 3.3747208775588566, + "tokens_seen": 2718681088 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014255566700100303, + "loss": 1.2038, + "theoretical_loss": 3.3747145508567797, + "tokens_seen": 2718746624 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014253961885656973, + "loss": 1.1904, + "theoretical_loss": 3.3747082243499085, + "tokens_seen": 2718812160 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014252357071213643, + "loss": 1.178, + "theoretical_loss": 3.374701898038232, + "tokens_seen": 2718877696 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014250752256770314, + "loss": 1.2629, + "theoretical_loss": 3.3746955719217397, + "tokens_seen": 2718943232 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001424914744232698, + "loss": 1.1714, + "theoretical_loss": 3.37468924600042, + "tokens_seen": 2719008768 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001424754262788365, + "loss": 1.2295, + "theoretical_loss": 3.3746829202742634, + "tokens_seen": 2719074304 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001424593781344032, + "loss": 1.2174, + "theoretical_loss": 3.374676594743258, + "tokens_seen": 2719139840 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001424433299899699, + "loss": 1.1419, + "theoretical_loss": 3.3746702694073942, + "tokens_seen": 2719205376 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001424272818455366, + "loss": 1.1575, + "theoretical_loss": 3.374663944266661, + "tokens_seen": 2719270912 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001424112337011033, + "loss": 1.2094, + "theoretical_loss": 3.374657619321047, + "tokens_seen": 2719336448 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014239518555667, + "loss": 1.1576, + "theoretical_loss": 3.3746512945705414, + "tokens_seen": 2719401984 + }, + { + "epoch": 1.2, + "learning_rate": 0.0001423791374122367, + "loss": 1.1854, + "theoretical_loss": 3.3746449700151344, + "tokens_seen": 2719467520 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014236308926780341, + "loss": 1.1287, + "theoretical_loss": 3.374638645654815, + "tokens_seen": 2719533056 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014234704112337011, + "loss": 1.1461, + "theoretical_loss": 3.374632321489572, + "tokens_seen": 2719598592 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014233099297893682, + "loss": 1.1389, + "theoretical_loss": 3.374625997519395, + "tokens_seen": 2719664128 + }, + { + "epoch": 1.2, + "objective/train/docs_used": 1004410, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2306292057037354, + "objective/train/theoretical_loss": 3.3746196737442737, + "objective/train/tokens_used": 2740189664, + "theoretical_loss": 3.3746196737442737, + "tokens_seen": 2719729664 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014231494483450352, + "loss": 1.2306, + "theoretical_loss": 3.3746196737442737, + "tokens_seen": 2719729664 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014229889669007022, + "loss": 1.2995, + "theoretical_loss": 3.374613350164197, + "tokens_seen": 2719795200 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014228284854563692, + "loss": 1.2382, + "theoretical_loss": 3.3746070267791537, + "tokens_seen": 2719860736 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014226680040120362, + "loss": 1.2312, + "theoretical_loss": 3.3746007035891337, + "tokens_seen": 2719926272 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014225075225677032, + "loss": 1.1197, + "theoretical_loss": 3.3745943805941265, + "tokens_seen": 2719991808 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014223470411233702, + "loss": 1.1034, + "theoretical_loss": 3.3745880577941207, + "tokens_seen": 2720057344 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014221865596790372, + "loss": 1.2131, + "theoretical_loss": 3.374581735189106, + "tokens_seen": 2720122880 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014220260782347042, + "loss": 1.1888, + "theoretical_loss": 3.3745754127790715, + "tokens_seen": 2720188416 + }, + { + "epoch": 1.2, + "learning_rate": 0.00014218655967903712, + "loss": 1.2424, + "theoretical_loss": 3.3745690905640062, + "tokens_seen": 2720253952 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014217051153460382, + "loss": 1.2411, + "theoretical_loss": 3.3745627685439006, + "tokens_seen": 2720319488 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014215446339017052, + "loss": 1.2382, + "theoretical_loss": 3.3745564467187426, + "tokens_seen": 2720385024 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014213841524573722, + "loss": 1.1115, + "theoretical_loss": 3.3745501250885224, + "tokens_seen": 2720450560 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014212236710130392, + "loss": 1.1476, + "theoretical_loss": 3.3745438036532285, + "tokens_seen": 2720516096 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001421063189568706, + "loss": 1.1733, + "theoretical_loss": 3.374537482412851, + "tokens_seen": 2720581632 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001420902708124373, + "loss": 1.2042, + "theoretical_loss": 3.374531161367379, + "tokens_seen": 2720647168 + }, + { + "epoch": 1.21, + "learning_rate": 0.000142074222668004, + "loss": 1.0999, + "theoretical_loss": 3.374524840516801, + "tokens_seen": 2720712704 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001420581745235707, + "loss": 1.1478, + "theoretical_loss": 3.3745185198611076, + "tokens_seen": 2720778240 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001420421263791374, + "loss": 1.2704, + "theoretical_loss": 3.3745121994002876, + "tokens_seen": 2720843776 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014202607823470413, + "loss": 1.1919, + "theoretical_loss": 3.3745058791343294, + "tokens_seen": 2720909312 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014201003009027083, + "loss": 1.1331, + "theoretical_loss": 3.3744995590632234, + "tokens_seen": 2720974848 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014199398194583753, + "loss": 1.1366, + "theoretical_loss": 3.3744932391869584, + "tokens_seen": 2721040384 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014197793380140423, + "loss": 1.2194, + "theoretical_loss": 3.3744869195055243, + "tokens_seen": 2721105920 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014196188565697093, + "loss": 1.1511, + "theoretical_loss": 3.3744806000189094, + "tokens_seen": 2721171456 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014194583751253763, + "loss": 1.2884, + "theoretical_loss": 3.3744742807271035, + "tokens_seen": 2721236992 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014192978936810433, + "loss": 1.1431, + "theoretical_loss": 3.3744679616300965, + "tokens_seen": 2721302528 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014191374122367103, + "loss": 1.1268, + "theoretical_loss": 3.3744616427278764, + "tokens_seen": 2721368064 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014189769307923773, + "loss": 1.1375, + "theoretical_loss": 3.374455324020434, + "tokens_seen": 2721433600 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014188164493480443, + "loss": 1.0385, + "theoretical_loss": 3.374449005507757, + "tokens_seen": 2721499136 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014186559679037113, + "loss": 1.2386, + "theoretical_loss": 3.3744426871898363, + "tokens_seen": 2721564672 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014184954864593783, + "loss": 1.2132, + "theoretical_loss": 3.37443636906666, + "tokens_seen": 2721630208 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014183350050150454, + "loss": 1.1568, + "theoretical_loss": 3.3744300511382184, + "tokens_seen": 2721695744 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014181745235707124, + "loss": 1.2198, + "theoretical_loss": 3.3744237334045, + "tokens_seen": 2721761280 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001418014042126379, + "loss": 1.1662, + "theoretical_loss": 3.3744174158654943, + "tokens_seen": 2721826816 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001417853560682046, + "loss": 1.094, + "theoretical_loss": 3.3744110985211906, + "tokens_seen": 2721892352 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001417693079237713, + "loss": 1.1502, + "theoretical_loss": 3.3744047813715787, + "tokens_seen": 2721957888 + }, + { + "epoch": 1.21, + "learning_rate": 0.000141753259779338, + "loss": 1.1692, + "theoretical_loss": 3.374398464416647, + "tokens_seen": 2722023424 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001417372116349047, + "loss": 1.2765, + "theoretical_loss": 3.374392147656386, + "tokens_seen": 2722088960 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001417211634904714, + "loss": 1.1959, + "theoretical_loss": 3.374385831090784, + "tokens_seen": 2722154496 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001417051153460381, + "loss": 1.0673, + "theoretical_loss": 3.3743795147198306, + "tokens_seen": 2722220032 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014168906720160481, + "loss": 1.0995, + "theoretical_loss": 3.3743731985435153, + "tokens_seen": 2722285568 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014167301905717151, + "loss": 1.1433, + "theoretical_loss": 3.3743668825618274, + "tokens_seen": 2722351104 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014165697091273821, + "loss": 1.1165, + "theoretical_loss": 3.3743605667747563, + "tokens_seen": 2722416640 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014164092276830492, + "loss": 1.2231, + "theoretical_loss": 3.3743542511822904, + "tokens_seen": 2722482176 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014162487462387162, + "loss": 1.1729, + "theoretical_loss": 3.3743479357844204, + "tokens_seen": 2722547712 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014160882647943832, + "loss": 1.1847, + "theoretical_loss": 3.3743416205811347, + "tokens_seen": 2722613248 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014159277833500502, + "loss": 1.1996, + "theoretical_loss": 3.3743353055724232, + "tokens_seen": 2722678784 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014157673019057172, + "loss": 1.2012, + "theoretical_loss": 3.3743289907582747, + "tokens_seen": 2722744320 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014156068204613842, + "loss": 1.1615, + "theoretical_loss": 3.3743226761386786, + "tokens_seen": 2722809856 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014154463390170512, + "loss": 1.2676, + "theoretical_loss": 3.3743163617136247, + "tokens_seen": 2722875392 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014152858575727182, + "loss": 1.1542, + "theoretical_loss": 3.3743100474831023, + "tokens_seen": 2722940928 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014151253761283852, + "loss": 1.2265, + "theoretical_loss": 3.3743037334470998, + "tokens_seen": 2723006464 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014149648946840522, + "loss": 1.2095, + "theoretical_loss": 3.374297419605607, + "tokens_seen": 2723072000 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014148044132397192, + "loss": 1.1911, + "theoretical_loss": 3.374291105958614, + "tokens_seen": 2723137536 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014146439317953862, + "loss": 1.1503, + "theoretical_loss": 3.374284792506109, + "tokens_seen": 2723203072 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014144834503510532, + "loss": 1.2263, + "theoretical_loss": 3.374278479248082, + "tokens_seen": 2723268608 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014143229689067202, + "loss": 1.2107, + "theoretical_loss": 3.3742721661845225, + "tokens_seen": 2723334144 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014141624874623872, + "loss": 1.1169, + "theoretical_loss": 3.374265853315419, + "tokens_seen": 2723399680 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014140020060180543, + "loss": 1.1973, + "theoretical_loss": 3.3742595406407614, + "tokens_seen": 2723465216 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014138415245737213, + "loss": 1.1771, + "theoretical_loss": 3.3742532281605393, + "tokens_seen": 2723530752 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014136810431293883, + "loss": 1.2018, + "theoretical_loss": 3.3742469158747412, + "tokens_seen": 2723596288 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014135205616850553, + "loss": 1.1712, + "theoretical_loss": 3.374240603783357, + "tokens_seen": 2723661824 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014133600802407223, + "loss": 1.1506, + "theoretical_loss": 3.3742342918863764, + "tokens_seen": 2723727360 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014131995987963893, + "loss": 1.2195, + "theoretical_loss": 3.374227980183788, + "tokens_seen": 2723792896 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014130391173520563, + "loss": 1.0957, + "theoretical_loss": 3.3742216686755815, + "tokens_seen": 2723858432 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014128786359077233, + "loss": 1.0987, + "theoretical_loss": 3.374215357361746, + "tokens_seen": 2723923968 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014127181544633903, + "loss": 1.2807, + "theoretical_loss": 3.374209046242271, + "tokens_seen": 2723989504 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014125576730190573, + "loss": 1.098, + "theoretical_loss": 3.374202735317146, + "tokens_seen": 2724055040 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014123971915747243, + "loss": 1.2026, + "theoretical_loss": 3.3741964245863603, + "tokens_seen": 2724120576 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014122367101303913, + "loss": 1.1922, + "theoretical_loss": 3.374190114049903, + "tokens_seen": 2724186112 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014120762286860583, + "loss": 1.2007, + "theoretical_loss": 3.3741838037077634, + "tokens_seen": 2724251648 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014119157472417253, + "loss": 1.1688, + "theoretical_loss": 3.374177493559931, + "tokens_seen": 2724317184 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014117552657973923, + "loss": 1.1849, + "theoretical_loss": 3.3741711836063955, + "tokens_seen": 2724382720 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014115947843530593, + "loss": 1.2693, + "theoretical_loss": 3.3741648738471453, + "tokens_seen": 2724448256 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014114343029087264, + "loss": 1.2084, + "theoretical_loss": 3.3741585642821708, + "tokens_seen": 2724513792 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014112738214643934, + "loss": 1.1756, + "theoretical_loss": 3.3741522549114604, + "tokens_seen": 2724579328 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014111133400200604, + "loss": 1.1696, + "theoretical_loss": 3.3741459457350045, + "tokens_seen": 2724644864 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001410952858575727, + "loss": 1.1932, + "theoretical_loss": 3.374139636752792, + "tokens_seen": 2724710400 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001410792377131394, + "loss": 1.0709, + "theoretical_loss": 3.3741333279648114, + "tokens_seen": 2724775936 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001410631895687061, + "loss": 1.0992, + "theoretical_loss": 3.3741270193710533, + "tokens_seen": 2724841472 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001410471414242728, + "loss": 1.1306, + "theoretical_loss": 3.374120710971506, + "tokens_seen": 2724907008 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001410310932798395, + "loss": 1.1556, + "theoretical_loss": 3.37411440276616, + "tokens_seen": 2724972544 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001410150451354062, + "loss": 1.208, + "theoretical_loss": 3.3741080947550035, + "tokens_seen": 2725038080 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014099899699097291, + "loss": 1.1832, + "theoretical_loss": 3.3741017869380268, + "tokens_seen": 2725103616 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014098294884653961, + "loss": 1.2256, + "theoretical_loss": 3.3740954793152187, + "tokens_seen": 2725169152 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014096690070210632, + "loss": 1.1792, + "theoretical_loss": 3.3740891718865686, + "tokens_seen": 2725234688 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014095085255767302, + "loss": 1.211, + "theoretical_loss": 3.374082864652066, + "tokens_seen": 2725300224 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014093480441323972, + "loss": 1.2269, + "theoretical_loss": 3.3740765576117, + "tokens_seen": 2725365760 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014091875626880642, + "loss": 1.1965, + "theoretical_loss": 3.3740702507654605, + "tokens_seen": 2725431296 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014090270812437312, + "loss": 1.2527, + "theoretical_loss": 3.374063944113336, + "tokens_seen": 2725496832 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014088665997993982, + "loss": 1.2629, + "theoretical_loss": 3.3740576376553166, + "tokens_seen": 2725562368 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014087061183550652, + "loss": 1.1444, + "theoretical_loss": 3.3740513313913914, + "tokens_seen": 2725627904 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014085456369107322, + "loss": 1.1941, + "theoretical_loss": 3.37404502532155, + "tokens_seen": 2725693440 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014083851554663995, + "loss": 1.1749, + "theoretical_loss": 3.3740387194457817, + "tokens_seen": 2725758976 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014082246740220665, + "loss": 1.1631, + "theoretical_loss": 3.374032413764075, + "tokens_seen": 2725824512 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014080641925777335, + "loss": 1.2062, + "theoretical_loss": 3.3740261082764205, + "tokens_seen": 2725890048 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014079037111334005, + "loss": 1.0557, + "theoretical_loss": 3.374019802982807, + "tokens_seen": 2725955584 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014077432296890672, + "loss": 1.2136, + "theoretical_loss": 3.3740134978832237, + "tokens_seen": 2726021120 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014075827482447342, + "loss": 1.1639, + "theoretical_loss": 3.3740071929776603, + "tokens_seen": 2726086656 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014074222668004012, + "loss": 1.2411, + "theoretical_loss": 3.3740008882661057, + "tokens_seen": 2726152192 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014072617853560682, + "loss": 1.2394, + "theoretical_loss": 3.3739945837485497, + "tokens_seen": 2726217728 + }, + { + "epoch": 1.21, + "objective/train/docs_used": 1006902, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1866517066955566, + "objective/train/theoretical_loss": 3.373988279424982, + "objective/train/tokens_used": 2746743264, + "theoretical_loss": 3.373988279424982, + "tokens_seen": 2726283264 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014071013039117353, + "loss": 1.1867, + "theoretical_loss": 3.373988279424982, + "tokens_seen": 2726283264 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014069408224674023, + "loss": 1.2034, + "theoretical_loss": 3.3739819752953912, + "tokens_seen": 2726348800 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014067803410230693, + "loss": 1.1857, + "theoretical_loss": 3.3739756713597666, + "tokens_seen": 2726414336 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014066198595787363, + "loss": 1.1669, + "theoretical_loss": 3.3739693676180984, + "tokens_seen": 2726479872 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014064593781344033, + "loss": 1.2199, + "theoretical_loss": 3.373963064070376, + "tokens_seen": 2726545408 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014062988966900703, + "loss": 1.1033, + "theoretical_loss": 3.3739567607165877, + "tokens_seen": 2726610944 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014061384152457373, + "loss": 1.2212, + "theoretical_loss": 3.3739504575567234, + "tokens_seen": 2726676480 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014059779338014043, + "loss": 1.2425, + "theoretical_loss": 3.373944154590773, + "tokens_seen": 2726742016 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014058174523570713, + "loss": 1.1882, + "theoretical_loss": 3.3739378518187246, + "tokens_seen": 2726807552 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014056569709127383, + "loss": 1.3319, + "theoretical_loss": 3.3739315492405693, + "tokens_seen": 2726873088 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014054964894684053, + "loss": 1.1661, + "theoretical_loss": 3.3739252468562952, + "tokens_seen": 2726938624 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014053360080240723, + "loss": 1.1565, + "theoretical_loss": 3.373918944665892, + "tokens_seen": 2727004160 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014051755265797393, + "loss": 1.1586, + "theoretical_loss": 3.373912642669349, + "tokens_seen": 2727069696 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014050150451354063, + "loss": 1.2872, + "theoretical_loss": 3.373906340866656, + "tokens_seen": 2727135232 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014048545636910733, + "loss": 1.1049, + "theoretical_loss": 3.3739000392578022, + "tokens_seen": 2727200768 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014046940822467403, + "loss": 1.15, + "theoretical_loss": 3.3738937378427765, + "tokens_seen": 2727266304 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014045336008024074, + "loss": 1.1163, + "theoretical_loss": 3.3738874366215685, + "tokens_seen": 2727331840 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014043731193580744, + "loss": 1.264, + "theoretical_loss": 3.3738811355941682, + "tokens_seen": 2727397376 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014042126379137414, + "loss": 1.2816, + "theoretical_loss": 3.3738748347605645, + "tokens_seen": 2727462912 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014040521564694084, + "loss": 1.0937, + "theoretical_loss": 3.3738685341207466, + "tokens_seen": 2727528448 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001403891675025075, + "loss": 1.2933, + "theoretical_loss": 3.373862233674704, + "tokens_seen": 2727593984 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001403731193580742, + "loss": 1.2257, + "theoretical_loss": 3.3738559334224263, + "tokens_seen": 2727659520 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001403570712136409, + "loss": 1.2427, + "theoretical_loss": 3.3738496333639025, + "tokens_seen": 2727725056 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001403410230692076, + "loss": 1.07, + "theoretical_loss": 3.3738433334991225, + "tokens_seen": 2727790592 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001403249749247743, + "loss": 1.3084, + "theoretical_loss": 3.3738370338280754, + "tokens_seen": 2727856128 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014030892678034101, + "loss": 1.242, + "theoretical_loss": 3.3738307343507508, + "tokens_seen": 2727921664 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014029287863590771, + "loss": 1.2617, + "theoretical_loss": 3.3738244350671374, + "tokens_seen": 2727987200 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014027683049147442, + "loss": 1.1906, + "theoretical_loss": 3.373818135977225, + "tokens_seen": 2728052736 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014026078234704112, + "loss": 1.1495, + "theoretical_loss": 3.3738118370810035, + "tokens_seen": 2728118272 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014024473420260782, + "loss": 1.1328, + "theoretical_loss": 3.3738055383784618, + "tokens_seen": 2728183808 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014022868605817454, + "loss": 1.1839, + "theoretical_loss": 3.3737992398695895, + "tokens_seen": 2728249344 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014021263791374124, + "loss": 1.112, + "theoretical_loss": 3.3737929415543757, + "tokens_seen": 2728314880 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014019658976930795, + "loss": 1.2028, + "theoretical_loss": 3.37378664343281, + "tokens_seen": 2728380416 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014018054162487465, + "loss": 1.1946, + "theoretical_loss": 3.3737803455048816, + "tokens_seen": 2728445952 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014016449348044135, + "loss": 1.2064, + "theoretical_loss": 3.3737740477705804, + "tokens_seen": 2728511488 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014014844533600805, + "loss": 1.2502, + "theoretical_loss": 3.373767750229895, + "tokens_seen": 2728577024 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014013239719157475, + "loss": 1.2022, + "theoretical_loss": 3.3737614528828153, + "tokens_seen": 2728642560 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014011634904714145, + "loss": 1.1512, + "theoretical_loss": 3.373755155729331, + "tokens_seen": 2728708096 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014010030090270815, + "loss": 1.1304, + "theoretical_loss": 3.3737488587694306, + "tokens_seen": 2728773632 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014008425275827485, + "loss": 1.2318, + "theoretical_loss": 3.3737425620031045, + "tokens_seen": 2728839168 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014006820461384152, + "loss": 1.1823, + "theoretical_loss": 3.3737362654303413, + "tokens_seen": 2728904704 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014005215646940822, + "loss": 1.3016, + "theoretical_loss": 3.373729969051131, + "tokens_seen": 2728970240 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014003610832497492, + "loss": 1.1343, + "theoretical_loss": 3.3737236728654625, + "tokens_seen": 2729035776 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014002006018054163, + "loss": 1.1908, + "theoretical_loss": 3.3737173768733255, + "tokens_seen": 2729101312 + }, + { + "epoch": 1.21, + "learning_rate": 0.00014000401203610833, + "loss": 1.2084, + "theoretical_loss": 3.3737110810747093, + "tokens_seen": 2729166848 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013998796389167503, + "loss": 1.1127, + "theoretical_loss": 3.373704785469603, + "tokens_seen": 2729232384 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013997191574724173, + "loss": 1.1825, + "theoretical_loss": 3.373698490057997, + "tokens_seen": 2729297920 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013995586760280843, + "loss": 1.1996, + "theoretical_loss": 3.37369219483988, + "tokens_seen": 2729363456 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013993981945837513, + "loss": 1.1566, + "theoretical_loss": 3.373685899815241, + "tokens_seen": 2729428992 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013992377131394183, + "loss": 1.1943, + "theoretical_loss": 3.37367960498407, + "tokens_seen": 2729494528 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013990772316950853, + "loss": 1.2237, + "theoretical_loss": 3.3736733103463568, + "tokens_seen": 2729560064 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013989167502507523, + "loss": 1.2985, + "theoretical_loss": 3.37366701590209, + "tokens_seen": 2729625600 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013987562688064193, + "loss": 1.141, + "theoretical_loss": 3.3736607216512593, + "tokens_seen": 2729691136 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013985957873620863, + "loss": 1.1885, + "theoretical_loss": 3.373654427593854, + "tokens_seen": 2729756672 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013984353059177533, + "loss": 1.0976, + "theoretical_loss": 3.3736481337298634, + "tokens_seen": 2729822208 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013982748244734203, + "loss": 1.1734, + "theoretical_loss": 3.3736418400592774, + "tokens_seen": 2729887744 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013981143430290873, + "loss": 1.1758, + "theoretical_loss": 3.3736355465820855, + "tokens_seen": 2729953280 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013979538615847543, + "loss": 1.2273, + "theoretical_loss": 3.3736292532982763, + "tokens_seen": 2730018816 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013977933801404214, + "loss": 1.2043, + "theoretical_loss": 3.3736229602078396, + "tokens_seen": 2730084352 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013976328986960884, + "loss": 1.1893, + "theoretical_loss": 3.3736166673107655, + "tokens_seen": 2730149888 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013974724172517554, + "loss": 1.1615, + "theoretical_loss": 3.3736103746070425, + "tokens_seen": 2730215424 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013973119358074224, + "loss": 1.2124, + "theoretical_loss": 3.3736040820966604, + "tokens_seen": 2730280960 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013971514543630894, + "loss": 1.1581, + "theoretical_loss": 3.373597789779608, + "tokens_seen": 2730346496 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013969909729187564, + "loss": 1.191, + "theoretical_loss": 3.373591497655876, + "tokens_seen": 2730412032 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001396830491474423, + "loss": 1.2024, + "theoretical_loss": 3.373585205725453, + "tokens_seen": 2730477568 + }, + { + "epoch": 1.21, + "learning_rate": 0.000139667001003009, + "loss": 1.1717, + "theoretical_loss": 3.373578913988328, + "tokens_seen": 2730543104 + }, + { + "epoch": 1.21, + "learning_rate": 0.0001396509528585757, + "loss": 1.2013, + "theoretical_loss": 3.3735726224444913, + "tokens_seen": 2730608640 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013963490471414244, + "loss": 1.3593, + "theoretical_loss": 3.373566331093932, + "tokens_seen": 2730674176 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013961885656970914, + "loss": 1.2312, + "theoretical_loss": 3.3735600399366397, + "tokens_seen": 2730739712 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013960280842527584, + "loss": 1.1627, + "theoretical_loss": 3.3735537489726033, + "tokens_seen": 2730805248 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013958676028084254, + "loss": 1.2413, + "theoretical_loss": 3.3735474582018123, + "tokens_seen": 2730870784 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013957071213640924, + "loss": 1.1018, + "theoretical_loss": 3.373541167624257, + "tokens_seen": 2730936320 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013955466399197594, + "loss": 1.1378, + "theoretical_loss": 3.373534877239926, + "tokens_seen": 2731001856 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013953861584754264, + "loss": 1.1831, + "theoretical_loss": 3.3735285870488085, + "tokens_seen": 2731067392 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013952256770310935, + "loss": 1.2042, + "theoretical_loss": 3.3735222970508945, + "tokens_seen": 2731132928 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013950651955867605, + "loss": 1.1895, + "theoretical_loss": 3.3735160072461734, + "tokens_seen": 2731198464 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013949047141424275, + "loss": 1.1981, + "theoretical_loss": 3.3735097176346347, + "tokens_seen": 2731264000 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013947442326980945, + "loss": 1.2167, + "theoretical_loss": 3.3735034282162673, + "tokens_seen": 2731329536 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013945837512537615, + "loss": 1.2388, + "theoretical_loss": 3.3734971389910613, + "tokens_seen": 2731395072 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013944232698094285, + "loss": 1.1866, + "theoretical_loss": 3.3734908499590057, + "tokens_seen": 2731460608 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013942627883650955, + "loss": 1.2018, + "theoretical_loss": 3.37348456112009, + "tokens_seen": 2731526144 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013941023069207625, + "loss": 1.1163, + "theoretical_loss": 3.3734782724743035, + "tokens_seen": 2731591680 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013939418254764295, + "loss": 1.1323, + "theoretical_loss": 3.373471984021636, + "tokens_seen": 2731657216 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013937813440320962, + "loss": 1.2159, + "theoretical_loss": 3.3734656957620768, + "tokens_seen": 2731722752 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013936208625877632, + "loss": 1.0978, + "theoretical_loss": 3.373459407695615, + "tokens_seen": 2731788288 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013934603811434303, + "loss": 1.297, + "theoretical_loss": 3.3734531198222406, + "tokens_seen": 2731853824 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013932998996990973, + "loss": 1.1281, + "theoretical_loss": 3.373446832141943, + "tokens_seen": 2731919360 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013931394182547643, + "loss": 1.1515, + "theoretical_loss": 3.373440544654711, + "tokens_seen": 2731984896 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013929789368104313, + "loss": 1.2503, + "theoretical_loss": 3.3734342573605343, + "tokens_seen": 2732050432 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013928184553660983, + "loss": 1.1186, + "theoretical_loss": 3.373427970259403, + "tokens_seen": 2732115968 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013926579739217653, + "loss": 1.1457, + "theoretical_loss": 3.3734216833513058, + "tokens_seen": 2732181504 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013924974924774323, + "loss": 1.2704, + "theoretical_loss": 3.3734153966362324, + "tokens_seen": 2732247040 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013923370110330993, + "loss": 1.1684, + "theoretical_loss": 3.373409110114172, + "tokens_seen": 2732312576 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013921765295887663, + "loss": 1.2583, + "theoretical_loss": 3.3734028237851144, + "tokens_seen": 2732378112 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013920160481444333, + "loss": 1.1, + "theoretical_loss": 3.3733965376490493, + "tokens_seen": 2732443648 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013918555667001003, + "loss": 1.2731, + "theoretical_loss": 3.373390251705965, + "tokens_seen": 2732509184 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013916950852557673, + "loss": 1.1085, + "theoretical_loss": 3.373383965955852, + "tokens_seen": 2732574720 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013915346038114343, + "loss": 1.1932, + "theoretical_loss": 3.3733776803986997, + "tokens_seen": 2732640256 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013913741223671013, + "loss": 1.2351, + "theoretical_loss": 3.3733713950344972, + "tokens_seen": 2732705792 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013912136409227683, + "loss": 1.1745, + "theoretical_loss": 3.373365109863234, + "tokens_seen": 2732771328 + }, + { + "epoch": 1.21, + "objective/train/docs_used": 1008946, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.178417682647705, + "objective/train/theoretical_loss": 3.3733588248848996, + "objective/train/tokens_used": 2753296864, + "theoretical_loss": 3.3733588248848996, + "tokens_seen": 2732836864 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013910531594784353, + "loss": 1.1784, + "theoretical_loss": 3.3733588248848996, + "tokens_seen": 2732836864 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013908926780341024, + "loss": 1.1925, + "theoretical_loss": 3.3733525400994835, + "tokens_seen": 2732902400 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013907321965897694, + "loss": 1.1684, + "theoretical_loss": 3.373346255506975, + "tokens_seen": 2732967936 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013905717151454364, + "loss": 1.2162, + "theoretical_loss": 3.3733399711073635, + "tokens_seen": 2733033472 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013904112337011034, + "loss": 1.2684, + "theoretical_loss": 3.3733336869006387, + "tokens_seen": 2733099008 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013902507522567704, + "loss": 1.2201, + "theoretical_loss": 3.3733274028867903, + "tokens_seen": 2733164544 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013900902708124374, + "loss": 1.1689, + "theoretical_loss": 3.3733211190658072, + "tokens_seen": 2733230080 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013899297893681044, + "loss": 1.2738, + "theoretical_loss": 3.3733148354376787, + "tokens_seen": 2733295616 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013897693079237714, + "loss": 1.2146, + "theoretical_loss": 3.373308552002395, + "tokens_seen": 2733361152 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013896088264794384, + "loss": 1.27, + "theoretical_loss": 3.373302268759945, + "tokens_seen": 2733426688 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013894483450351054, + "loss": 1.0866, + "theoretical_loss": 3.373295985710319, + "tokens_seen": 2733492224 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013892878635907724, + "loss": 1.1753, + "theoretical_loss": 3.3732897028535054, + "tokens_seen": 2733557760 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013891273821464394, + "loss": 1.278, + "theoretical_loss": 3.3732834201894937, + "tokens_seen": 2733623296 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013889669007021064, + "loss": 1.1974, + "theoretical_loss": 3.373277137718274, + "tokens_seen": 2733688832 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013888064192577734, + "loss": 1.2364, + "theoretical_loss": 3.3732708554398356, + "tokens_seen": 2733754368 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013886459378134404, + "loss": 1.2377, + "theoretical_loss": 3.373264573354168, + "tokens_seen": 2733819904 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013884854563691074, + "loss": 1.2126, + "theoretical_loss": 3.3732582914612603, + "tokens_seen": 2733885440 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013883249749247745, + "loss": 1.2119, + "theoretical_loss": 3.373252009761102, + "tokens_seen": 2733950976 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013881644934804415, + "loss": 1.3156, + "theoretical_loss": 3.373245728253683, + "tokens_seen": 2734016512 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013880040120361085, + "loss": 1.1304, + "theoretical_loss": 3.3732394469389924, + "tokens_seen": 2734082048 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013878435305917755, + "loss": 1.1958, + "theoretical_loss": 3.37323316581702, + "tokens_seen": 2734147584 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013876830491474425, + "loss": 1.1882, + "theoretical_loss": 3.373226884887755, + "tokens_seen": 2734213120 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013875225677031095, + "loss": 1.1849, + "theoretical_loss": 3.3732206041511867, + "tokens_seen": 2734278656 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013873620862587765, + "loss": 1.2658, + "theoretical_loss": 3.373214323607305, + "tokens_seen": 2734344192 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013872016048144435, + "loss": 1.1088, + "theoretical_loss": 3.373208043256099, + "tokens_seen": 2734409728 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013870411233701105, + "loss": 1.2175, + "theoretical_loss": 3.3732017630975584, + "tokens_seen": 2734475264 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013868806419257775, + "loss": 1.1563, + "theoretical_loss": 3.3731954831316724, + "tokens_seen": 2734540800 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013867201604814442, + "loss": 1.2463, + "theoretical_loss": 3.373189203358431, + "tokens_seen": 2734606336 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013865596790371113, + "loss": 1.2166, + "theoretical_loss": 3.3731829237778235, + "tokens_seen": 2734671872 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013863991975927783, + "loss": 1.2374, + "theoretical_loss": 3.3731766443898388, + "tokens_seen": 2734737408 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013862387161484453, + "loss": 1.2839, + "theoretical_loss": 3.373170365194467, + "tokens_seen": 2734802944 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013860782347041123, + "loss": 1.0618, + "theoretical_loss": 3.3731640861916974, + "tokens_seen": 2734868480 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013859177532597793, + "loss": 1.2057, + "theoretical_loss": 3.3731578073815194, + "tokens_seen": 2734934016 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013857572718154463, + "loss": 1.2476, + "theoretical_loss": 3.3731515287639224, + "tokens_seen": 2734999552 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013855967903711133, + "loss": 1.2212, + "theoretical_loss": 3.3731452503388963, + "tokens_seen": 2735065088 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013854363089267803, + "loss": 1.2538, + "theoretical_loss": 3.37313897210643, + "tokens_seen": 2735130624 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013852758274824473, + "loss": 1.1505, + "theoretical_loss": 3.3731326940665136, + "tokens_seen": 2735196160 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013851153460381143, + "loss": 1.1653, + "theoretical_loss": 3.373126416219136, + "tokens_seen": 2735261696 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013849548645937813, + "loss": 1.0993, + "theoretical_loss": 3.3731201385642873, + "tokens_seen": 2735327232 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013847943831494483, + "loss": 1.094, + "theoretical_loss": 3.373113861101956, + "tokens_seen": 2735392768 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013846339017051153, + "loss": 1.2514, + "theoretical_loss": 3.373107583832133, + "tokens_seen": 2735458304 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013844734202607823, + "loss": 1.1607, + "theoretical_loss": 3.373101306754806, + "tokens_seen": 2735523840 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013843129388164496, + "loss": 1.1863, + "theoretical_loss": 3.3730950298699662, + "tokens_seen": 2735589376 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013841524573721166, + "loss": 1.2333, + "theoretical_loss": 3.373088753177602, + "tokens_seen": 2735654912 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013839919759277836, + "loss": 1.133, + "theoretical_loss": 3.3730824766777037, + "tokens_seen": 2735720448 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013838314944834506, + "loss": 1.1916, + "theoretical_loss": 3.37307620037026, + "tokens_seen": 2735785984 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013836710130391176, + "loss": 1.1083, + "theoretical_loss": 3.373069924255261, + "tokens_seen": 2735851520 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013835105315947844, + "loss": 1.1819, + "theoretical_loss": 3.3730636483326952, + "tokens_seen": 2735917056 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013833500501504514, + "loss": 1.1962, + "theoretical_loss": 3.3730573726025534, + "tokens_seen": 2735982592 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013833500501504514, + "loss": 1.2057, + "theoretical_loss": 3.3730510970648244, + "tokens_seen": 2736048128 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013831895687061184, + "loss": 1.1866, + "theoretical_loss": 3.373044821719498, + "tokens_seen": 2736113664 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013830290872617854, + "loss": 1.1743, + "theoretical_loss": 3.373038546566563, + "tokens_seen": 2736179200 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013828686058174524, + "loss": 1.1851, + "theoretical_loss": 3.37303227160601, + "tokens_seen": 2736244736 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013827081243731194, + "loss": 1.2596, + "theoretical_loss": 3.3730259968378276, + "tokens_seen": 2736310272 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013825476429287864, + "loss": 1.1611, + "theoretical_loss": 3.373019722262005, + "tokens_seen": 2736375808 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013823871614844534, + "loss": 1.2087, + "theoretical_loss": 3.3730134478785327, + "tokens_seen": 2736441344 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013822266800401204, + "loss": 1.1345, + "theoretical_loss": 3.3730071736874, + "tokens_seen": 2736506880 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013820661985957874, + "loss": 1.1547, + "theoretical_loss": 3.373000899688596, + "tokens_seen": 2736572416 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013819057171514544, + "loss": 1.4053, + "theoretical_loss": 3.37299462588211, + "tokens_seen": 2736637952 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013817452357071214, + "loss": 1.2554, + "theoretical_loss": 3.3729883522679325, + "tokens_seen": 2736703488 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013815847542627884, + "loss": 1.1834, + "theoretical_loss": 3.372982078846052, + "tokens_seen": 2736769024 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013814242728184555, + "loss": 1.1871, + "theoretical_loss": 3.3729758056164583, + "tokens_seen": 2736834560 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013812637913741225, + "loss": 1.1736, + "theoretical_loss": 3.3729695325791407, + "tokens_seen": 2736900096 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013811033099297895, + "loss": 1.1981, + "theoretical_loss": 3.3729632597340893, + "tokens_seen": 2736965632 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013809428284854565, + "loss": 1.2878, + "theoretical_loss": 3.3729569870812934, + "tokens_seen": 2737031168 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013807823470411235, + "loss": 1.2011, + "theoretical_loss": 3.372950714620742, + "tokens_seen": 2737096704 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013806218655967905, + "loss": 1.1001, + "theoretical_loss": 3.3729444423524253, + "tokens_seen": 2737162240 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013804613841524575, + "loss": 1.2294, + "theoretical_loss": 3.372938170276332, + "tokens_seen": 2737227776 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013803009027081245, + "loss": 1.1041, + "theoretical_loss": 3.3729318983924523, + "tokens_seen": 2737293312 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013801404212637915, + "loss": 1.1852, + "theoretical_loss": 3.3729256267007757, + "tokens_seen": 2737358848 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013799799398194585, + "loss": 1.2433, + "theoretical_loss": 3.3729193552012915, + "tokens_seen": 2737424384 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013798194583751255, + "loss": 1.1765, + "theoretical_loss": 3.372913083893989, + "tokens_seen": 2737489920 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013796589769307923, + "loss": 1.2126, + "theoretical_loss": 3.372906812778858, + "tokens_seen": 2737555456 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013794984954864593, + "loss": 1.2559, + "theoretical_loss": 3.372900541855888, + "tokens_seen": 2737620992 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013793380140421263, + "loss": 1.1391, + "theoretical_loss": 3.3728942711250687, + "tokens_seen": 2737686528 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013791775325977933, + "loss": 1.394, + "theoretical_loss": 3.372888000586389, + "tokens_seen": 2737752064 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013790170511534603, + "loss": 1.2268, + "theoretical_loss": 3.372881730239839, + "tokens_seen": 2737817600 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013788565697091273, + "loss": 1.1367, + "theoretical_loss": 3.3728754600854076, + "tokens_seen": 2737883136 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013786960882647943, + "loss": 1.1617, + "theoretical_loss": 3.372869190123085, + "tokens_seen": 2737948672 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013785356068204613, + "loss": 1.084, + "theoretical_loss": 3.37286292035286, + "tokens_seen": 2738014208 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013783751253761286, + "loss": 1.0983, + "theoretical_loss": 3.372856650774723, + "tokens_seen": 2738079744 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013782146439317956, + "loss": 1.133, + "theoretical_loss": 3.372850381388663, + "tokens_seen": 2738145280 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013780541624874626, + "loss": 1.209, + "theoretical_loss": 3.3728441121946693, + "tokens_seen": 2738210816 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013778936810431296, + "loss": 1.2092, + "theoretical_loss": 3.372837843192732, + "tokens_seen": 2738276352 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013777331995987966, + "loss": 1.3213, + "theoretical_loss": 3.37283157438284, + "tokens_seen": 2738341888 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013775727181544636, + "loss": 1.1821, + "theoretical_loss": 3.3728253057649837, + "tokens_seen": 2738407424 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013774122367101306, + "loss": 1.0523, + "theoretical_loss": 3.3728190373391516, + "tokens_seen": 2738472960 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013772517552657976, + "loss": 1.1831, + "theoretical_loss": 3.372812769105334, + "tokens_seen": 2738538496 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013770912738214646, + "loss": 1.1368, + "theoretical_loss": 3.3728065010635193, + "tokens_seen": 2738604032 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013769307923771316, + "loss": 1.1284, + "theoretical_loss": 3.3728002332136984, + "tokens_seen": 2738669568 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013767703109327986, + "loss": 1.1662, + "theoretical_loss": 3.3727939655558603, + "tokens_seen": 2738735104 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013766098294884654, + "loss": 1.1739, + "theoretical_loss": 3.3727876980899945, + "tokens_seen": 2738800640 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013764493480441324, + "loss": 1.2493, + "theoretical_loss": 3.37278143081609, + "tokens_seen": 2738866176 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013762888665997994, + "loss": 1.2198, + "theoretical_loss": 3.372775163734137, + "tokens_seen": 2738931712 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013761283851554664, + "loss": 1.1005, + "theoretical_loss": 3.3727688968441254, + "tokens_seen": 2738997248 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013759679037111334, + "loss": 1.2505, + "theoretical_loss": 3.3727626301460436, + "tokens_seen": 2739062784 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013758074222668004, + "loss": 1.1582, + "theoretical_loss": 3.372756363639882, + "tokens_seen": 2739128320 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013756469408224674, + "loss": 1.1797, + "theoretical_loss": 3.3727500973256297, + "tokens_seen": 2739193856 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013754864593781344, + "loss": 1.1562, + "theoretical_loss": 3.3727438312032767, + "tokens_seen": 2739259392 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013753259779338014, + "loss": 1.1561, + "theoretical_loss": 3.3727375652728115, + "tokens_seen": 2739324928 + }, + { + "epoch": 1.21, + "objective/train/docs_used": 1011775, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1909780502319336, + "objective/train/theoretical_loss": 3.372731299534225, + "objective/train/tokens_used": 2759850464, + "theoretical_loss": 3.372731299534225, + "tokens_seen": 2739390464 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013751654964894684, + "loss": 1.191, + "theoretical_loss": 3.372731299534225, + "tokens_seen": 2739390464 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013750050150451354, + "loss": 1.2475, + "theoretical_loss": 3.372725033987506, + "tokens_seen": 2739456000 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013748445336008024, + "loss": 1.1693, + "theoretical_loss": 3.3727187686326436, + "tokens_seen": 2739521536 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013746840521564695, + "loss": 1.2093, + "theoretical_loss": 3.3727125034696286, + "tokens_seen": 2739587072 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013745235707121365, + "loss": 1.187, + "theoretical_loss": 3.3727062384984494, + "tokens_seen": 2739652608 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013743630892678035, + "loss": 1.1934, + "theoretical_loss": 3.3726999737190955, + "tokens_seen": 2739718144 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013742026078234705, + "loss": 1.276, + "theoretical_loss": 3.3726937091315574, + "tokens_seen": 2739783680 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013740421263791375, + "loss": 1.1692, + "theoretical_loss": 3.372687444735824, + "tokens_seen": 2739849216 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013738816449348045, + "loss": 1.1676, + "theoretical_loss": 3.3726811805318846, + "tokens_seen": 2739914752 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013737211634904715, + "loss": 1.2011, + "theoretical_loss": 3.3726749165197294, + "tokens_seen": 2739980288 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013735606820461385, + "loss": 1.1256, + "theoretical_loss": 3.372668652699348, + "tokens_seen": 2740045824 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013734002006018055, + "loss": 1.1196, + "theoretical_loss": 3.372662389070729, + "tokens_seen": 2740111360 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013732397191574725, + "loss": 1.1353, + "theoretical_loss": 3.3726561256338625, + "tokens_seen": 2740176896 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013730792377131395, + "loss": 1.2586, + "theoretical_loss": 3.372649862388738, + "tokens_seen": 2740242432 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013729187562688065, + "loss": 1.2679, + "theoretical_loss": 3.3726435993353454, + "tokens_seen": 2740307968 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013727582748244733, + "loss": 1.232, + "theoretical_loss": 3.372637336473674, + "tokens_seen": 2740373504 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013725977933801403, + "loss": 1.301, + "theoretical_loss": 3.372631073803713, + "tokens_seen": 2740439040 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013724373119358073, + "loss": 1.1158, + "theoretical_loss": 3.372624811325452, + "tokens_seen": 2740504576 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013722768304914745, + "loss": 1.1888, + "theoretical_loss": 3.3726185490388816, + "tokens_seen": 2740570112 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013721163490471416, + "loss": 1.2525, + "theoretical_loss": 3.37261228694399, + "tokens_seen": 2740635648 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013719558676028086, + "loss": 1.1806, + "theoretical_loss": 3.372606025040767, + "tokens_seen": 2740701184 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013717953861584756, + "loss": 1.1685, + "theoretical_loss": 3.372599763329203, + "tokens_seen": 2740766720 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013716349047141426, + "loss": 1.0926, + "theoretical_loss": 3.3725935018092867, + "tokens_seen": 2740832256 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013714744232698096, + "loss": 1.1567, + "theoretical_loss": 3.3725872404810078, + "tokens_seen": 2740897792 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013713139418254766, + "loss": 1.0905, + "theoretical_loss": 3.372580979344356, + "tokens_seen": 2740963328 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013711534603811436, + "loss": 1.1752, + "theoretical_loss": 3.372574718399321, + "tokens_seen": 2741028864 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013709929789368106, + "loss": 1.2524, + "theoretical_loss": 3.372568457645892, + "tokens_seen": 2741094400 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013708324974924776, + "loss": 1.2127, + "theoretical_loss": 3.372562197084059, + "tokens_seen": 2741159936 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013706720160481446, + "loss": 1.2306, + "theoretical_loss": 3.3725559367138116, + "tokens_seen": 2741225472 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013705115346038116, + "loss": 1.2312, + "theoretical_loss": 3.3725496765351384, + "tokens_seen": 2741291008 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013703510531594786, + "loss": 1.1545, + "theoretical_loss": 3.37254341654803, + "tokens_seen": 2741356544 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013701905717151456, + "loss": 1.2351, + "theoretical_loss": 3.3725371567524753, + "tokens_seen": 2741422080 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013700300902708126, + "loss": 1.2207, + "theoretical_loss": 3.372530897148464, + "tokens_seen": 2741487616 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013698696088264796, + "loss": 1.218, + "theoretical_loss": 3.3725246377359865, + "tokens_seen": 2741553152 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013697091273821466, + "loss": 1.1464, + "theoretical_loss": 3.3725183785150312, + "tokens_seen": 2741618688 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013695486459378134, + "loss": 1.217, + "theoretical_loss": 3.372512119485588, + "tokens_seen": 2741684224 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013693881644934804, + "loss": 1.2234, + "theoretical_loss": 3.372505860647647, + "tokens_seen": 2741749760 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013692276830491474, + "loss": 1.1355, + "theoretical_loss": 3.372499602001197, + "tokens_seen": 2741815296 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013690672016048144, + "loss": 1.2267, + "theoretical_loss": 3.372493343546228, + "tokens_seen": 2741880832 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013689067201604814, + "loss": 1.1337, + "theoretical_loss": 3.3724870852827293, + "tokens_seen": 2741946368 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013687462387161484, + "loss": 1.2523, + "theoretical_loss": 3.3724808272106905, + "tokens_seen": 2742011904 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013685857572718154, + "loss": 1.0563, + "theoretical_loss": 3.3724745693301017, + "tokens_seen": 2742077440 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013684252758274824, + "loss": 1.3015, + "theoretical_loss": 3.372468311640952, + "tokens_seen": 2742142976 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013682647943831494, + "loss": 1.2102, + "theoretical_loss": 3.3724620541432313, + "tokens_seen": 2742208512 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013681043129388164, + "loss": 1.2615, + "theoretical_loss": 3.3724557968369284, + "tokens_seen": 2742274048 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013679438314944834, + "loss": 1.2438, + "theoretical_loss": 3.3724495397220338, + "tokens_seen": 2742339584 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013677833500501505, + "loss": 1.1924, + "theoretical_loss": 3.3724432827985362, + "tokens_seen": 2742405120 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013676228686058175, + "loss": 1.2282, + "theoretical_loss": 3.372437026066426, + "tokens_seen": 2742470656 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013674623871614845, + "loss": 1.165, + "theoretical_loss": 3.3724307695256925, + "tokens_seen": 2742536192 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013673019057171515, + "loss": 1.2455, + "theoretical_loss": 3.372424513176325, + "tokens_seen": 2742601728 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013671414242728185, + "loss": 1.2028, + "theoretical_loss": 3.372418257018313, + "tokens_seen": 2742667264 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013669809428284855, + "loss": 1.2059, + "theoretical_loss": 3.3724120010516465, + "tokens_seen": 2742732800 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013668204613841525, + "loss": 1.2121, + "theoretical_loss": 3.372405745276315, + "tokens_seen": 2742798336 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013666599799398195, + "loss": 1.0571, + "theoretical_loss": 3.3723994896923077, + "tokens_seen": 2742863872 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013664994984954865, + "loss": 1.2255, + "theoretical_loss": 3.372393234299615, + "tokens_seen": 2742929408 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013663390170511535, + "loss": 1.1786, + "theoretical_loss": 3.3723869790982257, + "tokens_seen": 2742994944 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013661785356068205, + "loss": 1.2305, + "theoretical_loss": 3.372380724088129, + "tokens_seen": 2743060480 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013660180541624875, + "loss": 1.1481, + "theoretical_loss": 3.372374469269316, + "tokens_seen": 2743126016 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013658575727181545, + "loss": 1.2584, + "theoretical_loss": 3.372368214641775, + "tokens_seen": 2743191552 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013656970912738215, + "loss": 1.0644, + "theoretical_loss": 3.3723619602054957, + "tokens_seen": 2743257088 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013655366098294885, + "loss": 1.2889, + "theoretical_loss": 3.372355705960468, + "tokens_seen": 2743322624 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013653761283851555, + "loss": 1.1585, + "theoretical_loss": 3.3723494519066817, + "tokens_seen": 2743388160 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013652156469408226, + "loss": 1.2237, + "theoretical_loss": 3.372343198044126, + "tokens_seen": 2743453696 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013650551654964896, + "loss": 1.1905, + "theoretical_loss": 3.3723369443727904, + "tokens_seen": 2743519232 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013648946840521566, + "loss": 1.1954, + "theoretical_loss": 3.3723306908926647, + "tokens_seen": 2743584768 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013647342026078236, + "loss": 1.2477, + "theoretical_loss": 3.372324437603739, + "tokens_seen": 2743650304 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013645737211634906, + "loss": 1.1458, + "theoretical_loss": 3.3723181845060015, + "tokens_seen": 2743715840 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013644132397191576, + "loss": 1.2732, + "theoretical_loss": 3.372311931599443, + "tokens_seen": 2743781376 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013642527582748246, + "loss": 1.1379, + "theoretical_loss": 3.372305678884053, + "tokens_seen": 2743846912 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013640922768304916, + "loss": 1.1515, + "theoretical_loss": 3.3722994263598203, + "tokens_seen": 2743912448 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013639317953861586, + "loss": 1.1094, + "theoretical_loss": 3.3722931740267352, + "tokens_seen": 2743977984 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013637713139418256, + "loss": 1.2792, + "theoretical_loss": 3.372286921884787, + "tokens_seen": 2744043520 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013636108324974926, + "loss": 1.1498, + "theoretical_loss": 3.372280669933966, + "tokens_seen": 2744109056 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013634503510531596, + "loss": 1.1438, + "theoretical_loss": 3.3722744181742605, + "tokens_seen": 2744174592 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013632898696088266, + "loss": 1.2423, + "theoretical_loss": 3.372268166605661, + "tokens_seen": 2744240128 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013631293881644936, + "loss": 1.1182, + "theoretical_loss": 3.3722619152281568, + "tokens_seen": 2744305664 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013629689067201606, + "loss": 1.3236, + "theoretical_loss": 3.3722556640417376, + "tokens_seen": 2744371200 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013628084252758276, + "loss": 1.1986, + "theoretical_loss": 3.372249413046393, + "tokens_seen": 2744436736 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013626479438314947, + "loss": 1.1287, + "theoretical_loss": 3.3722431622421123, + "tokens_seen": 2744502272 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013624874623871614, + "loss": 1.2708, + "theoretical_loss": 3.3722369116288857, + "tokens_seen": 2744567808 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013623269809428284, + "loss": 1.1786, + "theoretical_loss": 3.3722306612067023, + "tokens_seen": 2744633344 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013621664994984954, + "loss": 1.1123, + "theoretical_loss": 3.372224410975552, + "tokens_seen": 2744698880 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013620060180541624, + "loss": 1.1361, + "theoretical_loss": 3.3722181609354243, + "tokens_seen": 2744764416 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013618455366098294, + "loss": 1.2224, + "theoretical_loss": 3.3722119110863082, + "tokens_seen": 2744829952 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013616850551654964, + "loss": 1.2884, + "theoretical_loss": 3.3722056614281946, + "tokens_seen": 2744895488 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013615245737211634, + "loss": 1.1178, + "theoretical_loss": 3.3721994119610716, + "tokens_seen": 2744961024 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013613640922768304, + "loss": 1.1599, + "theoretical_loss": 3.3721931626849297, + "tokens_seen": 2745026560 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013612036108324974, + "loss": 1.1952, + "theoretical_loss": 3.372186913599759, + "tokens_seen": 2745092096 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013610431293881644, + "loss": 1.1137, + "theoretical_loss": 3.372180664705548, + "tokens_seen": 2745157632 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013608826479438315, + "loss": 1.1869, + "theoretical_loss": 3.3721744160022866, + "tokens_seen": 2745223168 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013607221664994985, + "loss": 1.2517, + "theoretical_loss": 3.372168167489965, + "tokens_seen": 2745288704 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013605616850551655, + "loss": 1.1578, + "theoretical_loss": 3.372161919168572, + "tokens_seen": 2745354240 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013604012036108325, + "loss": 1.1891, + "theoretical_loss": 3.372155671038098, + "tokens_seen": 2745419776 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013602407221664998, + "loss": 1.2297, + "theoretical_loss": 3.372149423098532, + "tokens_seen": 2745485312 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013600802407221668, + "loss": 1.1784, + "theoretical_loss": 3.372143175349864, + "tokens_seen": 2745550848 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013599197592778338, + "loss": 1.1635, + "theoretical_loss": 3.372136927792083, + "tokens_seen": 2745616384 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013597592778335008, + "loss": 1.1678, + "theoretical_loss": 3.3721306804251796, + "tokens_seen": 2745681920 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013595987963891678, + "loss": 1.2819, + "theoretical_loss": 3.3721244332491422, + "tokens_seen": 2745747456 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013594383149448345, + "loss": 1.1753, + "theoretical_loss": 3.3721181862639615, + "tokens_seen": 2745812992 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013592778335005015, + "loss": 1.1308, + "theoretical_loss": 3.372111939469627, + "tokens_seen": 2745878528 + }, + { + "epoch": 1.21, + "objective/train/docs_used": 1014236, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1804070472717285, + "objective/train/theoretical_loss": 3.372105692866127, + "objective/train/tokens_used": 2766404064, + "theoretical_loss": 3.372105692866127, + "tokens_seen": 2745944064 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013591173520561685, + "loss": 1.1804, + "theoretical_loss": 3.372105692866127, + "tokens_seen": 2745944064 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013589568706118355, + "loss": 1.1612, + "theoretical_loss": 3.372099446453453, + "tokens_seen": 2746009600 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013587963891675025, + "loss": 1.142, + "theoretical_loss": 3.3720932002315935, + "tokens_seen": 2746075136 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013586359077231695, + "loss": 1.3104, + "theoretical_loss": 3.372086954200538, + "tokens_seen": 2746140672 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013584754262788365, + "loss": 1.2228, + "theoretical_loss": 3.3720807083602766, + "tokens_seen": 2746206208 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013583149448345036, + "loss": 1.1967, + "theoretical_loss": 3.372074462710799, + "tokens_seen": 2746271744 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013581544633901706, + "loss": 1.1645, + "theoretical_loss": 3.3720682172520946, + "tokens_seen": 2746337280 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013579939819458376, + "loss": 1.1372, + "theoretical_loss": 3.372061971984153, + "tokens_seen": 2746402816 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013578335005015046, + "loss": 1.2109, + "theoretical_loss": 3.3720557269069635, + "tokens_seen": 2746468352 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013576730190571716, + "loss": 1.2055, + "theoretical_loss": 3.3720494820205165, + "tokens_seen": 2746533888 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013575125376128386, + "loss": 1.222, + "theoretical_loss": 3.3720432373248013, + "tokens_seen": 2746599424 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013573520561685056, + "loss": 1.1497, + "theoretical_loss": 3.372036992819807, + "tokens_seen": 2746664960 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013571915747241726, + "loss": 1.2754, + "theoretical_loss": 3.372030748505524, + "tokens_seen": 2746730496 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013570310932798396, + "loss": 1.0973, + "theoretical_loss": 3.372024504381941, + "tokens_seen": 2746796032 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013568706118355066, + "loss": 1.1281, + "theoretical_loss": 3.3720182604490487, + "tokens_seen": 2746861568 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013567101303911736, + "loss": 1.2071, + "theoretical_loss": 3.3720120167068357, + "tokens_seen": 2746927104 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013565496489468406, + "loss": 1.1634, + "theoretical_loss": 3.3720057731552924, + "tokens_seen": 2746992640 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013563891675025076, + "loss": 1.196, + "theoretical_loss": 3.3719995297944085, + "tokens_seen": 2747058176 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013562286860581746, + "loss": 1.1481, + "theoretical_loss": 3.371993286624173, + "tokens_seen": 2747123712 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013560682046138416, + "loss": 1.1735, + "theoretical_loss": 3.371987043644576, + "tokens_seen": 2747189248 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013559077231695087, + "loss": 1.217, + "theoretical_loss": 3.371980800855607, + "tokens_seen": 2747254784 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013557472417251757, + "loss": 1.1773, + "theoretical_loss": 3.3719745582572553, + "tokens_seen": 2747320320 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013555867602808424, + "loss": 1.192, + "theoretical_loss": 3.371968315849511, + "tokens_seen": 2747385856 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013554262788365094, + "loss": 1.1273, + "theoretical_loss": 3.371962073632363, + "tokens_seen": 2747451392 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013552657973921764, + "loss": 1.26, + "theoretical_loss": 3.3719558316058023, + "tokens_seen": 2747516928 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013551053159478434, + "loss": 1.1439, + "theoretical_loss": 3.3719495897698177, + "tokens_seen": 2747582464 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013549448345035104, + "loss": 1.2728, + "theoretical_loss": 3.3719433481243986, + "tokens_seen": 2747648000 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013547843530591774, + "loss": 1.0943, + "theoretical_loss": 3.371937106669535, + "tokens_seen": 2747713536 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013546238716148444, + "loss": 1.3017, + "theoretical_loss": 3.3719308654052162, + "tokens_seen": 2747779072 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013544633901705114, + "loss": 1.2037, + "theoretical_loss": 3.3719246243314323, + "tokens_seen": 2747844608 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013543029087261787, + "loss": 1.1872, + "theoretical_loss": 3.3719183834481727, + "tokens_seen": 2747910144 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013541424272818457, + "loss": 1.1951, + "theoretical_loss": 3.371912142755427, + "tokens_seen": 2747975680 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013539819458375127, + "loss": 1.2324, + "theoretical_loss": 3.371905902253185, + "tokens_seen": 2748041216 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013538214643931797, + "loss": 1.2141, + "theoretical_loss": 3.371899661941436, + "tokens_seen": 2748106752 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013536609829488467, + "loss": 1.1799, + "theoretical_loss": 3.37189342182017, + "tokens_seen": 2748172288 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013535005015045137, + "loss": 1.2268, + "theoretical_loss": 3.3718871818893765, + "tokens_seen": 2748237824 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013533400200601808, + "loss": 1.3279, + "theoretical_loss": 3.3718809421490454, + "tokens_seen": 2748303360 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013531795386158478, + "loss": 1.1836, + "theoretical_loss": 3.371874702599166, + "tokens_seen": 2748368896 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013530190571715148, + "loss": 1.1947, + "theoretical_loss": 3.371868463239728, + "tokens_seen": 2748434432 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013528585757271818, + "loss": 1.2295, + "theoretical_loss": 3.3718622240707212, + "tokens_seen": 2748499968 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013526980942828488, + "loss": 1.1823, + "theoretical_loss": 3.371855985092135, + "tokens_seen": 2748565504 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013525376128385158, + "loss": 1.0956, + "theoretical_loss": 3.371849746303959, + "tokens_seen": 2748631040 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013523771313941825, + "loss": 1.2636, + "theoretical_loss": 3.371843507706184, + "tokens_seen": 2748696576 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013522166499498495, + "loss": 1.1934, + "theoretical_loss": 3.3718372692987977, + "tokens_seen": 2748762112 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013520561685055165, + "loss": 1.178, + "theoretical_loss": 3.371831031081791, + "tokens_seen": 2748827648 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013518956870611835, + "loss": 1.1518, + "theoretical_loss": 3.3718247930551537, + "tokens_seen": 2748893184 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013517352056168505, + "loss": 1.2061, + "theoretical_loss": 3.3718185552188746, + "tokens_seen": 2748958720 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013515747241725176, + "loss": 1.2489, + "theoretical_loss": 3.371812317572944, + "tokens_seen": 2749024256 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013514142427281846, + "loss": 1.2196, + "theoretical_loss": 3.3718060801173513, + "tokens_seen": 2749089792 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013512537612838516, + "loss": 1.2161, + "theoretical_loss": 3.371799842852086, + "tokens_seen": 2749155328 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013510932798395186, + "loss": 1.2767, + "theoretical_loss": 3.371793605777139, + "tokens_seen": 2749220864 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013509327983951856, + "loss": 1.1822, + "theoretical_loss": 3.3717873688924977, + "tokens_seen": 2749286400 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013507723169508526, + "loss": 1.1772, + "theoretical_loss": 3.3717811321981537, + "tokens_seen": 2749351936 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013506118355065196, + "loss": 1.2103, + "theoretical_loss": 3.3717748956940956, + "tokens_seen": 2749417472 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013504513540621866, + "loss": 1.3151, + "theoretical_loss": 3.3717686593803133, + "tokens_seen": 2749483008 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013502908726178536, + "loss": 1.0647, + "theoretical_loss": 3.371762423256797, + "tokens_seen": 2749548544 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013501303911735206, + "loss": 1.2401, + "theoretical_loss": 3.3717561873235358, + "tokens_seen": 2749614080 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013499699097291876, + "loss": 1.2503, + "theoretical_loss": 3.3717499515805196, + "tokens_seen": 2749679616 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013498094282848546, + "loss": 1.188, + "theoretical_loss": 3.3717437160277375, + "tokens_seen": 2749745152 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013496489468405216, + "loss": 1.1538, + "theoretical_loss": 3.37173748066518, + "tokens_seen": 2749810688 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013494884653961886, + "loss": 1.1601, + "theoretical_loss": 3.3717312454928363, + "tokens_seen": 2749876224 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013493279839518556, + "loss": 1.2464, + "theoretical_loss": 3.371725010510696, + "tokens_seen": 2749941760 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013491675025075226, + "loss": 1.2283, + "theoretical_loss": 3.371718775718749, + "tokens_seen": 2750007296 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013490070210631897, + "loss": 1.2247, + "theoretical_loss": 3.3717125411169846, + "tokens_seen": 2750072832 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013488465396188567, + "loss": 1.1617, + "theoretical_loss": 3.371706306705393, + "tokens_seen": 2750138368 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013486860581745237, + "loss": 1.2669, + "theoretical_loss": 3.3717000724839634, + "tokens_seen": 2750203904 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013485255767301904, + "loss": 1.2108, + "theoretical_loss": 3.371693838452686, + "tokens_seen": 2750269440 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013483650952858574, + "loss": 1.2526, + "theoretical_loss": 3.37168760461155, + "tokens_seen": 2750334976 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013482046138415247, + "loss": 1.2416, + "theoretical_loss": 3.371681370960545, + "tokens_seen": 2750400512 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013480441323971917, + "loss": 1.1667, + "theoretical_loss": 3.371675137499661, + "tokens_seen": 2750466048 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013478836509528587, + "loss": 1.2662, + "theoretical_loss": 3.371668904228888, + "tokens_seen": 2750531584 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013477231695085257, + "loss": 1.2137, + "theoretical_loss": 3.371662671148215, + "tokens_seen": 2750597120 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013475626880641927, + "loss": 1.2129, + "theoretical_loss": 3.3716564382576317, + "tokens_seen": 2750662656 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013474022066198597, + "loss": 1.1824, + "theoretical_loss": 3.371650205557128, + "tokens_seen": 2750728192 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013472417251755267, + "loss": 1.2508, + "theoretical_loss": 3.3716439730466936, + "tokens_seen": 2750793728 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013470812437311937, + "loss": 1.2118, + "theoretical_loss": 3.3716377407263183, + "tokens_seen": 2750859264 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013469207622868607, + "loss": 1.2756, + "theoretical_loss": 3.3716315085959914, + "tokens_seen": 2750924800 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013467602808425277, + "loss": 1.2683, + "theoretical_loss": 3.3716252766557027, + "tokens_seen": 2750990336 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013465997993981947, + "loss": 1.2597, + "theoretical_loss": 3.371619044905442, + "tokens_seen": 2751055872 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013464393179538618, + "loss": 1.1801, + "theoretical_loss": 3.371612813345199, + "tokens_seen": 2751121408 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013462788365095288, + "loss": 1.2434, + "theoretical_loss": 3.371606581974963, + "tokens_seen": 2751186944 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013461183550651958, + "loss": 1.1181, + "theoretical_loss": 3.3716003507947248, + "tokens_seen": 2751252480 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013459578736208628, + "loss": 1.26, + "theoretical_loss": 3.3715941198044725, + "tokens_seen": 2751318016 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013457973921765298, + "loss": 1.1915, + "theoretical_loss": 3.371587889004197, + "tokens_seen": 2751383552 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013456369107321968, + "loss": 1.2012, + "theoretical_loss": 3.3715816583938873, + "tokens_seen": 2751449088 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013454764292878638, + "loss": 1.2701, + "theoretical_loss": 3.3715754279735335, + "tokens_seen": 2751514624 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013453159478435305, + "loss": 1.2962, + "theoretical_loss": 3.371569197743125, + "tokens_seen": 2751580160 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013451554663991975, + "loss": 1.2658, + "theoretical_loss": 3.3715629677026513, + "tokens_seen": 2751645696 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013449949849548645, + "loss": 1.2324, + "theoretical_loss": 3.371556737852103, + "tokens_seen": 2751711232 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013448345035105315, + "loss": 1.355, + "theoretical_loss": 3.3715505081914685, + "tokens_seen": 2751776768 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013446740220661986, + "loss": 1.1528, + "theoretical_loss": 3.3715442787207386, + "tokens_seen": 2751842304 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013445135406218656, + "loss": 1.2247, + "theoretical_loss": 3.3715380494399025, + "tokens_seen": 2751907840 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013443530591775326, + "loss": 1.1608, + "theoretical_loss": 3.3715318203489497, + "tokens_seen": 2751973376 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013441925777331996, + "loss": 1.2464, + "theoretical_loss": 3.3715255914478703, + "tokens_seen": 2752038912 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013440320962888666, + "loss": 1.1012, + "theoretical_loss": 3.371519362736654, + "tokens_seen": 2752104448 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013438716148445336, + "loss": 1.1517, + "theoretical_loss": 3.37151313421529, + "tokens_seen": 2752169984 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013437111334002006, + "loss": 1.2236, + "theoretical_loss": 3.3715069058837686, + "tokens_seen": 2752235520 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013435506519558676, + "loss": 1.2348, + "theoretical_loss": 3.371500677742079, + "tokens_seen": 2752301056 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013433901705115346, + "loss": 1.2776, + "theoretical_loss": 3.3714944497902115, + "tokens_seen": 2752366592 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013432296890672016, + "loss": 1.1183, + "theoretical_loss": 3.371488222028155, + "tokens_seen": 2752432128 + }, + { + "epoch": 1.21, + "objective/train/docs_used": 1016528, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.213416337966919, + "objective/train/theoretical_loss": 3.371481994455899, + "objective/train/tokens_used": 2772957664, + "theoretical_loss": 3.371481994455899, + "tokens_seen": 2752497664 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013430692076228686, + "loss": 1.2134, + "theoretical_loss": 3.371481994455899, + "tokens_seen": 2752497664 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013429087261785356, + "loss": 1.2436, + "theoretical_loss": 3.3714757670734348, + "tokens_seen": 2752563200 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013427482447342026, + "loss": 1.0827, + "theoretical_loss": 3.3714695398807506, + "tokens_seen": 2752628736 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013425877632898696, + "loss": 1.166, + "theoretical_loss": 3.3714633128778364, + "tokens_seen": 2752694272 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013424272818455366, + "loss": 1.2138, + "theoretical_loss": 3.3714570860646824, + "tokens_seen": 2752759808 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013422668004012036, + "loss": 1.235, + "theoretical_loss": 3.371450859441278, + "tokens_seen": 2752825344 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013421063189568707, + "loss": 1.1368, + "theoretical_loss": 3.3714446330076124, + "tokens_seen": 2752890880 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013419458375125377, + "loss": 1.1141, + "theoretical_loss": 3.3714384067636765, + "tokens_seen": 2752956416 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013417853560682047, + "loss": 1.2303, + "theoretical_loss": 3.3714321807094585, + "tokens_seen": 2753021952 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013416248746238717, + "loss": 1.3175, + "theoretical_loss": 3.3714259548449492, + "tokens_seen": 2753087488 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013414643931795387, + "loss": 1.156, + "theoretical_loss": 3.371419729170138, + "tokens_seen": 2753153024 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013413039117352057, + "loss": 1.1307, + "theoretical_loss": 3.3714135036850146, + "tokens_seen": 2753218560 + }, + { + "epoch": 1.21, + "learning_rate": 0.00013411434302908727, + "loss": 1.1874, + "theoretical_loss": 3.3714072783895688, + "tokens_seen": 2753284096 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013409829488465397, + "loss": 1.3034, + "theoretical_loss": 3.37140105328379, + "tokens_seen": 2753349632 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013408224674022067, + "loss": 1.2186, + "theoretical_loss": 3.3713948283676682, + "tokens_seen": 2753415168 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013406619859578737, + "loss": 1.185, + "theoretical_loss": 3.371388603641193, + "tokens_seen": 2753480704 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013405015045135407, + "loss": 1.1687, + "theoretical_loss": 3.371382379104354, + "tokens_seen": 2753546240 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013403410230692077, + "loss": 1.2058, + "theoretical_loss": 3.371376154757141, + "tokens_seen": 2753611776 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013401805416248747, + "loss": 1.1697, + "theoretical_loss": 3.371369930599544, + "tokens_seen": 2753677312 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013400200601805417, + "loss": 1.24, + "theoretical_loss": 3.3713637066315525, + "tokens_seen": 2753742848 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013398595787362087, + "loss": 1.2027, + "theoretical_loss": 3.371357482853156, + "tokens_seen": 2753808384 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013396990972918757, + "loss": 1.0971, + "theoretical_loss": 3.3713512592643444, + "tokens_seen": 2753873920 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013395386158475428, + "loss": 1.2029, + "theoretical_loss": 3.371345035865107, + "tokens_seen": 2753939456 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013393781344032098, + "loss": 1.1686, + "theoretical_loss": 3.3713388126554347, + "tokens_seen": 2754004992 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013392176529588768, + "loss": 1.2246, + "theoretical_loss": 3.371332589635316, + "tokens_seen": 2754070528 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013390571715145438, + "loss": 1.1795, + "theoretical_loss": 3.371326366804741, + "tokens_seen": 2754136064 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013388966900702108, + "loss": 1.2511, + "theoretical_loss": 3.3713201441636995, + "tokens_seen": 2754201600 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013387362086258778, + "loss": 1.156, + "theoretical_loss": 3.371313921712181, + "tokens_seen": 2754267136 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013385757271815448, + "loss": 1.2163, + "theoretical_loss": 3.371307699450176, + "tokens_seen": 2754332672 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013384152457372115, + "loss": 1.1494, + "theoretical_loss": 3.3713014773776724, + "tokens_seen": 2754398208 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013382547642928785, + "loss": 1.0879, + "theoretical_loss": 3.3712952554946622, + "tokens_seen": 2754463744 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013380942828485455, + "loss": 1.2688, + "theoretical_loss": 3.3712890338011334, + "tokens_seen": 2754529280 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013379338014042125, + "loss": 1.2078, + "theoretical_loss": 3.371282812297077, + "tokens_seen": 2754594816 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013377733199598796, + "loss": 1.276, + "theoretical_loss": 3.3712765909824816, + "tokens_seen": 2754660352 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013376128385155466, + "loss": 1.2393, + "theoretical_loss": 3.3712703698573376, + "tokens_seen": 2754725888 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013374523570712136, + "loss": 1.163, + "theoretical_loss": 3.3712641489216346, + "tokens_seen": 2754791424 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013372918756268806, + "loss": 1.1857, + "theoretical_loss": 3.3712579281753623, + "tokens_seen": 2754856960 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013371313941825476, + "loss": 1.1221, + "theoretical_loss": 3.37125170761851, + "tokens_seen": 2754922496 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013369709127382146, + "loss": 1.0948, + "theoretical_loss": 3.371245487251068, + "tokens_seen": 2754988032 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013368104312938816, + "loss": 1.2111, + "theoretical_loss": 3.371239267073026, + "tokens_seen": 2755053568 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013366499498495486, + "loss": 1.152, + "theoretical_loss": 3.371233047084374, + "tokens_seen": 2755119104 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013364894684052156, + "loss": 1.2633, + "theoretical_loss": 3.3712268272851005, + "tokens_seen": 2755184640 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001336328986960883, + "loss": 1.2131, + "theoretical_loss": 3.3712206076751965, + "tokens_seen": 2755250176 + }, + { + "epoch": 1.22, + "learning_rate": 0.000133616850551655, + "loss": 1.2629, + "theoretical_loss": 3.371214388254651, + "tokens_seen": 2755315712 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001336008024072217, + "loss": 1.1284, + "theoretical_loss": 3.371208169023454, + "tokens_seen": 2755381248 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001335847542627884, + "loss": 1.2653, + "theoretical_loss": 3.3712019499815953, + "tokens_seen": 2755446784 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001335687061183551, + "loss": 1.1903, + "theoretical_loss": 3.3711957311290646, + "tokens_seen": 2755512320 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001335526579739218, + "loss": 1.1244, + "theoretical_loss": 3.3711895124658513, + "tokens_seen": 2755577856 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001335366098294885, + "loss": 1.2031, + "theoretical_loss": 3.3711832939919457, + "tokens_seen": 2755643392 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013352056168505517, + "loss": 1.3743, + "theoretical_loss": 3.371177075707337, + "tokens_seen": 2755708928 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013350451354062187, + "loss": 1.1465, + "theoretical_loss": 3.3711708576120154, + "tokens_seen": 2755774464 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013348846539618857, + "loss": 1.1712, + "theoretical_loss": 3.3711646397059707, + "tokens_seen": 2755840000 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013347241725175527, + "loss": 1.3865, + "theoretical_loss": 3.3711584219891924, + "tokens_seen": 2755905536 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013345636910732197, + "loss": 1.2011, + "theoretical_loss": 3.3711522044616697, + "tokens_seen": 2755971072 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013344032096288867, + "loss": 1.222, + "theoretical_loss": 3.371145987123393, + "tokens_seen": 2756036608 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013342427281845537, + "loss": 1.1735, + "theoretical_loss": 3.371139769974352, + "tokens_seen": 2756102144 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013340822467402207, + "loss": 1.2019, + "theoretical_loss": 3.371133553014536, + "tokens_seen": 2756167680 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013339217652958877, + "loss": 1.3244, + "theoretical_loss": 3.3711273362439353, + "tokens_seen": 2756233216 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013337612838515547, + "loss": 1.2565, + "theoretical_loss": 3.37112111966254, + "tokens_seen": 2756298752 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013336008024072217, + "loss": 1.1687, + "theoretical_loss": 3.3711149032703385, + "tokens_seen": 2756364288 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013334403209628887, + "loss": 1.1792, + "theoretical_loss": 3.3711086870673213, + "tokens_seen": 2756429824 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013332798395185557, + "loss": 1.1927, + "theoretical_loss": 3.3711024710534785, + "tokens_seen": 2756495360 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013331193580742227, + "loss": 1.1016, + "theoretical_loss": 3.371096255228799, + "tokens_seen": 2756560896 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013329588766298897, + "loss": 1.2363, + "theoretical_loss": 3.3710900395932732, + "tokens_seen": 2756626432 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013327983951855568, + "loss": 1.2409, + "theoretical_loss": 3.3710838241468912, + "tokens_seen": 2756691968 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013326379137412238, + "loss": 1.1163, + "theoretical_loss": 3.371077608889642, + "tokens_seen": 2756757504 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013324774322968908, + "loss": 1.2044, + "theoretical_loss": 3.371071393821515, + "tokens_seen": 2756823040 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013323169508525578, + "loss": 1.26, + "theoretical_loss": 3.3710651789425015, + "tokens_seen": 2756888576 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013321564694082248, + "loss": 1.2142, + "theoretical_loss": 3.3710589642525894, + "tokens_seen": 2756954112 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013319959879638918, + "loss": 1.2088, + "theoretical_loss": 3.3710527497517697, + "tokens_seen": 2757019648 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013318355065195588, + "loss": 1.2438, + "theoretical_loss": 3.3710465354400316, + "tokens_seen": 2757085184 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013316750250752258, + "loss": 1.1941, + "theoretical_loss": 3.3710403213173654, + "tokens_seen": 2757150720 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013315145436308928, + "loss": 1.1419, + "theoretical_loss": 3.3710341073837604, + "tokens_seen": 2757216256 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013313540621865595, + "loss": 1.1371, + "theoretical_loss": 3.371027893639206, + "tokens_seen": 2757281792 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013311935807422265, + "loss": 1.1312, + "theoretical_loss": 3.3710216800836927, + "tokens_seen": 2757347328 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013310330992978936, + "loss": 1.1917, + "theoretical_loss": 3.37101546671721, + "tokens_seen": 2757412864 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013308726178535606, + "loss": 1.1886, + "theoretical_loss": 3.3710092535397473, + "tokens_seen": 2757478400 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013307121364092276, + "loss": 1.2387, + "theoretical_loss": 3.371003040551295, + "tokens_seen": 2757543936 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013305516549648946, + "loss": 1.2384, + "theoretical_loss": 3.3709968277518425, + "tokens_seen": 2757609472 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013303911735205616, + "loss": 1.2662, + "theoretical_loss": 3.3709906151413795, + "tokens_seen": 2757675008 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013302306920762289, + "loss": 1.254, + "theoretical_loss": 3.370984402719896, + "tokens_seen": 2757740544 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013300702106318959, + "loss": 1.1148, + "theoretical_loss": 3.3709781904873815, + "tokens_seen": 2757806080 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001329909729187563, + "loss": 1.2621, + "theoretical_loss": 3.370971978443826, + "tokens_seen": 2757871616 + }, + { + "epoch": 1.22, + "learning_rate": 0.000132974924774323, + "loss": 1.2454, + "theoretical_loss": 3.3709657665892188, + "tokens_seen": 2757937152 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001329588766298897, + "loss": 1.2089, + "theoretical_loss": 3.3709595549235503, + "tokens_seen": 2758002688 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001329428284854564, + "loss": 1.232, + "theoretical_loss": 3.3709533434468097, + "tokens_seen": 2758068224 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001329267803410231, + "loss": 1.1075, + "theoretical_loss": 3.370947132158987, + "tokens_seen": 2758133760 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001329107321965898, + "loss": 1.2901, + "theoretical_loss": 3.3709409210600723, + "tokens_seen": 2758199296 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001328946840521565, + "loss": 1.1046, + "theoretical_loss": 3.370934710150055, + "tokens_seen": 2758264832 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001328786359077232, + "loss": 1.1835, + "theoretical_loss": 3.3709284994289246, + "tokens_seen": 2758330368 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001328625877632899, + "loss": 1.2566, + "theoretical_loss": 3.3709222888966717, + "tokens_seen": 2758395904 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001328465396188566, + "loss": 1.3408, + "theoretical_loss": 3.370916078553285, + "tokens_seen": 2758461440 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001328304914744233, + "loss": 1.2768, + "theoretical_loss": 3.370909868398755, + "tokens_seen": 2758526976 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013281444332998997, + "loss": 1.2372, + "theoretical_loss": 3.3709036584330714, + "tokens_seen": 2758592512 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013279839518555667, + "loss": 1.2091, + "theoretical_loss": 3.370897448656224, + "tokens_seen": 2758658048 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013278234704112337, + "loss": 1.2192, + "theoretical_loss": 3.370891239068202, + "tokens_seen": 2758723584 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013276629889669007, + "loss": 1.2219, + "theoretical_loss": 3.370885029668996, + "tokens_seen": 2758789120 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013275025075225677, + "loss": 1.087, + "theoretical_loss": 3.370878820458595, + "tokens_seen": 2758854656 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013273420260782347, + "loss": 1.2372, + "theoretical_loss": 3.37087261143699, + "tokens_seen": 2758920192 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013271815446339017, + "loss": 1.1696, + "theoretical_loss": 3.370866402604169, + "tokens_seen": 2758985728 + }, + { + "epoch": 1.22, + "objective/train/docs_used": 1018879, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1848057508468628, + "objective/train/theoretical_loss": 3.370860193960123, + "objective/train/tokens_used": 2779511264, + "theoretical_loss": 3.370860193960123, + "tokens_seen": 2759051264 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013270210631895687, + "loss": 1.1848, + "theoretical_loss": 3.370860193960123, + "tokens_seen": 2759051264 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013268605817452357, + "loss": 1.202, + "theoretical_loss": 3.3708539855048416, + "tokens_seen": 2759116800 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013267001003009027, + "loss": 1.2662, + "theoretical_loss": 3.3708477772383145, + "tokens_seen": 2759182336 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013265396188565697, + "loss": 1.1572, + "theoretical_loss": 3.370841569160531, + "tokens_seen": 2759247872 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013263791374122367, + "loss": 1.195, + "theoretical_loss": 3.3708353612714816, + "tokens_seen": 2759313408 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013262186559679037, + "loss": 1.2755, + "theoretical_loss": 3.3708291535711563, + "tokens_seen": 2759378944 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013260581745235707, + "loss": 1.1952, + "theoretical_loss": 3.370822946059544, + "tokens_seen": 2759444480 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013258976930792378, + "loss": 1.181, + "theoretical_loss": 3.3708167387366346, + "tokens_seen": 2759510016 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013257372116349048, + "loss": 1.1686, + "theoretical_loss": 3.3708105316024186, + "tokens_seen": 2759575552 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013255767301905718, + "loss": 1.2929, + "theoretical_loss": 3.370804324656885, + "tokens_seen": 2759641088 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013254162487462388, + "loss": 1.264, + "theoretical_loss": 3.3707981179000237, + "tokens_seen": 2759706624 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013252557673019058, + "loss": 1.2628, + "theoretical_loss": 3.370791911331825, + "tokens_seen": 2759772160 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013250952858575728, + "loss": 1.2283, + "theoretical_loss": 3.3707857049522785, + "tokens_seen": 2759837696 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013249348044132398, + "loss": 1.2004, + "theoretical_loss": 3.3707794987613737, + "tokens_seen": 2759903232 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013247743229689068, + "loss": 1.191, + "theoretical_loss": 3.3707732927591008, + "tokens_seen": 2759968768 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013246138415245738, + "loss": 1.1802, + "theoretical_loss": 3.370767086945449, + "tokens_seen": 2760034304 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013244533600802408, + "loss": 1.0433, + "theoretical_loss": 3.3707608813204084, + "tokens_seen": 2760099840 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013242928786359078, + "loss": 1.3176, + "theoretical_loss": 3.3707546758839695, + "tokens_seen": 2760165376 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013241323971915748, + "loss": 1.3288, + "theoretical_loss": 3.3707484706361206, + "tokens_seen": 2760230912 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013239719157472418, + "loss": 1.2629, + "theoretical_loss": 3.370742265576853, + "tokens_seen": 2760296448 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013238114343029088, + "loss": 1.3164, + "theoretical_loss": 3.370736060706155, + "tokens_seen": 2760361984 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013236509528585758, + "loss": 1.2415, + "theoretical_loss": 3.3707298560240178, + "tokens_seen": 2760427520 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013234904714142428, + "loss": 1.2051, + "theoretical_loss": 3.3707236515304304, + "tokens_seen": 2760493056 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013233299899699099, + "loss": 1.2395, + "theoretical_loss": 3.370717447225383, + "tokens_seen": 2760558592 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013231695085255769, + "loss": 1.1992, + "theoretical_loss": 3.3707112431088646, + "tokens_seen": 2760624128 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001323009027081244, + "loss": 1.2746, + "theoretical_loss": 3.3707050391808657, + "tokens_seen": 2760689664 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001322848545636911, + "loss": 1.2531, + "theoretical_loss": 3.3706988354413765, + "tokens_seen": 2760755200 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001322688064192578, + "loss": 1.2008, + "theoretical_loss": 3.370692631890386, + "tokens_seen": 2760820736 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001322527582748245, + "loss": 1.1652, + "theoretical_loss": 3.370686428527884, + "tokens_seen": 2760886272 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001322367101303912, + "loss": 1.2272, + "theoretical_loss": 3.3706802253538606, + "tokens_seen": 2760951808 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001322206619859579, + "loss": 1.1935, + "theoretical_loss": 3.3706740223683056, + "tokens_seen": 2761017344 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001322046138415246, + "loss": 1.3257, + "theoretical_loss": 3.3706678195712088, + "tokens_seen": 2761082880 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001321885656970913, + "loss": 1.2323, + "theoretical_loss": 3.3706616169625603, + "tokens_seen": 2761148416 + }, + { + "epoch": 1.22, + "learning_rate": 0.000132172517552658, + "loss": 1.2529, + "theoretical_loss": 3.370655414542349, + "tokens_seen": 2761213952 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001321564694082247, + "loss": 1.1066, + "theoretical_loss": 3.3706492123105654, + "tokens_seen": 2761279488 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001321404212637914, + "loss": 1.1957, + "theoretical_loss": 3.3706430102671994, + "tokens_seen": 2761345024 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001321243731193581, + "loss": 1.2073, + "theoretical_loss": 3.3706368084122404, + "tokens_seen": 2761410560 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013210832497492477, + "loss": 1.1, + "theoretical_loss": 3.3706306067456784, + "tokens_seen": 2761476096 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013209227683049147, + "loss": 1.2118, + "theoretical_loss": 3.370624405267503, + "tokens_seen": 2761541632 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013207622868605817, + "loss": 1.1114, + "theoretical_loss": 3.3706182039777044, + "tokens_seen": 2761607168 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013206018054162487, + "loss": 1.2714, + "theoretical_loss": 3.370612002876272, + "tokens_seen": 2761672704 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013204413239719157, + "loss": 1.2186, + "theoretical_loss": 3.3706058019631966, + "tokens_seen": 2761738240 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013202808425275827, + "loss": 1.2067, + "theoretical_loss": 3.3705996012384665, + "tokens_seen": 2761803776 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013201203610832497, + "loss": 1.2227, + "theoretical_loss": 3.3705934007020724, + "tokens_seen": 2761869312 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013199598796389167, + "loss": 1.1879, + "theoretical_loss": 3.3705872003540036, + "tokens_seen": 2761934848 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013197993981945837, + "loss": 1.1838, + "theoretical_loss": 3.3705810001942504, + "tokens_seen": 2762000384 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013196389167502507, + "loss": 1.1287, + "theoretical_loss": 3.3705748002228026, + "tokens_seen": 2762065920 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013194784353059177, + "loss": 1.2214, + "theoretical_loss": 3.37056860043965, + "tokens_seen": 2762131456 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013193179538615847, + "loss": 1.2396, + "theoretical_loss": 3.370562400844782, + "tokens_seen": 2762196992 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013191574724172517, + "loss": 1.2074, + "theoretical_loss": 3.370556201438189, + "tokens_seen": 2762262528 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013189969909729188, + "loss": 1.1832, + "theoretical_loss": 3.3705500022198605, + "tokens_seen": 2762328064 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013188365095285858, + "loss": 1.2334, + "theoretical_loss": 3.370543803189786, + "tokens_seen": 2762393600 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013186760280842528, + "loss": 1.1575, + "theoretical_loss": 3.370537604347956, + "tokens_seen": 2762459136 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013185155466399198, + "loss": 1.1452, + "theoretical_loss": 3.3705314056943596, + "tokens_seen": 2762524672 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013183550651955868, + "loss": 1.2311, + "theoretical_loss": 3.370525207228987, + "tokens_seen": 2762590208 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001318194583751254, + "loss": 1.1516, + "theoretical_loss": 3.3705190089518284, + "tokens_seen": 2762655744 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013180341023069208, + "loss": 1.2213, + "theoretical_loss": 3.370512810862873, + "tokens_seen": 2762721280 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013178736208625878, + "loss": 1.2644, + "theoretical_loss": 3.370506612962111, + "tokens_seen": 2762786816 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013177131394182548, + "loss": 1.186, + "theoretical_loss": 3.370500415249532, + "tokens_seen": 2762852352 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013175526579739218, + "loss": 1.2629, + "theoretical_loss": 3.370494217725126, + "tokens_seen": 2762917888 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013173921765295888, + "loss": 1.1962, + "theoretical_loss": 3.370488020388882, + "tokens_seen": 2762983424 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013172316950852558, + "loss": 1.1211, + "theoretical_loss": 3.3704818232407914, + "tokens_seen": 2763048960 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013170712136409228, + "loss": 1.263, + "theoretical_loss": 3.3704756262808426, + "tokens_seen": 2763114496 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013169107321965898, + "loss": 1.1382, + "theoretical_loss": 3.3704694295090265, + "tokens_seen": 2763180032 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013167502507522568, + "loss": 1.2116, + "theoretical_loss": 3.3704632329253323, + "tokens_seen": 2763245568 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013165897693079239, + "loss": 1.0903, + "theoretical_loss": 3.3704570365297495, + "tokens_seen": 2763311104 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013164292878635909, + "loss": 1.2466, + "theoretical_loss": 3.3704508403222686, + "tokens_seen": 2763376640 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013162688064192579, + "loss": 1.2219, + "theoretical_loss": 3.3704446443028795, + "tokens_seen": 2763442176 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001316108324974925, + "loss": 1.2273, + "theoretical_loss": 3.3704384484715715, + "tokens_seen": 2763507712 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001315947843530592, + "loss": 1.1843, + "theoretical_loss": 3.370432252828335, + "tokens_seen": 2763573248 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001315787362086259, + "loss": 1.1834, + "theoretical_loss": 3.370426057373159, + "tokens_seen": 2763638784 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001315626880641926, + "loss": 1.1467, + "theoretical_loss": 3.3704198621060337, + "tokens_seen": 2763704320 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001315466399197593, + "loss": 1.2933, + "theoretical_loss": 3.3704136670269493, + "tokens_seen": 2763769856 + }, + { + "epoch": 1.22, + "learning_rate": 0.000131530591775326, + "loss": 1.2709, + "theoretical_loss": 3.370407472135896, + "tokens_seen": 2763835392 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001315145436308927, + "loss": 1.2101, + "theoretical_loss": 3.370401277432862, + "tokens_seen": 2763900928 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001314984954864594, + "loss": 1.2186, + "theoretical_loss": 3.370395082917839, + "tokens_seen": 2763966464 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001314824473420261, + "loss": 1.2759, + "theoretical_loss": 3.3703888885908158, + "tokens_seen": 2764032000 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001314663991975928, + "loss": 1.1205, + "theoretical_loss": 3.370382694451782, + "tokens_seen": 2764097536 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001314503510531595, + "loss": 1.1082, + "theoretical_loss": 3.3703765005007282, + "tokens_seen": 2764163072 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001314343029087262, + "loss": 1.1544, + "theoretical_loss": 3.370370306737644, + "tokens_seen": 2764228608 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013141825476429287, + "loss": 1.2229, + "theoretical_loss": 3.3703641131625193, + "tokens_seen": 2764294144 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013140220661985957, + "loss": 1.221, + "theoretical_loss": 3.3703579197753433, + "tokens_seen": 2764359680 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013138615847542627, + "loss": 1.1962, + "theoretical_loss": 3.3703517265761067, + "tokens_seen": 2764425216 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013137011033099297, + "loss": 1.274, + "theoretical_loss": 3.370345533564799, + "tokens_seen": 2764490752 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013135406218655967, + "loss": 1.1594, + "theoretical_loss": 3.3703393407414097, + "tokens_seen": 2764556288 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013133801404212637, + "loss": 1.1366, + "theoretical_loss": 3.370333148105929, + "tokens_seen": 2764621824 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013132196589769307, + "loss": 1.2505, + "theoretical_loss": 3.3703269556583466, + "tokens_seen": 2764687360 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013130591775325977, + "loss": 1.1917, + "theoretical_loss": 3.370320763398653, + "tokens_seen": 2764752896 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013128986960882647, + "loss": 1.1148, + "theoretical_loss": 3.370314571326837, + "tokens_seen": 2764818432 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013127382146439317, + "loss": 1.1543, + "theoretical_loss": 3.370308379442889, + "tokens_seen": 2764883968 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013125777331995987, + "loss": 1.2389, + "theoretical_loss": 3.370302187746799, + "tokens_seen": 2764949504 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013124172517552657, + "loss": 1.1635, + "theoretical_loss": 3.370295996238557, + "tokens_seen": 2765015040 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001312256770310933, + "loss": 1.2026, + "theoretical_loss": 3.3702898049181518, + "tokens_seen": 2765080576 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013120962888666, + "loss": 1.0924, + "theoretical_loss": 3.370283613785574, + "tokens_seen": 2765146112 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001311935807422267, + "loss": 1.1448, + "theoretical_loss": 3.3702774228408137, + "tokens_seen": 2765211648 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001311775325977934, + "loss": 1.1656, + "theoretical_loss": 3.3702712320838604, + "tokens_seen": 2765277184 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001311614844533601, + "loss": 1.2217, + "theoretical_loss": 3.370265041514704, + "tokens_seen": 2765342720 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001311454363089268, + "loss": 1.3689, + "theoretical_loss": 3.3702588511333342, + "tokens_seen": 2765408256 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001311293881644935, + "loss": 1.1701, + "theoretical_loss": 3.370252660939741, + "tokens_seen": 2765473792 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001311133400200602, + "loss": 1.2391, + "theoretical_loss": 3.3702464709339144, + "tokens_seen": 2765539328 + }, + { + "epoch": 1.22, + "objective/train/docs_used": 1021278, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.161545991897583, + "objective/train/theoretical_loss": 3.370240281115844, + "objective/train/tokens_used": 2786064864, + "theoretical_loss": 3.370240281115844, + "tokens_seen": 2765604864 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013109729187562688, + "loss": 1.1615, + "theoretical_loss": 3.370240281115844, + "tokens_seen": 2765604864 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013108124373119358, + "loss": 1.1443, + "theoretical_loss": 3.3702340914855196, + "tokens_seen": 2765670400 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013106519558676028, + "loss": 1.1657, + "theoretical_loss": 3.3702279020429313, + "tokens_seen": 2765735936 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013104914744232698, + "loss": 1.1539, + "theoretical_loss": 3.370221712788069, + "tokens_seen": 2765801472 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013103309929789368, + "loss": 1.221, + "theoretical_loss": 3.3702155237209226, + "tokens_seen": 2765867008 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013101705115346038, + "loss": 1.092, + "theoretical_loss": 3.3702093348414817, + "tokens_seen": 2765932544 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013100100300902708, + "loss": 1.1375, + "theoretical_loss": 3.370203146149736, + "tokens_seen": 2765998080 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013098495486459378, + "loss": 1.133, + "theoretical_loss": 3.3701969576456756, + "tokens_seen": 2766063616 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013096890672016049, + "loss": 1.1064, + "theoretical_loss": 3.3701907693292905, + "tokens_seen": 2766129152 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013095285857572719, + "loss": 1.1662, + "theoretical_loss": 3.3701845812005704, + "tokens_seen": 2766194688 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001309368104312939, + "loss": 1.1583, + "theoretical_loss": 3.3701783932595055, + "tokens_seen": 2766260224 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001309207622868606, + "loss": 1.1965, + "theoretical_loss": 3.370172205506085, + "tokens_seen": 2766325760 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001309047141424273, + "loss": 1.1283, + "theoretical_loss": 3.3701660179402992, + "tokens_seen": 2766391296 + }, + { + "epoch": 1.22, + "learning_rate": 0.000130888665997994, + "loss": 1.1192, + "theoretical_loss": 3.3701598305621383, + "tokens_seen": 2766456832 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001308726178535607, + "loss": 1.1811, + "theoretical_loss": 3.3701536433715913, + "tokens_seen": 2766522368 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001308565697091274, + "loss": 1.2089, + "theoretical_loss": 3.3701474563686484, + "tokens_seen": 2766587904 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001308405215646941, + "loss": 1.2136, + "theoretical_loss": 3.3701412695533, + "tokens_seen": 2766653440 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001308244734202608, + "loss": 1.2751, + "theoretical_loss": 3.370135082925535, + "tokens_seen": 2766718976 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001308084252758275, + "loss": 1.1352, + "theoretical_loss": 3.370128896485345, + "tokens_seen": 2766784512 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001307923771313942, + "loss": 1.2238, + "theoretical_loss": 3.3701227102327174, + "tokens_seen": 2766850048 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001307763289869609, + "loss": 1.1779, + "theoretical_loss": 3.370116524167644, + "tokens_seen": 2766915584 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001307602808425276, + "loss": 1.1007, + "theoretical_loss": 3.370110338290114, + "tokens_seen": 2766981120 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001307442326980943, + "loss": 1.1662, + "theoretical_loss": 3.3701041526001174, + "tokens_seen": 2767046656 + }, + { + "epoch": 1.22, + "learning_rate": 0.000130728184553661, + "loss": 1.1659, + "theoretical_loss": 3.3700979670976436, + "tokens_seen": 2767112192 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013071213640922767, + "loss": 1.2882, + "theoretical_loss": 3.3700917817826834, + "tokens_seen": 2767177728 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013069608826479437, + "loss": 1.2567, + "theoretical_loss": 3.3700855966552257, + "tokens_seen": 2767243264 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013068004012036107, + "loss": 1.2441, + "theoretical_loss": 3.370079411715261, + "tokens_seen": 2767308800 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013066399197592777, + "loss": 1.1486, + "theoretical_loss": 3.370073226962779, + "tokens_seen": 2767374336 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013064794383149447, + "loss": 1.0803, + "theoretical_loss": 3.3700670423977694, + "tokens_seen": 2767439872 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013063189568706117, + "loss": 1.114, + "theoretical_loss": 3.3700608580202225, + "tokens_seen": 2767505408 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001306158475426279, + "loss": 1.1288, + "theoretical_loss": 3.3700546738301282, + "tokens_seen": 2767570944 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001305997993981946, + "loss": 1.2382, + "theoretical_loss": 3.3700484898274756, + "tokens_seen": 2767636480 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001305837512537613, + "loss": 1.2245, + "theoretical_loss": 3.3700423060122553, + "tokens_seen": 2767702016 + }, + { + "epoch": 1.22, + "learning_rate": 0.000130567703109328, + "loss": 1.1505, + "theoretical_loss": 3.3700361223844566, + "tokens_seen": 2767767552 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001305516549648947, + "loss": 1.3057, + "theoretical_loss": 3.37002993894407, + "tokens_seen": 2767833088 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001305356068204614, + "loss": 1.1947, + "theoretical_loss": 3.3700237556910855, + "tokens_seen": 2767898624 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001305195586760281, + "loss": 1.1352, + "theoretical_loss": 3.370017572625492, + "tokens_seen": 2767964160 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001305035105315948, + "loss": 1.1697, + "theoretical_loss": 3.3700113897472805, + "tokens_seen": 2768029696 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001304874623871615, + "loss": 1.1833, + "theoretical_loss": 3.37000520705644, + "tokens_seen": 2768095232 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001304714142427282, + "loss": 1.2235, + "theoretical_loss": 3.369999024552961, + "tokens_seen": 2768160768 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001304553660982949, + "loss": 1.119, + "theoretical_loss": 3.369992842236833, + "tokens_seen": 2768226304 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001304393179538616, + "loss": 1.1931, + "theoretical_loss": 3.3699866601080464, + "tokens_seen": 2768291840 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001304232698094283, + "loss": 1.1106, + "theoretical_loss": 3.3699804781665903, + "tokens_seen": 2768357376 + }, + { + "epoch": 1.22, + "learning_rate": 0.000130407221664995, + "loss": 1.1593, + "theoretical_loss": 3.3699742964124555, + "tokens_seen": 2768422912 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013039117352056168, + "loss": 1.224, + "theoretical_loss": 3.369968114845631, + "tokens_seen": 2768488448 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013037512537612838, + "loss": 1.225, + "theoretical_loss": 3.369961933466107, + "tokens_seen": 2768553984 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013035907723169508, + "loss": 1.2179, + "theoretical_loss": 3.369955752273874, + "tokens_seen": 2768619520 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013034302908726178, + "loss": 1.1561, + "theoretical_loss": 3.3699495712689207, + "tokens_seen": 2768685056 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013032698094282848, + "loss": 1.3867, + "theoretical_loss": 3.369943390451238, + "tokens_seen": 2768750592 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013031093279839518, + "loss": 1.1698, + "theoretical_loss": 3.369937209820816, + "tokens_seen": 2768816128 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013029488465396188, + "loss": 1.1402, + "theoretical_loss": 3.369931029377643, + "tokens_seen": 2768881664 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013027883650952859, + "loss": 1.2695, + "theoretical_loss": 3.369924849121711, + "tokens_seen": 2768947200 + }, + { + "epoch": 1.22, + "learning_rate": 0.00013026278836509529, + "loss": 1.1178, + "theoretical_loss": 3.3699186690530083, + "tokens_seen": 2769012736 + }, + { + "epoch": 1.22, + "learning_rate": 0.000130246740220662, + "loss": 1.2104, + "theoretical_loss": 3.3699124891715257, + "tokens_seen": 2769078272 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001302306920762287, + "loss": 1.2191, + "theoretical_loss": 3.369906309477252, + "tokens_seen": 2769143808 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001302146439317954, + "loss": 1.218, + "theoretical_loss": 3.369900129970179, + "tokens_seen": 2769209344 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001301985957873621, + "loss": 1.2352, + "theoretical_loss": 3.3698939506502947, + "tokens_seen": 2769274880 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001301825476429288, + "loss": 1.2557, + "theoretical_loss": 3.3698877715175897, + "tokens_seen": 2769340416 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001301664994984955, + "loss": 1.2637, + "theoretical_loss": 3.369881592572054, + "tokens_seen": 2769405952 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001301504513540622, + "loss": 1.2648, + "theoretical_loss": 3.369875413813678, + "tokens_seen": 2769471488 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001301344032096289, + "loss": 1.1299, + "theoretical_loss": 3.3698692352424504, + "tokens_seen": 2769537024 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001301183550651956, + "loss": 1.1547, + "theoretical_loss": 3.369863056858362, + "tokens_seen": 2769602560 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001301023069207623, + "loss": 1.15, + "theoretical_loss": 3.3698568786614027, + "tokens_seen": 2769668096 + }, + { + "epoch": 1.22, + "learning_rate": 0.000130086258776329, + "loss": 1.3021, + "theoretical_loss": 3.3698507006515617, + "tokens_seen": 2769733632 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001300702106318957, + "loss": 1.1398, + "theoretical_loss": 3.3698445228288296, + "tokens_seen": 2769799168 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001300541624874624, + "loss": 1.1344, + "theoretical_loss": 3.3698383451931964, + "tokens_seen": 2769864704 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001300381143430291, + "loss": 1.4011, + "theoretical_loss": 3.3698321677446508, + "tokens_seen": 2769930240 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001300220661985958, + "loss": 1.2014, + "theoretical_loss": 3.3698259904831844, + "tokens_seen": 2769995776 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001300060180541625, + "loss": 1.1431, + "theoretical_loss": 3.3698198134087862, + "tokens_seen": 2770061312 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001299899699097292, + "loss": 1.2059, + "theoretical_loss": 3.369813636521446, + "tokens_seen": 2770126848 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001299739217652959, + "loss": 1.2241, + "theoretical_loss": 3.369807459821154, + "tokens_seen": 2770192384 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001299578736208626, + "loss": 1.1381, + "theoretical_loss": 3.3698012833079, + "tokens_seen": 2770257920 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001299418254764293, + "loss": 1.3645, + "theoretical_loss": 3.369795106981674, + "tokens_seen": 2770323456 + }, + { + "epoch": 1.22, + "learning_rate": 0.000129925777331996, + "loss": 1.3014, + "theoretical_loss": 3.3697889308424656, + "tokens_seen": 2770388992 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001299097291875627, + "loss": 1.1419, + "theoretical_loss": 3.3697827548902652, + "tokens_seen": 2770454528 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001298936810431294, + "loss": 1.1546, + "theoretical_loss": 3.3697765791250625, + "tokens_seen": 2770520064 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001298776328986961, + "loss": 1.1601, + "theoretical_loss": 3.3697704035468474, + "tokens_seen": 2770585600 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001298615847542628, + "loss": 1.2743, + "theoretical_loss": 3.36976422815561, + "tokens_seen": 2770651136 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001298455366098295, + "loss": 1.2594, + "theoretical_loss": 3.36975805295134, + "tokens_seen": 2770716672 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001298294884653962, + "loss": 1.1762, + "theoretical_loss": 3.3697518779340268, + "tokens_seen": 2770782208 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001298134403209629, + "loss": 1.2107, + "theoretical_loss": 3.369745703103661, + "tokens_seen": 2770847744 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001297973921765296, + "loss": 1.1596, + "theoretical_loss": 3.369739528460233, + "tokens_seen": 2770913280 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001297813440320963, + "loss": 1.1353, + "theoretical_loss": 3.3697333540037313, + "tokens_seen": 2770978816 + }, + { + "epoch": 1.22, + "learning_rate": 0.000129765295887663, + "loss": 1.202, + "theoretical_loss": 3.3697271797341473, + "tokens_seen": 2771044352 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001297492477432297, + "loss": 1.218, + "theoretical_loss": 3.3697210056514697, + "tokens_seen": 2771109888 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001297331995987964, + "loss": 1.1705, + "theoretical_loss": 3.3697148317556893, + "tokens_seen": 2771175424 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001297171514543631, + "loss": 1.1924, + "theoretical_loss": 3.369708658046796, + "tokens_seen": 2771240960 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012970110330992978, + "loss": 1.1338, + "theoretical_loss": 3.3697024845247787, + "tokens_seen": 2771306496 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012968505516549648, + "loss": 1.2057, + "theoretical_loss": 3.3696963111896285, + "tokens_seen": 2771372032 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012966900702106318, + "loss": 1.1939, + "theoretical_loss": 3.3696901380413347, + "tokens_seen": 2771437568 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012965295887662988, + "loss": 1.2721, + "theoretical_loss": 3.3696839650798873, + "tokens_seen": 2771503104 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012963691073219658, + "loss": 1.1431, + "theoretical_loss": 3.3696777923052768, + "tokens_seen": 2771568640 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012962086258776328, + "loss": 1.2185, + "theoretical_loss": 3.369671619717492, + "tokens_seen": 2771634176 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012960481444332998, + "loss": 1.235, + "theoretical_loss": 3.3696654473165237, + "tokens_seen": 2771699712 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012958876629889669, + "loss": 1.1995, + "theoretical_loss": 3.3696592751023617, + "tokens_seen": 2771765248 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012957271815446339, + "loss": 1.1767, + "theoretical_loss": 3.369653103074996, + "tokens_seen": 2771830784 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001295566700100301, + "loss": 1.1776, + "theoretical_loss": 3.369646931234416, + "tokens_seen": 2771896320 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001295406218655968, + "loss": 1.2721, + "theoretical_loss": 3.369640759580612, + "tokens_seen": 2771961856 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001295245737211635, + "loss": 1.1713, + "theoretical_loss": 3.3696345881135743, + "tokens_seen": 2772027392 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001295085255767302, + "loss": 1.1267, + "theoretical_loss": 3.369628416833292, + "tokens_seen": 2772092928 + }, + { + "epoch": 1.22, + "objective/train/docs_used": 1023667, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.181121587753296, + "objective/train/theoretical_loss": 3.3696222457397558, + "objective/train/tokens_used": 2792618464, + "theoretical_loss": 3.3696222457397558, + "tokens_seen": 2772158464 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001294924774322969, + "loss": 1.1811, + "theoretical_loss": 3.3696222457397558, + "tokens_seen": 2772158464 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001294764292878636, + "loss": 1.2675, + "theoretical_loss": 3.369616074832955, + "tokens_seen": 2772224000 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001294603811434303, + "loss": 1.3817, + "theoretical_loss": 3.36960990411288, + "tokens_seen": 2772289536 + }, + { + "epoch": 1.22, + "learning_rate": 0.000129444332998997, + "loss": 1.0341, + "theoretical_loss": 3.369603733579521, + "tokens_seen": 2772355072 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012942828485456372, + "loss": 1.2905, + "theoretical_loss": 3.369597563232867, + "tokens_seen": 2772420608 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012941223671013042, + "loss": 1.2371, + "theoretical_loss": 3.3695913930729087, + "tokens_seen": 2772486144 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012939618856569712, + "loss": 1.2107, + "theoretical_loss": 3.369585223099636, + "tokens_seen": 2772551680 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001293801404212638, + "loss": 1.2391, + "theoretical_loss": 3.3695790533130383, + "tokens_seen": 2772617216 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001293640922768305, + "loss": 1.1744, + "theoretical_loss": 3.369572883713106, + "tokens_seen": 2772682752 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001293480441323972, + "loss": 1.2746, + "theoretical_loss": 3.369566714299829, + "tokens_seen": 2772748288 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001293319959879639, + "loss": 1.1447, + "theoretical_loss": 3.369560545073197, + "tokens_seen": 2772813824 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001293159478435306, + "loss": 1.2096, + "theoretical_loss": 3.3695543760332, + "tokens_seen": 2772879360 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001292998996990973, + "loss": 1.288, + "theoretical_loss": 3.3695482071798284, + "tokens_seen": 2772944896 + }, + { + "epoch": 1.22, + "learning_rate": 0.000129283851554664, + "loss": 1.2972, + "theoretical_loss": 3.369542038513072, + "tokens_seen": 2773010432 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001292678034102307, + "loss": 1.1011, + "theoretical_loss": 3.36953587003292, + "tokens_seen": 2773075968 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001292517552657974, + "loss": 1.1445, + "theoretical_loss": 3.3695297017393626, + "tokens_seen": 2773141504 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001292357071213641, + "loss": 1.1786, + "theoretical_loss": 3.3695235336323908, + "tokens_seen": 2773207040 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001292196589769308, + "loss": 1.1716, + "theoretical_loss": 3.3695173657119937, + "tokens_seen": 2773272576 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001292036108324975, + "loss": 1.179, + "theoretical_loss": 3.369511197978161, + "tokens_seen": 2773338112 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001291875626880642, + "loss": 1.2518, + "theoretical_loss": 3.369505030430883, + "tokens_seen": 2773403648 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001291715145436309, + "loss": 1.3208, + "theoretical_loss": 3.3694988630701497, + "tokens_seen": 2773469184 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001291554663991976, + "loss": 1.2911, + "theoretical_loss": 3.369492695895951, + "tokens_seen": 2773534720 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001291394182547643, + "loss": 1.2306, + "theoretical_loss": 3.369486528908277, + "tokens_seen": 2773600256 + }, + { + "epoch": 1.22, + "learning_rate": 0.000129123370110331, + "loss": 1.1994, + "theoretical_loss": 3.369480362107117, + "tokens_seen": 2773665792 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001291073219658977, + "loss": 1.1954, + "theoretical_loss": 3.3694741954924616, + "tokens_seen": 2773731328 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001290912738214644, + "loss": 1.228, + "theoretical_loss": 3.369468029064301, + "tokens_seen": 2773796864 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001290752256770311, + "loss": 1.1222, + "theoretical_loss": 3.3694618628226243, + "tokens_seen": 2773862400 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001290591775325978, + "loss": 1.1227, + "theoretical_loss": 3.369455696767422, + "tokens_seen": 2773927936 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001290431293881645, + "loss": 1.3042, + "theoretical_loss": 3.369449530898684, + "tokens_seen": 2773993472 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001290270812437312, + "loss": 1.1931, + "theoretical_loss": 3.3694433652164, + "tokens_seen": 2774059008 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001290110330992979, + "loss": 1.242, + "theoretical_loss": 3.3694371997205605, + "tokens_seen": 2774124544 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012899498495486458, + "loss": 1.2335, + "theoretical_loss": 3.369431034411155, + "tokens_seen": 2774190080 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012897893681043128, + "loss": 1.1355, + "theoretical_loss": 3.3694248692881734, + "tokens_seen": 2774255616 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012896288866599798, + "loss": 1.1614, + "theoretical_loss": 3.369418704351606, + "tokens_seen": 2774321152 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012894684052156468, + "loss": 1.2505, + "theoretical_loss": 3.3694125396014423, + "tokens_seen": 2774386688 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012893079237713138, + "loss": 1.1777, + "theoretical_loss": 3.369406375037673, + "tokens_seen": 2774452224 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012891474423269809, + "loss": 1.1049, + "theoretical_loss": 3.3694002106602876, + "tokens_seen": 2774517760 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012889869608826479, + "loss": 1.2743, + "theoretical_loss": 3.3693940464692758, + "tokens_seen": 2774583296 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012888264794383149, + "loss": 1.3552, + "theoretical_loss": 3.3693878824646277, + "tokens_seen": 2774648832 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001288665997993982, + "loss": 1.1942, + "theoretical_loss": 3.3693817186463337, + "tokens_seen": 2774714368 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001288505516549649, + "loss": 1.2049, + "theoretical_loss": 3.3693755550143836, + "tokens_seen": 2774779904 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001288345035105316, + "loss": 1.3102, + "theoretical_loss": 3.369369391568767, + "tokens_seen": 2774845440 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012881845536609832, + "loss": 1.2725, + "theoretical_loss": 3.3693632283094743, + "tokens_seen": 2774910976 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012880240722166502, + "loss": 1.2006, + "theoretical_loss": 3.369357065236495, + "tokens_seen": 2774976512 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012878635907723172, + "loss": 1.2235, + "theoretical_loss": 3.3693509023498196, + "tokens_seen": 2775042048 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012877031093279842, + "loss": 1.2209, + "theoretical_loss": 3.3693447396494376, + "tokens_seen": 2775107584 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012875426278836512, + "loss": 1.2597, + "theoretical_loss": 3.369338577135339, + "tokens_seen": 2775173120 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012873821464393182, + "loss": 1.1672, + "theoretical_loss": 3.3693324148075146, + "tokens_seen": 2775238656 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012872216649949852, + "loss": 1.0748, + "theoretical_loss": 3.369326252665953, + "tokens_seen": 2775304192 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012870611835506522, + "loss": 1.1744, + "theoretical_loss": 3.3693200907106453, + "tokens_seen": 2775369728 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012869007021063192, + "loss": 1.1097, + "theoretical_loss": 3.369313928941581, + "tokens_seen": 2775435264 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001286740220661986, + "loss": 1.1499, + "theoretical_loss": 3.36930776735875, + "tokens_seen": 2775500800 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001286579739217653, + "loss": 1.2611, + "theoretical_loss": 3.369301605962143, + "tokens_seen": 2775566336 + }, + { + "epoch": 1.22, + "learning_rate": 0.000128641925777332, + "loss": 1.1466, + "theoretical_loss": 3.3692954447517485, + "tokens_seen": 2775631872 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001286258776328987, + "loss": 1.2313, + "theoretical_loss": 3.3692892837275576, + "tokens_seen": 2775697408 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001286098294884654, + "loss": 1.1056, + "theoretical_loss": 3.36928312288956, + "tokens_seen": 2775762944 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001285937813440321, + "loss": 1.1716, + "theoretical_loss": 3.369276962237746, + "tokens_seen": 2775828480 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001285777331995988, + "loss": 1.2144, + "theoretical_loss": 3.3692708017721054, + "tokens_seen": 2775894016 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001285616850551655, + "loss": 1.1267, + "theoretical_loss": 3.3692646414926277, + "tokens_seen": 2775959552 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001285456369107322, + "loss": 1.1662, + "theoretical_loss": 3.3692584813993034, + "tokens_seen": 2776025088 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001285295887662989, + "loss": 1.2154, + "theoretical_loss": 3.369252321492122, + "tokens_seen": 2776090624 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001285135406218656, + "loss": 1.1767, + "theoretical_loss": 3.3692461617710743, + "tokens_seen": 2776156160 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001284974924774323, + "loss": 1.1795, + "theoretical_loss": 3.36924000223615, + "tokens_seen": 2776221696 + }, + { + "epoch": 1.22, + "learning_rate": 0.000128481444332999, + "loss": 1.1735, + "theoretical_loss": 3.3692338428873385, + "tokens_seen": 2776287232 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001284653961885657, + "loss": 1.2463, + "theoretical_loss": 3.3692276837246298, + "tokens_seen": 2776352768 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001284493480441324, + "loss": 1.1594, + "theoretical_loss": 3.369221524748015, + "tokens_seen": 2776418304 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001284332998996991, + "loss": 1.2832, + "theoretical_loss": 3.3692153659574826, + "tokens_seen": 2776483840 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001284172517552658, + "loss": 1.1424, + "theoretical_loss": 3.3692092073530238, + "tokens_seen": 2776549376 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001284012036108325, + "loss": 1.204, + "theoretical_loss": 3.3692030489346276, + "tokens_seen": 2776614912 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001283851554663992, + "loss": 1.167, + "theoretical_loss": 3.369196890702285, + "tokens_seen": 2776680448 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001283691073219659, + "loss": 1.1899, + "theoretical_loss": 3.3691907326559853, + "tokens_seen": 2776745984 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001283530591775326, + "loss": 1.2597, + "theoretical_loss": 3.3691845747957188, + "tokens_seen": 2776811520 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001283370110330993, + "loss": 1.162, + "theoretical_loss": 3.3691784171214754, + "tokens_seen": 2776877056 + }, + { + "epoch": 1.22, + "learning_rate": 0.000128320962888666, + "loss": 1.2491, + "theoretical_loss": 3.3691722596332445, + "tokens_seen": 2776942592 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001283049147442327, + "loss": 1.2489, + "theoretical_loss": 3.3691661023310173, + "tokens_seen": 2777008128 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012828886659979938, + "loss": 1.2007, + "theoretical_loss": 3.3691599452147827, + "tokens_seen": 2777073664 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012827281845536608, + "loss": 1.2759, + "theoretical_loss": 3.3691537882845313, + "tokens_seen": 2777139200 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012825677031093278, + "loss": 1.2168, + "theoretical_loss": 3.369147631540253, + "tokens_seen": 2777204736 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012824072216649948, + "loss": 1.1832, + "theoretical_loss": 3.3691414749819373, + "tokens_seen": 2777270272 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001282246740220662, + "loss": 1.1167, + "theoretical_loss": 3.369135318609575, + "tokens_seen": 2777335808 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001282086258776329, + "loss": 1.1654, + "theoretical_loss": 3.3691291624231554, + "tokens_seen": 2777401344 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012819257773319961, + "loss": 1.1853, + "theoretical_loss": 3.369123006422669, + "tokens_seen": 2777466880 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012817652958876631, + "loss": 1.2734, + "theoretical_loss": 3.3691168506081057, + "tokens_seen": 2777532416 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012816048144433301, + "loss": 1.2101, + "theoretical_loss": 3.3691106949794554, + "tokens_seen": 2777597952 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012814443329989972, + "loss": 1.2474, + "theoretical_loss": 3.369104539536708, + "tokens_seen": 2777663488 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012812838515546642, + "loss": 1.2284, + "theoretical_loss": 3.3690983842798534, + "tokens_seen": 2777729024 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012811233701103312, + "loss": 1.2319, + "theoretical_loss": 3.369092229208882, + "tokens_seen": 2777794560 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012809628886659982, + "loss": 1.2052, + "theoretical_loss": 3.369086074323783, + "tokens_seen": 2777860096 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012808024072216652, + "loss": 1.3003, + "theoretical_loss": 3.369079919624548, + "tokens_seen": 2777925632 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012806419257773322, + "loss": 1.1861, + "theoretical_loss": 3.3690737651111653, + "tokens_seen": 2777991168 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012804814443329992, + "loss": 1.1757, + "theoretical_loss": 3.3690676107836257, + "tokens_seen": 2778056704 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012803209628886662, + "loss": 1.2388, + "theoretical_loss": 3.3690614566419192, + "tokens_seen": 2778122240 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012801604814443332, + "loss": 1.0826, + "theoretical_loss": 3.3690553026860357, + "tokens_seen": 2778187776 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012800000000000002, + "loss": 1.1901, + "theoretical_loss": 3.3690491489159653, + "tokens_seen": 2778253312 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001279839518555667, + "loss": 1.219, + "theoretical_loss": 3.3690429953316974, + "tokens_seen": 2778318848 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001279679037111334, + "loss": 1.0966, + "theoretical_loss": 3.369036841933223, + "tokens_seen": 2778384384 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001279518555667001, + "loss": 1.2909, + "theoretical_loss": 3.3690306887205317, + "tokens_seen": 2778449920 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001279358074222668, + "loss": 1.2322, + "theoretical_loss": 3.3690245356936135, + "tokens_seen": 2778515456 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001279197592778335, + "loss": 1.2663, + "theoretical_loss": 3.3690183828524582, + "tokens_seen": 2778580992 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001279037111334002, + "loss": 1.2613, + "theoretical_loss": 3.3690122301970558, + "tokens_seen": 2778646528 + }, + { + "epoch": 1.22, + "objective/train/docs_used": 1026295, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1329652070999146, + "objective/train/theoretical_loss": 3.3690060777273967, + "objective/train/tokens_used": 2799172064, + "theoretical_loss": 3.3690060777273967, + "tokens_seen": 2778712064 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001278876629889669, + "loss": 1.133, + "theoretical_loss": 3.3690060777273967, + "tokens_seen": 2778712064 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001278716148445336, + "loss": 1.2096, + "theoretical_loss": 3.3689999254434704, + "tokens_seen": 2778777600 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001278555667001003, + "loss": 1.1508, + "theoretical_loss": 3.3689937733452675, + "tokens_seen": 2778843136 + }, + { + "epoch": 1.22, + "learning_rate": 0.000127839518555667, + "loss": 1.1422, + "theoretical_loss": 3.368987621432778, + "tokens_seen": 2778908672 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001278234704112337, + "loss": 1.1476, + "theoretical_loss": 3.368981469705991, + "tokens_seen": 2778974208 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001278074222668004, + "loss": 1.2498, + "theoretical_loss": 3.3689753181648974, + "tokens_seen": 2779039744 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001277913741223671, + "loss": 1.1668, + "theoretical_loss": 3.368969166809487, + "tokens_seen": 2779105280 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001277753259779338, + "loss": 1.2095, + "theoretical_loss": 3.36896301563975, + "tokens_seen": 2779170816 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001277592778335005, + "loss": 1.1941, + "theoretical_loss": 3.368956864655676, + "tokens_seen": 2779236352 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001277432296890672, + "loss": 1.3104, + "theoretical_loss": 3.3689507138572545, + "tokens_seen": 2779301888 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001277271815446339, + "loss": 1.16, + "theoretical_loss": 3.3689445632444768, + "tokens_seen": 2779367424 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001277111334002006, + "loss": 1.255, + "theoretical_loss": 3.3689384128173328, + "tokens_seen": 2779432960 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001276950852557673, + "loss": 1.1854, + "theoretical_loss": 3.368932262575812, + "tokens_seen": 2779498496 + }, + { + "epoch": 1.22, + "learning_rate": 0.000127679037111334, + "loss": 1.2878, + "theoretical_loss": 3.3689261125199037, + "tokens_seen": 2779564032 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001276629889669007, + "loss": 1.103, + "theoretical_loss": 3.368919962649599, + "tokens_seen": 2779629568 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001276469408224674, + "loss": 1.1895, + "theoretical_loss": 3.368913812964888, + "tokens_seen": 2779695104 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001276308926780341, + "loss": 1.2419, + "theoretical_loss": 3.36890766346576, + "tokens_seen": 2779760640 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001276148445336008, + "loss": 1.1593, + "theoretical_loss": 3.368901514152206, + "tokens_seen": 2779826176 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001275987963891675, + "loss": 1.2304, + "theoretical_loss": 3.3688953650242146, + "tokens_seen": 2779891712 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001275827482447342, + "loss": 1.224, + "theoretical_loss": 3.3688892160817776, + "tokens_seen": 2779957248 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001275667001003009, + "loss": 1.3386, + "theoretical_loss": 3.3688830673248833, + "tokens_seen": 2780022784 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001275506519558676, + "loss": 1.2079, + "theoretical_loss": 3.368876918753523, + "tokens_seen": 2780088320 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001275346038114343, + "loss": 1.2371, + "theoretical_loss": 3.3688707703676855, + "tokens_seen": 2780153856 + }, + { + "epoch": 1.22, + "learning_rate": 0.000127518555667001, + "loss": 1.1979, + "theoretical_loss": 3.368864622167362, + "tokens_seen": 2780219392 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012750250752256771, + "loss": 1.1486, + "theoretical_loss": 3.368858474152542, + "tokens_seen": 2780284928 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012748645937813441, + "loss": 1.2227, + "theoretical_loss": 3.3688523263232155, + "tokens_seen": 2780350464 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012747041123370112, + "loss": 1.188, + "theoretical_loss": 3.368846178679373, + "tokens_seen": 2780416000 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012745436308926782, + "loss": 1.1897, + "theoretical_loss": 3.368840031221004, + "tokens_seen": 2780481536 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012743831494483452, + "loss": 1.1306, + "theoretical_loss": 3.3688338839480982, + "tokens_seen": 2780547072 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012742226680040122, + "loss": 1.2149, + "theoretical_loss": 3.3688277368606467, + "tokens_seen": 2780612608 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012740621865596792, + "loss": 1.1561, + "theoretical_loss": 3.368821589958639, + "tokens_seen": 2780678144 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012739017051153462, + "loss": 1.2103, + "theoretical_loss": 3.368815443242065, + "tokens_seen": 2780743680 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012737412236710132, + "loss": 1.1237, + "theoretical_loss": 3.3688092967109147, + "tokens_seen": 2780809216 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012735807422266802, + "loss": 1.1782, + "theoretical_loss": 3.3688031503651783, + "tokens_seen": 2780874752 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012734202607823472, + "loss": 1.1364, + "theoretical_loss": 3.368797004204846, + "tokens_seen": 2780940288 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012732597793380142, + "loss": 1.2527, + "theoretical_loss": 3.3687908582299078, + "tokens_seen": 2781005824 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012730992978936812, + "loss": 1.0959, + "theoretical_loss": 3.368784712440353, + "tokens_seen": 2781071360 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012729388164493482, + "loss": 1.1494, + "theoretical_loss": 3.368778566836173, + "tokens_seen": 2781136896 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001272778335005015, + "loss": 1.2662, + "theoretical_loss": 3.3687724214173564, + "tokens_seen": 2781202432 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001272617853560682, + "loss": 1.2105, + "theoretical_loss": 3.3687662761838943, + "tokens_seen": 2781267968 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001272457372116349, + "loss": 1.2108, + "theoretical_loss": 3.368760131135776, + "tokens_seen": 2781333504 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001272296890672016, + "loss": 1.0896, + "theoretical_loss": 3.3687539862729925, + "tokens_seen": 2781399040 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001272136409227683, + "loss": 1.2263, + "theoretical_loss": 3.3687478415955328, + "tokens_seen": 2781464576 + }, + { + "epoch": 1.22, + "learning_rate": 0.000127197592778335, + "loss": 1.129, + "theoretical_loss": 3.3687416971033874, + "tokens_seen": 2781530112 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001271815446339017, + "loss": 1.155, + "theoretical_loss": 3.3687355527965464, + "tokens_seen": 2781595648 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001271654964894684, + "loss": 1.1512, + "theoretical_loss": 3.3687294086749997, + "tokens_seen": 2781661184 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001271494483450351, + "loss": 1.1769, + "theoretical_loss": 3.3687232647387377, + "tokens_seen": 2781726720 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001271334002006018, + "loss": 1.1559, + "theoretical_loss": 3.36871712098775, + "tokens_seen": 2781792256 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001271173520561685, + "loss": 1.1674, + "theoretical_loss": 3.368710977422027, + "tokens_seen": 2781857792 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001271013039117352, + "loss": 1.174, + "theoretical_loss": 3.368704834041558, + "tokens_seen": 2781923328 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001270852557673019, + "loss": 1.2316, + "theoretical_loss": 3.368698690846334, + "tokens_seen": 2781988864 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001270692076228686, + "loss": 1.1968, + "theoretical_loss": 3.368692547836345, + "tokens_seen": 2782054400 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001270531594784353, + "loss": 1.1849, + "theoretical_loss": 3.36868640501158, + "tokens_seen": 2782119936 + }, + { + "epoch": 1.22, + "learning_rate": 0.000127037111334002, + "loss": 1.1394, + "theoretical_loss": 3.3686802623720307, + "tokens_seen": 2782185472 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012702106318956873, + "loss": 1.1266, + "theoretical_loss": 3.3686741199176855, + "tokens_seen": 2782251008 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012700501504513543, + "loss": 1.1372, + "theoretical_loss": 3.3686679776485353, + "tokens_seen": 2782316544 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012698896690070213, + "loss": 1.242, + "theoretical_loss": 3.3686618355645703, + "tokens_seen": 2782382080 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012697291875626883, + "loss": 1.1209, + "theoretical_loss": 3.3686556936657803, + "tokens_seen": 2782447616 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001269568706118355, + "loss": 1.2975, + "theoretical_loss": 3.368649551952155, + "tokens_seen": 2782513152 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001269408224674022, + "loss": 1.3228, + "theoretical_loss": 3.368643410423685, + "tokens_seen": 2782578688 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001269247743229689, + "loss": 1.2924, + "theoretical_loss": 3.3686372690803603, + "tokens_seen": 2782644224 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001269087261785356, + "loss": 1.1265, + "theoretical_loss": 3.3686311279221703, + "tokens_seen": 2782709760 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001268926780341023, + "loss": 1.2258, + "theoretical_loss": 3.368624986949106, + "tokens_seen": 2782775296 + }, + { + "epoch": 1.22, + "learning_rate": 0.000126876629889669, + "loss": 1.2439, + "theoretical_loss": 3.3686188461611573, + "tokens_seen": 2782840832 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001268605817452357, + "loss": 1.14, + "theoretical_loss": 3.3686127055583137, + "tokens_seen": 2782906368 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001268445336008024, + "loss": 1.2426, + "theoretical_loss": 3.3686065651405652, + "tokens_seen": 2782971904 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001268284854563691, + "loss": 1.1913, + "theoretical_loss": 3.368600424907903, + "tokens_seen": 2783037440 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012681243731193581, + "loss": 1.2299, + "theoretical_loss": 3.3685942848603156, + "tokens_seen": 2783102976 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012679638916750251, + "loss": 1.1665, + "theoretical_loss": 3.368588144997794, + "tokens_seen": 2783168512 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012678034102306922, + "loss": 1.2111, + "theoretical_loss": 3.3685820053203286, + "tokens_seen": 2783234048 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012676429287863592, + "loss": 1.2235, + "theoretical_loss": 3.368575865827909, + "tokens_seen": 2783299584 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012674824473420262, + "loss": 1.2828, + "theoretical_loss": 3.3685697265205246, + "tokens_seen": 2783365120 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012673219658976932, + "loss": 1.2083, + "theoretical_loss": 3.3685635873981665, + "tokens_seen": 2783430656 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012671614844533602, + "loss": 1.2756, + "theoretical_loss": 3.3685574484608245, + "tokens_seen": 2783496192 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012670010030090272, + "loss": 1.1078, + "theoretical_loss": 3.3685513097084883, + "tokens_seen": 2783561728 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012668405215646942, + "loss": 1.1888, + "theoretical_loss": 3.3685451711411485, + "tokens_seen": 2783627264 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012666800401203612, + "loss": 1.1994, + "theoretical_loss": 3.3685390327587945, + "tokens_seen": 2783692800 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012665195586760282, + "loss": 1.2215, + "theoretical_loss": 3.368532894561417, + "tokens_seen": 2783758336 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012663590772316952, + "loss": 1.1558, + "theoretical_loss": 3.368526756549006, + "tokens_seen": 2783823872 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012661985957873622, + "loss": 1.1597, + "theoretical_loss": 3.368520618721551, + "tokens_seen": 2783889408 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012660381143430292, + "loss": 1.1399, + "theoretical_loss": 3.3685144810790426, + "tokens_seen": 2783954944 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012658776328986962, + "loss": 1.2331, + "theoretical_loss": 3.3685083436214707, + "tokens_seen": 2784020480 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001265717151454363, + "loss": 1.2427, + "theoretical_loss": 3.3685022063488255, + "tokens_seen": 2784086016 + }, + { + "epoch": 1.22, + "learning_rate": 0.000126555667001003, + "loss": 1.2238, + "theoretical_loss": 3.3684960692610972, + "tokens_seen": 2784151552 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001265396188565697, + "loss": 1.1689, + "theoretical_loss": 3.3684899323582753, + "tokens_seen": 2784217088 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001265235707121364, + "loss": 1.095, + "theoretical_loss": 3.3684837956403504, + "tokens_seen": 2784282624 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001265075225677031, + "loss": 1.1483, + "theoretical_loss": 3.3684776591073127, + "tokens_seen": 2784348160 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001264914744232698, + "loss": 1.2486, + "theoretical_loss": 3.3684715227591515, + "tokens_seen": 2784413696 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001264754262788365, + "loss": 1.1423, + "theoretical_loss": 3.3684653865958576, + "tokens_seen": 2784479232 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001264593781344032, + "loss": 1.2707, + "theoretical_loss": 3.368459250617421, + "tokens_seen": 2784544768 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001264433299899699, + "loss": 1.1872, + "theoretical_loss": 3.368453114823832, + "tokens_seen": 2784610304 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001264272818455366, + "loss": 1.1427, + "theoretical_loss": 3.3684469792150793, + "tokens_seen": 2784675840 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012641123370110333, + "loss": 1.2957, + "theoretical_loss": 3.368440843791155, + "tokens_seen": 2784741376 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012639518555667003, + "loss": 1.1136, + "theoretical_loss": 3.368434708552048, + "tokens_seen": 2784806912 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012637913741223673, + "loss": 1.2616, + "theoretical_loss": 3.3684285734977477, + "tokens_seen": 2784872448 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012636308926780343, + "loss": 1.0843, + "theoretical_loss": 3.368422438628246, + "tokens_seen": 2784937984 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012634704112337013, + "loss": 1.2391, + "theoretical_loss": 3.3684163039435315, + "tokens_seen": 2785003520 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012633099297893683, + "loss": 1.0419, + "theoretical_loss": 3.368410169443595, + "tokens_seen": 2785069056 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012631494483450353, + "loss": 1.2161, + "theoretical_loss": 3.368404035128427, + "tokens_seen": 2785134592 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012629889669007023, + "loss": 1.2301, + "theoretical_loss": 3.3683979009980165, + "tokens_seen": 2785200128 + }, + { + "epoch": 1.22, + "objective/train/docs_used": 1028369, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2570631504058838, + "objective/train/theoretical_loss": 3.368391767052354, + "objective/train/tokens_used": 2805725664, + "theoretical_loss": 3.368391767052354, + "tokens_seen": 2785265664 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012628284854563694, + "loss": 1.2571, + "theoretical_loss": 3.368391767052354, + "tokens_seen": 2785265664 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012626680040120364, + "loss": 1.2479, + "theoretical_loss": 3.3683856332914295, + "tokens_seen": 2785331200 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001262507522567703, + "loss": 1.2498, + "theoretical_loss": 3.3683794997152336, + "tokens_seen": 2785396736 + }, + { + "epoch": 1.22, + "learning_rate": 0.000126234704112337, + "loss": 1.3046, + "theoretical_loss": 3.3683733663237563, + "tokens_seen": 2785462272 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001262186559679037, + "loss": 1.1686, + "theoretical_loss": 3.368367233116987, + "tokens_seen": 2785527808 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001262026078234704, + "loss": 1.1456, + "theoretical_loss": 3.3683611000949165, + "tokens_seen": 2785593344 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001261865596790371, + "loss": 1.1493, + "theoretical_loss": 3.3683549672575346, + "tokens_seen": 2785658880 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001261705115346038, + "loss": 1.2272, + "theoretical_loss": 3.368348834604831, + "tokens_seen": 2785724416 + }, + { + "epoch": 1.22, + "learning_rate": 0.0001261544633901705, + "loss": 1.411, + "theoretical_loss": 3.368342702136797, + "tokens_seen": 2785789952 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012613841524573721, + "loss": 1.2665, + "theoretical_loss": 3.368336569853421, + "tokens_seen": 2785855488 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012612236710130391, + "loss": 1.2677, + "theoretical_loss": 3.3683304377546945, + "tokens_seen": 2785921024 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012610631895687061, + "loss": 1.1869, + "theoretical_loss": 3.3683243058406074, + "tokens_seen": 2785986560 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012609027081243732, + "loss": 1.1739, + "theoretical_loss": 3.368318174111149, + "tokens_seen": 2786052096 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012607422266800402, + "loss": 1.0606, + "theoretical_loss": 3.36831204256631, + "tokens_seen": 2786117632 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012605817452357072, + "loss": 1.1045, + "theoretical_loss": 3.3683059112060807, + "tokens_seen": 2786183168 + }, + { + "epoch": 1.22, + "learning_rate": 0.00012604212637913742, + "loss": 1.1381, + "theoretical_loss": 3.368299780030451, + "tokens_seen": 2786248704 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012602607823470412, + "loss": 1.162, + "theoretical_loss": 3.36829364903941, + "tokens_seen": 2786314240 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012601003009027082, + "loss": 1.3033, + "theoretical_loss": 3.36828751823295, + "tokens_seen": 2786379776 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012599398194583752, + "loss": 1.382, + "theoretical_loss": 3.368281387611059, + "tokens_seen": 2786445312 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012597793380140422, + "loss": 1.1066, + "theoretical_loss": 3.368275257173728, + "tokens_seen": 2786510848 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012596188565697092, + "loss": 1.1274, + "theoretical_loss": 3.3682691269209473, + "tokens_seen": 2786576384 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012594583751253762, + "loss": 1.2143, + "theoretical_loss": 3.3682629968527062, + "tokens_seen": 2786641920 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012592978936810432, + "loss": 1.3517, + "theoretical_loss": 3.368256866968996, + "tokens_seen": 2786707456 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012591374122367102, + "loss": 1.2883, + "theoretical_loss": 3.3682507372698054, + "tokens_seen": 2786772992 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012589769307923772, + "loss": 1.2274, + "theoretical_loss": 3.3682446077551256, + "tokens_seen": 2786838528 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012588164493480442, + "loss": 1.1916, + "theoretical_loss": 3.3682384784249466, + "tokens_seen": 2786904064 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001258655967903711, + "loss": 1.2462, + "theoretical_loss": 3.368232349279258, + "tokens_seen": 2786969600 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001258495486459378, + "loss": 1.2576, + "theoretical_loss": 3.36822622031805, + "tokens_seen": 2787035136 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001258335005015045, + "loss": 1.1806, + "theoretical_loss": 3.368220091541313, + "tokens_seen": 2787100672 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012581745235707123, + "loss": 1.1171, + "theoretical_loss": 3.368213962949037, + "tokens_seen": 2787166208 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012580140421263793, + "loss": 1.2685, + "theoretical_loss": 3.368207834541212, + "tokens_seen": 2787231744 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012578535606820463, + "loss": 1.1607, + "theoretical_loss": 3.3682017063178282, + "tokens_seen": 2787297280 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012576930792377133, + "loss": 1.1788, + "theoretical_loss": 3.368195578278876, + "tokens_seen": 2787362816 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012575325977933803, + "loss": 1.2278, + "theoretical_loss": 3.368189450424345, + "tokens_seen": 2787428352 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012573721163490473, + "loss": 1.1552, + "theoretical_loss": 3.3681833227542257, + "tokens_seen": 2787493888 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012572116349047143, + "loss": 1.2344, + "theoretical_loss": 3.368177195268508, + "tokens_seen": 2787559424 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012570511534603813, + "loss": 1.194, + "theoretical_loss": 3.368171067967182, + "tokens_seen": 2787624960 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012568906720160483, + "loss": 1.1899, + "theoretical_loss": 3.3681649408502374, + "tokens_seen": 2787690496 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012567301905717153, + "loss": 1.399, + "theoretical_loss": 3.3681588139176655, + "tokens_seen": 2787756032 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012565697091273823, + "loss": 1.2268, + "theoretical_loss": 3.3681526871694554, + "tokens_seen": 2787821568 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012564092276830493, + "loss": 1.1791, + "theoretical_loss": 3.368146560605598, + "tokens_seen": 2787887104 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012562487462387163, + "loss": 1.1155, + "theoretical_loss": 3.3681404342260826, + "tokens_seen": 2787952640 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012560882647943833, + "loss": 1.1604, + "theoretical_loss": 3.3681343080308994, + "tokens_seen": 2788018176 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012559277833500504, + "loss": 1.1864, + "theoretical_loss": 3.368128182020039, + "tokens_seen": 2788083712 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012557673019057174, + "loss": 1.1837, + "theoretical_loss": 3.3681220561934917, + "tokens_seen": 2788149248 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001255606820461384, + "loss": 1.1874, + "theoretical_loss": 3.368115930551247, + "tokens_seen": 2788214784 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001255446339017051, + "loss": 1.4046, + "theoretical_loss": 3.368109805093295, + "tokens_seen": 2788280320 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001255285857572718, + "loss": 1.2077, + "theoretical_loss": 3.3681036798196264, + "tokens_seen": 2788345856 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001255125376128385, + "loss": 1.2073, + "theoretical_loss": 3.368097554730231, + "tokens_seen": 2788411392 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001254964894684052, + "loss": 1.2163, + "theoretical_loss": 3.368091429825099, + "tokens_seen": 2788476928 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001254804413239719, + "loss": 1.2551, + "theoretical_loss": 3.3680853051042203, + "tokens_seen": 2788542464 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001254643931795386, + "loss": 1.2757, + "theoretical_loss": 3.3680791805675856, + "tokens_seen": 2788608000 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012544834503510531, + "loss": 1.2269, + "theoretical_loss": 3.368073056215184, + "tokens_seen": 2788673536 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012543229689067201, + "loss": 1.0992, + "theoretical_loss": 3.3680669320470065, + "tokens_seen": 2788739072 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012541624874623872, + "loss": 1.2689, + "theoretical_loss": 3.368060808063043, + "tokens_seen": 2788804608 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012540020060180542, + "loss": 1.0892, + "theoretical_loss": 3.3680546842632837, + "tokens_seen": 2788870144 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012538415245737212, + "loss": 1.2291, + "theoretical_loss": 3.368048560647719, + "tokens_seen": 2788935680 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012536810431293882, + "loss": 1.2886, + "theoretical_loss": 3.368042437216338, + "tokens_seen": 2789001216 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012535205616850552, + "loss": 1.1801, + "theoretical_loss": 3.3680363139691316, + "tokens_seen": 2789066752 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012533600802407222, + "loss": 1.2821, + "theoretical_loss": 3.36803019090609, + "tokens_seen": 2789132288 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012531995987963892, + "loss": 1.2542, + "theoretical_loss": 3.3680240680272036, + "tokens_seen": 2789197824 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012530391173520562, + "loss": 1.2193, + "theoretical_loss": 3.3680179453324617, + "tokens_seen": 2789263360 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012528786359077232, + "loss": 1.229, + "theoretical_loss": 3.3680118228218547, + "tokens_seen": 2789328896 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012527181544633902, + "loss": 1.1353, + "theoretical_loss": 3.368005700495373, + "tokens_seen": 2789394432 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012525576730190572, + "loss": 1.2799, + "theoretical_loss": 3.3679995783530066, + "tokens_seen": 2789459968 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012523971915747242, + "loss": 1.1224, + "theoretical_loss": 3.367993456394746, + "tokens_seen": 2789525504 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012522367101303912, + "loss": 1.2577, + "theoretical_loss": 3.3679873346205804, + "tokens_seen": 2789591040 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012520762286860582, + "loss": 1.2681, + "theoretical_loss": 3.367981213030501, + "tokens_seen": 2789656576 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012519157472417252, + "loss": 1.14, + "theoretical_loss": 3.3679750916244973, + "tokens_seen": 2789722112 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012517552657973922, + "loss": 1.3834, + "theoretical_loss": 3.3679689704025595, + "tokens_seen": 2789787648 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012515947843530593, + "loss": 1.2964, + "theoretical_loss": 3.3679628493646785, + "tokens_seen": 2789853184 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012514343029087263, + "loss": 1.2936, + "theoretical_loss": 3.367956728510843, + "tokens_seen": 2789918720 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012512738214643933, + "loss": 1.1258, + "theoretical_loss": 3.367950607841044, + "tokens_seen": 2789984256 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012511133400200603, + "loss": 1.1559, + "theoretical_loss": 3.367944487355272, + "tokens_seen": 2790049792 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012509528585757273, + "loss": 1.2499, + "theoretical_loss": 3.3679383670535166, + "tokens_seen": 2790115328 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012507923771313943, + "loss": 1.1723, + "theoretical_loss": 3.367932246935768, + "tokens_seen": 2790180864 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012506318956870613, + "loss": 1.2255, + "theoretical_loss": 3.3679261270020167, + "tokens_seen": 2790246400 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012504714142427283, + "loss": 1.2604, + "theoretical_loss": 3.367920007252252, + "tokens_seen": 2790311936 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012503109327983953, + "loss": 1.2564, + "theoretical_loss": 3.3679138876864654, + "tokens_seen": 2790377472 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012501504513540623, + "loss": 1.2092, + "theoretical_loss": 3.3679077683046454, + "tokens_seen": 2790443008 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012499899699097293, + "loss": 1.3653, + "theoretical_loss": 3.3679016491067832, + "tokens_seen": 2790508544 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012498294884653963, + "loss": 1.1871, + "theoretical_loss": 3.367895530092869, + "tokens_seen": 2790574080 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012496690070210633, + "loss": 1.1179, + "theoretical_loss": 3.367889411262893, + "tokens_seen": 2790639616 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012495085255767303, + "loss": 1.359, + "theoretical_loss": 3.367883292616845, + "tokens_seen": 2790705152 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012493480441323973, + "loss": 1.1416, + "theoretical_loss": 3.3678771741547147, + "tokens_seen": 2790770688 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012491875626880643, + "loss": 1.2586, + "theoretical_loss": 3.367871055876493, + "tokens_seen": 2790836224 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012490270812437314, + "loss": 1.0912, + "theoretical_loss": 3.36786493778217, + "tokens_seen": 2790901760 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012488665997993984, + "loss": 1.2826, + "theoretical_loss": 3.3678588198717354, + "tokens_seen": 2790967296 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012487061183550654, + "loss": 1.1718, + "theoretical_loss": 3.36785270214518, + "tokens_seen": 2791032832 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001248545636910732, + "loss": 1.1551, + "theoretical_loss": 3.367846584602493, + "tokens_seen": 2791098368 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001248385155466399, + "loss": 1.1968, + "theoretical_loss": 3.3678404672436657, + "tokens_seen": 2791163904 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001248224674022066, + "loss": 1.3822, + "theoretical_loss": 3.3678343500686876, + "tokens_seen": 2791229440 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001248064192577733, + "loss": 1.1933, + "theoretical_loss": 3.367828233077549, + "tokens_seen": 2791294976 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012479037111334, + "loss": 1.3035, + "theoretical_loss": 3.36782211627024, + "tokens_seen": 2791360512 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001247743229689067, + "loss": 1.4295, + "theoretical_loss": 3.3678159996467505, + "tokens_seen": 2791426048 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012475827482447341, + "loss": 1.3077, + "theoretical_loss": 3.3678098832070713, + "tokens_seen": 2791491584 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012474222668004011, + "loss": 1.3207, + "theoretical_loss": 3.3678037669511918, + "tokens_seen": 2791557120 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012472617853560682, + "loss": 1.1818, + "theoretical_loss": 3.367797650879103, + "tokens_seen": 2791622656 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012471013039117352, + "loss": 1.2236, + "theoretical_loss": 3.3677915349907943, + "tokens_seen": 2791688192 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012469408224674022, + "loss": 1.2631, + "theoretical_loss": 3.3677854192862564, + "tokens_seen": 2791753728 + }, + { + "epoch": 1.23, + "objective/train/docs_used": 1030663, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.304643154144287, + "objective/train/theoretical_loss": 3.367779303765479, + "objective/train/tokens_used": 2812279264, + "theoretical_loss": 3.367779303765479, + "tokens_seen": 2791819264 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012467803410230692, + "loss": 1.3046, + "theoretical_loss": 3.367779303765479, + "tokens_seen": 2791819264 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012466198595787362, + "loss": 1.2823, + "theoretical_loss": 3.367773188428453, + "tokens_seen": 2791884800 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012464593781344032, + "loss": 1.1072, + "theoretical_loss": 3.3677670732751674, + "tokens_seen": 2791950336 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012462988966900702, + "loss": 1.2991, + "theoretical_loss": 3.3677609583056136, + "tokens_seen": 2792015872 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012461384152457375, + "loss": 1.1181, + "theoretical_loss": 3.367754843519781, + "tokens_seen": 2792081408 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012459779338014045, + "loss": 1.276, + "theoretical_loss": 3.36774872891766, + "tokens_seen": 2792146944 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012458174523570715, + "loss": 1.2033, + "theoretical_loss": 3.367742614499241, + "tokens_seen": 2792212480 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012456569709127385, + "loss": 1.2514, + "theoretical_loss": 3.3677365002645137, + "tokens_seen": 2792278016 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012454964894684055, + "loss": 1.2638, + "theoretical_loss": 3.367730386213468, + "tokens_seen": 2792343552 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012453360080240722, + "loss": 1.209, + "theoretical_loss": 3.367724272346096, + "tokens_seen": 2792409088 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012451755265797392, + "loss": 1.2426, + "theoretical_loss": 3.367718158662385, + "tokens_seen": 2792474624 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012450150451354062, + "loss": 1.1751, + "theoretical_loss": 3.3677120451623273, + "tokens_seen": 2792540160 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012448545636910732, + "loss": 1.2457, + "theoretical_loss": 3.3677059318459124, + "tokens_seen": 2792605696 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012446940822467403, + "loss": 1.0437, + "theoretical_loss": 3.36769981871313, + "tokens_seen": 2792671232 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012445336008024073, + "loss": 1.2061, + "theoretical_loss": 3.367693705763971, + "tokens_seen": 2792736768 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012443731193580743, + "loss": 1.1597, + "theoretical_loss": 3.3676875929984256, + "tokens_seen": 2792802304 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012442126379137413, + "loss": 1.2986, + "theoretical_loss": 3.367681480416483, + "tokens_seen": 2792867840 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012440521564694083, + "loss": 1.2881, + "theoretical_loss": 3.367675368018135, + "tokens_seen": 2792933376 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012438916750250753, + "loss": 1.1791, + "theoretical_loss": 3.36766925580337, + "tokens_seen": 2792998912 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012437311935807423, + "loss": 1.2248, + "theoretical_loss": 3.3676631437721793, + "tokens_seen": 2793064448 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012435707121364093, + "loss": 1.2425, + "theoretical_loss": 3.3676570319245527, + "tokens_seen": 2793129984 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012434102306920763, + "loss": 1.2157, + "theoretical_loss": 3.3676509202604805, + "tokens_seen": 2793195520 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012432497492477433, + "loss": 1.1777, + "theoretical_loss": 3.3676448087799535, + "tokens_seen": 2793261056 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012430892678034103, + "loss": 1.1397, + "theoretical_loss": 3.3676386974829606, + "tokens_seen": 2793326592 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012429287863590773, + "loss": 1.263, + "theoretical_loss": 3.3676325863694925, + "tokens_seen": 2793392128 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012427683049147443, + "loss": 1.2094, + "theoretical_loss": 3.3676264754395397, + "tokens_seen": 2793457664 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012426078234704113, + "loss": 1.1614, + "theoretical_loss": 3.3676203646930922, + "tokens_seen": 2793523200 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012424473420260783, + "loss": 1.1298, + "theoretical_loss": 3.3676142541301406, + "tokens_seen": 2793588736 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012422868605817453, + "loss": 1.1956, + "theoretical_loss": 3.367608143750674, + "tokens_seen": 2793654272 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012421263791374124, + "loss": 1.2644, + "theoretical_loss": 3.3676020335546837, + "tokens_seen": 2793719808 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012419658976930794, + "loss": 1.2168, + "theoretical_loss": 3.367595923542159, + "tokens_seen": 2793785344 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012418054162487464, + "loss": 1.3048, + "theoretical_loss": 3.367589813713091, + "tokens_seen": 2793850880 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012416449348044134, + "loss": 1.194, + "theoretical_loss": 3.367583704067469, + "tokens_seen": 2793916416 + }, + { + "epoch": 1.23, + "learning_rate": 0.000124148445336008, + "loss": 1.074, + "theoretical_loss": 3.367577594605284, + "tokens_seen": 2793981952 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001241323971915747, + "loss": 1.1672, + "theoretical_loss": 3.3675714853265255, + "tokens_seen": 2794047488 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001241163490471414, + "loss": 1.1683, + "theoretical_loss": 3.367565376231184, + "tokens_seen": 2794113024 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001241003009027081, + "loss": 1.2893, + "theoretical_loss": 3.36755926731925, + "tokens_seen": 2794178560 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001240842527582748, + "loss": 1.227, + "theoretical_loss": 3.367553158590713, + "tokens_seen": 2794244096 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012406820461384151, + "loss": 1.2143, + "theoretical_loss": 3.3675470500455633, + "tokens_seen": 2794309632 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012405215646940821, + "loss": 1.2273, + "theoretical_loss": 3.367540941683792, + "tokens_seen": 2794375168 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012403610832497492, + "loss": 1.254, + "theoretical_loss": 3.3675348335053883, + "tokens_seen": 2794440704 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012402006018054164, + "loss": 1.2783, + "theoretical_loss": 3.3675287255103425, + "tokens_seen": 2794506240 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012400401203610834, + "loss": 1.1823, + "theoretical_loss": 3.3675226176986457, + "tokens_seen": 2794571776 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012398796389167504, + "loss": 1.125, + "theoretical_loss": 3.367516510070287, + "tokens_seen": 2794637312 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012397191574724175, + "loss": 1.2251, + "theoretical_loss": 3.367510402625257, + "tokens_seen": 2794702848 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012395586760280845, + "loss": 1.1056, + "theoretical_loss": 3.3675042953635463, + "tokens_seen": 2794768384 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012393981945837515, + "loss": 1.1594, + "theoretical_loss": 3.3674981882851442, + "tokens_seen": 2794833920 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012392377131394185, + "loss": 1.1993, + "theoretical_loss": 3.3674920813900417, + "tokens_seen": 2794899456 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012390772316950855, + "loss": 1.1982, + "theoretical_loss": 3.3674859746782286, + "tokens_seen": 2794964992 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012389167502507525, + "loss": 1.2542, + "theoretical_loss": 3.3674798681496956, + "tokens_seen": 2795030528 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012387562688064195, + "loss": 1.1707, + "theoretical_loss": 3.367473761804432, + "tokens_seen": 2795096064 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012385957873620865, + "loss": 1.2744, + "theoretical_loss": 3.367467655642429, + "tokens_seen": 2795161600 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012384353059177532, + "loss": 1.2698, + "theoretical_loss": 3.367461549663676, + "tokens_seen": 2795227136 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012382748244734202, + "loss": 1.1642, + "theoretical_loss": 3.3674554438681636, + "tokens_seen": 2795292672 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012381143430290872, + "loss": 1.3118, + "theoretical_loss": 3.367449338255882, + "tokens_seen": 2795358208 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012379538615847542, + "loss": 1.2469, + "theoretical_loss": 3.3674432328268216, + "tokens_seen": 2795423744 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012377933801404213, + "loss": 1.4387, + "theoretical_loss": 3.3674371275809722, + "tokens_seen": 2795489280 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012376328986960883, + "loss": 1.1728, + "theoretical_loss": 3.367431022518324, + "tokens_seen": 2795554816 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012374724172517553, + "loss": 1.2298, + "theoretical_loss": 3.3674249176388673, + "tokens_seen": 2795620352 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012373119358074223, + "loss": 1.1937, + "theoretical_loss": 3.3674188129425926, + "tokens_seen": 2795685888 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012371514543630893, + "loss": 1.1399, + "theoretical_loss": 3.36741270842949, + "tokens_seen": 2795751424 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012369909729187563, + "loss": 1.3119, + "theoretical_loss": 3.3674066040995494, + "tokens_seen": 2795816960 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012368304914744233, + "loss": 1.1995, + "theoretical_loss": 3.3674004999527614, + "tokens_seen": 2795882496 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012366700100300903, + "loss": 1.2023, + "theoretical_loss": 3.3673943959891157, + "tokens_seen": 2795948032 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012365095285857573, + "loss": 1.3, + "theoretical_loss": 3.367388292208603, + "tokens_seen": 2796013568 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012363490471414243, + "loss": 1.1771, + "theoretical_loss": 3.3673821886112134, + "tokens_seen": 2796079104 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012361885656970913, + "loss": 1.3217, + "theoretical_loss": 3.367376085196937, + "tokens_seen": 2796144640 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012360280842527583, + "loss": 1.2597, + "theoretical_loss": 3.3673699819657643, + "tokens_seen": 2796210176 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012358676028084253, + "loss": 1.2176, + "theoretical_loss": 3.3673638789176854, + "tokens_seen": 2796275712 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012357071213640923, + "loss": 1.2085, + "theoretical_loss": 3.36735777605269, + "tokens_seen": 2796341248 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012355466399197593, + "loss": 1.2211, + "theoretical_loss": 3.367351673370769, + "tokens_seen": 2796406784 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012353861584754264, + "loss": 1.2186, + "theoretical_loss": 3.367345570871912, + "tokens_seen": 2796472320 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012352256770310934, + "loss": 1.2494, + "theoretical_loss": 3.3673394685561098, + "tokens_seen": 2796537856 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012350651955867604, + "loss": 1.2701, + "theoretical_loss": 3.3673333664233525, + "tokens_seen": 2796603392 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012349047141424274, + "loss": 1.2119, + "theoretical_loss": 3.36732726447363, + "tokens_seen": 2796668928 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012347442326980944, + "loss": 1.2247, + "theoretical_loss": 3.367321162706933, + "tokens_seen": 2796734464 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001234583751253761, + "loss": 1.186, + "theoretical_loss": 3.367315061123251, + "tokens_seen": 2796800000 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001234423269809428, + "loss": 1.2655, + "theoretical_loss": 3.367308959722575, + "tokens_seen": 2796865536 + }, + { + "epoch": 1.23, + "learning_rate": 0.0001234262788365095, + "loss": 1.3259, + "theoretical_loss": 3.367302858504895, + "tokens_seen": 2796931072 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012341023069207624, + "loss": 1.2443, + "theoretical_loss": 3.367296757470201, + "tokens_seen": 2796996608 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012339418254764294, + "loss": 1.1856, + "theoretical_loss": 3.3672906566184833, + "tokens_seen": 2797062144 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012337813440320964, + "loss": 1.2333, + "theoretical_loss": 3.367284555949732, + "tokens_seen": 2797127680 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012336208625877634, + "loss": 1.2564, + "theoretical_loss": 3.367278455463938, + "tokens_seen": 2797193216 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012334603811434304, + "loss": 1.1787, + "theoretical_loss": 3.3672723551610906, + "tokens_seen": 2797258752 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012332998996990974, + "loss": 1.194, + "theoretical_loss": 3.3672662550411805, + "tokens_seen": 2797324288 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012331394182547644, + "loss": 1.3045, + "theoretical_loss": 3.367260155104198, + "tokens_seen": 2797389824 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012329789368104314, + "loss": 1.1956, + "theoretical_loss": 3.367254055350133, + "tokens_seen": 2797455360 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012328184553660985, + "loss": 1.182, + "theoretical_loss": 3.367247955778976, + "tokens_seen": 2797520896 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012326579739217655, + "loss": 1.2048, + "theoretical_loss": 3.3672418563907174, + "tokens_seen": 2797586432 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012324974924774325, + "loss": 1.2079, + "theoretical_loss": 3.367235757185347, + "tokens_seen": 2797651968 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012323370110330995, + "loss": 1.1817, + "theoretical_loss": 3.367229658162855, + "tokens_seen": 2797717504 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012321765295887665, + "loss": 1.2927, + "theoretical_loss": 3.3672235593232323, + "tokens_seen": 2797783040 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012320160481444335, + "loss": 1.3028, + "theoretical_loss": 3.367217460666468, + "tokens_seen": 2797848576 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012318555667001005, + "loss": 1.225, + "theoretical_loss": 3.367211362192554, + "tokens_seen": 2797914112 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012316950852557675, + "loss": 1.22, + "theoretical_loss": 3.367205263901479, + "tokens_seen": 2797979648 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012315346038114345, + "loss": 1.2511, + "theoretical_loss": 3.3671991657932336, + "tokens_seen": 2798045184 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012313741223671012, + "loss": 1.0377, + "theoretical_loss": 3.3671930678678086, + "tokens_seen": 2798110720 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012312136409227682, + "loss": 1.2602, + "theoretical_loss": 3.367186970125194, + "tokens_seen": 2798176256 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012310531594784353, + "loss": 1.2088, + "theoretical_loss": 3.36718087256538, + "tokens_seen": 2798241792 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012308926780341023, + "loss": 1.0996, + "theoretical_loss": 3.367174775188356, + "tokens_seen": 2798307328 + }, + { + "epoch": 1.23, + "objective/train/docs_used": 1033006, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2817031145095825, + "objective/train/theoretical_loss": 3.3671686779941137, + "objective/train/tokens_used": 2818832864, + "theoretical_loss": 3.3671686779941137, + "tokens_seen": 2798372864 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012307321965897693, + "loss": 1.2817, + "theoretical_loss": 3.3671686779941137, + "tokens_seen": 2798372864 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012305717151454363, + "loss": 1.1931, + "theoretical_loss": 3.3671625809826424, + "tokens_seen": 2798438400 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012304112337011033, + "loss": 1.2853, + "theoretical_loss": 3.3671564841539325, + "tokens_seen": 2798503936 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012302507522567703, + "loss": 1.191, + "theoretical_loss": 3.3671503875079747, + "tokens_seen": 2798569472 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012300902708124373, + "loss": 1.2212, + "theoretical_loss": 3.367144291044758, + "tokens_seen": 2798635008 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012299297893681043, + "loss": 1.2701, + "theoretical_loss": 3.3671381947642742, + "tokens_seen": 2798700544 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012297693079237713, + "loss": 1.1839, + "theoretical_loss": 3.367132098666513, + "tokens_seen": 2798766080 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012296088264794383, + "loss": 1.2504, + "theoretical_loss": 3.367126002751464, + "tokens_seen": 2798831616 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012294483450351053, + "loss": 1.0722, + "theoretical_loss": 3.3671199070191182, + "tokens_seen": 2798897152 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012292878635907723, + "loss": 1.1735, + "theoretical_loss": 3.3671138114694656, + "tokens_seen": 2798962688 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012291273821464393, + "loss": 1.2225, + "theoretical_loss": 3.3671077161024963, + "tokens_seen": 2799028224 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012289669007021063, + "loss": 1.2005, + "theoretical_loss": 3.3671016209182008, + "tokens_seen": 2799093760 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012288064192577733, + "loss": 1.2557, + "theoretical_loss": 3.367095525916569, + "tokens_seen": 2799159296 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012286459378134403, + "loss": 1.2081, + "theoretical_loss": 3.3670894310975914, + "tokens_seen": 2799224832 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012284854563691074, + "loss": 1.1678, + "theoretical_loss": 3.3670833364612585, + "tokens_seen": 2799290368 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012283249749247744, + "loss": 1.1134, + "theoretical_loss": 3.36707724200756, + "tokens_seen": 2799355904 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012281644934804414, + "loss": 1.2019, + "theoretical_loss": 3.3670711477364867, + "tokens_seen": 2799421440 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012280040120361084, + "loss": 1.242, + "theoretical_loss": 3.3670650536480284, + "tokens_seen": 2799486976 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012278435305917754, + "loss": 1.1047, + "theoretical_loss": 3.3670589597421756, + "tokens_seen": 2799552512 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012276830491474424, + "loss": 1.2483, + "theoretical_loss": 3.3670528660189185, + "tokens_seen": 2799618048 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012275225677031094, + "loss": 1.1524, + "theoretical_loss": 3.3670467724782474, + "tokens_seen": 2799683584 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012273620862587764, + "loss": 1.2321, + "theoretical_loss": 3.3670406791201524, + "tokens_seen": 2799749120 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012272016048144434, + "loss": 1.1607, + "theoretical_loss": 3.367034585944624, + "tokens_seen": 2799814656 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012270411233701104, + "loss": 1.1729, + "theoretical_loss": 3.367028492951652, + "tokens_seen": 2799880192 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012268806419257774, + "loss": 1.2781, + "theoretical_loss": 3.367022400141227, + "tokens_seen": 2799945728 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012267201604814444, + "loss": 1.1112, + "theoretical_loss": 3.3670163075133392, + "tokens_seen": 2800011264 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012265596790371114, + "loss": 1.1729, + "theoretical_loss": 3.367010215067979, + "tokens_seen": 2800076800 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012263991975927784, + "loss": 1.2585, + "theoretical_loss": 3.367004122805137, + "tokens_seen": 2800142336 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012262387161484454, + "loss": 1.1964, + "theoretical_loss": 3.366998030724802, + "tokens_seen": 2800207872 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012260782347041124, + "loss": 1.1465, + "theoretical_loss": 3.366991938826966, + "tokens_seen": 2800273408 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012259177532597795, + "loss": 1.22, + "theoretical_loss": 3.3669858471116183, + "tokens_seen": 2800338944 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012257572718154465, + "loss": 1.2228, + "theoretical_loss": 3.3669797555787495, + "tokens_seen": 2800404480 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012255967903711135, + "loss": 1.214, + "theoretical_loss": 3.36697366422835, + "tokens_seen": 2800470016 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012254363089267805, + "loss": 1.1442, + "theoretical_loss": 3.3669675730604096, + "tokens_seen": 2800535552 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012252758274824475, + "loss": 1.2066, + "theoretical_loss": 3.3669614820749185, + "tokens_seen": 2800601088 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012251153460381145, + "loss": 1.1919, + "theoretical_loss": 3.3669553912718677, + "tokens_seen": 2800666624 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012249548645937815, + "loss": 1.2418, + "theoretical_loss": 3.3669493006512465, + "tokens_seen": 2800732160 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012247943831494485, + "loss": 1.2375, + "theoretical_loss": 3.366943210213046, + "tokens_seen": 2800797696 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012246339017051155, + "loss": 1.2258, + "theoretical_loss": 3.366937119957256, + "tokens_seen": 2800863232 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012244734202607825, + "loss": 1.1469, + "theoretical_loss": 3.366931029883867, + "tokens_seen": 2800928768 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012243129388164492, + "loss": 1.2972, + "theoretical_loss": 3.3669249399928693, + "tokens_seen": 2800994304 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012241524573721163, + "loss": 1.218, + "theoretical_loss": 3.3669188502842533, + "tokens_seen": 2801059840 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012239919759277833, + "loss": 1.3056, + "theoretical_loss": 3.3669127607580087, + "tokens_seen": 2801125376 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012238314944834503, + "loss": 1.1116, + "theoretical_loss": 3.3669066714141263, + "tokens_seen": 2801190912 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012236710130391173, + "loss": 1.2146, + "theoretical_loss": 3.3669005822525957, + "tokens_seen": 2801256448 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012235105315947843, + "loss": 1.31, + "theoretical_loss": 3.366894493273408, + "tokens_seen": 2801321984 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012233500501504513, + "loss": 1.2594, + "theoretical_loss": 3.3668884044765535, + "tokens_seen": 2801387520 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012231895687061183, + "loss": 1.2065, + "theoretical_loss": 3.3668823158620214, + "tokens_seen": 2801453056 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012230290872617853, + "loss": 1.2039, + "theoretical_loss": 3.366876227429803, + "tokens_seen": 2801518592 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012228686058174523, + "loss": 1.1668, + "theoretical_loss": 3.3668701391798885, + "tokens_seen": 2801584128 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012227081243731193, + "loss": 1.1108, + "theoretical_loss": 3.3668640511122674, + "tokens_seen": 2801649664 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012225476429287863, + "loss": 1.1919, + "theoretical_loss": 3.3668579632269307, + "tokens_seen": 2801715200 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012223871614844533, + "loss": 1.1633, + "theoretical_loss": 3.3668518755238686, + "tokens_seen": 2801780736 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012222266800401203, + "loss": 1.1619, + "theoretical_loss": 3.3668457880030713, + "tokens_seen": 2801846272 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012220661985957876, + "loss": 1.0895, + "theoretical_loss": 3.3668397006645288, + "tokens_seen": 2801911808 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012219057171514546, + "loss": 1.2071, + "theoretical_loss": 3.366833613508232, + "tokens_seen": 2801977344 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012217452357071216, + "loss": 1.2474, + "theoretical_loss": 3.3668275265341703, + "tokens_seen": 2802042880 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012215847542627886, + "loss": 1.1782, + "theoretical_loss": 3.366821439742335, + "tokens_seen": 2802108416 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012214242728184556, + "loss": 1.1278, + "theoretical_loss": 3.3668153531327154, + "tokens_seen": 2802173952 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012212637913741224, + "loss": 1.2416, + "theoretical_loss": 3.3668092667053027, + "tokens_seen": 2802239488 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012211033099297894, + "loss": 1.2175, + "theoretical_loss": 3.3668031804600864, + "tokens_seen": 2802305024 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012209428284854564, + "loss": 1.2241, + "theoretical_loss": 3.3667970943970573, + "tokens_seen": 2802370560 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012207823470411234, + "loss": 1.1913, + "theoretical_loss": 3.3667910085162056, + "tokens_seen": 2802436096 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012206218655967905, + "loss": 1.2427, + "theoretical_loss": 3.366784922817521, + "tokens_seen": 2802501632 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012204613841524575, + "loss": 1.2888, + "theoretical_loss": 3.366778837300995, + "tokens_seen": 2802567168 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012203009027081244, + "loss": 1.221, + "theoretical_loss": 3.3667727519666166, + "tokens_seen": 2802632704 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012201404212637914, + "loss": 1.2465, + "theoretical_loss": 3.3667666668143768, + "tokens_seen": 2802698240 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012199799398194584, + "loss": 1.2197, + "theoretical_loss": 3.366760581844266, + "tokens_seen": 2802763776 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012198194583751254, + "loss": 1.2167, + "theoretical_loss": 3.3667544970562737, + "tokens_seen": 2802829312 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012196589769307924, + "loss": 1.167, + "theoretical_loss": 3.3667484124503915, + "tokens_seen": 2802894848 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012194984954864594, + "loss": 1.1771, + "theoretical_loss": 3.3667423280266084, + "tokens_seen": 2802960384 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012193380140421264, + "loss": 1.3689, + "theoretical_loss": 3.3667362437849153, + "tokens_seen": 2803025920 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012191775325977934, + "loss": 1.3141, + "theoretical_loss": 3.3667301597253023, + "tokens_seen": 2803091456 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012190170511534605, + "loss": 1.2478, + "theoretical_loss": 3.36672407584776, + "tokens_seen": 2803156992 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012188565697091275, + "loss": 1.2233, + "theoretical_loss": 3.3667179921522785, + "tokens_seen": 2803222528 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012186960882647945, + "loss": 1.2584, + "theoretical_loss": 3.366711908638848, + "tokens_seen": 2803288064 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012185356068204615, + "loss": 1.1713, + "theoretical_loss": 3.366705825307459, + "tokens_seen": 2803353600 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012183751253761283, + "loss": 1.1818, + "theoretical_loss": 3.3666997421581017, + "tokens_seen": 2803419136 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012182146439317954, + "loss": 1.1705, + "theoretical_loss": 3.366693659190766, + "tokens_seen": 2803484672 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012180541624874624, + "loss": 1.1882, + "theoretical_loss": 3.366687576405443, + "tokens_seen": 2803550208 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012178936810431294, + "loss": 1.2117, + "theoretical_loss": 3.3666814938021226, + "tokens_seen": 2803615744 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012177331995987964, + "loss": 1.1279, + "theoretical_loss": 3.366675411380795, + "tokens_seen": 2803681280 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012175727181544634, + "loss": 1.2605, + "theoretical_loss": 3.3666693291414505, + "tokens_seen": 2803746816 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012174122367101304, + "loss": 1.2131, + "theoretical_loss": 3.3666632470840794, + "tokens_seen": 2803812352 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012172517552657974, + "loss": 1.1807, + "theoretical_loss": 3.3666571652086725, + "tokens_seen": 2803877888 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012170912738214644, + "loss": 1.2346, + "theoretical_loss": 3.3666510835152192, + "tokens_seen": 2803943424 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012169307923771314, + "loss": 1.232, + "theoretical_loss": 3.3666450020037106, + "tokens_seen": 2804008960 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012167703109327984, + "loss": 1.149, + "theoretical_loss": 3.3666389206741365, + "tokens_seen": 2804074496 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012166098294884654, + "loss": 1.2211, + "theoretical_loss": 3.3666328395264875, + "tokens_seen": 2804140032 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012164493480441323, + "loss": 1.2079, + "theoretical_loss": 3.366626758560754, + "tokens_seen": 2804205568 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012162888665997993, + "loss": 1.1269, + "theoretical_loss": 3.366620677776926, + "tokens_seen": 2804271104 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012161283851554666, + "loss": 1.0601, + "theoretical_loss": 3.3666145971749937, + "tokens_seen": 2804336640 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012159679037111336, + "loss": 1.2389, + "theoretical_loss": 3.3666085167549475, + "tokens_seen": 2804402176 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012158074222668006, + "loss": 1.2263, + "theoretical_loss": 3.3666024365167786, + "tokens_seen": 2804467712 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012156469408224676, + "loss": 1.0885, + "theoretical_loss": 3.3665963564604757, + "tokens_seen": 2804533248 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012154864593781346, + "loss": 1.3701, + "theoretical_loss": 3.3665902765860305, + "tokens_seen": 2804598784 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012153259779338016, + "loss": 1.1582, + "theoretical_loss": 3.3665841968934327, + "tokens_seen": 2804664320 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012151654964894685, + "loss": 1.1478, + "theoretical_loss": 3.3665781173826725, + "tokens_seen": 2804729856 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012150050150451355, + "loss": 1.1885, + "theoretical_loss": 3.3665720380537407, + "tokens_seen": 2804795392 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012148445336008025, + "loss": 1.1912, + "theoretical_loss": 3.366565958906627, + "tokens_seen": 2804860928 + }, + { + "epoch": 1.23, + "objective/train/docs_used": 1035278, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1931676864624023, + "objective/train/theoretical_loss": 3.366559879941322, + "objective/train/tokens_used": 2825386464, + "theoretical_loss": 3.366559879941322, + "tokens_seen": 2804926464 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012146840521564695, + "loss": 1.1932, + "theoretical_loss": 3.366559879941322, + "tokens_seen": 2804926464 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012145235707121365, + "loss": 1.1498, + "theoretical_loss": 3.3665538011578167, + "tokens_seen": 2804992000 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012143630892678035, + "loss": 1.1919, + "theoretical_loss": 3.3665477225561, + "tokens_seen": 2805057536 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012142026078234705, + "loss": 1.2097, + "theoretical_loss": 3.3665416441361633, + "tokens_seen": 2805123072 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012140421263791375, + "loss": 1.1453, + "theoretical_loss": 3.3665355658979967, + "tokens_seen": 2805188608 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012138816449348045, + "loss": 1.1155, + "theoretical_loss": 3.36652948784159, + "tokens_seen": 2805254144 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012137211634904715, + "loss": 1.21, + "theoretical_loss": 3.366523409966934, + "tokens_seen": 2805319680 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012135606820461385, + "loss": 1.1846, + "theoretical_loss": 3.3665173322740194, + "tokens_seen": 2805385216 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012134002006018055, + "loss": 1.2653, + "theoretical_loss": 3.3665112547628357, + "tokens_seen": 2805450752 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012132397191574724, + "loss": 1.235, + "theoretical_loss": 3.366505177433374, + "tokens_seen": 2805516288 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012130792377131394, + "loss": 1.1976, + "theoretical_loss": 3.3664991002856235, + "tokens_seen": 2805581824 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012129187562688064, + "loss": 1.1242, + "theoretical_loss": 3.366493023319576, + "tokens_seen": 2805647360 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012127582748244734, + "loss": 1.1857, + "theoretical_loss": 3.36648694653522, + "tokens_seen": 2805712896 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012125977933801404, + "loss": 1.2765, + "theoretical_loss": 3.366480869932548, + "tokens_seen": 2805778432 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012124373119358074, + "loss": 1.169, + "theoretical_loss": 3.3664747935115487, + "tokens_seen": 2805843968 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012122768304914745, + "loss": 1.1846, + "theoretical_loss": 3.366468717272213, + "tokens_seen": 2805909504 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012121163490471415, + "loss": 1.1726, + "theoretical_loss": 3.366462641214531, + "tokens_seen": 2805975040 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012119558676028085, + "loss": 1.2468, + "theoretical_loss": 3.3664565653384937, + "tokens_seen": 2806040576 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012117953861584755, + "loss": 1.2298, + "theoretical_loss": 3.36645048964409, + "tokens_seen": 2806106112 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012116349047141425, + "loss": 1.2326, + "theoretical_loss": 3.366444414131312, + "tokens_seen": 2806171648 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012114744232698095, + "loss": 1.1105, + "theoretical_loss": 3.366438338800149, + "tokens_seen": 2806237184 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012113139418254764, + "loss": 1.238, + "theoretical_loss": 3.3664322636505912, + "tokens_seen": 2806302720 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012111534603811434, + "loss": 1.2563, + "theoretical_loss": 3.366426188682629, + "tokens_seen": 2806368256 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012109929789368104, + "loss": 1.1322, + "theoretical_loss": 3.3664201138962535, + "tokens_seen": 2806433792 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012108324974924774, + "loss": 1.1495, + "theoretical_loss": 3.3664140392914543, + "tokens_seen": 2806499328 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012106720160481444, + "loss": 1.1295, + "theoretical_loss": 3.3664079648682224, + "tokens_seen": 2806564864 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012105115346038114, + "loss": 1.1526, + "theoretical_loss": 3.366401890626547, + "tokens_seen": 2806630400 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012103510531594784, + "loss": 1.1696, + "theoretical_loss": 3.366395816566419, + "tokens_seen": 2806695936 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012101905717151457, + "loss": 1.18, + "theoretical_loss": 3.3663897426878293, + "tokens_seen": 2806761472 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012100300902708125, + "loss": 1.2551, + "theoretical_loss": 3.3663836689907676, + "tokens_seen": 2806827008 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012098696088264795, + "loss": 1.3083, + "theoretical_loss": 3.366377595475224, + "tokens_seen": 2806892544 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012097091273821466, + "loss": 1.2475, + "theoretical_loss": 3.36637152214119, + "tokens_seen": 2806958080 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012095486459378136, + "loss": 1.2431, + "theoretical_loss": 3.3663654489886548, + "tokens_seen": 2807023616 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012093881644934806, + "loss": 1.1807, + "theoretical_loss": 3.366359376017609, + "tokens_seen": 2807089152 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012092276830491476, + "loss": 1.222, + "theoretical_loss": 3.3663533032280433, + "tokens_seen": 2807154688 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012090672016048146, + "loss": 1.195, + "theoretical_loss": 3.3663472306199473, + "tokens_seen": 2807220224 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012089067201604816, + "loss": 1.2493, + "theoretical_loss": 3.3663411581933125, + "tokens_seen": 2807285760 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012087462387161486, + "loss": 1.2096, + "theoretical_loss": 3.366335085948128, + "tokens_seen": 2807351296 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012085857572718156, + "loss": 1.2916, + "theoretical_loss": 3.366329013884385, + "tokens_seen": 2807416832 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012084252758274826, + "loss": 1.1861, + "theoretical_loss": 3.3663229420020735, + "tokens_seen": 2807482368 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012082647943831496, + "loss": 1.2313, + "theoretical_loss": 3.366316870301184, + "tokens_seen": 2807547904 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012081043129388165, + "loss": 1.2338, + "theoretical_loss": 3.366310798781707, + "tokens_seen": 2807613440 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012079438314944835, + "loss": 1.1816, + "theoretical_loss": 3.366304727443632, + "tokens_seen": 2807678976 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012077833500501505, + "loss": 1.2038, + "theoretical_loss": 3.36629865628695, + "tokens_seen": 2807744512 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012076228686058175, + "loss": 1.1629, + "theoretical_loss": 3.3662925853116517, + "tokens_seen": 2807810048 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012074623871614845, + "loss": 1.2084, + "theoretical_loss": 3.3662865145177268, + "tokens_seen": 2807875584 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012073019057171515, + "loss": 1.2488, + "theoretical_loss": 3.3662804439051657, + "tokens_seen": 2807941120 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012071414242728185, + "loss": 1.2072, + "theoretical_loss": 3.3662743734739595, + "tokens_seen": 2808006656 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012069809428284855, + "loss": 1.1924, + "theoretical_loss": 3.3662683032240976, + "tokens_seen": 2808072192 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012068204613841525, + "loss": 1.265, + "theoretical_loss": 3.3662622331555707, + "tokens_seen": 2808137728 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012066599799398195, + "loss": 1.1762, + "theoretical_loss": 3.3662561632683694, + "tokens_seen": 2808203264 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012064994984954865, + "loss": 1.1234, + "theoretical_loss": 3.3662500935624835, + "tokens_seen": 2808268800 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012063390170511536, + "loss": 1.2586, + "theoretical_loss": 3.3662440240379037, + "tokens_seen": 2808334336 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012061785356068204, + "loss": 1.1549, + "theoretical_loss": 3.3662379546946206, + "tokens_seen": 2808399872 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012060180541624874, + "loss": 1.232, + "theoretical_loss": 3.366231885532624, + "tokens_seen": 2808465408 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012058575727181544, + "loss": 1.341, + "theoretical_loss": 3.366225816551905, + "tokens_seen": 2808530944 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012056970912738214, + "loss": 1.1885, + "theoretical_loss": 3.3662197477524534, + "tokens_seen": 2808596480 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012055366098294884, + "loss": 1.2351, + "theoretical_loss": 3.366213679134259, + "tokens_seen": 2808662016 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012053761283851555, + "loss": 1.2369, + "theoretical_loss": 3.366207610697314, + "tokens_seen": 2808727552 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012052156469408225, + "loss": 1.2788, + "theoretical_loss": 3.3662015424416065, + "tokens_seen": 2808793088 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012050551654964895, + "loss": 1.2791, + "theoretical_loss": 3.3661954743671285, + "tokens_seen": 2808858624 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012048946840521565, + "loss": 1.1997, + "theoretical_loss": 3.3661894064738695, + "tokens_seen": 2808924160 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012047342026078235, + "loss": 1.2808, + "theoretical_loss": 3.3661833387618203, + "tokens_seen": 2808989696 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012045737211634905, + "loss": 1.1644, + "theoretical_loss": 3.366177271230971, + "tokens_seen": 2809055232 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012044132397191575, + "loss": 1.1875, + "theoretical_loss": 3.3661712038813123, + "tokens_seen": 2809120768 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012042527582748244, + "loss": 1.2119, + "theoretical_loss": 3.3661651367128345, + "tokens_seen": 2809186304 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012040922768304916, + "loss": 1.214, + "theoretical_loss": 3.3661590697255273, + "tokens_seen": 2809251840 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012039317953861586, + "loss": 1.3122, + "theoretical_loss": 3.366153002919382, + "tokens_seen": 2809317376 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012037713139418257, + "loss": 1.2618, + "theoretical_loss": 3.3661469362943883, + "tokens_seen": 2809382912 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012036108324974927, + "loss": 1.122, + "theoretical_loss": 3.3661408698505366, + "tokens_seen": 2809448448 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012034503510531597, + "loss": 1.1513, + "theoretical_loss": 3.366134803587818, + "tokens_seen": 2809513984 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012032898696088267, + "loss": 1.258, + "theoretical_loss": 3.366128737506222, + "tokens_seen": 2809579520 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012031293881644935, + "loss": 1.1361, + "theoretical_loss": 3.3661226716057393, + "tokens_seen": 2809645056 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012029689067201605, + "loss": 1.1918, + "theoretical_loss": 3.36611660588636, + "tokens_seen": 2809710592 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012028084252758276, + "loss": 1.2152, + "theoretical_loss": 3.366110540348075, + "tokens_seen": 2809776128 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012026479438314946, + "loss": 1.1945, + "theoretical_loss": 3.3661044749908746, + "tokens_seen": 2809841664 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012024874623871616, + "loss": 1.1869, + "theoretical_loss": 3.366098409814749, + "tokens_seen": 2809907200 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012023269809428286, + "loss": 1.2049, + "theoretical_loss": 3.366092344819688, + "tokens_seen": 2809972736 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012021664994984956, + "loss": 1.1817, + "theoretical_loss": 3.3660862800056828, + "tokens_seen": 2810038272 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012020060180541626, + "loss": 1.161, + "theoretical_loss": 3.366080215372724, + "tokens_seen": 2810103808 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012018455366098296, + "loss": 1.1736, + "theoretical_loss": 3.3660741509208005, + "tokens_seen": 2810169344 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012016850551654966, + "loss": 1.1792, + "theoretical_loss": 3.3660680866499044, + "tokens_seen": 2810234880 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012015245737211636, + "loss": 1.2699, + "theoretical_loss": 3.3660620225600253, + "tokens_seen": 2810300416 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012013640922768306, + "loss": 1.2442, + "theoretical_loss": 3.366055958651153, + "tokens_seen": 2810365952 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012012036108324975, + "loss": 1.2202, + "theoretical_loss": 3.366049894923279, + "tokens_seen": 2810431488 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012010431293881645, + "loss": 1.2592, + "theoretical_loss": 3.366043831376393, + "tokens_seen": 2810497024 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012008826479438315, + "loss": 1.2, + "theoretical_loss": 3.3660377680104854, + "tokens_seen": 2810562560 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012007221664994985, + "loss": 1.244, + "theoretical_loss": 3.366031704825547, + "tokens_seen": 2810628096 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012005616850551655, + "loss": 1.2589, + "theoretical_loss": 3.366025641821567, + "tokens_seen": 2810693632 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012004012036108325, + "loss": 1.1419, + "theoretical_loss": 3.3660195789985377, + "tokens_seen": 2810759168 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012002407221664995, + "loss": 1.1433, + "theoretical_loss": 3.366013516356448, + "tokens_seen": 2810824704 + }, + { + "epoch": 1.23, + "learning_rate": 0.00012000802407221665, + "loss": 1.1435, + "theoretical_loss": 3.3660074538952887, + "tokens_seen": 2810890240 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011999197592778335, + "loss": 1.1209, + "theoretical_loss": 3.3660013916150504, + "tokens_seen": 2810955776 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011997592778335005, + "loss": 1.1959, + "theoretical_loss": 3.365995329515723, + "tokens_seen": 2811021312 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011995987963891675, + "loss": 1.3611, + "theoretical_loss": 3.365989267597297, + "tokens_seen": 2811086848 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011994383149448346, + "loss": 1.2033, + "theoretical_loss": 3.3659832058597634, + "tokens_seen": 2811152384 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011992778335005014, + "loss": 1.2741, + "theoretical_loss": 3.365977144303112, + "tokens_seen": 2811217920 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011991173520561684, + "loss": 1.1273, + "theoretical_loss": 3.3659710829273335, + "tokens_seen": 2811283456 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011989568706118354, + "loss": 1.2541, + "theoretical_loss": 3.3659650217324177, + "tokens_seen": 2811348992 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011987963891675024, + "loss": 1.1885, + "theoretical_loss": 3.365958960718356, + "tokens_seen": 2811414528 + }, + { + "epoch": 1.23, + "objective/train/docs_used": 1038077, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2164435386657715, + "objective/train/theoretical_loss": 3.365952899885138, + "objective/train/tokens_used": 2831940064, + "theoretical_loss": 3.365952899885138, + "tokens_seen": 2811480064 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011986359077231694, + "loss": 1.2164, + "theoretical_loss": 3.365952899885138, + "tokens_seen": 2811480064 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011984754262788365, + "loss": 1.2853, + "theoretical_loss": 3.3659468392327536, + "tokens_seen": 2811545600 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011983149448345035, + "loss": 1.2145, + "theoretical_loss": 3.3659407787611944, + "tokens_seen": 2811611136 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011981544633901707, + "loss": 1.2078, + "theoretical_loss": 3.3659347184704504, + "tokens_seen": 2811676672 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011979939819458376, + "loss": 1.23, + "theoretical_loss": 3.3659286583605117, + "tokens_seen": 2811742208 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011978335005015046, + "loss": 1.2534, + "theoretical_loss": 3.3659225984313688, + "tokens_seen": 2811807744 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011976730190571716, + "loss": 1.1752, + "theoretical_loss": 3.365916538683012, + "tokens_seen": 2811873280 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011975125376128386, + "loss": 1.1889, + "theoretical_loss": 3.3659104791154317, + "tokens_seen": 2811938816 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011973520561685056, + "loss": 1.2259, + "theoretical_loss": 3.365904419728619, + "tokens_seen": 2812004352 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011971915747241726, + "loss": 1.2396, + "theoretical_loss": 3.3658983605225634, + "tokens_seen": 2812069888 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011970310932798396, + "loss": 1.2582, + "theoretical_loss": 3.3658923014972553, + "tokens_seen": 2812135424 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011968706118355067, + "loss": 1.2562, + "theoretical_loss": 3.3658862426526857, + "tokens_seen": 2812200960 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011967101303911737, + "loss": 1.1816, + "theoretical_loss": 3.365880183988845, + "tokens_seen": 2812266496 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011965496489468407, + "loss": 1.2283, + "theoretical_loss": 3.365874125505723, + "tokens_seen": 2812332032 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011963891675025077, + "loss": 1.1658, + "theoretical_loss": 3.3658680672033103, + "tokens_seen": 2812397568 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011962286860581747, + "loss": 1.3839, + "theoretical_loss": 3.365862009081597, + "tokens_seen": 2812463104 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011960682046138416, + "loss": 1.1845, + "theoretical_loss": 3.3658559511405746, + "tokens_seen": 2812528640 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011959077231695086, + "loss": 1.1291, + "theoretical_loss": 3.3658498933802328, + "tokens_seen": 2812594176 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011957472417251756, + "loss": 1.163, + "theoretical_loss": 3.3658438358005616, + "tokens_seen": 2812659712 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011955867602808426, + "loss": 1.166, + "theoretical_loss": 3.365837778401552, + "tokens_seen": 2812725248 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011954262788365096, + "loss": 1.199, + "theoretical_loss": 3.365831721183194, + "tokens_seen": 2812790784 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011952657973921766, + "loss": 1.196, + "theoretical_loss": 3.3658256641454782, + "tokens_seen": 2812856320 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011951053159478436, + "loss": 1.1505, + "theoretical_loss": 3.3658196072883952, + "tokens_seen": 2812921856 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011949448345035106, + "loss": 1.1879, + "theoretical_loss": 3.365813550611935, + "tokens_seen": 2812987392 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011947843530591776, + "loss": 1.2494, + "theoretical_loss": 3.3658074941160887, + "tokens_seen": 2813052928 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011946238716148446, + "loss": 1.1688, + "theoretical_loss": 3.3658014378008456, + "tokens_seen": 2813118464 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011944633901705116, + "loss": 1.1893, + "theoretical_loss": 3.365795381666197, + "tokens_seen": 2813184000 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011943029087261786, + "loss": 1.2091, + "theoretical_loss": 3.365789325712133, + "tokens_seen": 2813249536 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011941424272818455, + "loss": 1.246, + "theoretical_loss": 3.3657832699386443, + "tokens_seen": 2813315072 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011939819458375125, + "loss": 1.2194, + "theoretical_loss": 3.365777214345721, + "tokens_seen": 2813380608 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011938214643931795, + "loss": 1.1979, + "theoretical_loss": 3.3657711589333537, + "tokens_seen": 2813446144 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011936609829488465, + "loss": 1.1285, + "theoretical_loss": 3.3657651037015324, + "tokens_seen": 2813511680 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011935005015045135, + "loss": 1.2307, + "theoretical_loss": 3.3657590486502476, + "tokens_seen": 2813577216 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011933400200601805, + "loss": 1.3153, + "theoretical_loss": 3.3657529937794903, + "tokens_seen": 2813642752 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011931795386158475, + "loss": 1.2108, + "theoretical_loss": 3.3657469390892505, + "tokens_seen": 2813708288 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011930190571715145, + "loss": 1.2921, + "theoretical_loss": 3.3657408845795183, + "tokens_seen": 2813773824 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011928585757271815, + "loss": 1.2611, + "theoretical_loss": 3.3657348302502843, + "tokens_seen": 2813839360 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011926980942828485, + "loss": 1.2477, + "theoretical_loss": 3.3657287761015398, + "tokens_seen": 2813904896 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011925376128385156, + "loss": 1.3133, + "theoretical_loss": 3.365722722133274, + "tokens_seen": 2813970432 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011923771313941826, + "loss": 1.1928, + "theoretical_loss": 3.3657166683454776, + "tokens_seen": 2814035968 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011922166499498494, + "loss": 1.0707, + "theoretical_loss": 3.3657106147381417, + "tokens_seen": 2814101504 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011920561685055167, + "loss": 1.2195, + "theoretical_loss": 3.365704561311256, + "tokens_seen": 2814167040 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011918956870611837, + "loss": 1.1581, + "theoretical_loss": 3.3656985080648107, + "tokens_seen": 2814232576 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011917352056168507, + "loss": 1.2226, + "theoretical_loss": 3.3656924549987974, + "tokens_seen": 2814298112 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011915747241725177, + "loss": 1.242, + "theoretical_loss": 3.3656864021132056, + "tokens_seen": 2814363648 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011914142427281847, + "loss": 1.2364, + "theoretical_loss": 3.365680349408026, + "tokens_seen": 2814429184 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011912537612838517, + "loss": 1.1868, + "theoretical_loss": 3.3656742968832485, + "tokens_seen": 2814494720 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011910932798395187, + "loss": 1.2311, + "theoretical_loss": 3.365668244538864, + "tokens_seen": 2814560256 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011909327983951856, + "loss": 1.2382, + "theoretical_loss": 3.365662192374863, + "tokens_seen": 2814625792 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011907723169508526, + "loss": 1.1849, + "theoretical_loss": 3.365656140391236, + "tokens_seen": 2814691328 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011906118355065196, + "loss": 1.2334, + "theoretical_loss": 3.365650088587973, + "tokens_seen": 2814756864 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011904513540621866, + "loss": 1.1146, + "theoretical_loss": 3.365644036965065, + "tokens_seen": 2814822400 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011902908726178536, + "loss": 1.0581, + "theoretical_loss": 3.365637985522502, + "tokens_seen": 2814887936 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011901303911735206, + "loss": 1.1472, + "theoretical_loss": 3.365631934260274, + "tokens_seen": 2814953472 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011899699097291877, + "loss": 1.2652, + "theoretical_loss": 3.3656258831783723, + "tokens_seen": 2815019008 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011898094282848547, + "loss": 1.2711, + "theoretical_loss": 3.3656198322767867, + "tokens_seen": 2815084544 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011896489468405217, + "loss": 1.1667, + "theoretical_loss": 3.365613781555508, + "tokens_seen": 2815150080 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011894884653961887, + "loss": 1.1375, + "theoretical_loss": 3.365607731014527, + "tokens_seen": 2815215616 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011893279839518557, + "loss": 1.2659, + "theoretical_loss": 3.365601680653833, + "tokens_seen": 2815281152 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011891675025075227, + "loss": 1.1567, + "theoretical_loss": 3.3655956304734174, + "tokens_seen": 2815346688 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011890070210631896, + "loss": 1.2716, + "theoretical_loss": 3.3655895804732703, + "tokens_seen": 2815412224 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011888465396188566, + "loss": 1.1583, + "theoretical_loss": 3.365583530653382, + "tokens_seen": 2815477760 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011886860581745236, + "loss": 1.2137, + "theoretical_loss": 3.365577481013743, + "tokens_seen": 2815543296 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011885255767301906, + "loss": 1.1823, + "theoretical_loss": 3.3655714315543444, + "tokens_seen": 2815608832 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011883650952858576, + "loss": 1.2054, + "theoretical_loss": 3.365565382275175, + "tokens_seen": 2815674368 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011882046138415246, + "loss": 1.1385, + "theoretical_loss": 3.3655593331762272, + "tokens_seen": 2815739904 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011880441323971916, + "loss": 1.308, + "theoretical_loss": 3.3655532842574902, + "tokens_seen": 2815805440 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011878836509528586, + "loss": 1.1555, + "theoretical_loss": 3.365547235518955, + "tokens_seen": 2815870976 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011877231695085256, + "loss": 1.2001, + "theoretical_loss": 3.3655411869606113, + "tokens_seen": 2815936512 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011875626880641926, + "loss": 1.221, + "theoretical_loss": 3.3655351385824503, + "tokens_seen": 2816002048 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011874022066198596, + "loss": 1.1582, + "theoretical_loss": 3.365529090384462, + "tokens_seen": 2816067584 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011872417251755266, + "loss": 1.2177, + "theoretical_loss": 3.3655230423666374, + "tokens_seen": 2816133120 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011870812437311935, + "loss": 1.161, + "theoretical_loss": 3.3655169945289662, + "tokens_seen": 2816198656 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011869207622868605, + "loss": 1.2418, + "theoretical_loss": 3.3655109468714395, + "tokens_seen": 2816264192 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011867602808425275, + "loss": 1.1571, + "theoretical_loss": 3.3655048993940473, + "tokens_seen": 2816329728 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011865997993981945, + "loss": 1.1792, + "theoretical_loss": 3.3654988520967803, + "tokens_seen": 2816395264 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011864393179538615, + "loss": 1.2152, + "theoretical_loss": 3.3654928049796284, + "tokens_seen": 2816460800 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011862788365095285, + "loss": 1.2619, + "theoretical_loss": 3.365486758042583, + "tokens_seen": 2816526336 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011861183550651958, + "loss": 1.2293, + "theoretical_loss": 3.3654807112856338, + "tokens_seen": 2816591872 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011859578736208628, + "loss": 1.1871, + "theoretical_loss": 3.3654746647087714, + "tokens_seen": 2816657408 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011857973921765297, + "loss": 1.2, + "theoretical_loss": 3.3654686183119864, + "tokens_seen": 2816722944 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011856369107321967, + "loss": 1.1634, + "theoretical_loss": 3.365462572095269, + "tokens_seen": 2816788480 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011854764292878637, + "loss": 1.0998, + "theoretical_loss": 3.36545652605861, + "tokens_seen": 2816854016 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011853159478435307, + "loss": 1.2839, + "theoretical_loss": 3.365450480201999, + "tokens_seen": 2816919552 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011851554663991977, + "loss": 1.1722, + "theoretical_loss": 3.365444434525428, + "tokens_seen": 2816985088 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011849949849548647, + "loss": 1.1837, + "theoretical_loss": 3.365438389028886, + "tokens_seen": 2817050624 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011848345035105317, + "loss": 1.1574, + "theoretical_loss": 3.3654323437123645, + "tokens_seen": 2817116160 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011846740220661987, + "loss": 1.2059, + "theoretical_loss": 3.365426298575853, + "tokens_seen": 2817181696 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011845135406218657, + "loss": 1.139, + "theoretical_loss": 3.3654202536193427, + "tokens_seen": 2817247232 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011843530591775327, + "loss": 1.1498, + "theoretical_loss": 3.3654142088428234, + "tokens_seen": 2817312768 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011841925777331997, + "loss": 1.1929, + "theoretical_loss": 3.3654081642462863, + "tokens_seen": 2817378304 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011840320962888668, + "loss": 1.2214, + "theoretical_loss": 3.3654021198297213, + "tokens_seen": 2817443840 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011838716148445336, + "loss": 1.1867, + "theoretical_loss": 3.365396075593119, + "tokens_seen": 2817509376 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011837111334002006, + "loss": 1.1608, + "theoretical_loss": 3.3653900315364695, + "tokens_seen": 2817574912 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011835506519558676, + "loss": 1.1989, + "theoretical_loss": 3.365383987659764, + "tokens_seen": 2817640448 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011833901705115346, + "loss": 1.1129, + "theoretical_loss": 3.3653779439629927, + "tokens_seen": 2817705984 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011832296890672017, + "loss": 1.096, + "theoretical_loss": 3.3653719004461458, + "tokens_seen": 2817771520 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011830692076228687, + "loss": 1.1528, + "theoretical_loss": 3.3653658571092144, + "tokens_seen": 2817837056 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011829087261785357, + "loss": 1.1778, + "theoretical_loss": 3.3653598139521876, + "tokens_seen": 2817902592 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011827482447342027, + "loss": 1.2055, + "theoretical_loss": 3.3653537709750574, + "tokens_seen": 2817968128 + }, + { + "epoch": 1.23, + "objective/train/docs_used": 1040355, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1814459562301636, + "objective/train/theoretical_loss": 3.3653477281778135, + "objective/train/tokens_used": 2838493664, + "theoretical_loss": 3.3653477281778135, + "tokens_seen": 2818033664 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011825877632898697, + "loss": 1.1814, + "theoretical_loss": 3.3653477281778135, + "tokens_seen": 2818033664 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011824272818455367, + "loss": 1.3694, + "theoretical_loss": 3.365341685560446, + "tokens_seen": 2818099200 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011822668004012037, + "loss": 1.2293, + "theoretical_loss": 3.3653356431229464, + "tokens_seen": 2818164736 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011821063189568707, + "loss": 1.1526, + "theoretical_loss": 3.365329600865304, + "tokens_seen": 2818230272 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011819458375125376, + "loss": 1.2609, + "theoretical_loss": 3.3653235587875105, + "tokens_seen": 2818295808 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011817853560682046, + "loss": 1.2408, + "theoretical_loss": 3.3653175168895553, + "tokens_seen": 2818361344 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011816248746238716, + "loss": 1.2562, + "theoretical_loss": 3.3653114751714295, + "tokens_seen": 2818426880 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011814643931795386, + "loss": 1.2138, + "theoretical_loss": 3.365305433633123, + "tokens_seen": 2818492416 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011813039117352056, + "loss": 1.2071, + "theoretical_loss": 3.365299392274627, + "tokens_seen": 2818557952 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011811434302908726, + "loss": 1.1564, + "theoretical_loss": 3.3652933510959313, + "tokens_seen": 2818623488 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011809829488465396, + "loss": 1.2097, + "theoretical_loss": 3.3652873100970266, + "tokens_seen": 2818689024 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011808224674022066, + "loss": 1.2663, + "theoretical_loss": 3.3652812692779035, + "tokens_seen": 2818754560 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011806619859578736, + "loss": 1.2133, + "theoretical_loss": 3.365275228638552, + "tokens_seen": 2818820096 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011805015045135406, + "loss": 1.2045, + "theoretical_loss": 3.3652691881789636, + "tokens_seen": 2818885632 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011803410230692076, + "loss": 1.3103, + "theoretical_loss": 3.3652631478991277, + "tokens_seen": 2818951168 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011801805416248746, + "loss": 1.1873, + "theoretical_loss": 3.3652571077990356, + "tokens_seen": 2819016704 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011800200601805418, + "loss": 1.3239, + "theoretical_loss": 3.365251067878677, + "tokens_seen": 2819082240 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011798595787362088, + "loss": 1.0872, + "theoretical_loss": 3.365245028138043, + "tokens_seen": 2819147776 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011796990972918758, + "loss": 1.186, + "theoretical_loss": 3.365238988577124, + "tokens_seen": 2819213312 + }, + { + "epoch": 1.23, + "learning_rate": 0.00011795386158475428, + "loss": 1.2221, + "theoretical_loss": 3.3652329491959097, + "tokens_seen": 2819278848 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011793781344032098, + "loss": 1.2002, + "theoretical_loss": 3.3652269099943917, + "tokens_seen": 2819344384 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011792176529588768, + "loss": 1.1194, + "theoretical_loss": 3.3652208709725597, + "tokens_seen": 2819409920 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011790571715145438, + "loss": 1.2392, + "theoretical_loss": 3.3652148321304045, + "tokens_seen": 2819475456 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011788966900702107, + "loss": 1.2682, + "theoretical_loss": 3.3652087934679162, + "tokens_seen": 2819540992 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011787362086258777, + "loss": 1.3152, + "theoretical_loss": 3.3652027549850856, + "tokens_seen": 2819606528 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011785757271815447, + "loss": 1.1717, + "theoretical_loss": 3.3651967166819037, + "tokens_seen": 2819672064 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011784152457372117, + "loss": 1.2054, + "theoretical_loss": 3.36519067855836, + "tokens_seen": 2819737600 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011782547642928787, + "loss": 1.1905, + "theoretical_loss": 3.365184640614445, + "tokens_seen": 2819803136 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011780942828485457, + "loss": 1.209, + "theoretical_loss": 3.3651786028501505, + "tokens_seen": 2819868672 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011779338014042127, + "loss": 1.1553, + "theoretical_loss": 3.365172565265466, + "tokens_seen": 2819934208 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011777733199598797, + "loss": 1.1641, + "theoretical_loss": 3.3651665278603815, + "tokens_seen": 2819999744 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011776128385155467, + "loss": 1.1901, + "theoretical_loss": 3.3651604906348886, + "tokens_seen": 2820065280 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011774523570712137, + "loss": 1.1607, + "theoretical_loss": 3.3651544535889766, + "tokens_seen": 2820130816 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011772918756268808, + "loss": 1.2208, + "theoretical_loss": 3.3651484167226373, + "tokens_seen": 2820196352 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011771313941825478, + "loss": 1.1438, + "theoretical_loss": 3.36514238003586, + "tokens_seen": 2820261888 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011769709127382146, + "loss": 1.1973, + "theoretical_loss": 3.365136343528636, + "tokens_seen": 2820327424 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011768104312938816, + "loss": 1.1035, + "theoretical_loss": 3.365130307200955, + "tokens_seen": 2820392960 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011766499498495486, + "loss": 1.2127, + "theoretical_loss": 3.3651242710528084, + "tokens_seen": 2820458496 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011764894684052156, + "loss": 1.2292, + "theoretical_loss": 3.3651182350841866, + "tokens_seen": 2820524032 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011763289869608827, + "loss": 1.1643, + "theoretical_loss": 3.365112199295079, + "tokens_seen": 2820589568 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011761685055165497, + "loss": 1.2264, + "theoretical_loss": 3.365106163685477, + "tokens_seen": 2820655104 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011760080240722167, + "loss": 1.1908, + "theoretical_loss": 3.365100128255371, + "tokens_seen": 2820720640 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011758475426278837, + "loss": 1.1394, + "theoretical_loss": 3.365094093004752, + "tokens_seen": 2820786176 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011756870611835507, + "loss": 1.1097, + "theoretical_loss": 3.365088057933609, + "tokens_seen": 2820851712 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011755265797392177, + "loss": 1.2526, + "theoretical_loss": 3.365082023041934, + "tokens_seen": 2820917248 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011753660982948847, + "loss": 1.2858, + "theoretical_loss": 3.3650759883297168, + "tokens_seen": 2820982784 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011752056168505517, + "loss": 1.2268, + "theoretical_loss": 3.3650699537969477, + "tokens_seen": 2821048320 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011750451354062186, + "loss": 1.1724, + "theoretical_loss": 3.365063919443618, + "tokens_seen": 2821113856 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011748846539618856, + "loss": 1.1299, + "theoretical_loss": 3.3650578852697173, + "tokens_seen": 2821179392 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011747241725175526, + "loss": 1.2683, + "theoretical_loss": 3.3650518512752363, + "tokens_seen": 2821244928 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011745636910732196, + "loss": 1.067, + "theoretical_loss": 3.365045817460166, + "tokens_seen": 2821310464 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011744032096288866, + "loss": 1.1953, + "theoretical_loss": 3.3650397838244968, + "tokens_seen": 2821376000 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011742427281845536, + "loss": 1.1199, + "theoretical_loss": 3.3650337503682186, + "tokens_seen": 2821441536 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011740822467402209, + "loss": 1.1632, + "theoretical_loss": 3.3650277170913223, + "tokens_seen": 2821507072 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011739217652958879, + "loss": 1.1257, + "theoretical_loss": 3.3650216839937985, + "tokens_seen": 2821572608 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011737612838515548, + "loss": 1.191, + "theoretical_loss": 3.3650156510756375, + "tokens_seen": 2821638144 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011736008024072218, + "loss": 1.2295, + "theoretical_loss": 3.36500961833683, + "tokens_seen": 2821703680 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011734403209628888, + "loss": 1.1539, + "theoretical_loss": 3.365003585777366, + "tokens_seen": 2821769216 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011732798395185558, + "loss": 1.2128, + "theoretical_loss": 3.3649975533972367, + "tokens_seen": 2821834752 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011731193580742228, + "loss": 1.1293, + "theoretical_loss": 3.364991521196432, + "tokens_seen": 2821900288 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011729588766298898, + "loss": 1.204, + "theoretical_loss": 3.364985489174943, + "tokens_seen": 2821965824 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011727983951855568, + "loss": 1.1583, + "theoretical_loss": 3.36497945733276, + "tokens_seen": 2822031360 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011726379137412238, + "loss": 1.2827, + "theoretical_loss": 3.364973425669873, + "tokens_seen": 2822096896 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011724774322968908, + "loss": 1.2329, + "theoretical_loss": 3.3649673941862734, + "tokens_seen": 2822162432 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011723169508525578, + "loss": 1.1005, + "theoretical_loss": 3.3649613628819512, + "tokens_seen": 2822227968 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011721564694082248, + "loss": 1.2412, + "theoretical_loss": 3.3649553317568963, + "tokens_seen": 2822293504 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011719959879638918, + "loss": 1.2186, + "theoretical_loss": 3.3649493008111, + "tokens_seen": 2822359040 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011718355065195587, + "loss": 1.1693, + "theoretical_loss": 3.364943270044553, + "tokens_seen": 2822424576 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011716750250752257, + "loss": 1.2332, + "theoretical_loss": 3.3649372394572454, + "tokens_seen": 2822490112 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011715145436308927, + "loss": 1.2503, + "theoretical_loss": 3.364931209049168, + "tokens_seen": 2822555648 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011713540621865597, + "loss": 1.1243, + "theoretical_loss": 3.3649251788203105, + "tokens_seen": 2822621184 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011711935807422267, + "loss": 1.2725, + "theoretical_loss": 3.3649191487706647, + "tokens_seen": 2822686720 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011710330992978937, + "loss": 1.1584, + "theoretical_loss": 3.36491311890022, + "tokens_seen": 2822752256 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011708726178535607, + "loss": 1.2727, + "theoretical_loss": 3.364907089208967, + "tokens_seen": 2822817792 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011707121364092277, + "loss": 1.2245, + "theoretical_loss": 3.364901059696897, + "tokens_seen": 2822883328 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011705516549648947, + "loss": 1.2003, + "theoretical_loss": 3.364895030364, + "tokens_seen": 2822948864 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011703911735205618, + "loss": 1.2556, + "theoretical_loss": 3.3648890012102664, + "tokens_seen": 2823014400 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011702306920762288, + "loss": 1.0667, + "theoretical_loss": 3.3648829722356868, + "tokens_seen": 2823079936 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011700702106318958, + "loss": 1.2033, + "theoretical_loss": 3.364876943440252, + "tokens_seen": 2823145472 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011699097291875626, + "loss": 1.1667, + "theoretical_loss": 3.3648709148239524, + "tokens_seen": 2823211008 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011697492477432296, + "loss": 1.1255, + "theoretical_loss": 3.364864886386778, + "tokens_seen": 2823276544 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011695887662988966, + "loss": 1.2332, + "theoretical_loss": 3.36485885812872, + "tokens_seen": 2823342080 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011694282848545637, + "loss": 1.2038, + "theoretical_loss": 3.364852830049769, + "tokens_seen": 2823407616 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011692678034102307, + "loss": 1.1644, + "theoretical_loss": 3.364846802149915, + "tokens_seen": 2823473152 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011691073219658977, + "loss": 1.1151, + "theoretical_loss": 3.3648407744291484, + "tokens_seen": 2823538688 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011689468405215647, + "loss": 1.1863, + "theoretical_loss": 3.3648347468874604, + "tokens_seen": 2823604224 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011687863590772317, + "loss": 1.2451, + "theoretical_loss": 3.3648287195248407, + "tokens_seen": 2823669760 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011686258776328987, + "loss": 1.242, + "theoretical_loss": 3.364822692341281, + "tokens_seen": 2823735296 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011684653961885657, + "loss": 1.1818, + "theoretical_loss": 3.364816665336771, + "tokens_seen": 2823800832 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011683049147442327, + "loss": 1.2469, + "theoretical_loss": 3.364810638511301, + "tokens_seen": 2823866368 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011681444332998998, + "loss": 1.1435, + "theoretical_loss": 3.364804611864862, + "tokens_seen": 2823931904 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011679839518555668, + "loss": 1.1596, + "theoretical_loss": 3.3647985853974443, + "tokens_seen": 2823997440 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011678234704112339, + "loss": 1.3027, + "theoretical_loss": 3.3647925591090386, + "tokens_seen": 2824062976 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011676629889669009, + "loss": 1.1425, + "theoretical_loss": 3.364786532999635, + "tokens_seen": 2824128512 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011675025075225679, + "loss": 1.1457, + "theoretical_loss": 3.364780507069225, + "tokens_seen": 2824194048 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011673420260782349, + "loss": 1.1842, + "theoretical_loss": 3.364774481317798, + "tokens_seen": 2824259584 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011671815446339019, + "loss": 1.2343, + "theoretical_loss": 3.3647684557453457, + "tokens_seen": 2824325120 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011670210631895689, + "loss": 1.2336, + "theoretical_loss": 3.3647624303518575, + "tokens_seen": 2824390656 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011668605817452359, + "loss": 1.2036, + "theoretical_loss": 3.3647564051373244, + "tokens_seen": 2824456192 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011667001003009028, + "loss": 1.0854, + "theoretical_loss": 3.364750380101737, + "tokens_seen": 2824521728 + }, + { + "epoch": 1.24, + "objective/train/docs_used": 1042672, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2352032661437988, + "objective/train/theoretical_loss": 3.364744355245086, + "objective/train/tokens_used": 2845047264, + "theoretical_loss": 3.364744355245086, + "tokens_seen": 2824587264 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011665396188565698, + "loss": 1.2352, + "theoretical_loss": 3.364744355245086, + "tokens_seen": 2824587264 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011663791374122368, + "loss": 1.1519, + "theoretical_loss": 3.3647383305673615, + "tokens_seen": 2824652800 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011662186559679038, + "loss": 1.3044, + "theoretical_loss": 3.3647323060685546, + "tokens_seen": 2824718336 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011660581745235708, + "loss": 1.2358, + "theoretical_loss": 3.3647262817486547, + "tokens_seen": 2824783872 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011658976930792378, + "loss": 1.1824, + "theoretical_loss": 3.3647202576076536, + "tokens_seen": 2824849408 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011657372116349048, + "loss": 1.1219, + "theoretical_loss": 3.3647142336455413, + "tokens_seen": 2824914944 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011655767301905718, + "loss": 1.2113, + "theoretical_loss": 3.3647082098623087, + "tokens_seen": 2824980480 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011654162487462388, + "loss": 1.2883, + "theoretical_loss": 3.3647021862579454, + "tokens_seen": 2825046016 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011652557673019058, + "loss": 1.1197, + "theoretical_loss": 3.364696162832443, + "tokens_seen": 2825111552 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011650952858575728, + "loss": 1.1662, + "theoretical_loss": 3.3646901395857913, + "tokens_seen": 2825177088 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011649348044132398, + "loss": 1.2048, + "theoretical_loss": 3.3646841165179815, + "tokens_seen": 2825242624 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011647743229689067, + "loss": 1.1723, + "theoretical_loss": 3.3646780936290037, + "tokens_seen": 2825308160 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011646138415245737, + "loss": 1.2073, + "theoretical_loss": 3.3646720709188487, + "tokens_seen": 2825373696 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011644533600802407, + "loss": 1.1292, + "theoretical_loss": 3.3646660483875066, + "tokens_seen": 2825439232 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011642928786359077, + "loss": 1.2585, + "theoretical_loss": 3.3646600260349686, + "tokens_seen": 2825504768 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011641323971915747, + "loss": 1.1671, + "theoretical_loss": 3.364654003861224, + "tokens_seen": 2825570304 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011639719157472417, + "loss": 1.1044, + "theoretical_loss": 3.3646479818662653, + "tokens_seen": 2825635840 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011638114343029087, + "loss": 1.2629, + "theoretical_loss": 3.364641960050081, + "tokens_seen": 2825701376 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011636509528585757, + "loss": 1.255, + "theoretical_loss": 3.364635938412663, + "tokens_seen": 2825766912 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011634904714142428, + "loss": 1.0929, + "theoretical_loss": 3.3646299169540015, + "tokens_seen": 2825832448 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011633299899699098, + "loss": 1.2775, + "theoretical_loss": 3.364623895674087, + "tokens_seen": 2825897984 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011631695085255768, + "loss": 1.1031, + "theoretical_loss": 3.36461787457291, + "tokens_seen": 2825963520 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011630090270812438, + "loss": 1.238, + "theoretical_loss": 3.3646118536504614, + "tokens_seen": 2826029056 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011628485456369106, + "loss": 1.1171, + "theoretical_loss": 3.364605832906731, + "tokens_seen": 2826094592 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011626880641925777, + "loss": 1.2347, + "theoretical_loss": 3.36459981234171, + "tokens_seen": 2826160128 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011625275827482447, + "loss": 1.1624, + "theoretical_loss": 3.3645937919553885, + "tokens_seen": 2826225664 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011623671013039117, + "loss": 1.1173, + "theoretical_loss": 3.3645877717477575, + "tokens_seen": 2826291200 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011622066198595787, + "loss": 1.2344, + "theoretical_loss": 3.3645817517188075, + "tokens_seen": 2826356736 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001162046138415246, + "loss": 1.1799, + "theoretical_loss": 3.3645757318685288, + "tokens_seen": 2826422272 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001161885656970913, + "loss": 1.1311, + "theoretical_loss": 3.364569712196912, + "tokens_seen": 2826487808 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011617251755265798, + "loss": 1.2051, + "theoretical_loss": 3.364563692703948, + "tokens_seen": 2826553344 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011615646940822468, + "loss": 1.1842, + "theoretical_loss": 3.3645576733896267, + "tokens_seen": 2826618880 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011614042126379138, + "loss": 1.1679, + "theoretical_loss": 3.364551654253939, + "tokens_seen": 2826684416 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011612437311935808, + "loss": 1.0906, + "theoretical_loss": 3.3645456352968757, + "tokens_seen": 2826749952 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011610832497492478, + "loss": 1.1868, + "theoretical_loss": 3.364539616518427, + "tokens_seen": 2826815488 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011609227683049149, + "loss": 1.2309, + "theoretical_loss": 3.3645335979185838, + "tokens_seen": 2826881024 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011607622868605819, + "loss": 1.1532, + "theoretical_loss": 3.364527579497336, + "tokens_seen": 2826946560 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011606018054162489, + "loss": 1.2369, + "theoretical_loss": 3.3645215612546755, + "tokens_seen": 2827012096 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011604413239719159, + "loss": 1.1005, + "theoretical_loss": 3.364515543190591, + "tokens_seen": 2827077632 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011602808425275829, + "loss": 1.1424, + "theoretical_loss": 3.364509525305075, + "tokens_seen": 2827143168 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011601203610832499, + "loss": 1.1784, + "theoretical_loss": 3.3645035075981164, + "tokens_seen": 2827208704 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011599598796389169, + "loss": 1.2273, + "theoretical_loss": 3.3644974900697067, + "tokens_seen": 2827274240 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011597993981945838, + "loss": 1.1515, + "theoretical_loss": 3.364491472719836, + "tokens_seen": 2827339776 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011596389167502508, + "loss": 1.1038, + "theoretical_loss": 3.3644854555484955, + "tokens_seen": 2827405312 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011594784353059178, + "loss": 1.1788, + "theoretical_loss": 3.364479438555675, + "tokens_seen": 2827470848 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011593179538615848, + "loss": 1.2883, + "theoretical_loss": 3.3644734217413657, + "tokens_seen": 2827536384 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011591574724172518, + "loss": 1.2038, + "theoretical_loss": 3.364467405105558, + "tokens_seen": 2827601920 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011589969909729188, + "loss": 1.1939, + "theoretical_loss": 3.364461388648242, + "tokens_seen": 2827667456 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011588365095285858, + "loss": 1.2105, + "theoretical_loss": 3.3644553723694086, + "tokens_seen": 2827732992 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011586760280842528, + "loss": 1.2127, + "theoretical_loss": 3.364449356269049, + "tokens_seen": 2827798528 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011585155466399198, + "loss": 1.2436, + "theoretical_loss": 3.3644433403471528, + "tokens_seen": 2827864064 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011583550651955868, + "loss": 1.2344, + "theoretical_loss": 3.3644373246037107, + "tokens_seen": 2827929600 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011581945837512538, + "loss": 1.1917, + "theoretical_loss": 3.3644313090387135, + "tokens_seen": 2827995136 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011580341023069208, + "loss": 1.2836, + "theoretical_loss": 3.364425293652152, + "tokens_seen": 2828060672 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011578736208625877, + "loss": 1.2311, + "theoretical_loss": 3.364419278444017, + "tokens_seen": 2828126208 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011577131394182547, + "loss": 1.1267, + "theoretical_loss": 3.3644132634142982, + "tokens_seen": 2828191744 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011575526579739217, + "loss": 1.1753, + "theoretical_loss": 3.3644072485629866, + "tokens_seen": 2828257280 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011573921765295887, + "loss": 1.1449, + "theoretical_loss": 3.3644012338900726, + "tokens_seen": 2828322816 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011572316950852557, + "loss": 1.1774, + "theoretical_loss": 3.364395219395547, + "tokens_seen": 2828388352 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011570712136409227, + "loss": 1.1961, + "theoretical_loss": 3.3643892050794006, + "tokens_seen": 2828453888 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011569107321965897, + "loss": 1.3305, + "theoretical_loss": 3.3643831909416235, + "tokens_seen": 2828519424 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011567502507522567, + "loss": 1.1476, + "theoretical_loss": 3.364377176982207, + "tokens_seen": 2828584960 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011565897693079238, + "loss": 1.0056, + "theoretical_loss": 3.3643711632011404, + "tokens_seen": 2828650496 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011564292878635908, + "loss": 1.2252, + "theoretical_loss": 3.3643651495984157, + "tokens_seen": 2828716032 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011562688064192578, + "loss": 1.2264, + "theoretical_loss": 3.364359136174022, + "tokens_seen": 2828781568 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011561083249749249, + "loss": 1.1774, + "theoretical_loss": 3.3643531229279513, + "tokens_seen": 2828847104 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011559478435305919, + "loss": 1.1419, + "theoretical_loss": 3.3643471098601934, + "tokens_seen": 2828912640 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011557873620862589, + "loss": 1.205, + "theoretical_loss": 3.3643410969707395, + "tokens_seen": 2828978176 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011556268806419259, + "loss": 1.2595, + "theoretical_loss": 3.3643350842595794, + "tokens_seen": 2829043712 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001155466399197593, + "loss": 1.2712, + "theoretical_loss": 3.364329071726704, + "tokens_seen": 2829109248 + }, + { + "epoch": 1.24, + "learning_rate": 0.000115530591775326, + "loss": 1.2294, + "theoretical_loss": 3.364323059372104, + "tokens_seen": 2829174784 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001155145436308927, + "loss": 1.1288, + "theoretical_loss": 3.3643170471957697, + "tokens_seen": 2829240320 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001154984954864594, + "loss": 1.0652, + "theoretical_loss": 3.3643110351976917, + "tokens_seen": 2829305856 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001154824473420261, + "loss": 1.272, + "theoretical_loss": 3.3643050233778613, + "tokens_seen": 2829371392 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011546639919759278, + "loss": 1.121, + "theoretical_loss": 3.3642990117362683, + "tokens_seen": 2829436928 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011545035105315948, + "loss": 1.1384, + "theoretical_loss": 3.3642930002729035, + "tokens_seen": 2829502464 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011543430290872618, + "loss": 1.1005, + "theoretical_loss": 3.3642869889877574, + "tokens_seen": 2829568000 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011541825476429289, + "loss": 1.2479, + "theoretical_loss": 3.364280977880821, + "tokens_seen": 2829633536 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011540220661985959, + "loss": 1.1712, + "theoretical_loss": 3.3642749669520846, + "tokens_seen": 2829699072 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011538615847542629, + "loss": 1.1587, + "theoretical_loss": 3.364268956201539, + "tokens_seen": 2829764608 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011537011033099299, + "loss": 1.1733, + "theoretical_loss": 3.364262945629174, + "tokens_seen": 2829830144 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011535406218655969, + "loss": 1.2343, + "theoretical_loss": 3.364256935234981, + "tokens_seen": 2829895680 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011533801404212639, + "loss": 1.2108, + "theoretical_loss": 3.3642509250189505, + "tokens_seen": 2829961216 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011532196589769309, + "loss": 1.1144, + "theoretical_loss": 3.3642449149810734, + "tokens_seen": 2830026752 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011530591775325979, + "loss": 1.227, + "theoretical_loss": 3.364238905121339, + "tokens_seen": 2830092288 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011528986960882649, + "loss": 1.2057, + "theoretical_loss": 3.364232895439739, + "tokens_seen": 2830157824 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011527382146439318, + "loss": 1.1675, + "theoretical_loss": 3.3642268859362643, + "tokens_seen": 2830223360 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011525777331995988, + "loss": 1.1877, + "theoretical_loss": 3.3642208766109043, + "tokens_seen": 2830288896 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011524172517552658, + "loss": 1.1083, + "theoretical_loss": 3.3642148674636507, + "tokens_seen": 2830354432 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011522567703109328, + "loss": 1.2068, + "theoretical_loss": 3.3642088584944934, + "tokens_seen": 2830419968 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011520962888665998, + "loss": 1.2322, + "theoretical_loss": 3.3642028497034233, + "tokens_seen": 2830485504 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011519358074222668, + "loss": 1.2496, + "theoretical_loss": 3.364196841090431, + "tokens_seen": 2830551040 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011517753259779338, + "loss": 1.1764, + "theoretical_loss": 3.364190832655507, + "tokens_seen": 2830616576 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011516148445336008, + "loss": 1.2128, + "theoretical_loss": 3.3641848243986416, + "tokens_seen": 2830682112 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011514543630892678, + "loss": 1.1686, + "theoretical_loss": 3.3641788163198263, + "tokens_seen": 2830747648 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011512938816449348, + "loss": 1.2167, + "theoretical_loss": 3.3641728084190508, + "tokens_seen": 2830813184 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011511334002006018, + "loss": 1.1346, + "theoretical_loss": 3.364166800696306, + "tokens_seen": 2830878720 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011509729187562688, + "loss": 1.2615, + "theoretical_loss": 3.3641607931515827, + "tokens_seen": 2830944256 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011508124373119357, + "loss": 1.1849, + "theoretical_loss": 3.3641547857848715, + "tokens_seen": 2831009792 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011506519558676027, + "loss": 1.1965, + "theoretical_loss": 3.364148778596163, + "tokens_seen": 2831075328 + }, + { + "epoch": 1.24, + "objective/train/docs_used": 1044909, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2661100625991821, + "objective/train/theoretical_loss": 3.3641427715854473, + "objective/train/tokens_used": 2851600864, + "theoretical_loss": 3.3641427715854473, + "tokens_seen": 2831140864 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011504914744232697, + "loss": 1.2661, + "theoretical_loss": 3.3641427715854473, + "tokens_seen": 2831140864 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011503309929789367, + "loss": 1.1974, + "theoretical_loss": 3.364136764752715, + "tokens_seen": 2831206400 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011501705115346037, + "loss": 1.1964, + "theoretical_loss": 3.3641307580979576, + "tokens_seen": 2831271936 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001150010030090271, + "loss": 1.2096, + "theoretical_loss": 3.364124751621165, + "tokens_seen": 2831337472 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001149849548645938, + "loss": 1.2809, + "theoretical_loss": 3.3641187453223282, + "tokens_seen": 2831403008 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001149689067201605, + "loss": 1.2196, + "theoretical_loss": 3.3641127392014374, + "tokens_seen": 2831468544 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011495285857572719, + "loss": 1.2768, + "theoretical_loss": 3.3641067332584838, + "tokens_seen": 2831534080 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011493681043129389, + "loss": 1.1835, + "theoretical_loss": 3.364100727493457, + "tokens_seen": 2831599616 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011492076228686059, + "loss": 1.185, + "theoretical_loss": 3.3640947219063486, + "tokens_seen": 2831665152 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011490471414242729, + "loss": 1.1788, + "theoretical_loss": 3.364088716497149, + "tokens_seen": 2831730688 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011488866599799399, + "loss": 1.2606, + "theoretical_loss": 3.3640827112658482, + "tokens_seen": 2831796224 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011487261785356069, + "loss": 1.2636, + "theoretical_loss": 3.3640767062124373, + "tokens_seen": 2831861760 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001148565697091274, + "loss": 1.2197, + "theoretical_loss": 3.364070701336907, + "tokens_seen": 2831927296 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001148405215646941, + "loss": 1.1696, + "theoretical_loss": 3.364064696639248, + "tokens_seen": 2831992832 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001148244734202608, + "loss": 1.3194, + "theoretical_loss": 3.3640586921194506, + "tokens_seen": 2832058368 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001148084252758275, + "loss": 1.1743, + "theoretical_loss": 3.364052687777505, + "tokens_seen": 2832123904 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001147923771313942, + "loss": 1.1872, + "theoretical_loss": 3.3640466836134033, + "tokens_seen": 2832189440 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001147763289869609, + "loss": 1.2657, + "theoretical_loss": 3.3640406796271347, + "tokens_seen": 2832254976 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011476028084252758, + "loss": 1.2216, + "theoretical_loss": 3.36403467581869, + "tokens_seen": 2832320512 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011474423269809428, + "loss": 1.2339, + "theoretical_loss": 3.3640286721880606, + "tokens_seen": 2832386048 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011472818455366099, + "loss": 1.1588, + "theoretical_loss": 3.3640226687352364, + "tokens_seen": 2832451584 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011471213640922769, + "loss": 1.1402, + "theoretical_loss": 3.364016665460208, + "tokens_seen": 2832517120 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011469608826479439, + "loss": 1.2279, + "theoretical_loss": 3.3640106623629666, + "tokens_seen": 2832582656 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011468004012036109, + "loss": 1.165, + "theoretical_loss": 3.3640046594435025, + "tokens_seen": 2832648192 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011466399197592779, + "loss": 1.2902, + "theoretical_loss": 3.3639986567018063, + "tokens_seen": 2832713728 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011464794383149449, + "loss": 1.2111, + "theoretical_loss": 3.3639926541378684, + "tokens_seen": 2832779264 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011463189568706119, + "loss": 1.2439, + "theoretical_loss": 3.36398665175168, + "tokens_seen": 2832844800 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011461584754262789, + "loss": 1.2132, + "theoretical_loss": 3.3639806495432305, + "tokens_seen": 2832910336 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011459979939819459, + "loss": 1.2081, + "theoretical_loss": 3.3639746475125123, + "tokens_seen": 2832975872 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011458375125376129, + "loss": 1.2397, + "theoretical_loss": 3.363968645659515, + "tokens_seen": 2833041408 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011456770310932798, + "loss": 1.2444, + "theoretical_loss": 3.363962643984229, + "tokens_seen": 2833106944 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011455165496489468, + "loss": 1.0626, + "theoretical_loss": 3.3639566424866456, + "tokens_seen": 2833172480 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011453560682046138, + "loss": 1.1074, + "theoretical_loss": 3.363950641166755, + "tokens_seen": 2833238016 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011451955867602808, + "loss": 1.1263, + "theoretical_loss": 3.3639446400245476, + "tokens_seen": 2833303552 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011450351053159478, + "loss": 1.0961, + "theoretical_loss": 3.363938639060015, + "tokens_seen": 2833369088 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011448746238716148, + "loss": 1.2357, + "theoretical_loss": 3.3639326382731465, + "tokens_seen": 2833434624 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011447141424272818, + "loss": 1.2304, + "theoretical_loss": 3.363926637663934, + "tokens_seen": 2833500160 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011445536609829488, + "loss": 1.1232, + "theoretical_loss": 3.3639206372323676, + "tokens_seen": 2833565696 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011443931795386158, + "loss": 1.2424, + "theoretical_loss": 3.3639146369784374, + "tokens_seen": 2833631232 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011442326980942828, + "loss": 1.2648, + "theoretical_loss": 3.363908636902135, + "tokens_seen": 2833696768 + }, + { + "epoch": 1.24, + "learning_rate": 0.000114407221664995, + "loss": 1.1928, + "theoretical_loss": 3.36390263700345, + "tokens_seen": 2833762304 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001143911735205617, + "loss": 1.2634, + "theoretical_loss": 3.363896637282374, + "tokens_seen": 2833827840 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001143751253761284, + "loss": 1.202, + "theoretical_loss": 3.363890637738897, + "tokens_seen": 2833893376 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001143590772316951, + "loss": 1.0952, + "theoretical_loss": 3.36388463837301, + "tokens_seen": 2833958912 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001143430290872618, + "loss": 1.3057, + "theoretical_loss": 3.3638786391847035, + "tokens_seen": 2834024448 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001143269809428285, + "loss": 1.128, + "theoretical_loss": 3.3638726401739683, + "tokens_seen": 2834089984 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001143109327983952, + "loss": 1.1511, + "theoretical_loss": 3.3638666413407945, + "tokens_seen": 2834155520 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001142948846539619, + "loss": 1.2343, + "theoretical_loss": 3.3638606426851734, + "tokens_seen": 2834221056 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001142788365095286, + "loss": 1.2059, + "theoretical_loss": 3.363854644207095, + "tokens_seen": 2834286592 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011426278836509529, + "loss": 1.2681, + "theoretical_loss": 3.363848645906551, + "tokens_seen": 2834352128 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011424674022066199, + "loss": 1.2533, + "theoretical_loss": 3.363842647783531, + "tokens_seen": 2834417664 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011423069207622869, + "loss": 1.2135, + "theoretical_loss": 3.3638366498380257, + "tokens_seen": 2834483200 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011421464393179539, + "loss": 1.2111, + "theoretical_loss": 3.363830652070026, + "tokens_seen": 2834548736 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011419859578736209, + "loss": 1.1948, + "theoretical_loss": 3.3638246544795227, + "tokens_seen": 2834614272 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001141825476429288, + "loss": 1.2326, + "theoretical_loss": 3.3638186570665067, + "tokens_seen": 2834679808 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001141664994984955, + "loss": 1.1817, + "theoretical_loss": 3.363812659830968, + "tokens_seen": 2834745344 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001141504513540622, + "loss": 1.1439, + "theoretical_loss": 3.3638066627728973, + "tokens_seen": 2834810880 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001141344032096289, + "loss": 1.1705, + "theoretical_loss": 3.3638006658922857, + "tokens_seen": 2834876416 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001141183550651956, + "loss": 1.1539, + "theoretical_loss": 3.363794669189123, + "tokens_seen": 2834941952 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001141023069207623, + "loss": 1.2708, + "theoretical_loss": 3.363788672663401, + "tokens_seen": 2835007488 + }, + { + "epoch": 1.24, + "learning_rate": 0.000114086258776329, + "loss": 1.1905, + "theoretical_loss": 3.3637826763151097, + "tokens_seen": 2835073024 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011407021063189568, + "loss": 1.1331, + "theoretical_loss": 3.3637766801442397, + "tokens_seen": 2835138560 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011405416248746238, + "loss": 1.1419, + "theoretical_loss": 3.363770684150782, + "tokens_seen": 2835204096 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011403811434302909, + "loss": 1.1739, + "theoretical_loss": 3.3637646883347267, + "tokens_seen": 2835269632 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011402206619859579, + "loss": 1.1512, + "theoretical_loss": 3.3637586926960648, + "tokens_seen": 2835335168 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011400601805416249, + "loss": 1.1993, + "theoretical_loss": 3.363752697234787, + "tokens_seen": 2835400704 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011398996990972919, + "loss": 1.2002, + "theoretical_loss": 3.3637467019508844, + "tokens_seen": 2835466240 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011397392176529589, + "loss": 1.115, + "theoretical_loss": 3.3637407068443466, + "tokens_seen": 2835531776 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011395787362086259, + "loss": 1.1964, + "theoretical_loss": 3.363734711915165, + "tokens_seen": 2835597312 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011394182547642929, + "loss": 1.1284, + "theoretical_loss": 3.3637287171633297, + "tokens_seen": 2835662848 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011392577733199599, + "loss": 1.2786, + "theoretical_loss": 3.3637227225888324, + "tokens_seen": 2835728384 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011390972918756269, + "loss": 1.1468, + "theoretical_loss": 3.3637167281916627, + "tokens_seen": 2835793920 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011389368104312939, + "loss": 1.0756, + "theoretical_loss": 3.3637107339718115, + "tokens_seen": 2835859456 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011387763289869608, + "loss": 1.2083, + "theoretical_loss": 3.363704739929269, + "tokens_seen": 2835924992 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011386158475426278, + "loss": 1.1501, + "theoretical_loss": 3.3636987460640273, + "tokens_seen": 2835990528 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011384553660982948, + "loss": 1.1136, + "theoretical_loss": 3.363692752376076, + "tokens_seen": 2836056064 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011382948846539618, + "loss": 1.1585, + "theoretical_loss": 3.3636867588654056, + "tokens_seen": 2836121600 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011381344032096291, + "loss": 1.1733, + "theoretical_loss": 3.3636807655320076, + "tokens_seen": 2836187136 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011379739217652961, + "loss": 1.1894, + "theoretical_loss": 3.3636747723758718, + "tokens_seen": 2836252672 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011378134403209631, + "loss": 1.2785, + "theoretical_loss": 3.3636687793969893, + "tokens_seen": 2836318208 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011376529588766301, + "loss": 1.1858, + "theoretical_loss": 3.363662786595351, + "tokens_seen": 2836383744 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001137492477432297, + "loss": 1.1377, + "theoretical_loss": 3.363656793970947, + "tokens_seen": 2836449280 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001137331995987964, + "loss": 1.1527, + "theoretical_loss": 3.363650801523768, + "tokens_seen": 2836514816 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001137171514543631, + "loss": 1.2045, + "theoretical_loss": 3.363644809253805, + "tokens_seen": 2836580352 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001137011033099298, + "loss": 1.1206, + "theoretical_loss": 3.363638817161049, + "tokens_seen": 2836645888 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001136850551654965, + "loss": 1.1232, + "theoretical_loss": 3.3636328252454897, + "tokens_seen": 2836711424 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001136690070210632, + "loss": 1.1864, + "theoretical_loss": 3.3636268335071184, + "tokens_seen": 2836776960 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001136529588766299, + "loss": 1.3226, + "theoretical_loss": 3.3636208419459255, + "tokens_seen": 2836842496 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001136369107321966, + "loss": 1.2642, + "theoretical_loss": 3.3636148505619023, + "tokens_seen": 2836908032 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001136208625877633, + "loss": 1.2271, + "theoretical_loss": 3.3636088593550384, + "tokens_seen": 2836973568 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011360481444333, + "loss": 1.1481, + "theoretical_loss": 3.3636028683253256, + "tokens_seen": 2837039104 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001135887662988967, + "loss": 1.1813, + "theoretical_loss": 3.3635968774727534, + "tokens_seen": 2837104640 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001135727181544634, + "loss": 1.2495, + "theoretical_loss": 3.3635908867973137, + "tokens_seen": 2837170176 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011355667001003009, + "loss": 1.2412, + "theoretical_loss": 3.3635848962989963, + "tokens_seen": 2837235712 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011354062186559679, + "loss": 1.2299, + "theoretical_loss": 3.363578905977792, + "tokens_seen": 2837301248 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011352457372116349, + "loss": 1.3366, + "theoretical_loss": 3.363572915833692, + "tokens_seen": 2837366784 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011350852557673019, + "loss": 1.2487, + "theoretical_loss": 3.363566925866686, + "tokens_seen": 2837432320 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001134924774322969, + "loss": 1.2021, + "theoretical_loss": 3.3635609360767655, + "tokens_seen": 2837497856 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001134764292878636, + "loss": 1.1317, + "theoretical_loss": 3.363554946463921, + "tokens_seen": 2837563392 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001134603811434303, + "loss": 1.1801, + "theoretical_loss": 3.363548957028143, + "tokens_seen": 2837628928 + }, + { + "epoch": 1.24, + "objective/train/docs_used": 1047645, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1553617715835571, + "objective/train/theoretical_loss": 3.3635429677694226, + "objective/train/tokens_used": 2858154464, + "theoretical_loss": 3.3635429677694226, + "tokens_seen": 2837694464 + }, + { + "epoch": 1.24, + "learning_rate": 0.000113444332998997, + "loss": 1.1554, + "theoretical_loss": 3.3635429677694226, + "tokens_seen": 2837694464 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001134282848545637, + "loss": 1.1813, + "theoretical_loss": 3.36353697868775, + "tokens_seen": 2837760000 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001134122367101304, + "loss": 1.2156, + "theoretical_loss": 3.363530989783116, + "tokens_seen": 2837825536 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001133961885656971, + "loss": 1.2197, + "theoretical_loss": 3.3635250010555113, + "tokens_seen": 2837891072 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001133801404212638, + "loss": 1.1587, + "theoretical_loss": 3.3635190125049266, + "tokens_seen": 2837956608 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011336409227683049, + "loss": 1.1918, + "theoretical_loss": 3.3635130241313522, + "tokens_seen": 2838022144 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011334804413239719, + "loss": 1.2653, + "theoretical_loss": 3.36350703593478, + "tokens_seen": 2838087680 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011333199598796389, + "loss": 1.1939, + "theoretical_loss": 3.363501047915199, + "tokens_seen": 2838153216 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011331594784353059, + "loss": 1.3406, + "theoretical_loss": 3.363495060072601, + "tokens_seen": 2838218752 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011329989969909729, + "loss": 1.3303, + "theoretical_loss": 3.3634890724069764, + "tokens_seen": 2838284288 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011328385155466399, + "loss": 1.1967, + "theoretical_loss": 3.3634830849183164, + "tokens_seen": 2838349824 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011326780341023069, + "loss": 1.1952, + "theoretical_loss": 3.3634770976066104, + "tokens_seen": 2838415360 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011325175526579739, + "loss": 1.0868, + "theoretical_loss": 3.36347111047185, + "tokens_seen": 2838480896 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011323570712136409, + "loss": 1.1113, + "theoretical_loss": 3.3634651235140263, + "tokens_seen": 2838546432 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011321965897693079, + "loss": 1.1382, + "theoretical_loss": 3.3634591367331286, + "tokens_seen": 2838611968 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001132036108324975, + "loss": 1.1909, + "theoretical_loss": 3.363453150129149, + "tokens_seen": 2838677504 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001131875626880642, + "loss": 1.2423, + "theoretical_loss": 3.3634471637020775, + "tokens_seen": 2838743040 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001131715145436309, + "loss": 1.1759, + "theoretical_loss": 3.3634411774519046, + "tokens_seen": 2838808576 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011315546639919761, + "loss": 1.2265, + "theoretical_loss": 3.3634351913786213, + "tokens_seen": 2838874112 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011313941825476431, + "loss": 1.2035, + "theoretical_loss": 3.363429205482219, + "tokens_seen": 2838939648 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011312337011033101, + "loss": 1.2606, + "theoretical_loss": 3.3634232197626868, + "tokens_seen": 2839005184 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011310732196589771, + "loss": 1.1543, + "theoretical_loss": 3.363417234220017, + "tokens_seen": 2839070720 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011309127382146441, + "loss": 1.2323, + "theoretical_loss": 3.3634112488541987, + "tokens_seen": 2839136256 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011307522567703111, + "loss": 1.0864, + "theoretical_loss": 3.3634052636652236, + "tokens_seen": 2839201792 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011305917753259781, + "loss": 1.2668, + "theoretical_loss": 3.3633992786530826, + "tokens_seen": 2839267328 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001130431293881645, + "loss": 1.2043, + "theoretical_loss": 3.363393293817766, + "tokens_seen": 2839332864 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001130270812437312, + "loss": 1.1801, + "theoretical_loss": 3.3633873091592643, + "tokens_seen": 2839398400 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001130110330992979, + "loss": 1.1708, + "theoretical_loss": 3.3633813246775683, + "tokens_seen": 2839463936 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001129949849548646, + "loss": 1.2933, + "theoretical_loss": 3.363375340372669, + "tokens_seen": 2839529472 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001129789368104313, + "loss": 1.2075, + "theoretical_loss": 3.363369356244557, + "tokens_seen": 2839595008 + }, + { + "epoch": 1.24, + "learning_rate": 0.000112962888665998, + "loss": 1.202, + "theoretical_loss": 3.3633633722932226, + "tokens_seen": 2839660544 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001129468405215647, + "loss": 1.1468, + "theoretical_loss": 3.363357388518657, + "tokens_seen": 2839726080 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001129307923771314, + "loss": 1.1704, + "theoretical_loss": 3.363351404920851, + "tokens_seen": 2839791616 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001129147442326981, + "loss": 1.0679, + "theoretical_loss": 3.3633454214997944, + "tokens_seen": 2839857152 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001128986960882648, + "loss": 1.2707, + "theoretical_loss": 3.363339438255479, + "tokens_seen": 2839922688 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001128826479438315, + "loss": 1.2224, + "theoretical_loss": 3.3633334551878944, + "tokens_seen": 2839988224 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001128665997993982, + "loss": 1.1619, + "theoretical_loss": 3.3633274722970325, + "tokens_seen": 2840053760 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011285055165496489, + "loss": 1.105, + "theoretical_loss": 3.363321489582883, + "tokens_seen": 2840119296 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011283450351053159, + "loss": 1.1848, + "theoretical_loss": 3.3633155070454372, + "tokens_seen": 2840184832 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011281845536609829, + "loss": 1.2201, + "theoretical_loss": 3.3633095246846856, + "tokens_seen": 2840250368 + }, + { + "epoch": 1.24, + "learning_rate": 0.000112802407221665, + "loss": 1.2707, + "theoretical_loss": 3.363303542500619, + "tokens_seen": 2840315904 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001127863590772317, + "loss": 1.1243, + "theoretical_loss": 3.3632975604932276, + "tokens_seen": 2840381440 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001127703109327984, + "loss": 1.1222, + "theoretical_loss": 3.363291578662503, + "tokens_seen": 2840446976 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001127542627883651, + "loss": 1.1347, + "theoretical_loss": 3.363285597008435, + "tokens_seen": 2840512512 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001127382146439318, + "loss": 1.2974, + "theoretical_loss": 3.363279615531015, + "tokens_seen": 2840578048 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001127221664994985, + "loss": 1.203, + "theoretical_loss": 3.3632736342302336, + "tokens_seen": 2840643584 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001127061183550652, + "loss": 1.0877, + "theoretical_loss": 3.3632676531060812, + "tokens_seen": 2840709120 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001126900702106319, + "loss": 1.1693, + "theoretical_loss": 3.3632616721585484, + "tokens_seen": 2840774656 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001126740220661986, + "loss": 1.1826, + "theoretical_loss": 3.3632556913876264, + "tokens_seen": 2840840192 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011265797392176529, + "loss": 1.2094, + "theoretical_loss": 3.3632497107933057, + "tokens_seen": 2840905728 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011264192577733199, + "loss": 1.2134, + "theoretical_loss": 3.363243730375577, + "tokens_seen": 2840971264 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011262587763289869, + "loss": 1.1929, + "theoretical_loss": 3.363237750134431, + "tokens_seen": 2841036800 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011260982948846541, + "loss": 1.1892, + "theoretical_loss": 3.3632317700698584, + "tokens_seen": 2841102336 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011259378134403212, + "loss": 1.1971, + "theoretical_loss": 3.36322579018185, + "tokens_seen": 2841167872 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011257773319959882, + "loss": 1.2639, + "theoretical_loss": 3.3632198104703965, + "tokens_seen": 2841233408 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011256168505516552, + "loss": 1.1895, + "theoretical_loss": 3.3632138309354884, + "tokens_seen": 2841298944 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001125456369107322, + "loss": 1.1682, + "theoretical_loss": 3.3632078515771164, + "tokens_seen": 2841364480 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001125295887662989, + "loss": 1.0889, + "theoretical_loss": 3.3632018723952717, + "tokens_seen": 2841430016 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001125135406218656, + "loss": 1.3086, + "theoretical_loss": 3.3631958933899444, + "tokens_seen": 2841495552 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001124974924774323, + "loss": 1.2913, + "theoretical_loss": 3.363189914561126, + "tokens_seen": 2841561088 + }, + { + "epoch": 1.24, + "learning_rate": 0.000112481444332999, + "loss": 1.1388, + "theoretical_loss": 3.3631839359088067, + "tokens_seen": 2841626624 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011246539618856571, + "loss": 1.1849, + "theoretical_loss": 3.3631779574329768, + "tokens_seen": 2841692160 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011244934804413241, + "loss": 1.1134, + "theoretical_loss": 3.363171979133628, + "tokens_seen": 2841757696 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011243329989969911, + "loss": 1.2367, + "theoretical_loss": 3.36316600101075, + "tokens_seen": 2841823232 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011241725175526581, + "loss": 1.1107, + "theoretical_loss": 3.363160023064334, + "tokens_seen": 2841888768 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011240120361083251, + "loss": 1.1206, + "theoretical_loss": 3.3631540452943716, + "tokens_seen": 2841954304 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011238515546639921, + "loss": 1.1414, + "theoretical_loss": 3.363148067700852, + "tokens_seen": 2842019840 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011236910732196591, + "loss": 1.189, + "theoretical_loss": 3.3631420902837665, + "tokens_seen": 2842085376 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001123530591775326, + "loss": 1.3119, + "theoretical_loss": 3.3631361130431063, + "tokens_seen": 2842150912 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001123370110330993, + "loss": 1.185, + "theoretical_loss": 3.3631301359788615, + "tokens_seen": 2842216448 + }, + { + "epoch": 1.24, + "learning_rate": 0.000112320962888666, + "loss": 1.1852, + "theoretical_loss": 3.363124159091023, + "tokens_seen": 2842281984 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001123049147442327, + "loss": 1.1664, + "theoretical_loss": 3.3631181823795817, + "tokens_seen": 2842347520 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001122888665997994, + "loss": 1.1344, + "theoretical_loss": 3.363112205844528, + "tokens_seen": 2842413056 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001122728184553661, + "loss": 1.1305, + "theoretical_loss": 3.3631062294858536, + "tokens_seen": 2842478592 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001122567703109328, + "loss": 1.3311, + "theoretical_loss": 3.363100253303548, + "tokens_seen": 2842544128 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001122407221664995, + "loss": 1.2734, + "theoretical_loss": 3.363094277297602, + "tokens_seen": 2842609664 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001122246740220662, + "loss": 1.1666, + "theoretical_loss": 3.3630883014680073, + "tokens_seen": 2842675200 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001122086258776329, + "loss": 1.1149, + "theoretical_loss": 3.363082325814754, + "tokens_seen": 2842740736 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001121925777331996, + "loss": 1.2051, + "theoretical_loss": 3.363076350337833, + "tokens_seen": 2842806272 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001121765295887663, + "loss": 1.1687, + "theoretical_loss": 3.3630703750372346, + "tokens_seen": 2842871808 + }, + { + "epoch": 1.24, + "learning_rate": 0.000112160481444333, + "loss": 1.0873, + "theoretical_loss": 3.3630643999129504, + "tokens_seen": 2842937344 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011214443329989969, + "loss": 1.1807, + "theoretical_loss": 3.36305842496497, + "tokens_seen": 2843002880 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001121283851554664, + "loss": 1.3089, + "theoretical_loss": 3.363052450193285, + "tokens_seen": 2843068416 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001121123370110331, + "loss": 1.2642, + "theoretical_loss": 3.3630464755978857, + "tokens_seen": 2843133952 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001120962888665998, + "loss": 1.1365, + "theoretical_loss": 3.363040501178763, + "tokens_seen": 2843199488 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001120802407221665, + "loss": 1.2432, + "theoretical_loss": 3.3630345269359077, + "tokens_seen": 2843265024 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001120641925777332, + "loss": 1.0856, + "theoretical_loss": 3.3630285528693107, + "tokens_seen": 2843330560 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001120481444332999, + "loss": 1.1959, + "theoretical_loss": 3.3630225789789625, + "tokens_seen": 2843396096 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001120320962888666, + "loss": 1.2055, + "theoretical_loss": 3.3630166052648534, + "tokens_seen": 2843461632 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001120160481444333, + "loss": 1.1356, + "theoretical_loss": 3.363010631726975, + "tokens_seen": 2843527168 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011200000000000001, + "loss": 1.2725, + "theoretical_loss": 3.3630046583653175, + "tokens_seen": 2843592704 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011198395185556671, + "loss": 1.1325, + "theoretical_loss": 3.3629986851798717, + "tokens_seen": 2843658240 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011196790371113341, + "loss": 1.2305, + "theoretical_loss": 3.3629927121706285, + "tokens_seen": 2843723776 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011195185556670011, + "loss": 1.2437, + "theoretical_loss": 3.3629867393375785, + "tokens_seen": 2843789312 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011193580742226681, + "loss": 1.1438, + "theoretical_loss": 3.362980766680713, + "tokens_seen": 2843854848 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011191975927783352, + "loss": 1.2048, + "theoretical_loss": 3.3629747942000217, + "tokens_seen": 2843920384 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011190371113340022, + "loss": 1.2414, + "theoretical_loss": 3.3629688218954956, + "tokens_seen": 2843985920 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011188766298896692, + "loss": 1.162, + "theoretical_loss": 3.3629628497671264, + "tokens_seen": 2844051456 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011187161484453362, + "loss": 1.2403, + "theoretical_loss": 3.362956877814904, + "tokens_seen": 2844116992 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011185556670010032, + "loss": 1.232, + "theoretical_loss": 3.3629509060388187, + "tokens_seen": 2844182528 + }, + { + "epoch": 1.24, + "objective/train/docs_used": 1050027, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1540961265563965, + "objective/train/theoretical_loss": 3.3629449344388624, + "objective/train/tokens_used": 2864708064, + "theoretical_loss": 3.3629449344388624, + "tokens_seen": 2844248064 + }, + { + "epoch": 1.24, + "learning_rate": 0.000111839518555667, + "loss": 1.1541, + "theoretical_loss": 3.3629449344388624, + "tokens_seen": 2844248064 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001118234704112337, + "loss": 1.1896, + "theoretical_loss": 3.3629389630150253, + "tokens_seen": 2844313600 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001118074222668004, + "loss": 1.3134, + "theoretical_loss": 3.3629329917672983, + "tokens_seen": 2844379136 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001117913741223671, + "loss": 1.207, + "theoretical_loss": 3.362927020695672, + "tokens_seen": 2844444672 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011177532597793381, + "loss": 1.1303, + "theoretical_loss": 3.3629210498001365, + "tokens_seen": 2844510208 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011175927783350051, + "loss": 1.188, + "theoretical_loss": 3.362915079080684, + "tokens_seen": 2844575744 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011174322968906721, + "loss": 1.1242, + "theoretical_loss": 3.362909108537304, + "tokens_seen": 2844641280 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011172718154463391, + "loss": 1.2558, + "theoretical_loss": 3.362903138169988, + "tokens_seen": 2844706816 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011171113340020061, + "loss": 1.1928, + "theoretical_loss": 3.3628971679787263, + "tokens_seen": 2844772352 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011169508525576731, + "loss": 1.2067, + "theoretical_loss": 3.36289119796351, + "tokens_seen": 2844837888 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011167903711133401, + "loss": 1.359, + "theoretical_loss": 3.362885228124329, + "tokens_seen": 2844903424 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011166298896690071, + "loss": 1.2238, + "theoretical_loss": 3.3628792584611755, + "tokens_seen": 2844968960 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001116469408224674, + "loss": 1.1376, + "theoretical_loss": 3.362873288974039, + "tokens_seen": 2845034496 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001116308926780341, + "loss": 1.2404, + "theoretical_loss": 3.362867319662911, + "tokens_seen": 2845100032 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001116148445336008, + "loss": 1.1545, + "theoretical_loss": 3.362861350527782, + "tokens_seen": 2845165568 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001115987963891675, + "loss": 1.2119, + "theoretical_loss": 3.3628553815686426, + "tokens_seen": 2845231104 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001115827482447342, + "loss": 1.3071, + "theoretical_loss": 3.3628494127854838, + "tokens_seen": 2845296640 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001115667001003009, + "loss": 1.1581, + "theoretical_loss": 3.362843444178296, + "tokens_seen": 2845362176 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001115506519558676, + "loss": 1.2165, + "theoretical_loss": 3.3628374757470705, + "tokens_seen": 2845427712 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001115346038114343, + "loss": 1.1577, + "theoretical_loss": 3.362831507491798, + "tokens_seen": 2845493248 + }, + { + "epoch": 1.24, + "learning_rate": 0.000111518555667001, + "loss": 1.2799, + "theoretical_loss": 3.3628255394124684, + "tokens_seen": 2845558784 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001115025075225677, + "loss": 1.2467, + "theoretical_loss": 3.362819571509074, + "tokens_seen": 2845624320 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001114864593781344, + "loss": 1.182, + "theoretical_loss": 3.362813603781604, + "tokens_seen": 2845689856 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001114704112337011, + "loss": 1.1749, + "theoretical_loss": 3.3628076362300496, + "tokens_seen": 2845755392 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011145436308926779, + "loss": 1.2137, + "theoretical_loss": 3.3628016688544022, + "tokens_seen": 2845820928 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001114383149448345, + "loss": 1.2669, + "theoretical_loss": 3.3627957016546524, + "tokens_seen": 2845886464 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001114222668004012, + "loss": 1.196, + "theoretical_loss": 3.3627897346307902, + "tokens_seen": 2845952000 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011140621865596792, + "loss": 1.0467, + "theoretical_loss": 3.3627837677828074, + "tokens_seen": 2846017536 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011139017051153462, + "loss": 1.2433, + "theoretical_loss": 3.3627778011106937, + "tokens_seen": 2846083072 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011137412236710132, + "loss": 1.1788, + "theoretical_loss": 3.362771834614441, + "tokens_seen": 2846148608 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011135807422266802, + "loss": 1.1639, + "theoretical_loss": 3.362765868294039, + "tokens_seen": 2846214144 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011134202607823472, + "loss": 1.1883, + "theoretical_loss": 3.362759902149479, + "tokens_seen": 2846279680 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011132597793380141, + "loss": 1.1748, + "theoretical_loss": 3.362753936180752, + "tokens_seen": 2846345216 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011130992978936811, + "loss": 1.1872, + "theoretical_loss": 3.3627479703878485, + "tokens_seen": 2846410752 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011129388164493481, + "loss": 1.2619, + "theoretical_loss": 3.362742004770759, + "tokens_seen": 2846476288 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011127783350050151, + "loss": 1.2031, + "theoretical_loss": 3.3627360393294747, + "tokens_seen": 2846541824 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011126178535606821, + "loss": 1.0794, + "theoretical_loss": 3.362730074063986, + "tokens_seen": 2846607360 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011124573721163491, + "loss": 1.2794, + "theoretical_loss": 3.362724108974284, + "tokens_seen": 2846672896 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011122968906720162, + "loss": 1.2072, + "theoretical_loss": 3.3627181440603593, + "tokens_seen": 2846738432 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011121364092276832, + "loss": 1.2528, + "theoretical_loss": 3.3627121793222026, + "tokens_seen": 2846803968 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011119759277833502, + "loss": 1.1078, + "theoretical_loss": 3.3627062147598052, + "tokens_seen": 2846869504 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011118154463390172, + "loss": 1.2185, + "theoretical_loss": 3.362700250373157, + "tokens_seen": 2846935040 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011116549648946842, + "loss": 1.1629, + "theoretical_loss": 3.3626942861622497, + "tokens_seen": 2847000576 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011114944834503512, + "loss": 1.2384, + "theoretical_loss": 3.362688322127074, + "tokens_seen": 2847066112 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001111334002006018, + "loss": 1.165, + "theoretical_loss": 3.3626823582676195, + "tokens_seen": 2847131648 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001111173520561685, + "loss": 1.242, + "theoretical_loss": 3.362676394583878, + "tokens_seen": 2847197184 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001111013039117352, + "loss": 1.2314, + "theoretical_loss": 3.36267043107584, + "tokens_seen": 2847262720 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011108525576730191, + "loss": 1.1193, + "theoretical_loss": 3.362664467743496, + "tokens_seen": 2847328256 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011106920762286861, + "loss": 1.1729, + "theoretical_loss": 3.362658504586838, + "tokens_seen": 2847393792 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011105315947843531, + "loss": 1.1733, + "theoretical_loss": 3.362652541605855, + "tokens_seen": 2847459328 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011103711133400201, + "loss": 1.2082, + "theoretical_loss": 3.3626465788005397, + "tokens_seen": 2847524864 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011102106318956871, + "loss": 1.2577, + "theoretical_loss": 3.362640616170881, + "tokens_seen": 2847590400 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011100501504513541, + "loss": 1.2827, + "theoretical_loss": 3.3626346537168708, + "tokens_seen": 2847655936 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011098896690070211, + "loss": 1.0995, + "theoretical_loss": 3.3626286914384997, + "tokens_seen": 2847721472 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011097291875626881, + "loss": 1.1284, + "theoretical_loss": 3.3626227293357585, + "tokens_seen": 2847787008 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011095687061183551, + "loss": 1.2286, + "theoretical_loss": 3.3626167674086376, + "tokens_seen": 2847852544 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001109408224674022, + "loss": 1.2824, + "theoretical_loss": 3.362610805657128, + "tokens_seen": 2847918080 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001109247743229689, + "loss": 1.1708, + "theoretical_loss": 3.362604844081221, + "tokens_seen": 2847983616 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001109087261785356, + "loss": 1.2807, + "theoretical_loss": 3.362598882680907, + "tokens_seen": 2848049152 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001108926780341023, + "loss": 1.1491, + "theoretical_loss": 3.3625929214561765, + "tokens_seen": 2848114688 + }, + { + "epoch": 1.24, + "learning_rate": 0.000110876629889669, + "loss": 1.1468, + "theoretical_loss": 3.362586960407021, + "tokens_seen": 2848180224 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001108605817452357, + "loss": 1.1906, + "theoretical_loss": 3.3625809995334297, + "tokens_seen": 2848245760 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001108445336008024, + "loss": 1.1537, + "theoretical_loss": 3.3625750388353954, + "tokens_seen": 2848311296 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001108284854563691, + "loss": 1.2111, + "theoretical_loss": 3.362569078312908, + "tokens_seen": 2848376832 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001108124373119358, + "loss": 1.1398, + "theoretical_loss": 3.362563117965958, + "tokens_seen": 2848442368 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011079638916750252, + "loss": 1.1971, + "theoretical_loss": 3.362557157794537, + "tokens_seen": 2848507904 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011078034102306922, + "loss": 1.1142, + "theoretical_loss": 3.3625511977986347, + "tokens_seen": 2848573440 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011076429287863592, + "loss": 1.2241, + "theoretical_loss": 3.3625452379782423, + "tokens_seen": 2848638976 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011074824473420262, + "loss": 1.1595, + "theoretical_loss": 3.3625392783333514, + "tokens_seen": 2848704512 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011073219658976932, + "loss": 1.1731, + "theoretical_loss": 3.3625333188639517, + "tokens_seen": 2848770048 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011071614844533602, + "loss": 1.222, + "theoretical_loss": 3.3625273595700347, + "tokens_seen": 2848835584 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011070010030090272, + "loss": 1.0259, + "theoretical_loss": 3.362521400451591, + "tokens_seen": 2848901120 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011068405215646942, + "loss": 1.1849, + "theoretical_loss": 3.362515441508611, + "tokens_seen": 2848966656 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011066800401203612, + "loss": 1.2086, + "theoretical_loss": 3.3625094827410864, + "tokens_seen": 2849032192 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011065195586760282, + "loss": 1.2625, + "theoretical_loss": 3.362503524149007, + "tokens_seen": 2849097728 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011063590772316953, + "loss": 1.0725, + "theoretical_loss": 3.3624975657323644, + "tokens_seen": 2849163264 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011061985957873621, + "loss": 1.1665, + "theoretical_loss": 3.3624916074911484, + "tokens_seen": 2849228800 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011060381143430291, + "loss": 1.2152, + "theoretical_loss": 3.362485649425351, + "tokens_seen": 2849294336 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011058776328986961, + "loss": 1.2295, + "theoretical_loss": 3.362479691534962, + "tokens_seen": 2849359872 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011057171514543631, + "loss": 1.1941, + "theoretical_loss": 3.3624737338199733, + "tokens_seen": 2849425408 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011055566700100301, + "loss": 1.2149, + "theoretical_loss": 3.3624677762803747, + "tokens_seen": 2849490944 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011053961885656972, + "loss": 1.1758, + "theoretical_loss": 3.3624618189161573, + "tokens_seen": 2849556480 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011052357071213642, + "loss": 1.1559, + "theoretical_loss": 3.362455861727312, + "tokens_seen": 2849622016 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011050752256770312, + "loss": 1.2257, + "theoretical_loss": 3.3624499047138294, + "tokens_seen": 2849687552 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011049147442326982, + "loss": 1.2139, + "theoretical_loss": 3.362443947875701, + "tokens_seen": 2849753088 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011047542627883652, + "loss": 1.3003, + "theoretical_loss": 3.3624379912129165, + "tokens_seen": 2849818624 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011045937813440322, + "loss": 1.1439, + "theoretical_loss": 3.3624320347254675, + "tokens_seen": 2849884160 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011044332998996992, + "loss": 1.1652, + "theoretical_loss": 3.3624260784133444, + "tokens_seen": 2849949696 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001104272818455366, + "loss": 1.2464, + "theoretical_loss": 3.3624201222765384, + "tokens_seen": 2850015232 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011041123370110331, + "loss": 1.2569, + "theoretical_loss": 3.36241416631504, + "tokens_seen": 2850080768 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011039518555667001, + "loss": 1.1579, + "theoretical_loss": 3.3624082105288404, + "tokens_seen": 2850146304 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011037913741223671, + "loss": 1.1407, + "theoretical_loss": 3.3624022549179298, + "tokens_seen": 2850211840 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011036308926780341, + "loss": 1.1631, + "theoretical_loss": 3.3623962994822993, + "tokens_seen": 2850277376 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011034704112337011, + "loss": 1.1675, + "theoretical_loss": 3.36239034422194, + "tokens_seen": 2850342912 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011033099297893681, + "loss": 1.3261, + "theoretical_loss": 3.3623843891368423, + "tokens_seen": 2850408448 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011031494483450351, + "loss": 1.1927, + "theoretical_loss": 3.362378434226997, + "tokens_seen": 2850473984 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011029889669007021, + "loss": 1.2857, + "theoretical_loss": 3.3623724794923953, + "tokens_seen": 2850539520 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011028284854563691, + "loss": 1.268, + "theoretical_loss": 3.362366524933028, + "tokens_seen": 2850605056 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011026680040120361, + "loss": 1.1859, + "theoretical_loss": 3.3623605705488853, + "tokens_seen": 2850670592 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011025075225677031, + "loss": 1.0698, + "theoretical_loss": 3.3623546163399585, + "tokens_seen": 2850736128 + }, + { + "epoch": 1.24, + "objective/train/docs_used": 1052446, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1708232164382935, + "objective/train/theoretical_loss": 3.3623486623062386, + "objective/train/tokens_used": 2871261664, + "theoretical_loss": 3.3623486623062386, + "tokens_seen": 2850801664 + }, + { + "epoch": 1.24, + "learning_rate": 0.000110234704112337, + "loss": 1.1708, + "theoretical_loss": 3.3623486623062386, + "tokens_seen": 2850801664 + }, + { + "epoch": 1.24, + "learning_rate": 0.0001102186559679037, + "loss": 1.0924, + "theoretical_loss": 3.362342708447716, + "tokens_seen": 2850867200 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011020260782347043, + "loss": 1.3239, + "theoretical_loss": 3.3623367547643817, + "tokens_seen": 2850932736 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011018655967903713, + "loss": 1.2091, + "theoretical_loss": 3.3623308012562263, + "tokens_seen": 2850998272 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011017051153460383, + "loss": 1.0633, + "theoretical_loss": 3.362324847923241, + "tokens_seen": 2851063808 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011015446339017053, + "loss": 1.2709, + "theoretical_loss": 3.362318894765416, + "tokens_seen": 2851129344 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011013841524573723, + "loss": 1.2212, + "theoretical_loss": 3.362312941782743, + "tokens_seen": 2851194880 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011012236710130392, + "loss": 1.1357, + "theoretical_loss": 3.3623069889752126, + "tokens_seen": 2851260416 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011010631895687062, + "loss": 1.2085, + "theoretical_loss": 3.362301036342815, + "tokens_seen": 2851325952 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011009027081243732, + "loss": 1.2334, + "theoretical_loss": 3.3622950838855417, + "tokens_seen": 2851391488 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011007422266800402, + "loss": 1.2982, + "theoretical_loss": 3.362289131603383, + "tokens_seen": 2851457024 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011005817452357072, + "loss": 1.1772, + "theoretical_loss": 3.36228317949633, + "tokens_seen": 2851522560 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011004212637913742, + "loss": 1.1838, + "theoretical_loss": 3.3622772275643733, + "tokens_seen": 2851588096 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011002607823470412, + "loss": 1.1851, + "theoretical_loss": 3.362271275807504, + "tokens_seen": 2851653632 + }, + { + "epoch": 1.24, + "learning_rate": 0.00011001003009027082, + "loss": 1.1909, + "theoretical_loss": 3.3622653242257132, + "tokens_seen": 2851719168 + }, + { + "epoch": 1.24, + "learning_rate": 0.00010999398194583752, + "loss": 1.1213, + "theoretical_loss": 3.362259372818991, + "tokens_seen": 2851784704 + }, + { + "epoch": 1.24, + "learning_rate": 0.00010997793380140422, + "loss": 1.2112, + "theoretical_loss": 3.3622534215873285, + "tokens_seen": 2851850240 + }, + { + "epoch": 1.24, + "learning_rate": 0.00010996188565697092, + "loss": 1.2617, + "theoretical_loss": 3.362247470530717, + "tokens_seen": 2851915776 + }, + { + "epoch": 1.24, + "learning_rate": 0.00010994583751253763, + "loss": 1.1345, + "theoretical_loss": 3.362241519649147, + "tokens_seen": 2851981312 + }, + { + "epoch": 1.24, + "learning_rate": 0.00010992978936810431, + "loss": 1.2073, + "theoretical_loss": 3.362235568942609, + "tokens_seen": 2852046848 + }, + { + "epoch": 1.24, + "learning_rate": 0.00010991374122367101, + "loss": 1.1588, + "theoretical_loss": 3.362229618411094, + "tokens_seen": 2852112384 + }, + { + "epoch": 1.24, + "learning_rate": 0.00010989769307923771, + "loss": 1.1871, + "theoretical_loss": 3.362223668054593, + "tokens_seen": 2852177920 + }, + { + "epoch": 1.24, + "learning_rate": 0.00010988164493480441, + "loss": 1.2159, + "theoretical_loss": 3.362217717873097, + "tokens_seen": 2852243456 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010986559679037111, + "loss": 1.1337, + "theoretical_loss": 3.3622117678665964, + "tokens_seen": 2852308992 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010984954864593782, + "loss": 1.2614, + "theoretical_loss": 3.3622058180350827, + "tokens_seen": 2852374528 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010983350050150452, + "loss": 1.2252, + "theoretical_loss": 3.3621998683785455, + "tokens_seen": 2852440064 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010981745235707122, + "loss": 1.2437, + "theoretical_loss": 3.3621939188969767, + "tokens_seen": 2852505600 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010980140421263792, + "loss": 1.1736, + "theoretical_loss": 3.362187969590367, + "tokens_seen": 2852571136 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010978535606820462, + "loss": 1.2729, + "theoretical_loss": 3.3621820204587074, + "tokens_seen": 2852636672 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010976930792377132, + "loss": 1.2257, + "theoretical_loss": 3.3621760715019877, + "tokens_seen": 2852702208 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010975325977933802, + "loss": 1.1593, + "theoretical_loss": 3.3621701227201997, + "tokens_seen": 2852767744 + }, + { + "epoch": 1.25, + "learning_rate": 0.0001097372116349047, + "loss": 1.2533, + "theoretical_loss": 3.3621641741133343, + "tokens_seen": 2852833280 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010972116349047141, + "loss": 1.1313, + "theoretical_loss": 3.3621582256813816, + "tokens_seen": 2852898816 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010970511534603811, + "loss": 1.2078, + "theoretical_loss": 3.3621522774243333, + "tokens_seen": 2852964352 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010968906720160481, + "loss": 1.1397, + "theoretical_loss": 3.3621463293421794, + "tokens_seen": 2853029888 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010967301905717151, + "loss": 1.238, + "theoretical_loss": 3.362140381434911, + "tokens_seen": 2853095424 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010965697091273821, + "loss": 1.1722, + "theoretical_loss": 3.3621344337025194, + "tokens_seen": 2853160960 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010964092276830491, + "loss": 1.2469, + "theoretical_loss": 3.362128486144995, + "tokens_seen": 2853226496 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010962487462387161, + "loss": 1.1117, + "theoretical_loss": 3.362122538762329, + "tokens_seen": 2853292032 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010960882647943833, + "loss": 1.1719, + "theoretical_loss": 3.3621165915545115, + "tokens_seen": 2853357568 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010959277833500503, + "loss": 1.1618, + "theoretical_loss": 3.3621106445215343, + "tokens_seen": 2853423104 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010957673019057173, + "loss": 1.2282, + "theoretical_loss": 3.362104697663388, + "tokens_seen": 2853488640 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010956068204613843, + "loss": 1.1376, + "theoretical_loss": 3.3620987509800626, + "tokens_seen": 2853554176 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010954463390170513, + "loss": 1.2571, + "theoretical_loss": 3.36209280447155, + "tokens_seen": 2853619712 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010952858575727183, + "loss": 1.1518, + "theoretical_loss": 3.3620868581378405, + "tokens_seen": 2853685248 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010951253761283853, + "loss": 1.1669, + "theoretical_loss": 3.362080911978925, + "tokens_seen": 2853750784 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010949648946840523, + "loss": 1.2514, + "theoretical_loss": 3.362074965994795, + "tokens_seen": 2853816320 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010948044132397193, + "loss": 1.1935, + "theoretical_loss": 3.36206902018544, + "tokens_seen": 2853881856 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010946439317953863, + "loss": 1.2475, + "theoretical_loss": 3.362063074550852, + "tokens_seen": 2853947392 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010944834503510533, + "loss": 1.1645, + "theoretical_loss": 3.3620571290910215, + "tokens_seen": 2854012928 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010943229689067203, + "loss": 1.1375, + "theoretical_loss": 3.3620511838059395, + "tokens_seen": 2854078464 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010941624874623872, + "loss": 1.2699, + "theoretical_loss": 3.3620452386955963, + "tokens_seen": 2854144000 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010940020060180542, + "loss": 1.186, + "theoretical_loss": 3.3620392937599832, + "tokens_seen": 2854209536 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010938415245737212, + "loss": 1.1536, + "theoretical_loss": 3.3620333489990912, + "tokens_seen": 2854275072 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010936810431293882, + "loss": 1.3329, + "theoretical_loss": 3.3620274044129106, + "tokens_seen": 2854340608 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010935205616850552, + "loss": 1.3081, + "theoretical_loss": 3.362021460001433, + "tokens_seen": 2854406144 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010933600802407222, + "loss": 1.2144, + "theoretical_loss": 3.3620155157646487, + "tokens_seen": 2854471680 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010931995987963892, + "loss": 1.2041, + "theoretical_loss": 3.362009571702549, + "tokens_seen": 2854537216 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010930391173520562, + "loss": 1.1678, + "theoretical_loss": 3.362003627815124, + "tokens_seen": 2854602752 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010928786359077232, + "loss": 1.2074, + "theoretical_loss": 3.3619976841023647, + "tokens_seen": 2854668288 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010927181544633902, + "loss": 1.3381, + "theoretical_loss": 3.361991740564263, + "tokens_seen": 2854733824 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010925576730190573, + "loss": 1.2091, + "theoretical_loss": 3.3619857972008087, + "tokens_seen": 2854799360 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010923971915747243, + "loss": 1.2117, + "theoretical_loss": 3.361979854011993, + "tokens_seen": 2854864896 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010922367101303911, + "loss": 1.2014, + "theoretical_loss": 3.361973910997807, + "tokens_seen": 2854930432 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010920762286860581, + "loss": 1.1302, + "theoretical_loss": 3.3619679681582406, + "tokens_seen": 2854995968 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010919157472417251, + "loss": 1.1851, + "theoretical_loss": 3.361962025493286, + "tokens_seen": 2855061504 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010917552657973922, + "loss": 1.2077, + "theoretical_loss": 3.3619560830029336, + "tokens_seen": 2855127040 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010915947843530592, + "loss": 1.2348, + "theoretical_loss": 3.3619501406871737, + "tokens_seen": 2855192576 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010914343029087262, + "loss": 1.2398, + "theoretical_loss": 3.3619441985459977, + "tokens_seen": 2855258112 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010912738214643932, + "loss": 1.2265, + "theoretical_loss": 3.3619382565793963, + "tokens_seen": 2855323648 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010911133400200602, + "loss": 1.16, + "theoretical_loss": 3.36193231478736, + "tokens_seen": 2855389184 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010909528585757272, + "loss": 1.1349, + "theoretical_loss": 3.3619263731698807, + "tokens_seen": 2855454720 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010907923771313942, + "loss": 1.2319, + "theoretical_loss": 3.361920431726948, + "tokens_seen": 2855520256 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010906318956870612, + "loss": 1.1278, + "theoretical_loss": 3.3619144904585543, + "tokens_seen": 2855585792 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010904714142427282, + "loss": 1.2185, + "theoretical_loss": 3.3619085493646885, + "tokens_seen": 2855651328 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010903109327983951, + "loss": 1.1461, + "theoretical_loss": 3.361902608445343, + "tokens_seen": 2855716864 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010901504513540621, + "loss": 1.1681, + "theoretical_loss": 3.3618966677005084, + "tokens_seen": 2855782400 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010899899699097294, + "loss": 1.1603, + "theoretical_loss": 3.361890727130175, + "tokens_seen": 2855847936 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010898294884653964, + "loss": 1.2374, + "theoretical_loss": 3.3618847867343344, + "tokens_seen": 2855913472 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010896690070210634, + "loss": 1.2474, + "theoretical_loss": 3.3618788465129765, + "tokens_seen": 2855979008 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010895085255767304, + "loss": 1.2025, + "theoretical_loss": 3.3618729064660933, + "tokens_seen": 2856044544 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010893480441323974, + "loss": 1.2623, + "theoretical_loss": 3.361866966593675, + "tokens_seen": 2856110080 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010891875626880644, + "loss": 1.1159, + "theoretical_loss": 3.3618610268957125, + "tokens_seen": 2856175616 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010890270812437313, + "loss": 1.2253, + "theoretical_loss": 3.361855087372197, + "tokens_seen": 2856241152 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010888665997993983, + "loss": 1.1952, + "theoretical_loss": 3.361849148023119, + "tokens_seen": 2856306688 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010887061183550653, + "loss": 1.1808, + "theoretical_loss": 3.3618432088484695, + "tokens_seen": 2856372224 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010885456369107323, + "loss": 1.1531, + "theoretical_loss": 3.3618372698482393, + "tokens_seen": 2856437760 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010883851554663993, + "loss": 1.2231, + "theoretical_loss": 3.3618313310224197, + "tokens_seen": 2856503296 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010882246740220663, + "loss": 1.2385, + "theoretical_loss": 3.361825392371001, + "tokens_seen": 2856568832 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010880641925777333, + "loss": 1.1331, + "theoretical_loss": 3.3618194538939745, + "tokens_seen": 2856634368 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010879037111334003, + "loss": 1.1903, + "theoretical_loss": 3.361813515591331, + "tokens_seen": 2856699904 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010877432296890673, + "loss": 1.1472, + "theoretical_loss": 3.361807577463061, + "tokens_seen": 2856765440 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010875827482447343, + "loss": 1.1501, + "theoretical_loss": 3.361801639509156, + "tokens_seen": 2856830976 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010874222668004013, + "loss": 1.0875, + "theoretical_loss": 3.3617957017296067, + "tokens_seen": 2856896512 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010872617853560683, + "loss": 1.1802, + "theoretical_loss": 3.3617897641244037, + "tokens_seen": 2856962048 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010871013039117352, + "loss": 1.0728, + "theoretical_loss": 3.3617838266935376, + "tokens_seen": 2857027584 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010869408224674022, + "loss": 1.1966, + "theoretical_loss": 3.361777889437, + "tokens_seen": 2857093120 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010867803410230692, + "loss": 1.1641, + "theoretical_loss": 3.3617719523547818, + "tokens_seen": 2857158656 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010866198595787362, + "loss": 1.1683, + "theoretical_loss": 3.361766015446873, + "tokens_seen": 2857224192 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010864593781344032, + "loss": 1.1902, + "theoretical_loss": 3.3617600787132655, + "tokens_seen": 2857289728 + }, + { + "epoch": 1.25, + "objective/train/docs_used": 1054959, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2752203941345215, + "objective/train/theoretical_loss": 3.3617541421539494, + "objective/train/tokens_used": 2877815264, + "theoretical_loss": 3.3617541421539494, + "tokens_seen": 2857355264 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010862988966900702, + "loss": 1.2752, + "theoretical_loss": 3.3617541421539494, + "tokens_seen": 2857355264 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010861384152457372, + "loss": 1.0481, + "theoretical_loss": 3.361748205768916, + "tokens_seen": 2857420800 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010859779338014042, + "loss": 1.2583, + "theoretical_loss": 3.3617422695581562, + "tokens_seen": 2857486336 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010858174523570713, + "loss": 1.1783, + "theoretical_loss": 3.361736333521661, + "tokens_seen": 2857551872 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010856569709127383, + "loss": 1.1288, + "theoretical_loss": 3.3617303976594206, + "tokens_seen": 2857617408 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010854964894684053, + "loss": 1.1957, + "theoretical_loss": 3.361724461971427, + "tokens_seen": 2857682944 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010853360080240723, + "loss": 1.2442, + "theoretical_loss": 3.36171852645767, + "tokens_seen": 2857748480 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010851755265797391, + "loss": 1.1542, + "theoretical_loss": 3.361712591118141, + "tokens_seen": 2857814016 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010850150451354061, + "loss": 1.1856, + "theoretical_loss": 3.361706655952831, + "tokens_seen": 2857879552 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010848545636910732, + "loss": 1.2499, + "theoretical_loss": 3.3617007209617307, + "tokens_seen": 2857945088 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010846940822467402, + "loss": 1.2068, + "theoretical_loss": 3.361694786144831, + "tokens_seen": 2858010624 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010845336008024072, + "loss": 1.1292, + "theoretical_loss": 3.361688851502123, + "tokens_seen": 2858076160 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010843731193580742, + "loss": 1.1357, + "theoretical_loss": 3.361682917033597, + "tokens_seen": 2858141696 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010842126379137412, + "loss": 1.2854, + "theoretical_loss": 3.3616769827392448, + "tokens_seen": 2858207232 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010840521564694083, + "loss": 1.1977, + "theoretical_loss": 3.361671048619056, + "tokens_seen": 2858272768 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010838916750250753, + "loss": 1.1697, + "theoretical_loss": 3.3616651146730234, + "tokens_seen": 2858338304 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010837311935807423, + "loss": 1.2043, + "theoretical_loss": 3.3616591809011362, + "tokens_seen": 2858403840 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010835707121364093, + "loss": 1.2705, + "theoretical_loss": 3.3616532473033858, + "tokens_seen": 2858469376 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010834102306920763, + "loss": 1.237, + "theoretical_loss": 3.3616473138797636, + "tokens_seen": 2858534912 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010832497492477434, + "loss": 1.1664, + "theoretical_loss": 3.3616413806302594, + "tokens_seen": 2858600448 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010830892678034104, + "loss": 1.2842, + "theoretical_loss": 3.3616354475548653, + "tokens_seen": 2858665984 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010829287863590774, + "loss": 1.1841, + "theoretical_loss": 3.3616295146535715, + "tokens_seen": 2858731520 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010827683049147444, + "loss": 1.17, + "theoretical_loss": 3.361623581926369, + "tokens_seen": 2858797056 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010826078234704114, + "loss": 1.1616, + "theoretical_loss": 3.3616176493732493, + "tokens_seen": 2858862592 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010824473420260784, + "loss": 1.2049, + "theoretical_loss": 3.3616117169942026, + "tokens_seen": 2858928128 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010822868605817454, + "loss": 1.256, + "theoretical_loss": 3.3616057847892193, + "tokens_seen": 2858993664 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010821263791374123, + "loss": 1.1786, + "theoretical_loss": 3.3615998527582915, + "tokens_seen": 2859059200 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010819658976930793, + "loss": 1.2407, + "theoretical_loss": 3.36159392090141, + "tokens_seen": 2859124736 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010818054162487463, + "loss": 1.1129, + "theoretical_loss": 3.361587989218565, + "tokens_seen": 2859190272 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010816449348044133, + "loss": 1.2662, + "theoretical_loss": 3.3615820577097475, + "tokens_seen": 2859255808 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010814844533600803, + "loss": 1.1776, + "theoretical_loss": 3.3615761263749486, + "tokens_seen": 2859321344 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010813239719157473, + "loss": 1.1518, + "theoretical_loss": 3.3615701952141595, + "tokens_seen": 2859386880 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010811634904714143, + "loss": 1.1791, + "theoretical_loss": 3.3615642642273706, + "tokens_seen": 2859452416 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010810030090270813, + "loss": 1.1991, + "theoretical_loss": 3.3615583334145733, + "tokens_seen": 2859517952 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010808425275827483, + "loss": 1.2226, + "theoretical_loss": 3.361552402775758, + "tokens_seen": 2859583488 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010806820461384153, + "loss": 1.167, + "theoretical_loss": 3.361546472310916, + "tokens_seen": 2859649024 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010805215646940823, + "loss": 1.1253, + "theoretical_loss": 3.361540542020038, + "tokens_seen": 2859714560 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010803610832497493, + "loss": 1.3037, + "theoretical_loss": 3.3615346119031146, + "tokens_seen": 2859780096 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010802006018054162, + "loss": 1.1487, + "theoretical_loss": 3.3615286819601375, + "tokens_seen": 2859845632 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010800401203610832, + "loss": 1.1777, + "theoretical_loss": 3.361522752191097, + "tokens_seen": 2859911168 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010798796389167502, + "loss": 1.2772, + "theoretical_loss": 3.361516822595984, + "tokens_seen": 2859976704 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010797191574724172, + "loss": 1.1553, + "theoretical_loss": 3.36151089317479, + "tokens_seen": 2860042240 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010795586760280842, + "loss": 1.2618, + "theoretical_loss": 3.361504963927506, + "tokens_seen": 2860107776 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010793981945837512, + "loss": 1.1837, + "theoretical_loss": 3.3614990348541216, + "tokens_seen": 2860173312 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010792377131394182, + "loss": 1.1253, + "theoretical_loss": 3.361493105954629, + "tokens_seen": 2860238848 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010790772316950852, + "loss": 1.2327, + "theoretical_loss": 3.3614871772290185, + "tokens_seen": 2860304384 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010789167502507523, + "loss": 1.1933, + "theoretical_loss": 3.361481248677281, + "tokens_seen": 2860369920 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010787562688064193, + "loss": 1.1809, + "theoretical_loss": 3.361475320299408, + "tokens_seen": 2860435456 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010785957873620863, + "loss": 1.2206, + "theoretical_loss": 3.36146939209539, + "tokens_seen": 2860500992 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010784353059177533, + "loss": 1.2458, + "theoretical_loss": 3.361463464065218, + "tokens_seen": 2860566528 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010782748244734201, + "loss": 1.2643, + "theoretical_loss": 3.3614575362088823, + "tokens_seen": 2860632064 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010781143430290871, + "loss": 1.2032, + "theoretical_loss": 3.361451608526375, + "tokens_seen": 2860697600 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010779538615847544, + "loss": 1.2228, + "theoretical_loss": 3.361445681017686, + "tokens_seen": 2860763136 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010777933801404214, + "loss": 1.1894, + "theoretical_loss": 3.3614397536828067, + "tokens_seen": 2860828672 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010776328986960884, + "loss": 1.1602, + "theoretical_loss": 3.3614338265217283, + "tokens_seen": 2860894208 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010774724172517554, + "loss": 1.1496, + "theoretical_loss": 3.361427899534441, + "tokens_seen": 2860959744 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010773119358074225, + "loss": 1.1794, + "theoretical_loss": 3.3614219727209362, + "tokens_seen": 2861025280 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010771514543630895, + "loss": 1.3198, + "theoretical_loss": 3.361416046081205, + "tokens_seen": 2861090816 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010769909729187563, + "loss": 1.1424, + "theoretical_loss": 3.3614101196152375, + "tokens_seen": 2861156352 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010768304914744233, + "loss": 1.2744, + "theoretical_loss": 3.3614041933230254, + "tokens_seen": 2861221888 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010766700100300903, + "loss": 1.2027, + "theoretical_loss": 3.3613982672045597, + "tokens_seen": 2861287424 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010765095285857573, + "loss": 1.2268, + "theoretical_loss": 3.3613923412598306, + "tokens_seen": 2861352960 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010763490471414244, + "loss": 1.2366, + "theoretical_loss": 3.3613864154888295, + "tokens_seen": 2861418496 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010761885656970914, + "loss": 1.167, + "theoretical_loss": 3.361380489891548, + "tokens_seen": 2861484032 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010760280842527584, + "loss": 1.2795, + "theoretical_loss": 3.3613745644679756, + "tokens_seen": 2861549568 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010758676028084254, + "loss": 1.3053, + "theoretical_loss": 3.361368639218104, + "tokens_seen": 2861615104 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010757071213640924, + "loss": 1.2552, + "theoretical_loss": 3.361362714141924, + "tokens_seen": 2861680640 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010755466399197594, + "loss": 1.3362, + "theoretical_loss": 3.3613567892394265, + "tokens_seen": 2861746176 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010753861584754264, + "loss": 1.1715, + "theoretical_loss": 3.361350864510603, + "tokens_seen": 2861811712 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010752256770310934, + "loss": 1.2254, + "theoretical_loss": 3.3613449399554436, + "tokens_seen": 2861877248 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010750651955867603, + "loss": 1.2464, + "theoretical_loss": 3.3613390155739395, + "tokens_seen": 2861942784 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010749047141424273, + "loss": 1.3321, + "theoretical_loss": 3.3613330913660824, + "tokens_seen": 2862008320 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010747442326980943, + "loss": 1.2004, + "theoretical_loss": 3.361327167331862, + "tokens_seen": 2862073856 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010745837512537613, + "loss": 1.186, + "theoretical_loss": 3.36132124347127, + "tokens_seen": 2862139392 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010744232698094283, + "loss": 1.2883, + "theoretical_loss": 3.361315319784297, + "tokens_seen": 2862204928 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010742627883650953, + "loss": 1.2052, + "theoretical_loss": 3.3613093962709337, + "tokens_seen": 2862270464 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010741023069207623, + "loss": 1.2293, + "theoretical_loss": 3.361303472931172, + "tokens_seen": 2862336000 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010739418254764293, + "loss": 1.2123, + "theoretical_loss": 3.361297549765002, + "tokens_seen": 2862401536 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010737813440320963, + "loss": 1.0971, + "theoretical_loss": 3.361291626772415, + "tokens_seen": 2862467072 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010736208625877633, + "loss": 1.143, + "theoretical_loss": 3.361285703953402, + "tokens_seen": 2862532608 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010734603811434303, + "loss": 1.201, + "theoretical_loss": 3.3612797813079536, + "tokens_seen": 2862598144 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010732998996990973, + "loss": 1.1852, + "theoretical_loss": 3.3612738588360607, + "tokens_seen": 2862663680 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010731394182547642, + "loss": 1.2272, + "theoretical_loss": 3.3612679365377147, + "tokens_seen": 2862729216 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010729789368104312, + "loss": 1.1331, + "theoretical_loss": 3.361262014412906, + "tokens_seen": 2862794752 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010728184553660982, + "loss": 1.299, + "theoretical_loss": 3.361256092461626, + "tokens_seen": 2862860288 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010726579739217652, + "loss": 1.2571, + "theoretical_loss": 3.3612501706838653, + "tokens_seen": 2862925824 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010724974924774322, + "loss": 1.1329, + "theoretical_loss": 3.3612442490796153, + "tokens_seen": 2862991360 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010723370110330992, + "loss": 1.277, + "theoretical_loss": 3.3612383276488664, + "tokens_seen": 2863056896 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010721765295887662, + "loss": 1.2763, + "theoretical_loss": 3.36123240639161, + "tokens_seen": 2863122432 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010720160481444335, + "loss": 1.1688, + "theoretical_loss": 3.361226485307837, + "tokens_seen": 2863187968 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010718555667001004, + "loss": 1.2585, + "theoretical_loss": 3.361220564397538, + "tokens_seen": 2863253504 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010716950852557674, + "loss": 1.2042, + "theoretical_loss": 3.3612146436607038, + "tokens_seen": 2863319040 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010715346038114344, + "loss": 1.2529, + "theoretical_loss": 3.361208723097326, + "tokens_seen": 2863384576 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010713741223671014, + "loss": 1.2083, + "theoretical_loss": 3.3612028027073952, + "tokens_seen": 2863450112 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010712136409227684, + "loss": 1.287, + "theoretical_loss": 3.3611968824909026, + "tokens_seen": 2863515648 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010710531594784354, + "loss": 1.1551, + "theoretical_loss": 3.361190962447839, + "tokens_seen": 2863581184 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010708926780341024, + "loss": 1.1506, + "theoretical_loss": 3.361185042578195, + "tokens_seen": 2863646720 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010707321965897694, + "loss": 1.2655, + "theoretical_loss": 3.3611791228819614, + "tokens_seen": 2863712256 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010705717151454364, + "loss": 1.2817, + "theoretical_loss": 3.3611732033591304, + "tokens_seen": 2863777792 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010704112337011035, + "loss": 1.2244, + "theoretical_loss": 3.361167284009692, + "tokens_seen": 2863843328 + }, + { + "epoch": 1.25, + "objective/train/docs_used": 1057336, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2573106288909912, + "objective/train/theoretical_loss": 3.361161364833637, + "objective/train/tokens_used": 2884368864, + "theoretical_loss": 3.361161364833637, + "tokens_seen": 2863908864 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010702507522567705, + "loss": 1.2573, + "theoretical_loss": 3.361161364833637, + "tokens_seen": 2863908864 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010700902708124375, + "loss": 1.2885, + "theoretical_loss": 3.3611554458309567, + "tokens_seen": 2863974400 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010699297893681043, + "loss": 1.2672, + "theoretical_loss": 3.3611495270016425, + "tokens_seen": 2864039936 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010697693079237713, + "loss": 1.1288, + "theoretical_loss": 3.3611436083456843, + "tokens_seen": 2864105472 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010696088264794383, + "loss": 1.2889, + "theoretical_loss": 3.3611376898630736, + "tokens_seen": 2864171008 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010694483450351054, + "loss": 1.2426, + "theoretical_loss": 3.3611317715538016, + "tokens_seen": 2864236544 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010692878635907724, + "loss": 1.1576, + "theoretical_loss": 3.361125853417859, + "tokens_seen": 2864302080 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010691273821464394, + "loss": 1.1931, + "theoretical_loss": 3.361119935455237, + "tokens_seen": 2864367616 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010689669007021064, + "loss": 1.2303, + "theoretical_loss": 3.3611140176659258, + "tokens_seen": 2864433152 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010688064192577734, + "loss": 1.3041, + "theoretical_loss": 3.3611081000499174, + "tokens_seen": 2864498688 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010686459378134404, + "loss": 1.2478, + "theoretical_loss": 3.361102182607202, + "tokens_seen": 2864564224 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010684854563691074, + "loss": 1.122, + "theoretical_loss": 3.361096265337771, + "tokens_seen": 2864629760 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010683249749247744, + "loss": 1.21, + "theoretical_loss": 3.361090348241615, + "tokens_seen": 2864695296 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010681644934804414, + "loss": 1.1741, + "theoretical_loss": 3.3610844313187256, + "tokens_seen": 2864760832 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010680040120361083, + "loss": 1.2836, + "theoretical_loss": 3.361078514569093, + "tokens_seen": 2864826368 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010678435305917753, + "loss": 1.2521, + "theoretical_loss": 3.3610725979927083, + "tokens_seen": 2864891904 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010676830491474423, + "loss": 1.1473, + "theoretical_loss": 3.3610666815895627, + "tokens_seen": 2864957440 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010675225677031093, + "loss": 1.1563, + "theoretical_loss": 3.3610607653596474, + "tokens_seen": 2865022976 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010673620862587763, + "loss": 1.1875, + "theoretical_loss": 3.361054849302953, + "tokens_seen": 2865088512 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010672016048144433, + "loss": 1.1891, + "theoretical_loss": 3.3610489334194704, + "tokens_seen": 2865154048 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010670411233701103, + "loss": 1.2239, + "theoretical_loss": 3.3610430177091906, + "tokens_seen": 2865219584 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010668806419257773, + "loss": 1.1619, + "theoretical_loss": 3.361037102172105, + "tokens_seen": 2865285120 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010667201604814443, + "loss": 1.0677, + "theoretical_loss": 3.361031186808204, + "tokens_seen": 2865350656 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010665596790371113, + "loss": 1.2373, + "theoretical_loss": 3.361025271617479, + "tokens_seen": 2865416192 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010663991975927783, + "loss": 1.203, + "theoretical_loss": 3.3610193565999205, + "tokens_seen": 2865481728 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010662387161484453, + "loss": 1.1072, + "theoretical_loss": 3.36101344175552, + "tokens_seen": 2865547264 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010660782347041122, + "loss": 1.3183, + "theoretical_loss": 3.361007527084268, + "tokens_seen": 2865612800 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010659177532597795, + "loss": 1.1821, + "theoretical_loss": 3.361001612586156, + "tokens_seen": 2865678336 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010657572718154465, + "loss": 1.1576, + "theoretical_loss": 3.360995698261174, + "tokens_seen": 2865743872 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010655967903711135, + "loss": 1.2463, + "theoretical_loss": 3.3609897841093144, + "tokens_seen": 2865809408 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010654363089267805, + "loss": 1.2367, + "theoretical_loss": 3.360983870130567, + "tokens_seen": 2865874944 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010652758274824475, + "loss": 1.1729, + "theoretical_loss": 3.360977956324924, + "tokens_seen": 2865940480 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010651153460381145, + "loss": 1.14, + "theoretical_loss": 3.3609720426923744, + "tokens_seen": 2866006016 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010649548645937814, + "loss": 1.3157, + "theoretical_loss": 3.360966129232911, + "tokens_seen": 2866071552 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010647943831494484, + "loss": 1.2081, + "theoretical_loss": 3.360960215946524, + "tokens_seen": 2866137088 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010646339017051154, + "loss": 1.1496, + "theoretical_loss": 3.3609543028332043, + "tokens_seen": 2866202624 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010644734202607824, + "loss": 1.2883, + "theoretical_loss": 3.3609483898929433, + "tokens_seen": 2866268160 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010643129388164494, + "loss": 1.1864, + "theoretical_loss": 3.3609424771257315, + "tokens_seen": 2866333696 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010641524573721164, + "loss": 1.2184, + "theoretical_loss": 3.3609365645315603, + "tokens_seen": 2866399232 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010639919759277834, + "loss": 1.2005, + "theoretical_loss": 3.3609306521104205, + "tokens_seen": 2866464768 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010638314944834504, + "loss": 1.2205, + "theoretical_loss": 3.3609247398623032, + "tokens_seen": 2866530304 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010636710130391174, + "loss": 1.3886, + "theoretical_loss": 3.360918827787199, + "tokens_seen": 2866595840 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010635105315947845, + "loss": 1.1171, + "theoretical_loss": 3.3609129158850997, + "tokens_seen": 2866661376 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010633500501504515, + "loss": 1.2857, + "theoretical_loss": 3.360907004155995, + "tokens_seen": 2866726912 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010631895687061185, + "loss": 1.1764, + "theoretical_loss": 3.3609010925998772, + "tokens_seen": 2866792448 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010630290872617853, + "loss": 1.2708, + "theoretical_loss": 3.360895181216736, + "tokens_seen": 2866857984 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010628686058174523, + "loss": 1.1045, + "theoretical_loss": 3.360889270006564, + "tokens_seen": 2866923520 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010627081243731194, + "loss": 1.1371, + "theoretical_loss": 3.3608833589693505, + "tokens_seen": 2866989056 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010625476429287864, + "loss": 1.142, + "theoretical_loss": 3.3608774481050876, + "tokens_seen": 2867054592 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010623871614844534, + "loss": 1.1484, + "theoretical_loss": 3.360871537413766, + "tokens_seen": 2867120128 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010622266800401204, + "loss": 1.2089, + "theoretical_loss": 3.3608656268953765, + "tokens_seen": 2867185664 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010620661985957874, + "loss": 1.2124, + "theoretical_loss": 3.36085971654991, + "tokens_seen": 2867251200 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010619057171514544, + "loss": 1.13, + "theoretical_loss": 3.3608538063773583, + "tokens_seen": 2867316736 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010617452357071214, + "loss": 1.1744, + "theoretical_loss": 3.3608478963777113, + "tokens_seen": 2867382272 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010615847542627884, + "loss": 1.1871, + "theoretical_loss": 3.3608419865509607, + "tokens_seen": 2867447808 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010614242728184554, + "loss": 1.1242, + "theoretical_loss": 3.360836076897097, + "tokens_seen": 2867513344 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010612637913741224, + "loss": 1.2095, + "theoretical_loss": 3.3608301674161116, + "tokens_seen": 2867578880 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010611033099297893, + "loss": 1.2715, + "theoretical_loss": 3.3608242581079955, + "tokens_seen": 2867644416 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010609428284854563, + "loss": 1.2426, + "theoretical_loss": 3.3608183489727397, + "tokens_seen": 2867709952 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010607823470411233, + "loss": 1.2761, + "theoretical_loss": 3.3608124400103345, + "tokens_seen": 2867775488 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010606218655967903, + "loss": 1.1305, + "theoretical_loss": 3.360806531220772, + "tokens_seen": 2867841024 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010604613841524573, + "loss": 1.1745, + "theoretical_loss": 3.3608006226040423, + "tokens_seen": 2867906560 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010603009027081243, + "loss": 1.182, + "theoretical_loss": 3.3607947141601366, + "tokens_seen": 2867972096 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010601404212637913, + "loss": 1.1834, + "theoretical_loss": 3.3607888058890465, + "tokens_seen": 2868037632 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010599799398194586, + "loss": 1.1057, + "theoretical_loss": 3.360782897790762, + "tokens_seen": 2868103168 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010598194583751255, + "loss": 1.2414, + "theoretical_loss": 3.360776989865275, + "tokens_seen": 2868168704 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010596589769307925, + "loss": 1.2044, + "theoretical_loss": 3.360771082112576, + "tokens_seen": 2868234240 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010594984954864595, + "loss": 1.1454, + "theoretical_loss": 3.360765174532656, + "tokens_seen": 2868299776 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010593380140421265, + "loss": 1.1722, + "theoretical_loss": 3.3607592671255064, + "tokens_seen": 2868365312 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010591775325977935, + "loss": 1.2412, + "theoretical_loss": 3.3607533598911177, + "tokens_seen": 2868430848 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010590170511534605, + "loss": 1.1539, + "theoretical_loss": 3.360747452829481, + "tokens_seen": 2868496384 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010588565697091275, + "loss": 1.2527, + "theoretical_loss": 3.3607415459405874, + "tokens_seen": 2868561920 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010586960882647945, + "loss": 1.1757, + "theoretical_loss": 3.360735639224428, + "tokens_seen": 2868627456 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010585356068204615, + "loss": 1.1869, + "theoretical_loss": 3.360729732680994, + "tokens_seen": 2868692992 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010583751253761285, + "loss": 1.1042, + "theoretical_loss": 3.360723826310276, + "tokens_seen": 2868758528 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010582146439317955, + "loss": 1.2876, + "theoretical_loss": 3.360717920112265, + "tokens_seen": 2868824064 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010580541624874625, + "loss": 1.1739, + "theoretical_loss": 3.360712014086952, + "tokens_seen": 2868889600 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010578936810431294, + "loss": 1.216, + "theoretical_loss": 3.3607061082343286, + "tokens_seen": 2868955136 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010577331995987964, + "loss": 1.2085, + "theoretical_loss": 3.360700202554385, + "tokens_seen": 2869020672 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010575727181544634, + "loss": 1.2106, + "theoretical_loss": 3.3606942970471123, + "tokens_seen": 2869086208 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010574122367101304, + "loss": 1.2756, + "theoretical_loss": 3.360688391712502, + "tokens_seen": 2869151744 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010572517552657974, + "loss": 1.1082, + "theoretical_loss": 3.360682486550545, + "tokens_seen": 2869217280 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010570912738214644, + "loss": 1.2261, + "theoretical_loss": 3.3606765815612323, + "tokens_seen": 2869282816 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010569307923771314, + "loss": 1.1978, + "theoretical_loss": 3.3606706767445544, + "tokens_seen": 2869348352 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010567703109327985, + "loss": 1.0668, + "theoretical_loss": 3.360664772100503, + "tokens_seen": 2869413888 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010566098294884655, + "loss": 1.2349, + "theoretical_loss": 3.3606588676290685, + "tokens_seen": 2869479424 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010564493480441325, + "loss": 1.1817, + "theoretical_loss": 3.360652963330242, + "tokens_seen": 2869544960 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010562888665997995, + "loss": 1.1131, + "theoretical_loss": 3.3606470592040156, + "tokens_seen": 2869610496 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010561283851554665, + "loss": 1.1786, + "theoretical_loss": 3.360641155250379, + "tokens_seen": 2869676032 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010559679037111333, + "loss": 1.1628, + "theoretical_loss": 3.3606352514693234, + "tokens_seen": 2869741568 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010558074222668004, + "loss": 1.2028, + "theoretical_loss": 3.3606293478608404, + "tokens_seen": 2869807104 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010556469408224674, + "loss": 1.2285, + "theoretical_loss": 3.3606234444249203, + "tokens_seen": 2869872640 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010554864593781344, + "loss": 1.1774, + "theoretical_loss": 3.360617541161555, + "tokens_seen": 2869938176 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010553259779338014, + "loss": 1.1058, + "theoretical_loss": 3.3606116380707345, + "tokens_seen": 2870003712 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010551654964894684, + "loss": 1.1279, + "theoretical_loss": 3.360605735152451, + "tokens_seen": 2870069248 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010550050150451354, + "loss": 1.0757, + "theoretical_loss": 3.3605998324066944, + "tokens_seen": 2870134784 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010548445336008024, + "loss": 1.0629, + "theoretical_loss": 3.360593929833456, + "tokens_seen": 2870200320 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010546840521564694, + "loss": 1.2533, + "theoretical_loss": 3.360588027432727, + "tokens_seen": 2870265856 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010545235707121364, + "loss": 1.2696, + "theoretical_loss": 3.3605821252044987, + "tokens_seen": 2870331392 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010543630892678034, + "loss": 1.1848, + "theoretical_loss": 3.3605762231487617, + "tokens_seen": 2870396928 + }, + { + "epoch": 1.25, + "objective/train/docs_used": 1059708, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2104249000549316, + "objective/train/theoretical_loss": 3.3605703212655076, + "objective/train/tokens_used": 2890922464, + "theoretical_loss": 3.3605703212655076, + "tokens_seen": 2870462464 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010542026078234704, + "loss": 1.2104, + "theoretical_loss": 3.3605703212655076, + "tokens_seen": 2870462464 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010540421263791376, + "loss": 1.1844, + "theoretical_loss": 3.360564419554726, + "tokens_seen": 2870528000 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010538816449348046, + "loss": 1.2383, + "theoretical_loss": 3.36055851801641, + "tokens_seen": 2870593536 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010537211634904716, + "loss": 1.1709, + "theoretical_loss": 3.3605526166505486, + "tokens_seen": 2870659072 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010535606820461386, + "loss": 1.2516, + "theoretical_loss": 3.3605467154571342, + "tokens_seen": 2870724608 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010534002006018056, + "loss": 1.1834, + "theoretical_loss": 3.3605408144361575, + "tokens_seen": 2870790144 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010532397191574726, + "loss": 1.1922, + "theoretical_loss": 3.360534913587609, + "tokens_seen": 2870855680 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010530792377131396, + "loss": 1.1425, + "theoretical_loss": 3.3605290129114804, + "tokens_seen": 2870921216 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010529187562688066, + "loss": 1.1516, + "theoretical_loss": 3.360523112407762, + "tokens_seen": 2870986752 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010527582748244735, + "loss": 1.3085, + "theoretical_loss": 3.3605172120764455, + "tokens_seen": 2871052288 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010525977933801405, + "loss": 1.1505, + "theoretical_loss": 3.360511311917522, + "tokens_seen": 2871117824 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010524373119358075, + "loss": 1.1632, + "theoretical_loss": 3.3605054119309816, + "tokens_seen": 2871183360 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010522768304914745, + "loss": 1.2068, + "theoretical_loss": 3.3604995121168164, + "tokens_seen": 2871248896 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010521163490471415, + "loss": 1.2438, + "theoretical_loss": 3.3604936124750173, + "tokens_seen": 2871314432 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010519558676028085, + "loss": 1.291, + "theoretical_loss": 3.3604877130055746, + "tokens_seen": 2871379968 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010517953861584755, + "loss": 1.1708, + "theoretical_loss": 3.3604818137084793, + "tokens_seen": 2871445504 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010516349047141425, + "loss": 1.2553, + "theoretical_loss": 3.3604759145837235, + "tokens_seen": 2871511040 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010514744232698095, + "loss": 1.1526, + "theoretical_loss": 3.3604700156312974, + "tokens_seen": 2871576576 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010513139418254765, + "loss": 1.1638, + "theoretical_loss": 3.360464116851192, + "tokens_seen": 2871642112 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010511534603811435, + "loss": 1.1085, + "theoretical_loss": 3.3604582182433993, + "tokens_seen": 2871707648 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010509929789368105, + "loss": 1.3014, + "theoretical_loss": 3.360452319807909, + "tokens_seen": 2871773184 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010508324974924774, + "loss": 1.211, + "theoretical_loss": 3.360446421544713, + "tokens_seen": 2871838720 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010506720160481444, + "loss": 1.2016, + "theoretical_loss": 3.3604405234538017, + "tokens_seen": 2871904256 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010505115346038114, + "loss": 1.1276, + "theoretical_loss": 3.360434625535167, + "tokens_seen": 2871969792 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010503510531594784, + "loss": 1.1568, + "theoretical_loss": 3.3604287277887988, + "tokens_seen": 2872035328 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010501905717151454, + "loss": 1.1791, + "theoretical_loss": 3.3604228302146897, + "tokens_seen": 2872100864 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010500300902708124, + "loss": 1.1107, + "theoretical_loss": 3.360416932812829, + "tokens_seen": 2872166400 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010498696088264795, + "loss": 1.1134, + "theoretical_loss": 3.360411035583209, + "tokens_seen": 2872231936 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010497091273821465, + "loss": 1.2713, + "theoretical_loss": 3.3604051385258202, + "tokens_seen": 2872297472 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010495486459378135, + "loss": 1.1695, + "theoretical_loss": 3.360399241640654, + "tokens_seen": 2872363008 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010493881644934805, + "loss": 1.1926, + "theoretical_loss": 3.360393344927701, + "tokens_seen": 2872428544 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010492276830491475, + "loss": 1.1335, + "theoretical_loss": 3.360387448386952, + "tokens_seen": 2872494080 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010490672016048145, + "loss": 1.1039, + "theoretical_loss": 3.360381552018399, + "tokens_seen": 2872559616 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010489067201604814, + "loss": 1.2326, + "theoretical_loss": 3.360375655822032, + "tokens_seen": 2872625152 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010487462387161484, + "loss": 1.1848, + "theoretical_loss": 3.360369759797843, + "tokens_seen": 2872690688 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010485857572718154, + "loss": 1.1281, + "theoretical_loss": 3.3603638639458224, + "tokens_seen": 2872756224 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010484252758274824, + "loss": 1.1649, + "theoretical_loss": 3.3603579682659617, + "tokens_seen": 2872821760 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010482647943831494, + "loss": 1.1916, + "theoretical_loss": 3.3603520727582517, + "tokens_seen": 2872887296 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010481043129388164, + "loss": 1.1771, + "theoretical_loss": 3.360346177422683, + "tokens_seen": 2872952832 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010479438314944837, + "loss": 1.2905, + "theoretical_loss": 3.3603402822592474, + "tokens_seen": 2873018368 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010477833500501507, + "loss": 1.1882, + "theoretical_loss": 3.3603343872679354, + "tokens_seen": 2873083904 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010476228686058175, + "loss": 1.1732, + "theoretical_loss": 3.3603284924487387, + "tokens_seen": 2873149440 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010474623871614845, + "loss": 1.208, + "theoretical_loss": 3.3603225978016478, + "tokens_seen": 2873214976 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010473019057171516, + "loss": 1.2361, + "theoretical_loss": 3.360316703326654, + "tokens_seen": 2873280512 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010471414242728186, + "loss": 1.2058, + "theoretical_loss": 3.3603108090237477, + "tokens_seen": 2873346048 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010469809428284856, + "loss": 1.1952, + "theoretical_loss": 3.3603049148929207, + "tokens_seen": 2873411584 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010468204613841526, + "loss": 1.2043, + "theoretical_loss": 3.360299020934164, + "tokens_seen": 2873477120 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010466599799398196, + "loss": 1.2366, + "theoretical_loss": 3.360293127147468, + "tokens_seen": 2873542656 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010464994984954866, + "loss": 1.2138, + "theoretical_loss": 3.360287233532825, + "tokens_seen": 2873608192 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010463390170511536, + "loss": 1.2272, + "theoretical_loss": 3.360281340090225, + "tokens_seen": 2873673728 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010461785356068206, + "loss": 1.1444, + "theoretical_loss": 3.360275446819659, + "tokens_seen": 2873739264 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010460180541624876, + "loss": 1.1232, + "theoretical_loss": 3.360269553721119, + "tokens_seen": 2873804800 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010458575727181546, + "loss": 1.2314, + "theoretical_loss": 3.360263660794595, + "tokens_seen": 2873870336 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010456970912738215, + "loss": 1.3065, + "theoretical_loss": 3.3602577680400785, + "tokens_seen": 2873935872 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010455366098294885, + "loss": 1.2601, + "theoretical_loss": 3.360251875457561, + "tokens_seen": 2874001408 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010453761283851555, + "loss": 1.1869, + "theoretical_loss": 3.3602459830470326, + "tokens_seen": 2874066944 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010452156469408225, + "loss": 1.1763, + "theoretical_loss": 3.3602400908084853, + "tokens_seen": 2874132480 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010450551654964895, + "loss": 1.2238, + "theoretical_loss": 3.3602341987419093, + "tokens_seen": 2874198016 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010448946840521565, + "loss": 1.2348, + "theoretical_loss": 3.3602283068472967, + "tokens_seen": 2874263552 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010447342026078235, + "loss": 1.2067, + "theoretical_loss": 3.3602224151246376, + "tokens_seen": 2874329088 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010445737211634905, + "loss": 1.2551, + "theoretical_loss": 3.3602165235739236, + "tokens_seen": 2874394624 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010444132397191575, + "loss": 1.2599, + "theoretical_loss": 3.3602106321951455, + "tokens_seen": 2874460160 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010442527582748245, + "loss": 1.2492, + "theoretical_loss": 3.3602047409882942, + "tokens_seen": 2874525696 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010440922768304915, + "loss": 1.2109, + "theoretical_loss": 3.3601988499533615, + "tokens_seen": 2874591232 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010439317953861586, + "loss": 1.2273, + "theoretical_loss": 3.3601929590903374, + "tokens_seen": 2874656768 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010437713139418254, + "loss": 1.1815, + "theoretical_loss": 3.360187068399214, + "tokens_seen": 2874722304 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010436108324974924, + "loss": 1.1002, + "theoretical_loss": 3.3601811778799817, + "tokens_seen": 2874787840 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010434503510531594, + "loss": 1.2039, + "theoretical_loss": 3.360175287532632, + "tokens_seen": 2874853376 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010432898696088264, + "loss": 1.2874, + "theoretical_loss": 3.3601693973571556, + "tokens_seen": 2874918912 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010431293881644934, + "loss": 1.2712, + "theoretical_loss": 3.3601635073535436, + "tokens_seen": 2874984448 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010429689067201605, + "loss": 1.2449, + "theoretical_loss": 3.3601576175217875, + "tokens_seen": 2875049984 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010428084252758275, + "loss": 1.2053, + "theoretical_loss": 3.3601517278618775, + "tokens_seen": 2875115520 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010426479438314945, + "loss": 1.1781, + "theoretical_loss": 3.3601458383738056, + "tokens_seen": 2875181056 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010424874623871615, + "loss": 1.2072, + "theoretical_loss": 3.3601399490575625, + "tokens_seen": 2875246592 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010423269809428285, + "loss": 1.1449, + "theoretical_loss": 3.360134059913139, + "tokens_seen": 2875312128 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010421664994984955, + "loss": 1.1774, + "theoretical_loss": 3.3601281709405266, + "tokens_seen": 2875377664 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010420060180541626, + "loss": 1.2123, + "theoretical_loss": 3.360122282139716, + "tokens_seen": 2875443200 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010418455366098296, + "loss": 1.1798, + "theoretical_loss": 3.360116393510699, + "tokens_seen": 2875508736 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010416850551654966, + "loss": 1.2484, + "theoretical_loss": 3.3601105050534654, + "tokens_seen": 2875574272 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010415245737211636, + "loss": 1.2472, + "theoretical_loss": 3.3601046167680075, + "tokens_seen": 2875639808 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010413640922768307, + "loss": 1.2304, + "theoretical_loss": 3.360098728654316, + "tokens_seen": 2875705344 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010412036108324977, + "loss": 1.0603, + "theoretical_loss": 3.3600928407123813, + "tokens_seen": 2875770880 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010410431293881647, + "loss": 1.0805, + "theoretical_loss": 3.3600869529421957, + "tokens_seen": 2875836416 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010408826479438317, + "loss": 1.1896, + "theoretical_loss": 3.360081065343749, + "tokens_seen": 2875901952 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010407221664994985, + "loss": 1.1949, + "theoretical_loss": 3.3600751779170337, + "tokens_seen": 2875967488 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010405616850551655, + "loss": 1.2189, + "theoretical_loss": 3.3600692906620395, + "tokens_seen": 2876033024 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010404012036108326, + "loss": 1.2157, + "theoretical_loss": 3.360063403578758, + "tokens_seen": 2876098560 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010402407221664996, + "loss": 1.2255, + "theoretical_loss": 3.3600575166671804, + "tokens_seen": 2876164096 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010400802407221666, + "loss": 1.1986, + "theoretical_loss": 3.360051629927298, + "tokens_seen": 2876229632 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010399197592778336, + "loss": 1.0928, + "theoretical_loss": 3.360045743359101, + "tokens_seen": 2876295168 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010397592778335006, + "loss": 1.2279, + "theoretical_loss": 3.3600398569625813, + "tokens_seen": 2876360704 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010395987963891676, + "loss": 1.1409, + "theoretical_loss": 3.3600339707377302, + "tokens_seen": 2876426240 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010394383149448346, + "loss": 1.1969, + "theoretical_loss": 3.3600280846845383, + "tokens_seen": 2876491776 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010392778335005016, + "loss": 1.2222, + "theoretical_loss": 3.360022198802996, + "tokens_seen": 2876557312 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010391173520561686, + "loss": 1.1629, + "theoretical_loss": 3.360016313093096, + "tokens_seen": 2876622848 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010389568706118356, + "loss": 1.115, + "theoretical_loss": 3.360010427554828, + "tokens_seen": 2876688384 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010387963891675025, + "loss": 1.1314, + "theoretical_loss": 3.3600045421881832, + "tokens_seen": 2876753920 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010386359077231695, + "loss": 1.3304, + "theoretical_loss": 3.3599986569931537, + "tokens_seen": 2876819456 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010384754262788365, + "loss": 1.2135, + "theoretical_loss": 3.3599927719697296, + "tokens_seen": 2876884992 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010383149448345035, + "loss": 1.1729, + "theoretical_loss": 3.3599868871179024, + "tokens_seen": 2876950528 + }, + { + "epoch": 1.25, + "objective/train/docs_used": 1062065, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.152599573135376, + "objective/train/theoretical_loss": 3.3599810024376633, + "objective/train/tokens_used": 2897476064, + "theoretical_loss": 3.3599810024376633, + "tokens_seen": 2877016064 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010381544633901705, + "loss": 1.1526, + "theoretical_loss": 3.3599810024376633, + "tokens_seen": 2877016064 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010379939819458375, + "loss": 1.1633, + "theoretical_loss": 3.359975117929003, + "tokens_seen": 2877081600 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010378335005015045, + "loss": 1.2273, + "theoretical_loss": 3.359969233591913, + "tokens_seen": 2877147136 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010376730190571715, + "loss": 1.17, + "theoretical_loss": 3.359963349426384, + "tokens_seen": 2877212672 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010375125376128385, + "loss": 1.1586, + "theoretical_loss": 3.359957465432407, + "tokens_seen": 2877278208 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010373520561685055, + "loss": 1.2292, + "theoretical_loss": 3.359951581609974, + "tokens_seen": 2877343744 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010371915747241725, + "loss": 1.2237, + "theoretical_loss": 3.359945697959075, + "tokens_seen": 2877409280 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010370310932798396, + "loss": 1.174, + "theoretical_loss": 3.3599398144797017, + "tokens_seen": 2877474816 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010368706118355064, + "loss": 1.2626, + "theoretical_loss": 3.3599339311718452, + "tokens_seen": 2877540352 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010367101303911734, + "loss": 1.1573, + "theoretical_loss": 3.359928048035496, + "tokens_seen": 2877605888 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010365496489468404, + "loss": 1.1355, + "theoretical_loss": 3.359922165070646, + "tokens_seen": 2877671424 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010363891675025074, + "loss": 1.261, + "theoretical_loss": 3.359916282277286, + "tokens_seen": 2877736960 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010362286860581744, + "loss": 1.1671, + "theoretical_loss": 3.3599103996554067, + "tokens_seen": 2877802496 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010360682046138415, + "loss": 1.1601, + "theoretical_loss": 3.359904517205, + "tokens_seen": 2877868032 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010359077231695087, + "loss": 1.1546, + "theoretical_loss": 3.359898634926056, + "tokens_seen": 2877933568 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010357472417251757, + "loss": 1.173, + "theoretical_loss": 3.3598927528185665, + "tokens_seen": 2877999104 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010355867602808426, + "loss": 1.2321, + "theoretical_loss": 3.3598868708825225, + "tokens_seen": 2878064640 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010354262788365096, + "loss": 1.3059, + "theoretical_loss": 3.359880989117915, + "tokens_seen": 2878130176 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010352657973921766, + "loss": 1.2435, + "theoretical_loss": 3.359875107524735, + "tokens_seen": 2878195712 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010351053159478436, + "loss": 1.2062, + "theoretical_loss": 3.3598692261029734, + "tokens_seen": 2878261248 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010349448345035106, + "loss": 1.2223, + "theoretical_loss": 3.3598633448526223, + "tokens_seen": 2878326784 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010347843530591776, + "loss": 1.1697, + "theoretical_loss": 3.359857463773672, + "tokens_seen": 2878392320 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010346238716148446, + "loss": 1.1861, + "theoretical_loss": 3.3598515828661135, + "tokens_seen": 2878457856 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010344633901705117, + "loss": 1.2416, + "theoretical_loss": 3.359845702129938, + "tokens_seen": 2878523392 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010343029087261787, + "loss": 1.1844, + "theoretical_loss": 3.359839821565137, + "tokens_seen": 2878588928 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010341424272818457, + "loss": 1.1405, + "theoretical_loss": 3.359833941171701, + "tokens_seen": 2878654464 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010339819458375127, + "loss": 1.1646, + "theoretical_loss": 3.3598280609496216, + "tokens_seen": 2878720000 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010338214643931797, + "loss": 1.1049, + "theoretical_loss": 3.35982218089889, + "tokens_seen": 2878785536 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010336609829488466, + "loss": 1.196, + "theoretical_loss": 3.359816301019497, + "tokens_seen": 2878851072 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010335005015045136, + "loss": 1.197, + "theoretical_loss": 3.3598104213114333, + "tokens_seen": 2878916608 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010333400200601806, + "loss": 1.1206, + "theoretical_loss": 3.359804541774691, + "tokens_seen": 2878982144 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010331795386158476, + "loss": 1.133, + "theoretical_loss": 3.3597986624092604, + "tokens_seen": 2879047680 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010330190571715146, + "loss": 1.1501, + "theoretical_loss": 3.359792783215133, + "tokens_seen": 2879113216 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010328585757271816, + "loss": 1.1279, + "theoretical_loss": 3.3597869041922994, + "tokens_seen": 2879178752 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010326980942828486, + "loss": 1.172, + "theoretical_loss": 3.3597810253407516, + "tokens_seen": 2879244288 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010325376128385156, + "loss": 1.145, + "theoretical_loss": 3.3597751466604797, + "tokens_seen": 2879309824 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010323771313941826, + "loss": 1.3307, + "theoretical_loss": 3.3597692681514757, + "tokens_seen": 2879375360 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010322166499498496, + "loss": 1.2295, + "theoretical_loss": 3.3597633898137302, + "tokens_seen": 2879440896 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010320561685055166, + "loss": 1.1004, + "theoretical_loss": 3.3597575116472345, + "tokens_seen": 2879506432 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010318956870611836, + "loss": 1.2777, + "theoretical_loss": 3.35975163365198, + "tokens_seen": 2879571968 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010317352056168505, + "loss": 1.2386, + "theoretical_loss": 3.359745755827957, + "tokens_seen": 2879637504 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010315747241725175, + "loss": 1.2154, + "theoretical_loss": 3.359739878175157, + "tokens_seen": 2879703040 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010314142427281845, + "loss": 1.2452, + "theoretical_loss": 3.3597340006935714, + "tokens_seen": 2879768576 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010312537612838515, + "loss": 1.1563, + "theoretical_loss": 3.3597281233831913, + "tokens_seen": 2879834112 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010310932798395185, + "loss": 1.2646, + "theoretical_loss": 3.3597222462440075, + "tokens_seen": 2879899648 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010309327983951855, + "loss": 1.2564, + "theoretical_loss": 3.3597163692760112, + "tokens_seen": 2879965184 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010307723169508525, + "loss": 1.2204, + "theoretical_loss": 3.3597104924791936, + "tokens_seen": 2880030720 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010306118355065195, + "loss": 1.0986, + "theoretical_loss": 3.3597046158535457, + "tokens_seen": 2880096256 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010304513540621865, + "loss": 1.1645, + "theoretical_loss": 3.359698739399059, + "tokens_seen": 2880161792 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010302908726178535, + "loss": 1.1678, + "theoretical_loss": 3.359692863115724, + "tokens_seen": 2880227328 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010301303911735206, + "loss": 1.121, + "theoretical_loss": 3.3596869870035326, + "tokens_seen": 2880292864 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010299699097291877, + "loss": 1.2623, + "theoretical_loss": 3.3596811110624754, + "tokens_seen": 2880358400 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010298094282848547, + "loss": 1.2161, + "theoretical_loss": 3.359675235292543, + "tokens_seen": 2880423936 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010296489468405217, + "loss": 1.1204, + "theoretical_loss": 3.3596693596937275, + "tokens_seen": 2880489472 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010294884653961887, + "loss": 1.2047, + "theoretical_loss": 3.35966348426602, + "tokens_seen": 2880555008 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010293279839518557, + "loss": 1.2278, + "theoretical_loss": 3.359657609009411, + "tokens_seen": 2880620544 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010291675025075227, + "loss": 1.2132, + "theoretical_loss": 3.359651733923892, + "tokens_seen": 2880686080 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010290070210631897, + "loss": 1.1491, + "theoretical_loss": 3.359645859009454, + "tokens_seen": 2880751616 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010288465396188567, + "loss": 1.1566, + "theoretical_loss": 3.359639984266088, + "tokens_seen": 2880817152 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010286860581745237, + "loss": 1.1216, + "theoretical_loss": 3.359634109693785, + "tokens_seen": 2880882688 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010285255767301906, + "loss": 1.2189, + "theoretical_loss": 3.3596282352925364, + "tokens_seen": 2880948224 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010283650952858576, + "loss": 1.168, + "theoretical_loss": 3.3596223610623337, + "tokens_seen": 2881013760 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010282046138415246, + "loss": 1.1646, + "theoretical_loss": 3.3596164870031675, + "tokens_seen": 2881079296 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010280441323971916, + "loss": 1.2622, + "theoretical_loss": 3.3596106131150294, + "tokens_seen": 2881144832 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010278836509528586, + "loss": 1.1944, + "theoretical_loss": 3.35960473939791, + "tokens_seen": 2881210368 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010277231695085257, + "loss": 1.2252, + "theoretical_loss": 3.359598865851801, + "tokens_seen": 2881275904 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010275626880641927, + "loss": 1.1498, + "theoretical_loss": 3.3595929924766925, + "tokens_seen": 2881341440 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010274022066198597, + "loss": 1.1362, + "theoretical_loss": 3.3595871192725766, + "tokens_seen": 2881406976 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010272417251755267, + "loss": 1.2133, + "theoretical_loss": 3.3595812462394443, + "tokens_seen": 2881472512 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010270812437311937, + "loss": 1.1347, + "theoretical_loss": 3.3595753733772864, + "tokens_seen": 2881538048 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010269207622868607, + "loss": 1.1117, + "theoretical_loss": 3.3595695006860944, + "tokens_seen": 2881603584 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010267602808425277, + "loss": 1.1095, + "theoretical_loss": 3.359563628165859, + "tokens_seen": 2881669120 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010265997993981946, + "loss": 1.2088, + "theoretical_loss": 3.3595577558165717, + "tokens_seen": 2881734656 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010264393179538616, + "loss": 1.2574, + "theoretical_loss": 3.3595518836382237, + "tokens_seen": 2881800192 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010262788365095286, + "loss": 1.234, + "theoretical_loss": 3.3595460116308056, + "tokens_seen": 2881865728 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010261183550651956, + "loss": 1.2557, + "theoretical_loss": 3.3595401397943094, + "tokens_seen": 2881931264 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010259578736208626, + "loss": 1.1767, + "theoretical_loss": 3.3595342681287255, + "tokens_seen": 2881996800 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010257973921765296, + "loss": 1.2289, + "theoretical_loss": 3.3595283966340452, + "tokens_seen": 2882062336 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010256369107321966, + "loss": 1.1578, + "theoretical_loss": 3.3595225253102594, + "tokens_seen": 2882127872 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010254764292878636, + "loss": 1.1805, + "theoretical_loss": 3.35951665415736, + "tokens_seen": 2882193408 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010253159478435306, + "loss": 1.1601, + "theoretical_loss": 3.3595107831753377, + "tokens_seen": 2882258944 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010251554663991976, + "loss": 1.1809, + "theoretical_loss": 3.3595049123641836, + "tokens_seen": 2882324480 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010249949849548646, + "loss": 1.0946, + "theoretical_loss": 3.359499041723889, + "tokens_seen": 2882390016 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010248345035105316, + "loss": 1.251, + "theoretical_loss": 3.3594931712544445, + "tokens_seen": 2882455552 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010246740220661985, + "loss": 1.259, + "theoretical_loss": 3.3594873009558417, + "tokens_seen": 2882521088 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010245135406218655, + "loss": 1.1657, + "theoretical_loss": 3.359481430828072, + "tokens_seen": 2882586624 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010243530591775325, + "loss": 1.2619, + "theoretical_loss": 3.359475560871126, + "tokens_seen": 2882652160 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010241925777331995, + "loss": 1.0747, + "theoretical_loss": 3.3594696910849953, + "tokens_seen": 2882717696 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010240320962888665, + "loss": 1.1649, + "theoretical_loss": 3.359463821469671, + "tokens_seen": 2882783232 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010238716148445338, + "loss": 1.2204, + "theoretical_loss": 3.3594579520251435, + "tokens_seen": 2882848768 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010237111334002008, + "loss": 1.2908, + "theoretical_loss": 3.359452082751405, + "tokens_seen": 2882914304 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010235506519558677, + "loss": 1.1492, + "theoretical_loss": 3.359446213648446, + "tokens_seen": 2882979840 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010233901705115347, + "loss": 1.2924, + "theoretical_loss": 3.359440344716258, + "tokens_seen": 2883045376 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010232296890672017, + "loss": 1.0994, + "theoretical_loss": 3.3594344759548322, + "tokens_seen": 2883110912 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010230692076228687, + "loss": 1.1762, + "theoretical_loss": 3.3594286073641593, + "tokens_seen": 2883176448 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010229087261785357, + "loss": 1.1853, + "theoretical_loss": 3.3594227389442306, + "tokens_seen": 2883241984 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010227482447342027, + "loss": 1.1967, + "theoretical_loss": 3.3594168706950374, + "tokens_seen": 2883307520 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010225877632898697, + "loss": 1.3011, + "theoretical_loss": 3.3594110026165707, + "tokens_seen": 2883373056 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010224272818455367, + "loss": 1.1505, + "theoretical_loss": 3.3594051347088216, + "tokens_seen": 2883438592 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010222668004012037, + "loss": 1.2103, + "theoretical_loss": 3.359399266971782, + "tokens_seen": 2883504128 + }, + { + "epoch": 1.25, + "objective/train/docs_used": 1064408, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.248471975326538, + "objective/train/theoretical_loss": 3.359393399405442, + "objective/train/tokens_used": 2904029664, + "theoretical_loss": 3.359393399405442, + "tokens_seen": 2883569664 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010221063189568707, + "loss": 1.2485, + "theoretical_loss": 3.359393399405442, + "tokens_seen": 2883569664 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010219458375125377, + "loss": 1.3054, + "theoretical_loss": 3.359387532009793, + "tokens_seen": 2883635200 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010217853560682047, + "loss": 1.2041, + "theoretical_loss": 3.3593816647848267, + "tokens_seen": 2883700736 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010216248746238716, + "loss": 1.1913, + "theoretical_loss": 3.359375797730534, + "tokens_seen": 2883766272 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010214643931795386, + "loss": 1.2913, + "theoretical_loss": 3.3593699308469054, + "tokens_seen": 2883831808 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010213039117352056, + "loss": 1.1531, + "theoretical_loss": 3.359364064133933, + "tokens_seen": 2883897344 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010211434302908726, + "loss": 1.2564, + "theoretical_loss": 3.359358197591608, + "tokens_seen": 2883962880 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010209829488465396, + "loss": 1.2829, + "theoretical_loss": 3.3593523312199207, + "tokens_seen": 2884028416 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010208224674022067, + "loss": 1.1565, + "theoretical_loss": 3.359346465018863, + "tokens_seen": 2884093952 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010206619859578737, + "loss": 1.198, + "theoretical_loss": 3.3593405989884255, + "tokens_seen": 2884159488 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010205015045135407, + "loss": 1.2096, + "theoretical_loss": 3.359334733128599, + "tokens_seen": 2884225024 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010203410230692077, + "loss": 1.1035, + "theoretical_loss": 3.3593288674393764, + "tokens_seen": 2884290560 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010201805416248747, + "loss": 1.1998, + "theoretical_loss": 3.3593230019207474, + "tokens_seen": 2884356096 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010200200601805417, + "loss": 1.2248, + "theoretical_loss": 3.3593171365727033, + "tokens_seen": 2884421632 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010198595787362087, + "loss": 1.2561, + "theoretical_loss": 3.359311271395236, + "tokens_seen": 2884487168 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010196990972918756, + "loss": 1.2227, + "theoretical_loss": 3.3593054063883354, + "tokens_seen": 2884552704 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010195386158475426, + "loss": 1.1168, + "theoretical_loss": 3.359299541551994, + "tokens_seen": 2884618240 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010193781344032096, + "loss": 1.1561, + "theoretical_loss": 3.359293676886202, + "tokens_seen": 2884683776 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010192176529588766, + "loss": 1.1551, + "theoretical_loss": 3.359287812390951, + "tokens_seen": 2884749312 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010190571715145436, + "loss": 1.1439, + "theoretical_loss": 3.359281948066232, + "tokens_seen": 2884814848 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010188966900702106, + "loss": 1.28, + "theoretical_loss": 3.359276083912037, + "tokens_seen": 2884880384 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010187362086258776, + "loss": 1.1747, + "theoretical_loss": 3.3592702199283555, + "tokens_seen": 2884945920 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010185757271815446, + "loss": 1.2158, + "theoretical_loss": 3.35926435611518, + "tokens_seen": 2885011456 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010184152457372116, + "loss": 1.2251, + "theoretical_loss": 3.359258492472501, + "tokens_seen": 2885076992 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010182547642928786, + "loss": 1.1628, + "theoretical_loss": 3.35925262900031, + "tokens_seen": 2885142528 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010180942828485456, + "loss": 1.226, + "theoretical_loss": 3.3592467656985985, + "tokens_seen": 2885208064 + }, + { + "epoch": 1.25, + "learning_rate": 0.00010179338014042128, + "loss": 1.0982, + "theoretical_loss": 3.3592409025673575, + "tokens_seen": 2885273600 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010177733199598798, + "loss": 1.2367, + "theoretical_loss": 3.3592350396065775, + "tokens_seen": 2885339136 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010176128385155468, + "loss": 1.1404, + "theoretical_loss": 3.35922917681625, + "tokens_seen": 2885404672 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010174523570712138, + "loss": 1.1823, + "theoretical_loss": 3.3592233141963668, + "tokens_seen": 2885470208 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010172918756268808, + "loss": 1.1981, + "theoretical_loss": 3.359217451746918, + "tokens_seen": 2885535744 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010171313941825478, + "loss": 1.1407, + "theoretical_loss": 3.3592115894678956, + "tokens_seen": 2885601280 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010169709127382148, + "loss": 1.1518, + "theoretical_loss": 3.3592057273592903, + "tokens_seen": 2885666816 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010168104312938818, + "loss": 1.1137, + "theoretical_loss": 3.359199865421094, + "tokens_seen": 2885732352 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010166499498495488, + "loss": 1.2494, + "theoretical_loss": 3.3591940036532972, + "tokens_seen": 2885797888 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010164894684052157, + "loss": 1.068, + "theoretical_loss": 3.359188142055891, + "tokens_seen": 2885863424 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010163289869608827, + "loss": 1.1722, + "theoretical_loss": 3.3591822806288674, + "tokens_seen": 2885928960 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010161685055165497, + "loss": 1.2426, + "theoretical_loss": 3.3591764193722167, + "tokens_seen": 2885994496 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010160080240722167, + "loss": 1.2182, + "theoretical_loss": 3.35917055828593, + "tokens_seen": 2886060032 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010158475426278837, + "loss": 1.2899, + "theoretical_loss": 3.3591646973699993, + "tokens_seen": 2886125568 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010156870611835507, + "loss": 1.1704, + "theoretical_loss": 3.359158836624415, + "tokens_seen": 2886191104 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010155265797392177, + "loss": 1.1909, + "theoretical_loss": 3.359152976049169, + "tokens_seen": 2886256640 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010153660982948847, + "loss": 1.1572, + "theoretical_loss": 3.359147115644252, + "tokens_seen": 2886322176 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010152056168505517, + "loss": 1.2826, + "theoretical_loss": 3.3591412554096554, + "tokens_seen": 2886387712 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010150451354062187, + "loss": 1.1923, + "theoretical_loss": 3.35913539534537, + "tokens_seen": 2886453248 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010148846539618858, + "loss": 1.1606, + "theoretical_loss": 3.3591295354513875, + "tokens_seen": 2886518784 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010147241725175528, + "loss": 1.1218, + "theoretical_loss": 3.359123675727699, + "tokens_seen": 2886584320 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010145636910732196, + "loss": 1.3207, + "theoretical_loss": 3.359117816174295, + "tokens_seen": 2886649856 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010144032096288866, + "loss": 1.2244, + "theoretical_loss": 3.3591119567911676, + "tokens_seen": 2886715392 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010142427281845536, + "loss": 1.1078, + "theoretical_loss": 3.359106097578308, + "tokens_seen": 2886780928 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010140822467402206, + "loss": 1.1772, + "theoretical_loss": 3.3591002385357065, + "tokens_seen": 2886846464 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010139217652958877, + "loss": 1.2996, + "theoretical_loss": 3.359094379663355, + "tokens_seen": 2886912000 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010137612838515547, + "loss": 1.2246, + "theoretical_loss": 3.3590885209612438, + "tokens_seen": 2886977536 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010136008024072217, + "loss": 1.1698, + "theoretical_loss": 3.3590826624293655, + "tokens_seen": 2887043072 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010134403209628887, + "loss": 1.2629, + "theoretical_loss": 3.35907680406771, + "tokens_seen": 2887108608 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010132798395185557, + "loss": 1.0141, + "theoretical_loss": 3.3590709458762693, + "tokens_seen": 2887174144 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010131193580742227, + "loss": 1.1208, + "theoretical_loss": 3.3590650878550345, + "tokens_seen": 2887239680 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010129588766298897, + "loss": 1.224, + "theoretical_loss": 3.3590592300039965, + "tokens_seen": 2887305216 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010127983951855567, + "loss": 1.0619, + "theoretical_loss": 3.359053372323147, + "tokens_seen": 2887370752 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010126379137412236, + "loss": 1.3157, + "theoretical_loss": 3.359047514812476, + "tokens_seen": 2887436288 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010124774322968906, + "loss": 1.2623, + "theoretical_loss": 3.359041657471976, + "tokens_seen": 2887501824 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010123169508525576, + "loss": 1.152, + "theoretical_loss": 3.3590358003016374, + "tokens_seen": 2887567360 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010121564694082246, + "loss": 1.2343, + "theoretical_loss": 3.359029943301452, + "tokens_seen": 2887632896 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010119959879638919, + "loss": 1.1264, + "theoretical_loss": 3.3590240864714107, + "tokens_seen": 2887698432 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010118355065195589, + "loss": 1.1447, + "theoretical_loss": 3.3590182298115043, + "tokens_seen": 2887763968 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010116750250752259, + "loss": 1.2499, + "theoretical_loss": 3.3590123733217245, + "tokens_seen": 2887829504 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010115145436308929, + "loss": 1.1598, + "theoretical_loss": 3.3590065170020624, + "tokens_seen": 2887895040 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010113540621865598, + "loss": 1.1908, + "theoretical_loss": 3.3590006608525096, + "tokens_seen": 2887960576 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010111935807422268, + "loss": 1.2485, + "theoretical_loss": 3.3589948048730562, + "tokens_seen": 2888026112 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010110330992978938, + "loss": 1.1493, + "theoretical_loss": 3.3589889490636944, + "tokens_seen": 2888091648 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010108726178535608, + "loss": 1.1349, + "theoretical_loss": 3.358983093424415, + "tokens_seen": 2888157184 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010107121364092278, + "loss": 1.2456, + "theoretical_loss": 3.3589772379552096, + "tokens_seen": 2888222720 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010105516549648948, + "loss": 1.2519, + "theoretical_loss": 3.358971382656069, + "tokens_seen": 2888288256 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010103911735205618, + "loss": 1.1485, + "theoretical_loss": 3.3589655275269843, + "tokens_seen": 2888353792 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010102306920762288, + "loss": 1.2294, + "theoretical_loss": 3.3589596725679467, + "tokens_seen": 2888419328 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010100702106318958, + "loss": 1.1655, + "theoretical_loss": 3.3589538177789477, + "tokens_seen": 2888484864 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010099097291875628, + "loss": 1.1889, + "theoretical_loss": 3.3589479631599786, + "tokens_seen": 2888550400 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010097492477432298, + "loss": 1.1909, + "theoretical_loss": 3.3589421087110303, + "tokens_seen": 2888615936 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010095887662988968, + "loss": 1.0967, + "theoretical_loss": 3.358936254432094, + "tokens_seen": 2888681472 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010094282848545637, + "loss": 1.1506, + "theoretical_loss": 3.358930400323161, + "tokens_seen": 2888747008 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010092678034102307, + "loss": 1.2136, + "theoretical_loss": 3.3589245463842223, + "tokens_seen": 2888812544 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010091073219658977, + "loss": 1.1466, + "theoretical_loss": 3.3589186926152697, + "tokens_seen": 2888878080 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010089468405215647, + "loss": 1.1792, + "theoretical_loss": 3.358912839016294, + "tokens_seen": 2888943616 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010087863590772317, + "loss": 1.1599, + "theoretical_loss": 3.3589069855872857, + "tokens_seen": 2889009152 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010086258776328987, + "loss": 1.1285, + "theoretical_loss": 3.3589011323282376, + "tokens_seen": 2889074688 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010084653961885657, + "loss": 1.1694, + "theoretical_loss": 3.3588952792391398, + "tokens_seen": 2889140224 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010083049147442327, + "loss": 1.1269, + "theoretical_loss": 3.3588894263199833, + "tokens_seen": 2889205760 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010081444332998997, + "loss": 1.2005, + "theoretical_loss": 3.3588835735707603, + "tokens_seen": 2889271296 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010079839518555668, + "loss": 1.2371, + "theoretical_loss": 3.3588777209914618, + "tokens_seen": 2889336832 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010078234704112338, + "loss": 1.1761, + "theoretical_loss": 3.358871868582078, + "tokens_seen": 2889402368 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010076629889669008, + "loss": 1.2302, + "theoretical_loss": 3.358866016342601, + "tokens_seen": 2889467904 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010075025075225676, + "loss": 1.1322, + "theoretical_loss": 3.358860164273022, + "tokens_seen": 2889533440 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010073420260782346, + "loss": 1.2865, + "theoretical_loss": 3.3588543123733317, + "tokens_seen": 2889598976 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010071815446339016, + "loss": 1.2272, + "theoretical_loss": 3.358848460643522, + "tokens_seen": 2889664512 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010070210631895687, + "loss": 1.2191, + "theoretical_loss": 3.3588426090835837, + "tokens_seen": 2889730048 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010068605817452357, + "loss": 1.3177, + "theoretical_loss": 3.358836757693508, + "tokens_seen": 2889795584 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010067001003009027, + "loss": 1.1925, + "theoretical_loss": 3.3588309064732864, + "tokens_seen": 2889861120 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010065396188565697, + "loss": 1.2038, + "theoretical_loss": 3.3588250554229098, + "tokens_seen": 2889926656 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010063791374122367, + "loss": 1.1796, + "theoretical_loss": 3.3588192045423693, + "tokens_seen": 2889992192 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010062186559679037, + "loss": 1.2037, + "theoretical_loss": 3.358813353831657, + "tokens_seen": 2890057728 + }, + { + "epoch": 1.26, + "objective/train/docs_used": 1067172, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1935980319976807, + "objective/train/theoretical_loss": 3.3588075032907625, + "objective/train/tokens_used": 2910583264, + "theoretical_loss": 3.3588075032907625, + "tokens_seen": 2890123264 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010060581745235707, + "loss": 1.1936, + "theoretical_loss": 3.3588075032907625, + "tokens_seen": 2890123264 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010058976930792378, + "loss": 1.2008, + "theoretical_loss": 3.358801652919679, + "tokens_seen": 2890188800 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010057372116349048, + "loss": 1.1242, + "theoretical_loss": 3.358795802718396, + "tokens_seen": 2890254336 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010055767301905718, + "loss": 1.1875, + "theoretical_loss": 3.3587899526869056, + "tokens_seen": 2890319872 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010054162487462389, + "loss": 1.191, + "theoretical_loss": 3.358784102825199, + "tokens_seen": 2890385408 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010052557673019059, + "loss": 1.1841, + "theoretical_loss": 3.3587782531332673, + "tokens_seen": 2890450944 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010050952858575729, + "loss": 1.1648, + "theoretical_loss": 3.3587724036111015, + "tokens_seen": 2890516480 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010049348044132399, + "loss": 1.2349, + "theoretical_loss": 3.3587665542586933, + "tokens_seen": 2890582016 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010047743229689069, + "loss": 1.3152, + "theoretical_loss": 3.3587607050760333, + "tokens_seen": 2890647552 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010046138415245739, + "loss": 1.1734, + "theoretical_loss": 3.358754856063113, + "tokens_seen": 2890713088 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010044533600802408, + "loss": 1.2669, + "theoretical_loss": 3.358749007219924, + "tokens_seen": 2890778624 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010042928786359078, + "loss": 1.2338, + "theoretical_loss": 3.3587431585464573, + "tokens_seen": 2890844160 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010041323971915748, + "loss": 1.2328, + "theoretical_loss": 3.358737310042704, + "tokens_seen": 2890909696 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010039719157472418, + "loss": 1.1914, + "theoretical_loss": 3.358731461708656, + "tokens_seen": 2890975232 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010038114343029088, + "loss": 1.194, + "theoretical_loss": 3.358725613544303, + "tokens_seen": 2891040768 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010036509528585758, + "loss": 1.2052, + "theoretical_loss": 3.3587197655496372, + "tokens_seen": 2891106304 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010034904714142428, + "loss": 1.0986, + "theoretical_loss": 3.35871391772465, + "tokens_seen": 2891171840 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010033299899699098, + "loss": 1.1312, + "theoretical_loss": 3.3587080700693326, + "tokens_seen": 2891237376 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010031695085255768, + "loss": 1.1659, + "theoretical_loss": 3.358702222583676, + "tokens_seen": 2891302912 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010030090270812438, + "loss": 1.1211, + "theoretical_loss": 3.3586963752676713, + "tokens_seen": 2891368448 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010028485456369108, + "loss": 1.1759, + "theoretical_loss": 3.35869052812131, + "tokens_seen": 2891433984 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010026880641925778, + "loss": 1.1261, + "theoretical_loss": 3.358684681144583, + "tokens_seen": 2891499520 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010025275827482447, + "loss": 1.2527, + "theoretical_loss": 3.358678834337482, + "tokens_seen": 2891565056 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010023671013039117, + "loss": 1.1522, + "theoretical_loss": 3.358672987699998, + "tokens_seen": 2891630592 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010022066198595787, + "loss": 1.194, + "theoretical_loss": 3.3586671412321225, + "tokens_seen": 2891696128 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010020461384152457, + "loss": 1.2103, + "theoretical_loss": 3.358661294933846, + "tokens_seen": 2891761664 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010018856569709127, + "loss": 1.1406, + "theoretical_loss": 3.3586554488051603, + "tokens_seen": 2891827200 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010017251755265797, + "loss": 1.1844, + "theoretical_loss": 3.358649602846057, + "tokens_seen": 2891892736 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010015646940822467, + "loss": 1.2106, + "theoretical_loss": 3.3586437570565266, + "tokens_seen": 2891958272 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010014042126379137, + "loss": 1.1935, + "theoretical_loss": 3.3586379114365603, + "tokens_seen": 2892023808 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010012437311935807, + "loss": 1.1766, + "theoretical_loss": 3.35863206598615, + "tokens_seen": 2892089344 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010010832497492478, + "loss": 1.1776, + "theoretical_loss": 3.3586262207052866, + "tokens_seen": 2892154880 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010009227683049148, + "loss": 1.194, + "theoretical_loss": 3.3586203755939614, + "tokens_seen": 2892220416 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010007622868605818, + "loss": 1.2018, + "theoretical_loss": 3.3586145306521655, + "tokens_seen": 2892285952 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010006018054162486, + "loss": 1.203, + "theoretical_loss": 3.3586086858798905, + "tokens_seen": 2892351488 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010004413239719156, + "loss": 1.1099, + "theoretical_loss": 3.358602841277127, + "tokens_seen": 2892417024 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010002808425275827, + "loss": 1.1998, + "theoretical_loss": 3.3585969968438665, + "tokens_seen": 2892482560 + }, + { + "epoch": 1.26, + "learning_rate": 0.00010001203610832497, + "loss": 1.1747, + "theoretical_loss": 3.3585911525801007, + "tokens_seen": 2892548096 + }, + { + "epoch": 1.26, + "learning_rate": 9.999598796389168e-05, + "loss": 1.2375, + "theoretical_loss": 3.3585853084858206, + "tokens_seen": 2892613632 + }, + { + "epoch": 1.26, + "learning_rate": 9.997993981945838e-05, + "loss": 1.1854, + "theoretical_loss": 3.358579464561017, + "tokens_seen": 2892679168 + }, + { + "epoch": 1.26, + "learning_rate": 9.996389167502508e-05, + "loss": 1.2399, + "theoretical_loss": 3.358573620805682, + "tokens_seen": 2892744704 + }, + { + "epoch": 1.26, + "learning_rate": 9.994784353059178e-05, + "loss": 1.1622, + "theoretical_loss": 3.3585677772198057, + "tokens_seen": 2892810240 + }, + { + "epoch": 1.26, + "learning_rate": 9.993179538615848e-05, + "loss": 1.1649, + "theoretical_loss": 3.3585619338033803, + "tokens_seen": 2892875776 + }, + { + "epoch": 1.26, + "learning_rate": 9.991574724172518e-05, + "loss": 1.1823, + "theoretical_loss": 3.3585560905563967, + "tokens_seen": 2892941312 + }, + { + "epoch": 1.26, + "learning_rate": 9.989969909729187e-05, + "loss": 1.1307, + "theoretical_loss": 3.3585502474788465, + "tokens_seen": 2893006848 + }, + { + "epoch": 1.26, + "learning_rate": 9.988365095285857e-05, + "loss": 1.1591, + "theoretical_loss": 3.3585444045707202, + "tokens_seen": 2893072384 + }, + { + "epoch": 1.26, + "learning_rate": 9.986760280842527e-05, + "loss": 1.177, + "theoretical_loss": 3.3585385618320096, + "tokens_seen": 2893137920 + }, + { + "epoch": 1.26, + "learning_rate": 9.985155466399197e-05, + "loss": 1.2079, + "theoretical_loss": 3.3585327192627057, + "tokens_seen": 2893203456 + }, + { + "epoch": 1.26, + "learning_rate": 9.983550651955869e-05, + "loss": 1.249, + "theoretical_loss": 3.3585268768628005, + "tokens_seen": 2893268992 + }, + { + "epoch": 1.26, + "learning_rate": 9.981945837512539e-05, + "loss": 1.2315, + "theoretical_loss": 3.3585210346322842, + "tokens_seen": 2893334528 + }, + { + "epoch": 1.26, + "learning_rate": 9.980341023069209e-05, + "loss": 1.2045, + "theoretical_loss": 3.358515192571148, + "tokens_seen": 2893400064 + }, + { + "epoch": 1.26, + "learning_rate": 9.978736208625879e-05, + "loss": 1.132, + "theoretical_loss": 3.3585093506793844, + "tokens_seen": 2893465600 + }, + { + "epoch": 1.26, + "learning_rate": 9.977131394182549e-05, + "loss": 1.1186, + "theoretical_loss": 3.358503508956984, + "tokens_seen": 2893531136 + }, + { + "epoch": 1.26, + "learning_rate": 9.975526579739219e-05, + "loss": 1.1171, + "theoretical_loss": 3.3584976674039373, + "tokens_seen": 2893596672 + }, + { + "epoch": 1.26, + "learning_rate": 9.973921765295888e-05, + "loss": 1.1942, + "theoretical_loss": 3.3584918260202365, + "tokens_seen": 2893662208 + }, + { + "epoch": 1.26, + "learning_rate": 9.972316950852558e-05, + "loss": 1.185, + "theoretical_loss": 3.3584859848058723, + "tokens_seen": 2893727744 + }, + { + "epoch": 1.26, + "learning_rate": 9.970712136409228e-05, + "loss": 1.1678, + "theoretical_loss": 3.3584801437608367, + "tokens_seen": 2893793280 + }, + { + "epoch": 1.26, + "learning_rate": 9.969107321965898e-05, + "loss": 1.2169, + "theoretical_loss": 3.3584743028851203, + "tokens_seen": 2893858816 + }, + { + "epoch": 1.26, + "learning_rate": 9.967502507522568e-05, + "loss": 1.138, + "theoretical_loss": 3.3584684621787146, + "tokens_seen": 2893924352 + }, + { + "epoch": 1.26, + "learning_rate": 9.965897693079238e-05, + "loss": 1.1337, + "theoretical_loss": 3.3584626216416105, + "tokens_seen": 2893989888 + }, + { + "epoch": 1.26, + "learning_rate": 9.964292878635908e-05, + "loss": 1.1813, + "theoretical_loss": 3.3584567812737998, + "tokens_seen": 2894055424 + }, + { + "epoch": 1.26, + "learning_rate": 9.962688064192578e-05, + "loss": 1.1751, + "theoretical_loss": 3.358450941075273, + "tokens_seen": 2894120960 + }, + { + "epoch": 1.26, + "learning_rate": 9.961083249749248e-05, + "loss": 1.1498, + "theoretical_loss": 3.358445101046023, + "tokens_seen": 2894186496 + }, + { + "epoch": 1.26, + "learning_rate": 9.959478435305918e-05, + "loss": 1.1697, + "theoretical_loss": 3.3584392611860387, + "tokens_seen": 2894252032 + }, + { + "epoch": 1.26, + "learning_rate": 9.957873620862588e-05, + "loss": 1.1872, + "theoretical_loss": 3.358433421495313, + "tokens_seen": 2894317568 + }, + { + "epoch": 1.26, + "learning_rate": 9.956268806419258e-05, + "loss": 1.1174, + "theoretical_loss": 3.3584275819738374, + "tokens_seen": 2894383104 + }, + { + "epoch": 1.26, + "learning_rate": 9.954663991975928e-05, + "loss": 1.0425, + "theoretical_loss": 3.358421742621602, + "tokens_seen": 2894448640 + }, + { + "epoch": 1.26, + "learning_rate": 9.953059177532598e-05, + "loss": 1.124, + "theoretical_loss": 3.3584159034385985, + "tokens_seen": 2894514176 + }, + { + "epoch": 1.26, + "learning_rate": 9.951454363089269e-05, + "loss": 1.2761, + "theoretical_loss": 3.3584100644248185, + "tokens_seen": 2894579712 + }, + { + "epoch": 1.26, + "learning_rate": 9.949849548645939e-05, + "loss": 1.2909, + "theoretical_loss": 3.3584042255802524, + "tokens_seen": 2894645248 + }, + { + "epoch": 1.26, + "learning_rate": 9.948244734202609e-05, + "loss": 1.1539, + "theoretical_loss": 3.358398386904893, + "tokens_seen": 2894710784 + }, + { + "epoch": 1.26, + "learning_rate": 9.946639919759279e-05, + "loss": 1.2105, + "theoretical_loss": 3.35839254839873, + "tokens_seen": 2894776320 + }, + { + "epoch": 1.26, + "learning_rate": 9.945035105315949e-05, + "loss": 1.2048, + "theoretical_loss": 3.3583867100617555, + "tokens_seen": 2894841856 + }, + { + "epoch": 1.26, + "learning_rate": 9.943430290872619e-05, + "loss": 1.2576, + "theoretical_loss": 3.35838087189396, + "tokens_seen": 2894907392 + }, + { + "epoch": 1.26, + "learning_rate": 9.941825476429289e-05, + "loss": 1.1831, + "theoretical_loss": 3.358375033895336, + "tokens_seen": 2894972928 + }, + { + "epoch": 1.26, + "learning_rate": 9.940220661985959e-05, + "loss": 1.1959, + "theoretical_loss": 3.3583691960658744, + "tokens_seen": 2895038464 + }, + { + "epoch": 1.26, + "learning_rate": 9.938615847542628e-05, + "loss": 1.0531, + "theoretical_loss": 3.3583633584055654, + "tokens_seen": 2895104000 + }, + { + "epoch": 1.26, + "learning_rate": 9.937011033099298e-05, + "loss": 1.2318, + "theoretical_loss": 3.358357520914401, + "tokens_seen": 2895169536 + }, + { + "epoch": 1.26, + "learning_rate": 9.935406218655968e-05, + "loss": 1.2219, + "theoretical_loss": 3.3583516835923732, + "tokens_seen": 2895235072 + }, + { + "epoch": 1.26, + "learning_rate": 9.933801404212638e-05, + "loss": 1.1991, + "theoretical_loss": 3.358345846439472, + "tokens_seen": 2895300608 + }, + { + "epoch": 1.26, + "learning_rate": 9.932196589769308e-05, + "loss": 1.1564, + "theoretical_loss": 3.35834000945569, + "tokens_seen": 2895366144 + }, + { + "epoch": 1.26, + "learning_rate": 9.930591775325978e-05, + "loss": 1.2303, + "theoretical_loss": 3.358334172641017, + "tokens_seen": 2895431680 + }, + { + "epoch": 1.26, + "learning_rate": 9.928986960882648e-05, + "loss": 1.1686, + "theoretical_loss": 3.3583283359954454, + "tokens_seen": 2895497216 + }, + { + "epoch": 1.26, + "learning_rate": 9.927382146439318e-05, + "loss": 1.2388, + "theoretical_loss": 3.358322499518966, + "tokens_seen": 2895562752 + }, + { + "epoch": 1.26, + "learning_rate": 9.925777331995988e-05, + "loss": 1.2616, + "theoretical_loss": 3.3583166632115704, + "tokens_seen": 2895628288 + }, + { + "epoch": 1.26, + "learning_rate": 9.92417251755266e-05, + "loss": 1.1758, + "theoretical_loss": 3.3583108270732493, + "tokens_seen": 2895693824 + }, + { + "epoch": 1.26, + "learning_rate": 9.922567703109328e-05, + "loss": 1.2111, + "theoretical_loss": 3.3583049911039944, + "tokens_seen": 2895759360 + }, + { + "epoch": 1.26, + "learning_rate": 9.920962888665998e-05, + "loss": 1.2073, + "theoretical_loss": 3.3582991553037966, + "tokens_seen": 2895824896 + }, + { + "epoch": 1.26, + "learning_rate": 9.919358074222668e-05, + "loss": 1.1685, + "theoretical_loss": 3.3582933196726477, + "tokens_seen": 2895890432 + }, + { + "epoch": 1.26, + "learning_rate": 9.917753259779339e-05, + "loss": 1.1409, + "theoretical_loss": 3.358287484210539, + "tokens_seen": 2895955968 + }, + { + "epoch": 1.26, + "learning_rate": 9.916148445336009e-05, + "loss": 1.2029, + "theoretical_loss": 3.358281648917461, + "tokens_seen": 2896021504 + }, + { + "epoch": 1.26, + "learning_rate": 9.914543630892679e-05, + "loss": 1.392, + "theoretical_loss": 3.358275813793406, + "tokens_seen": 2896087040 + }, + { + "epoch": 1.26, + "learning_rate": 9.912938816449349e-05, + "loss": 1.1029, + "theoretical_loss": 3.358269978838364, + "tokens_seen": 2896152576 + }, + { + "epoch": 1.26, + "learning_rate": 9.911334002006019e-05, + "loss": 1.277, + "theoretical_loss": 3.358264144052328, + "tokens_seen": 2896218112 + }, + { + "epoch": 1.26, + "learning_rate": 9.909729187562689e-05, + "loss": 1.1422, + "theoretical_loss": 3.358258309435288, + "tokens_seen": 2896283648 + }, + { + "epoch": 1.26, + "learning_rate": 9.908124373119359e-05, + "loss": 1.0839, + "theoretical_loss": 3.3582524749872356, + "tokens_seen": 2896349184 + }, + { + "epoch": 1.26, + "learning_rate": 9.906519558676029e-05, + "loss": 1.2736, + "theoretical_loss": 3.3582466407081615, + "tokens_seen": 2896414720 + }, + { + "epoch": 1.26, + "learning_rate": 9.904914744232699e-05, + "loss": 1.2608, + "theoretical_loss": 3.358240806598058, + "tokens_seen": 2896480256 + }, + { + "epoch": 1.26, + "learning_rate": 9.903309929789368e-05, + "loss": 1.1213, + "theoretical_loss": 3.3582349726569163, + "tokens_seen": 2896545792 + }, + { + "epoch": 1.26, + "learning_rate": 9.901705115346038e-05, + "loss": 1.1278, + "theoretical_loss": 3.358229138884727, + "tokens_seen": 2896611328 + }, + { + "epoch": 1.26, + "objective/train/docs_used": 1069490, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.280707597732544, + "objective/train/theoretical_loss": 3.358223305281482, + "objective/train/tokens_used": 2917136864, + "theoretical_loss": 3.358223305281482, + "tokens_seen": 2896676864 + }, + { + "epoch": 1.26, + "learning_rate": 9.900100300902708e-05, + "loss": 1.2807, + "theoretical_loss": 3.358223305281482, + "tokens_seen": 2896676864 + }, + { + "epoch": 1.26, + "learning_rate": 9.898495486459378e-05, + "loss": 1.1888, + "theoretical_loss": 3.358217471847172, + "tokens_seen": 2896742400 + }, + { + "epoch": 1.26, + "learning_rate": 9.896890672016048e-05, + "loss": 1.2615, + "theoretical_loss": 3.3582116385817886, + "tokens_seen": 2896807936 + }, + { + "epoch": 1.26, + "learning_rate": 9.895285857572718e-05, + "loss": 1.1905, + "theoretical_loss": 3.3582058054853237, + "tokens_seen": 2896873472 + }, + { + "epoch": 1.26, + "learning_rate": 9.89368104312939e-05, + "loss": 1.198, + "theoretical_loss": 3.358199972557767, + "tokens_seen": 2896939008 + }, + { + "epoch": 1.26, + "learning_rate": 9.89207622868606e-05, + "loss": 1.223, + "theoretical_loss": 3.3581941397991115, + "tokens_seen": 2897004544 + }, + { + "epoch": 1.26, + "learning_rate": 9.89047141424273e-05, + "loss": 1.2166, + "theoretical_loss": 3.3581883072093475, + "tokens_seen": 2897070080 + }, + { + "epoch": 1.26, + "learning_rate": 9.8888665997994e-05, + "loss": 1.2578, + "theoretical_loss": 3.3581824747884665, + "tokens_seen": 2897135616 + }, + { + "epoch": 1.26, + "learning_rate": 9.887261785356068e-05, + "loss": 1.2162, + "theoretical_loss": 3.35817664253646, + "tokens_seen": 2897201152 + }, + { + "epoch": 1.26, + "learning_rate": 9.885656970912738e-05, + "loss": 1.2725, + "theoretical_loss": 3.358170810453319, + "tokens_seen": 2897266688 + }, + { + "epoch": 1.26, + "learning_rate": 9.884052156469408e-05, + "loss": 1.154, + "theoretical_loss": 3.358164978539035, + "tokens_seen": 2897332224 + }, + { + "epoch": 1.26, + "learning_rate": 9.882447342026079e-05, + "loss": 1.1206, + "theoretical_loss": 3.358159146793599, + "tokens_seen": 2897397760 + }, + { + "epoch": 1.26, + "learning_rate": 9.880842527582749e-05, + "loss": 1.1257, + "theoretical_loss": 3.358153315217003, + "tokens_seen": 2897463296 + }, + { + "epoch": 1.26, + "learning_rate": 9.879237713139419e-05, + "loss": 1.1949, + "theoretical_loss": 3.3581474838092373, + "tokens_seen": 2897528832 + }, + { + "epoch": 1.26, + "learning_rate": 9.877632898696089e-05, + "loss": 1.1614, + "theoretical_loss": 3.3581416525702936, + "tokens_seen": 2897594368 + }, + { + "epoch": 1.26, + "learning_rate": 9.876028084252759e-05, + "loss": 1.1903, + "theoretical_loss": 3.3581358215001638, + "tokens_seen": 2897659904 + }, + { + "epoch": 1.26, + "learning_rate": 9.874423269809429e-05, + "loss": 1.2987, + "theoretical_loss": 3.3581299905988384, + "tokens_seen": 2897725440 + }, + { + "epoch": 1.26, + "learning_rate": 9.872818455366099e-05, + "loss": 1.2154, + "theoretical_loss": 3.3581241598663087, + "tokens_seen": 2897790976 + }, + { + "epoch": 1.26, + "learning_rate": 9.871213640922769e-05, + "loss": 1.2225, + "theoretical_loss": 3.3581183293025667, + "tokens_seen": 2897856512 + }, + { + "epoch": 1.26, + "learning_rate": 9.869608826479439e-05, + "loss": 1.2936, + "theoretical_loss": 3.358112498907603, + "tokens_seen": 2897922048 + }, + { + "epoch": 1.26, + "learning_rate": 9.868004012036108e-05, + "loss": 1.1786, + "theoretical_loss": 3.358106668681409, + "tokens_seen": 2897987584 + }, + { + "epoch": 1.26, + "learning_rate": 9.866399197592778e-05, + "loss": 1.1796, + "theoretical_loss": 3.358100838623977, + "tokens_seen": 2898053120 + }, + { + "epoch": 1.26, + "learning_rate": 9.864794383149449e-05, + "loss": 1.2382, + "theoretical_loss": 3.3580950087352965, + "tokens_seen": 2898118656 + }, + { + "epoch": 1.26, + "learning_rate": 9.86318956870612e-05, + "loss": 1.2844, + "theoretical_loss": 3.35808917901536, + "tokens_seen": 2898184192 + }, + { + "epoch": 1.26, + "learning_rate": 9.86158475426279e-05, + "loss": 1.1993, + "theoretical_loss": 3.3580833494641587, + "tokens_seen": 2898249728 + }, + { + "epoch": 1.26, + "learning_rate": 9.85997993981946e-05, + "loss": 1.0794, + "theoretical_loss": 3.3580775200816837, + "tokens_seen": 2898315264 + }, + { + "epoch": 1.26, + "learning_rate": 9.85837512537613e-05, + "loss": 1.1672, + "theoretical_loss": 3.3580716908679262, + "tokens_seen": 2898380800 + }, + { + "epoch": 1.26, + "learning_rate": 9.8567703109328e-05, + "loss": 1.2042, + "theoretical_loss": 3.358065861822878, + "tokens_seen": 2898446336 + }, + { + "epoch": 1.26, + "learning_rate": 9.85516549648947e-05, + "loss": 1.1034, + "theoretical_loss": 3.35806003294653, + "tokens_seen": 2898511872 + }, + { + "epoch": 1.26, + "learning_rate": 9.853560682046138e-05, + "loss": 1.1842, + "theoretical_loss": 3.358054204238873, + "tokens_seen": 2898577408 + }, + { + "epoch": 1.26, + "learning_rate": 9.851955867602808e-05, + "loss": 1.1879, + "theoretical_loss": 3.3580483756999, + "tokens_seen": 2898642944 + }, + { + "epoch": 1.26, + "learning_rate": 9.850351053159478e-05, + "loss": 1.0996, + "theoretical_loss": 3.3580425473296, + "tokens_seen": 2898708480 + }, + { + "epoch": 1.26, + "learning_rate": 9.848746238716149e-05, + "loss": 1.2188, + "theoretical_loss": 3.3580367191279663, + "tokens_seen": 2898774016 + }, + { + "epoch": 1.26, + "learning_rate": 9.847141424272819e-05, + "loss": 1.1513, + "theoretical_loss": 3.3580308910949888, + "tokens_seen": 2898839552 + }, + { + "epoch": 1.26, + "learning_rate": 9.845536609829489e-05, + "loss": 1.1836, + "theoretical_loss": 3.35802506323066, + "tokens_seen": 2898905088 + }, + { + "epoch": 1.26, + "learning_rate": 9.843931795386159e-05, + "loss": 1.2265, + "theoretical_loss": 3.3580192355349703, + "tokens_seen": 2898970624 + }, + { + "epoch": 1.26, + "learning_rate": 9.842326980942829e-05, + "loss": 1.1297, + "theoretical_loss": 3.358013408007911, + "tokens_seen": 2899036160 + }, + { + "epoch": 1.26, + "learning_rate": 9.840722166499499e-05, + "loss": 1.2039, + "theoretical_loss": 3.358007580649474, + "tokens_seen": 2899101696 + }, + { + "epoch": 1.26, + "learning_rate": 9.839117352056169e-05, + "loss": 1.1969, + "theoretical_loss": 3.3580017534596505, + "tokens_seen": 2899167232 + }, + { + "epoch": 1.26, + "learning_rate": 9.837512537612839e-05, + "loss": 1.1552, + "theoretical_loss": 3.3579959264384316, + "tokens_seen": 2899232768 + }, + { + "epoch": 1.26, + "learning_rate": 9.835907723169509e-05, + "loss": 1.1453, + "theoretical_loss": 3.357990099585808, + "tokens_seen": 2899298304 + }, + { + "epoch": 1.26, + "learning_rate": 9.834302908726179e-05, + "loss": 1.2145, + "theoretical_loss": 3.3579842729017724, + "tokens_seen": 2899363840 + }, + { + "epoch": 1.26, + "learning_rate": 9.832698094282849e-05, + "loss": 1.2692, + "theoretical_loss": 3.357978446386315, + "tokens_seen": 2899429376 + }, + { + "epoch": 1.26, + "learning_rate": 9.831093279839519e-05, + "loss": 1.1484, + "theoretical_loss": 3.357972620039428, + "tokens_seen": 2899494912 + }, + { + "epoch": 1.26, + "learning_rate": 9.829488465396189e-05, + "loss": 1.1554, + "theoretical_loss": 3.3579667938611015, + "tokens_seen": 2899560448 + }, + { + "epoch": 1.26, + "learning_rate": 9.82788365095286e-05, + "loss": 1.1128, + "theoretical_loss": 3.3579609678513282, + "tokens_seen": 2899625984 + }, + { + "epoch": 1.26, + "learning_rate": 9.82627883650953e-05, + "loss": 1.2614, + "theoretical_loss": 3.3579551420100984, + "tokens_seen": 2899691520 + }, + { + "epoch": 1.26, + "learning_rate": 9.8246740220662e-05, + "loss": 1.275, + "theoretical_loss": 3.3579493163374035, + "tokens_seen": 2899757056 + }, + { + "epoch": 1.26, + "learning_rate": 9.82306920762287e-05, + "loss": 1.2051, + "theoretical_loss": 3.3579434908332355, + "tokens_seen": 2899822592 + }, + { + "epoch": 1.26, + "learning_rate": 9.82146439317954e-05, + "loss": 1.1291, + "theoretical_loss": 3.3579376654975848, + "tokens_seen": 2899888128 + }, + { + "epoch": 1.26, + "learning_rate": 9.81985957873621e-05, + "loss": 1.1689, + "theoretical_loss": 3.3579318403304437, + "tokens_seen": 2899953664 + }, + { + "epoch": 1.26, + "learning_rate": 9.818254764292878e-05, + "loss": 1.1376, + "theoretical_loss": 3.3579260153318025, + "tokens_seen": 2900019200 + }, + { + "epoch": 1.26, + "learning_rate": 9.816649949849548e-05, + "loss": 1.2474, + "theoretical_loss": 3.357920190501653, + "tokens_seen": 2900084736 + }, + { + "epoch": 1.26, + "learning_rate": 9.815045135406219e-05, + "loss": 1.144, + "theoretical_loss": 3.357914365839987, + "tokens_seen": 2900150272 + }, + { + "epoch": 1.26, + "learning_rate": 9.813440320962889e-05, + "loss": 1.3164, + "theoretical_loss": 3.357908541346795, + "tokens_seen": 2900215808 + }, + { + "epoch": 1.26, + "learning_rate": 9.811835506519559e-05, + "loss": 1.1501, + "theoretical_loss": 3.357902717022069, + "tokens_seen": 2900281344 + }, + { + "epoch": 1.26, + "learning_rate": 9.810230692076229e-05, + "loss": 1.1366, + "theoretical_loss": 3.3578968928658, + "tokens_seen": 2900346880 + }, + { + "epoch": 1.26, + "learning_rate": 9.808625877632899e-05, + "loss": 1.2289, + "theoretical_loss": 3.3578910688779793, + "tokens_seen": 2900412416 + }, + { + "epoch": 1.26, + "learning_rate": 9.807021063189569e-05, + "loss": 1.0655, + "theoretical_loss": 3.357885245058598, + "tokens_seen": 2900477952 + }, + { + "epoch": 1.26, + "learning_rate": 9.805416248746239e-05, + "loss": 1.1619, + "theoretical_loss": 3.357879421407648, + "tokens_seen": 2900543488 + }, + { + "epoch": 1.26, + "learning_rate": 9.80381143430291e-05, + "loss": 1.2319, + "theoretical_loss": 3.3578735979251197, + "tokens_seen": 2900609024 + }, + { + "epoch": 1.26, + "learning_rate": 9.802206619859579e-05, + "loss": 1.3429, + "theoretical_loss": 3.3578677746110053, + "tokens_seen": 2900674560 + }, + { + "epoch": 1.26, + "learning_rate": 9.800601805416249e-05, + "loss": 1.1811, + "theoretical_loss": 3.357861951465296, + "tokens_seen": 2900740096 + }, + { + "epoch": 1.26, + "learning_rate": 9.798996990972919e-05, + "loss": 1.1211, + "theoretical_loss": 3.357856128487983, + "tokens_seen": 2900805632 + }, + { + "epoch": 1.26, + "learning_rate": 9.797392176529589e-05, + "loss": 1.0926, + "theoretical_loss": 3.3578503056790576, + "tokens_seen": 2900871168 + }, + { + "epoch": 1.26, + "learning_rate": 9.795787362086259e-05, + "loss": 1.2552, + "theoretical_loss": 3.357844483038511, + "tokens_seen": 2900936704 + }, + { + "epoch": 1.26, + "learning_rate": 9.79418254764293e-05, + "loss": 1.185, + "theoretical_loss": 3.3578386605663346, + "tokens_seen": 2901002240 + }, + { + "epoch": 1.26, + "learning_rate": 9.7925777331996e-05, + "loss": 1.1501, + "theoretical_loss": 3.35783283826252, + "tokens_seen": 2901067776 + }, + { + "epoch": 1.26, + "learning_rate": 9.79097291875627e-05, + "loss": 1.2145, + "theoretical_loss": 3.3578270161270583, + "tokens_seen": 2901133312 + }, + { + "epoch": 1.26, + "learning_rate": 9.78936810431294e-05, + "loss": 1.1803, + "theoretical_loss": 3.35782119415994, + "tokens_seen": 2901198848 + }, + { + "epoch": 1.26, + "learning_rate": 9.78776328986961e-05, + "loss": 1.0997, + "theoretical_loss": 3.3578153723611583, + "tokens_seen": 2901264384 + }, + { + "epoch": 1.26, + "learning_rate": 9.78615847542628e-05, + "loss": 1.2498, + "theoretical_loss": 3.3578095507307033, + "tokens_seen": 2901329920 + }, + { + "epoch": 1.26, + "learning_rate": 9.78455366098295e-05, + "loss": 1.1632, + "theoretical_loss": 3.357803729268566, + "tokens_seen": 2901395456 + }, + { + "epoch": 1.26, + "learning_rate": 9.782948846539618e-05, + "loss": 1.2502, + "theoretical_loss": 3.3577979079747386, + "tokens_seen": 2901460992 + }, + { + "epoch": 1.26, + "learning_rate": 9.781344032096288e-05, + "loss": 1.1787, + "theoretical_loss": 3.357792086849212, + "tokens_seen": 2901526528 + }, + { + "epoch": 1.26, + "learning_rate": 9.779739217652959e-05, + "loss": 1.1311, + "theoretical_loss": 3.357786265891978, + "tokens_seen": 2901592064 + }, + { + "epoch": 1.26, + "learning_rate": 9.778134403209629e-05, + "loss": 1.2483, + "theoretical_loss": 3.3577804451030273, + "tokens_seen": 2901657600 + }, + { + "epoch": 1.26, + "learning_rate": 9.776529588766299e-05, + "loss": 1.263, + "theoretical_loss": 3.357774624482351, + "tokens_seen": 2901723136 + }, + { + "epoch": 1.26, + "learning_rate": 9.774924774322969e-05, + "loss": 1.1318, + "theoretical_loss": 3.357768804029942, + "tokens_seen": 2901788672 + }, + { + "epoch": 1.26, + "learning_rate": 9.77331995987964e-05, + "loss": 1.2051, + "theoretical_loss": 3.35776298374579, + "tokens_seen": 2901854208 + }, + { + "epoch": 1.26, + "learning_rate": 9.77171514543631e-05, + "loss": 1.1708, + "theoretical_loss": 3.357757163629887, + "tokens_seen": 2901919744 + }, + { + "epoch": 1.26, + "learning_rate": 9.77011033099298e-05, + "loss": 1.1005, + "theoretical_loss": 3.3577513436822235, + "tokens_seen": 2901985280 + }, + { + "epoch": 1.26, + "learning_rate": 9.76850551654965e-05, + "loss": 1.0911, + "theoretical_loss": 3.3577455239027922, + "tokens_seen": 2902050816 + }, + { + "epoch": 1.26, + "learning_rate": 9.766900702106319e-05, + "loss": 1.177, + "theoretical_loss": 3.3577397042915837, + "tokens_seen": 2902116352 + }, + { + "epoch": 1.26, + "learning_rate": 9.765295887662989e-05, + "loss": 1.1791, + "theoretical_loss": 3.3577338848485896, + "tokens_seen": 2902181888 + }, + { + "epoch": 1.26, + "learning_rate": 9.763691073219659e-05, + "loss": 1.1341, + "theoretical_loss": 3.3577280655738013, + "tokens_seen": 2902247424 + }, + { + "epoch": 1.26, + "learning_rate": 9.762086258776329e-05, + "loss": 1.2569, + "theoretical_loss": 3.357722246467209, + "tokens_seen": 2902312960 + }, + { + "epoch": 1.26, + "learning_rate": 9.760481444332999e-05, + "loss": 1.1181, + "theoretical_loss": 3.357716427528806, + "tokens_seen": 2902378496 + }, + { + "epoch": 1.26, + "learning_rate": 9.75887662988967e-05, + "loss": 1.3244, + "theoretical_loss": 3.357710608758582, + "tokens_seen": 2902444032 + }, + { + "epoch": 1.26, + "learning_rate": 9.75727181544634e-05, + "loss": 1.1607, + "theoretical_loss": 3.357704790156529, + "tokens_seen": 2902509568 + }, + { + "epoch": 1.26, + "learning_rate": 9.75566700100301e-05, + "loss": 1.1359, + "theoretical_loss": 3.3576989717226384, + "tokens_seen": 2902575104 + }, + { + "epoch": 1.26, + "learning_rate": 9.75406218655968e-05, + "loss": 1.1946, + "theoretical_loss": 3.3576931534569017, + "tokens_seen": 2902640640 + }, + { + "epoch": 1.26, + "learning_rate": 9.75245737211635e-05, + "loss": 1.1597, + "theoretical_loss": 3.35768733535931, + "tokens_seen": 2902706176 + }, + { + "epoch": 1.26, + "learning_rate": 9.75085255767302e-05, + "loss": 1.1948, + "theoretical_loss": 3.357681517429854, + "tokens_seen": 2902771712 + }, + { + "epoch": 1.26, + "learning_rate": 9.74924774322969e-05, + "loss": 1.1009, + "theoretical_loss": 3.357675699668526, + "tokens_seen": 2902837248 + }, + { + "epoch": 1.26, + "learning_rate": 9.747642928786358e-05, + "loss": 1.1906, + "theoretical_loss": 3.3576698820753172, + "tokens_seen": 2902902784 + }, + { + "epoch": 1.26, + "learning_rate": 9.746038114343029e-05, + "loss": 1.2213, + "theoretical_loss": 3.3576640646502187, + "tokens_seen": 2902968320 + }, + { + "epoch": 1.26, + "learning_rate": 9.7444332998997e-05, + "loss": 1.218, + "theoretical_loss": 3.3576582473932217, + "tokens_seen": 2903033856 + }, + { + "epoch": 1.26, + "learning_rate": 9.74282848545637e-05, + "loss": 1.2786, + "theoretical_loss": 3.3576524303043183, + "tokens_seen": 2903099392 + }, + { + "epoch": 1.26, + "learning_rate": 9.74122367101304e-05, + "loss": 1.1472, + "theoretical_loss": 3.3576466133834986, + "tokens_seen": 2903164928 + }, + { + "epoch": 1.26, + "objective/train/docs_used": 1071717, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1781606674194336, + "objective/train/theoretical_loss": 3.357640796630755, + "objective/train/tokens_used": 2923690464, + "theoretical_loss": 3.357640796630755, + "tokens_seen": 2903230464 + }, + { + "epoch": 1.26, + "learning_rate": 9.73961885656971e-05, + "loss": 1.1782, + "theoretical_loss": 3.357640796630755, + "tokens_seen": 2903230464 + }, + { + "epoch": 1.26, + "learning_rate": 9.73801404212638e-05, + "loss": 1.2068, + "theoretical_loss": 3.3576349800460785, + "tokens_seen": 2903296000 + }, + { + "epoch": 1.26, + "learning_rate": 9.73640922768305e-05, + "loss": 1.2403, + "theoretical_loss": 3.3576291636294604, + "tokens_seen": 2903361536 + }, + { + "epoch": 1.26, + "learning_rate": 9.73480441323972e-05, + "loss": 1.048, + "theoretical_loss": 3.357623347380892, + "tokens_seen": 2903427072 + }, + { + "epoch": 1.26, + "learning_rate": 9.73319959879639e-05, + "loss": 1.1588, + "theoretical_loss": 3.357617531300365, + "tokens_seen": 2903492608 + }, + { + "epoch": 1.26, + "learning_rate": 9.731594784353059e-05, + "loss": 1.2041, + "theoretical_loss": 3.35761171538787, + "tokens_seen": 2903558144 + }, + { + "epoch": 1.26, + "learning_rate": 9.729989969909729e-05, + "loss": 1.2187, + "theoretical_loss": 3.3576058996433993, + "tokens_seen": 2903623680 + }, + { + "epoch": 1.26, + "learning_rate": 9.728385155466399e-05, + "loss": 1.1421, + "theoretical_loss": 3.3576000840669438, + "tokens_seen": 2903689216 + }, + { + "epoch": 1.26, + "learning_rate": 9.726780341023069e-05, + "loss": 1.1714, + "theoretical_loss": 3.3575942686584948, + "tokens_seen": 2903754752 + }, + { + "epoch": 1.26, + "learning_rate": 9.72517552657974e-05, + "loss": 1.1059, + "theoretical_loss": 3.357588453418044, + "tokens_seen": 2903820288 + }, + { + "epoch": 1.26, + "learning_rate": 9.72357071213641e-05, + "loss": 1.1644, + "theoretical_loss": 3.357582638345582, + "tokens_seen": 2903885824 + }, + { + "epoch": 1.26, + "learning_rate": 9.72196589769308e-05, + "loss": 1.2251, + "theoretical_loss": 3.357576823441101, + "tokens_seen": 2903951360 + }, + { + "epoch": 1.26, + "learning_rate": 9.72036108324975e-05, + "loss": 1.1182, + "theoretical_loss": 3.357571008704592, + "tokens_seen": 2904016896 + }, + { + "epoch": 1.26, + "learning_rate": 9.71875626880642e-05, + "loss": 1.1987, + "theoretical_loss": 3.357565194136046, + "tokens_seen": 2904082432 + }, + { + "epoch": 1.26, + "learning_rate": 9.71715145436309e-05, + "loss": 1.0404, + "theoretical_loss": 3.357559379735455, + "tokens_seen": 2904147968 + }, + { + "epoch": 1.26, + "learning_rate": 9.71554663991976e-05, + "loss": 1.2452, + "theoretical_loss": 3.3575535655028097, + "tokens_seen": 2904213504 + }, + { + "epoch": 1.26, + "learning_rate": 9.71394182547643e-05, + "loss": 1.1748, + "theoretical_loss": 3.3575477514381022, + "tokens_seen": 2904279040 + }, + { + "epoch": 1.26, + "learning_rate": 9.7123370110331e-05, + "loss": 1.1502, + "theoretical_loss": 3.3575419375413236, + "tokens_seen": 2904344576 + }, + { + "epoch": 1.26, + "learning_rate": 9.71073219658977e-05, + "loss": 1.1858, + "theoretical_loss": 3.357536123812465, + "tokens_seen": 2904410112 + }, + { + "epoch": 1.26, + "learning_rate": 9.70912738214644e-05, + "loss": 1.1357, + "theoretical_loss": 3.3575303102515175, + "tokens_seen": 2904475648 + }, + { + "epoch": 1.26, + "learning_rate": 9.70752256770311e-05, + "loss": 1.181, + "theoretical_loss": 3.357524496858473, + "tokens_seen": 2904541184 + }, + { + "epoch": 1.26, + "learning_rate": 9.70591775325978e-05, + "loss": 1.1932, + "theoretical_loss": 3.357518683633323, + "tokens_seen": 2904606720 + }, + { + "epoch": 1.26, + "learning_rate": 9.70431293881645e-05, + "loss": 1.1885, + "theoretical_loss": 3.3575128705760586, + "tokens_seen": 2904672256 + }, + { + "epoch": 1.26, + "learning_rate": 9.70270812437312e-05, + "loss": 1.121, + "theoretical_loss": 3.3575070576866715, + "tokens_seen": 2904737792 + }, + { + "epoch": 1.26, + "learning_rate": 9.70110330992979e-05, + "loss": 1.1767, + "theoretical_loss": 3.3575012449651522, + "tokens_seen": 2904803328 + }, + { + "epoch": 1.26, + "learning_rate": 9.69949849548646e-05, + "loss": 1.2058, + "theoretical_loss": 3.3574954324114925, + "tokens_seen": 2904868864 + }, + { + "epoch": 1.26, + "learning_rate": 9.69789368104313e-05, + "loss": 1.1551, + "theoretical_loss": 3.3574896200256843, + "tokens_seen": 2904934400 + }, + { + "epoch": 1.26, + "learning_rate": 9.696288866599799e-05, + "loss": 1.2601, + "theoretical_loss": 3.3574838078077183, + "tokens_seen": 2904999936 + }, + { + "epoch": 1.26, + "learning_rate": 9.694684052156469e-05, + "loss": 1.2057, + "theoretical_loss": 3.357477995757586, + "tokens_seen": 2905065472 + }, + { + "epoch": 1.26, + "learning_rate": 9.693079237713139e-05, + "loss": 1.1594, + "theoretical_loss": 3.357472183875279, + "tokens_seen": 2905131008 + }, + { + "epoch": 1.26, + "learning_rate": 9.69147442326981e-05, + "loss": 1.1408, + "theoretical_loss": 3.3574663721607885, + "tokens_seen": 2905196544 + }, + { + "epoch": 1.26, + "learning_rate": 9.68986960882648e-05, + "loss": 1.1435, + "theoretical_loss": 3.357460560614106, + "tokens_seen": 2905262080 + }, + { + "epoch": 1.26, + "learning_rate": 9.68826479438315e-05, + "loss": 1.2474, + "theoretical_loss": 3.3574547492352225, + "tokens_seen": 2905327616 + }, + { + "epoch": 1.26, + "learning_rate": 9.68665997993982e-05, + "loss": 1.0967, + "theoretical_loss": 3.3574489380241297, + "tokens_seen": 2905393152 + }, + { + "epoch": 1.26, + "learning_rate": 9.68505516549649e-05, + "loss": 1.0977, + "theoretical_loss": 3.3574431269808187, + "tokens_seen": 2905458688 + }, + { + "epoch": 1.26, + "learning_rate": 9.683450351053161e-05, + "loss": 1.2773, + "theoretical_loss": 3.3574373161052815, + "tokens_seen": 2905524224 + }, + { + "epoch": 1.26, + "learning_rate": 9.681845536609831e-05, + "loss": 1.1632, + "theoretical_loss": 3.357431505397509, + "tokens_seen": 2905589760 + }, + { + "epoch": 1.26, + "learning_rate": 9.6802407221665e-05, + "loss": 1.2774, + "theoretical_loss": 3.3574256948574925, + "tokens_seen": 2905655296 + }, + { + "epoch": 1.26, + "learning_rate": 9.67863590772317e-05, + "loss": 1.2566, + "theoretical_loss": 3.357419884485223, + "tokens_seen": 2905720832 + }, + { + "epoch": 1.26, + "learning_rate": 9.67703109327984e-05, + "loss": 1.2221, + "theoretical_loss": 3.357414074280693, + "tokens_seen": 2905786368 + }, + { + "epoch": 1.26, + "learning_rate": 9.67542627883651e-05, + "loss": 1.0755, + "theoretical_loss": 3.357408264243893, + "tokens_seen": 2905851904 + }, + { + "epoch": 1.26, + "learning_rate": 9.67382146439318e-05, + "loss": 1.1772, + "theoretical_loss": 3.3574024543748147, + "tokens_seen": 2905917440 + }, + { + "epoch": 1.26, + "learning_rate": 9.67221664994985e-05, + "loss": 1.2422, + "theoretical_loss": 3.3573966446734493, + "tokens_seen": 2905982976 + }, + { + "epoch": 1.26, + "learning_rate": 9.67061183550652e-05, + "loss": 1.2696, + "theoretical_loss": 3.3573908351397885, + "tokens_seen": 2906048512 + }, + { + "epoch": 1.26, + "learning_rate": 9.66900702106319e-05, + "loss": 1.1751, + "theoretical_loss": 3.3573850257738234, + "tokens_seen": 2906114048 + }, + { + "epoch": 1.26, + "learning_rate": 9.66740220661986e-05, + "loss": 1.1511, + "theoretical_loss": 3.3573792165755454, + "tokens_seen": 2906179584 + }, + { + "epoch": 1.26, + "learning_rate": 9.66579739217653e-05, + "loss": 1.2045, + "theoretical_loss": 3.3573734075449457, + "tokens_seen": 2906245120 + }, + { + "epoch": 1.26, + "learning_rate": 9.6641925777332e-05, + "loss": 1.1311, + "theoretical_loss": 3.357367598682016, + "tokens_seen": 2906310656 + }, + { + "epoch": 1.26, + "learning_rate": 9.66258776328987e-05, + "loss": 1.2754, + "theoretical_loss": 3.3573617899867476, + "tokens_seen": 2906376192 + }, + { + "epoch": 1.26, + "learning_rate": 9.660982948846539e-05, + "loss": 1.1916, + "theoretical_loss": 3.357355981459132, + "tokens_seen": 2906441728 + }, + { + "epoch": 1.26, + "learning_rate": 9.659378134403209e-05, + "loss": 1.2272, + "theoretical_loss": 3.35735017309916, + "tokens_seen": 2906507264 + }, + { + "epoch": 1.26, + "learning_rate": 9.657773319959879e-05, + "loss": 1.2385, + "theoretical_loss": 3.3573443649068238, + "tokens_seen": 2906572800 + }, + { + "epoch": 1.26, + "learning_rate": 9.65616850551655e-05, + "loss": 1.084, + "theoretical_loss": 3.357338556882114, + "tokens_seen": 2906638336 + }, + { + "epoch": 1.26, + "learning_rate": 9.654563691073221e-05, + "loss": 1.1566, + "theoretical_loss": 3.357332749025023, + "tokens_seen": 2906703872 + }, + { + "epoch": 1.26, + "learning_rate": 9.652958876629891e-05, + "loss": 1.1389, + "theoretical_loss": 3.357326941335541, + "tokens_seen": 2906769408 + }, + { + "epoch": 1.26, + "learning_rate": 9.651354062186561e-05, + "loss": 1.2033, + "theoretical_loss": 3.35732113381366, + "tokens_seen": 2906834944 + }, + { + "epoch": 1.26, + "learning_rate": 9.649749247743231e-05, + "loss": 1.0955, + "theoretical_loss": 3.3573153264593714, + "tokens_seen": 2906900480 + }, + { + "epoch": 1.26, + "learning_rate": 9.648144433299901e-05, + "loss": 1.2023, + "theoretical_loss": 3.3573095192726665, + "tokens_seen": 2906966016 + }, + { + "epoch": 1.26, + "learning_rate": 9.64653961885657e-05, + "loss": 1.2509, + "theoretical_loss": 3.3573037122535365, + "tokens_seen": 2907031552 + }, + { + "epoch": 1.26, + "learning_rate": 9.64493480441324e-05, + "loss": 1.1202, + "theoretical_loss": 3.3572979054019734, + "tokens_seen": 2907097088 + }, + { + "epoch": 1.26, + "learning_rate": 9.64332998996991e-05, + "loss": 1.1288, + "theoretical_loss": 3.3572920987179677, + "tokens_seen": 2907162624 + }, + { + "epoch": 1.26, + "learning_rate": 9.64172517552658e-05, + "loss": 1.2004, + "theoretical_loss": 3.3572862922015116, + "tokens_seen": 2907228160 + }, + { + "epoch": 1.26, + "learning_rate": 9.64012036108325e-05, + "loss": 1.1551, + "theoretical_loss": 3.357280485852596, + "tokens_seen": 2907293696 + }, + { + "epoch": 1.26, + "learning_rate": 9.63851554663992e-05, + "loss": 1.2078, + "theoretical_loss": 3.3572746796712125, + "tokens_seen": 2907359232 + }, + { + "epoch": 1.26, + "learning_rate": 9.63691073219659e-05, + "loss": 1.2149, + "theoretical_loss": 3.3572688736573526, + "tokens_seen": 2907424768 + }, + { + "epoch": 1.26, + "learning_rate": 9.63530591775326e-05, + "loss": 1.1166, + "theoretical_loss": 3.357263067811007, + "tokens_seen": 2907490304 + }, + { + "epoch": 1.26, + "learning_rate": 9.63370110330993e-05, + "loss": 1.2668, + "theoretical_loss": 3.357257262132168, + "tokens_seen": 2907555840 + }, + { + "epoch": 1.26, + "learning_rate": 9.6320962888666e-05, + "loss": 1.2345, + "theoretical_loss": 3.3572514566208267, + "tokens_seen": 2907621376 + }, + { + "epoch": 1.26, + "learning_rate": 9.63049147442327e-05, + "loss": 1.1784, + "theoretical_loss": 3.3572456512769744, + "tokens_seen": 2907686912 + }, + { + "epoch": 1.26, + "learning_rate": 9.62888665997994e-05, + "loss": 1.1724, + "theoretical_loss": 3.357239846100602, + "tokens_seen": 2907752448 + }, + { + "epoch": 1.26, + "learning_rate": 9.627281845536609e-05, + "loss": 1.2083, + "theoretical_loss": 3.3572340410917016, + "tokens_seen": 2907817984 + }, + { + "epoch": 1.26, + "learning_rate": 9.625677031093279e-05, + "loss": 1.2388, + "theoretical_loss": 3.357228236250265, + "tokens_seen": 2907883520 + }, + { + "epoch": 1.26, + "learning_rate": 9.62407221664995e-05, + "loss": 1.2848, + "theoretical_loss": 3.357222431576282, + "tokens_seen": 2907949056 + }, + { + "epoch": 1.26, + "learning_rate": 9.622467402206621e-05, + "loss": 1.0937, + "theoretical_loss": 3.357216627069745, + "tokens_seen": 2908014592 + }, + { + "epoch": 1.26, + "learning_rate": 9.620862587763291e-05, + "loss": 1.2419, + "theoretical_loss": 3.357210822730646, + "tokens_seen": 2908080128 + }, + { + "epoch": 1.26, + "learning_rate": 9.619257773319961e-05, + "loss": 1.2669, + "theoretical_loss": 3.3572050185589752, + "tokens_seen": 2908145664 + }, + { + "epoch": 1.26, + "learning_rate": 9.617652958876631e-05, + "loss": 1.2612, + "theoretical_loss": 3.357199214554725, + "tokens_seen": 2908211200 + }, + { + "epoch": 1.26, + "learning_rate": 9.616048144433301e-05, + "loss": 1.1475, + "theoretical_loss": 3.357193410717886, + "tokens_seen": 2908276736 + }, + { + "epoch": 1.26, + "learning_rate": 9.614443329989971e-05, + "loss": 1.1644, + "theoretical_loss": 3.3571876070484503, + "tokens_seen": 2908342272 + }, + { + "epoch": 1.26, + "learning_rate": 9.612838515546641e-05, + "loss": 1.1695, + "theoretical_loss": 3.3571818035464087, + "tokens_seen": 2908407808 + }, + { + "epoch": 1.26, + "learning_rate": 9.61123370110331e-05, + "loss": 1.2144, + "theoretical_loss": 3.357176000211753, + "tokens_seen": 2908473344 + }, + { + "epoch": 1.26, + "learning_rate": 9.60962888665998e-05, + "loss": 1.2669, + "theoretical_loss": 3.3571701970444745, + "tokens_seen": 2908538880 + }, + { + "epoch": 1.26, + "learning_rate": 9.60802407221665e-05, + "loss": 1.2588, + "theoretical_loss": 3.357164394044564, + "tokens_seen": 2908604416 + }, + { + "epoch": 1.26, + "learning_rate": 9.60641925777332e-05, + "loss": 1.1547, + "theoretical_loss": 3.357158591212014, + "tokens_seen": 2908669952 + }, + { + "epoch": 1.26, + "learning_rate": 9.60481444332999e-05, + "loss": 1.2007, + "theoretical_loss": 3.3571527885468155, + "tokens_seen": 2908735488 + }, + { + "epoch": 1.26, + "learning_rate": 9.60320962888666e-05, + "loss": 1.1884, + "theoretical_loss": 3.357146986048959, + "tokens_seen": 2908801024 + }, + { + "epoch": 1.26, + "learning_rate": 9.60160481444333e-05, + "loss": 1.1697, + "theoretical_loss": 3.3571411837184373, + "tokens_seen": 2908866560 + }, + { + "epoch": 1.26, + "learning_rate": 9.6e-05, + "loss": 1.228, + "theoretical_loss": 3.357135381555241, + "tokens_seen": 2908932096 + }, + { + "epoch": 1.26, + "learning_rate": 9.59839518555667e-05, + "loss": 1.1736, + "theoretical_loss": 3.357129579559362, + "tokens_seen": 2908997632 + }, + { + "epoch": 1.26, + "learning_rate": 9.59679037111334e-05, + "loss": 1.1991, + "theoretical_loss": 3.357123777730791, + "tokens_seen": 2909063168 + }, + { + "epoch": 1.26, + "learning_rate": 9.59518555667001e-05, + "loss": 1.3373, + "theoretical_loss": 3.35711797606952, + "tokens_seen": 2909128704 + }, + { + "epoch": 1.26, + "learning_rate": 9.59358074222668e-05, + "loss": 1.1652, + "theoretical_loss": 3.3571121745755397, + "tokens_seen": 2909194240 + }, + { + "epoch": 1.26, + "learning_rate": 9.59197592778335e-05, + "loss": 1.2696, + "theoretical_loss": 3.3571063732488424, + "tokens_seen": 2909259776 + }, + { + "epoch": 1.26, + "learning_rate": 9.59037111334002e-05, + "loss": 1.2296, + "theoretical_loss": 3.357100572089419, + "tokens_seen": 2909325312 + }, + { + "epoch": 1.26, + "learning_rate": 9.588766298896691e-05, + "loss": 1.1552, + "theoretical_loss": 3.357094771097261, + "tokens_seen": 2909390848 + }, + { + "epoch": 1.26, + "learning_rate": 9.587161484453361e-05, + "loss": 1.2025, + "theoretical_loss": 3.35708897027236, + "tokens_seen": 2909456384 + }, + { + "epoch": 1.26, + "learning_rate": 9.585556670010031e-05, + "loss": 1.3247, + "theoretical_loss": 3.357083169614707, + "tokens_seen": 2909521920 + }, + { + "epoch": 1.26, + "learning_rate": 9.583951855566701e-05, + "loss": 1.2699, + "theoretical_loss": 3.357077369124294, + "tokens_seen": 2909587456 + }, + { + "epoch": 1.26, + "learning_rate": 9.582347041123371e-05, + "loss": 1.1979, + "theoretical_loss": 3.3570715688011115, + "tokens_seen": 2909652992 + }, + { + "epoch": 1.26, + "learning_rate": 9.580742226680041e-05, + "loss": 1.1963, + "theoretical_loss": 3.357065768645152, + "tokens_seen": 2909718528 + }, + { + "epoch": 1.26, + "objective/train/docs_used": 1073986, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.195838451385498, + "objective/train/theoretical_loss": 3.357059968656406, + "objective/train/tokens_used": 2930244064, + "theoretical_loss": 3.357059968656406, + "tokens_seen": 2909784064 + }, + { + "epoch": 1.26, + "learning_rate": 9.579137412236711e-05, + "loss": 1.1958, + "theoretical_loss": 3.357059968656406, + "tokens_seen": 2909784064 + }, + { + "epoch": 1.26, + "learning_rate": 9.577532597793381e-05, + "loss": 1.1775, + "theoretical_loss": 3.3570541688348654, + "tokens_seen": 2909849600 + }, + { + "epoch": 1.26, + "learning_rate": 9.57592778335005e-05, + "loss": 1.1307, + "theoretical_loss": 3.3570483691805215, + "tokens_seen": 2909915136 + }, + { + "epoch": 1.26, + "learning_rate": 9.57432296890672e-05, + "loss": 1.3044, + "theoretical_loss": 3.357042569693366, + "tokens_seen": 2909980672 + }, + { + "epoch": 1.26, + "learning_rate": 9.57271815446339e-05, + "loss": 1.2312, + "theoretical_loss": 3.35703677037339, + "tokens_seen": 2910046208 + }, + { + "epoch": 1.26, + "learning_rate": 9.57111334002006e-05, + "loss": 1.2225, + "theoretical_loss": 3.357030971220585, + "tokens_seen": 2910111744 + }, + { + "epoch": 1.26, + "learning_rate": 9.56950852557673e-05, + "loss": 1.2053, + "theoretical_loss": 3.357025172234942, + "tokens_seen": 2910177280 + }, + { + "epoch": 1.26, + "learning_rate": 9.5679037111334e-05, + "loss": 1.1235, + "theoretical_loss": 3.3570193734164526, + "tokens_seen": 2910242816 + }, + { + "epoch": 1.26, + "learning_rate": 9.56629889669007e-05, + "loss": 1.2573, + "theoretical_loss": 3.3570135747651086, + "tokens_seen": 2910308352 + }, + { + "epoch": 1.26, + "learning_rate": 9.56469408224674e-05, + "loss": 1.3255, + "theoretical_loss": 3.3570077762809016, + "tokens_seen": 2910373888 + }, + { + "epoch": 1.26, + "learning_rate": 9.563089267803412e-05, + "loss": 1.1306, + "theoretical_loss": 3.3570019779638227, + "tokens_seen": 2910439424 + }, + { + "epoch": 1.26, + "learning_rate": 9.561484453360082e-05, + "loss": 1.1732, + "theoretical_loss": 3.356996179813863, + "tokens_seen": 2910504960 + }, + { + "epoch": 1.26, + "learning_rate": 9.55987963891675e-05, + "loss": 1.301, + "theoretical_loss": 3.356990381831014, + "tokens_seen": 2910570496 + }, + { + "epoch": 1.26, + "learning_rate": 9.55827482447342e-05, + "loss": 1.1034, + "theoretical_loss": 3.3569845840152674, + "tokens_seen": 2910636032 + }, + { + "epoch": 1.26, + "learning_rate": 9.55667001003009e-05, + "loss": 1.2362, + "theoretical_loss": 3.3569787863666147, + "tokens_seen": 2910701568 + }, + { + "epoch": 1.26, + "learning_rate": 9.55506519558676e-05, + "loss": 1.1256, + "theoretical_loss": 3.356972988885047, + "tokens_seen": 2910767104 + }, + { + "epoch": 1.26, + "learning_rate": 9.553460381143431e-05, + "loss": 1.1715, + "theoretical_loss": 3.356967191570556, + "tokens_seen": 2910832640 + }, + { + "epoch": 1.26, + "learning_rate": 9.551855566700101e-05, + "loss": 1.1541, + "theoretical_loss": 3.356961394423133, + "tokens_seen": 2910898176 + }, + { + "epoch": 1.26, + "learning_rate": 9.550250752256771e-05, + "loss": 1.1711, + "theoretical_loss": 3.356955597442769, + "tokens_seen": 2910963712 + }, + { + "epoch": 1.26, + "learning_rate": 9.548645937813441e-05, + "loss": 1.1537, + "theoretical_loss": 3.3569498006294567, + "tokens_seen": 2911029248 + }, + { + "epoch": 1.26, + "learning_rate": 9.547041123370111e-05, + "loss": 1.263, + "theoretical_loss": 3.356944003983186, + "tokens_seen": 2911094784 + }, + { + "epoch": 1.26, + "learning_rate": 9.545436308926781e-05, + "loss": 1.1172, + "theoretical_loss": 3.3569382075039496, + "tokens_seen": 2911160320 + }, + { + "epoch": 1.26, + "learning_rate": 9.543831494483451e-05, + "loss": 1.2111, + "theoretical_loss": 3.3569324111917376, + "tokens_seen": 2911225856 + }, + { + "epoch": 1.26, + "learning_rate": 9.542226680040121e-05, + "loss": 1.3277, + "theoretical_loss": 3.356926615046542, + "tokens_seen": 2911291392 + }, + { + "epoch": 1.26, + "learning_rate": 9.54062186559679e-05, + "loss": 1.2803, + "theoretical_loss": 3.356920819068355, + "tokens_seen": 2911356928 + }, + { + "epoch": 1.26, + "learning_rate": 9.53901705115346e-05, + "loss": 1.1551, + "theoretical_loss": 3.3569150232571676, + "tokens_seen": 2911422464 + }, + { + "epoch": 1.26, + "learning_rate": 9.53741223671013e-05, + "loss": 1.2402, + "theoretical_loss": 3.3569092276129706, + "tokens_seen": 2911488000 + }, + { + "epoch": 1.26, + "learning_rate": 9.5358074222668e-05, + "loss": 1.305, + "theoretical_loss": 3.356903432135756, + "tokens_seen": 2911553536 + }, + { + "epoch": 1.26, + "learning_rate": 9.534202607823471e-05, + "loss": 1.2084, + "theoretical_loss": 3.356897636825515, + "tokens_seen": 2911619072 + }, + { + "epoch": 1.26, + "learning_rate": 9.532597793380142e-05, + "loss": 1.2082, + "theoretical_loss": 3.356891841682239, + "tokens_seen": 2911684608 + }, + { + "epoch": 1.26, + "learning_rate": 9.530992978936812e-05, + "loss": 1.2327, + "theoretical_loss": 3.35688604670592, + "tokens_seen": 2911750144 + }, + { + "epoch": 1.26, + "learning_rate": 9.529388164493482e-05, + "loss": 1.1337, + "theoretical_loss": 3.356880251896549, + "tokens_seen": 2911815680 + }, + { + "epoch": 1.26, + "learning_rate": 9.527783350050152e-05, + "loss": 1.2695, + "theoretical_loss": 3.3568744572541167, + "tokens_seen": 2911881216 + }, + { + "epoch": 1.26, + "learning_rate": 9.526178535606822e-05, + "loss": 1.1411, + "theoretical_loss": 3.356868662778616, + "tokens_seen": 2911946752 + }, + { + "epoch": 1.26, + "learning_rate": 9.52457372116349e-05, + "loss": 1.1016, + "theoretical_loss": 3.3568628684700372, + "tokens_seen": 2912012288 + }, + { + "epoch": 1.26, + "learning_rate": 9.52296890672016e-05, + "loss": 1.2179, + "theoretical_loss": 3.356857074328372, + "tokens_seen": 2912077824 + }, + { + "epoch": 1.26, + "learning_rate": 9.52136409227683e-05, + "loss": 1.1565, + "theoretical_loss": 3.3568512803536126, + "tokens_seen": 2912143360 + }, + { + "epoch": 1.26, + "learning_rate": 9.519759277833501e-05, + "loss": 1.1593, + "theoretical_loss": 3.3568454865457493, + "tokens_seen": 2912208896 + }, + { + "epoch": 1.26, + "learning_rate": 9.518154463390171e-05, + "loss": 1.217, + "theoretical_loss": 3.3568396929047744, + "tokens_seen": 2912274432 + }, + { + "epoch": 1.26, + "learning_rate": 9.516549648946841e-05, + "loss": 1.1541, + "theoretical_loss": 3.3568338994306783, + "tokens_seen": 2912339968 + }, + { + "epoch": 1.26, + "learning_rate": 9.514944834503511e-05, + "loss": 1.2566, + "theoretical_loss": 3.3568281061234537, + "tokens_seen": 2912405504 + }, + { + "epoch": 1.26, + "learning_rate": 9.513340020060181e-05, + "loss": 1.2054, + "theoretical_loss": 3.356822312983091, + "tokens_seen": 2912471040 + }, + { + "epoch": 1.26, + "learning_rate": 9.511735205616851e-05, + "loss": 1.1797, + "theoretical_loss": 3.3568165200095827, + "tokens_seen": 2912536576 + }, + { + "epoch": 1.26, + "learning_rate": 9.510130391173521e-05, + "loss": 1.2385, + "theoretical_loss": 3.3568107272029195, + "tokens_seen": 2912602112 + }, + { + "epoch": 1.26, + "learning_rate": 9.508525576730191e-05, + "loss": 1.283, + "theoretical_loss": 3.3568049345630926, + "tokens_seen": 2912667648 + }, + { + "epoch": 1.26, + "learning_rate": 9.506920762286861e-05, + "loss": 1.1662, + "theoretical_loss": 3.356799142090094, + "tokens_seen": 2912733184 + }, + { + "epoch": 1.26, + "learning_rate": 9.50531594784353e-05, + "loss": 1.1304, + "theoretical_loss": 3.356793349783915, + "tokens_seen": 2912798720 + }, + { + "epoch": 1.26, + "learning_rate": 9.503711133400201e-05, + "loss": 1.2209, + "theoretical_loss": 3.356787557644547, + "tokens_seen": 2912864256 + }, + { + "epoch": 1.26, + "learning_rate": 9.502106318956871e-05, + "loss": 1.139, + "theoretical_loss": 3.3567817656719816, + "tokens_seen": 2912929792 + }, + { + "epoch": 1.26, + "learning_rate": 9.500501504513541e-05, + "loss": 1.1057, + "theoretical_loss": 3.3567759738662097, + "tokens_seen": 2912995328 + }, + { + "epoch": 1.26, + "learning_rate": 9.498896690070212e-05, + "loss": 1.2324, + "theoretical_loss": 3.3567701822272236, + "tokens_seen": 2913060864 + }, + { + "epoch": 1.26, + "learning_rate": 9.497291875626882e-05, + "loss": 1.3227, + "theoretical_loss": 3.3567643907550138, + "tokens_seen": 2913126400 + }, + { + "epoch": 1.26, + "learning_rate": 9.495687061183552e-05, + "loss": 1.3324, + "theoretical_loss": 3.3567585994495723, + "tokens_seen": 2913191936 + }, + { + "epoch": 1.26, + "learning_rate": 9.494082246740222e-05, + "loss": 1.2442, + "theoretical_loss": 3.3567528083108904, + "tokens_seen": 2913257472 + }, + { + "epoch": 1.26, + "learning_rate": 9.492477432296892e-05, + "loss": 1.1302, + "theoretical_loss": 3.35674701733896, + "tokens_seen": 2913323008 + }, + { + "epoch": 1.26, + "learning_rate": 9.490872617853562e-05, + "loss": 1.1439, + "theoretical_loss": 3.3567412265337717, + "tokens_seen": 2913388544 + }, + { + "epoch": 1.26, + "learning_rate": 9.48926780341023e-05, + "loss": 1.206, + "theoretical_loss": 3.3567354358953176, + "tokens_seen": 2913454080 + }, + { + "epoch": 1.26, + "learning_rate": 9.4876629889669e-05, + "loss": 1.1535, + "theoretical_loss": 3.3567296454235893, + "tokens_seen": 2913519616 + }, + { + "epoch": 1.26, + "learning_rate": 9.486058174523571e-05, + "loss": 1.2099, + "theoretical_loss": 3.3567238551185774, + "tokens_seen": 2913585152 + }, + { + "epoch": 1.26, + "learning_rate": 9.484453360080241e-05, + "loss": 1.3144, + "theoretical_loss": 3.3567180649802744, + "tokens_seen": 2913650688 + }, + { + "epoch": 1.26, + "learning_rate": 9.482848545636911e-05, + "loss": 1.1618, + "theoretical_loss": 3.356712275008671, + "tokens_seen": 2913716224 + }, + { + "epoch": 1.26, + "learning_rate": 9.481243731193581e-05, + "loss": 1.2574, + "theoretical_loss": 3.3567064852037585, + "tokens_seen": 2913781760 + }, + { + "epoch": 1.26, + "learning_rate": 9.479638916750251e-05, + "loss": 1.3067, + "theoretical_loss": 3.3567006955655287, + "tokens_seen": 2913847296 + }, + { + "epoch": 1.26, + "learning_rate": 9.478034102306921e-05, + "loss": 1.2473, + "theoretical_loss": 3.3566949060939737, + "tokens_seen": 2913912832 + }, + { + "epoch": 1.26, + "learning_rate": 9.476429287863591e-05, + "loss": 1.2447, + "theoretical_loss": 3.356689116789084, + "tokens_seen": 2913978368 + }, + { + "epoch": 1.26, + "learning_rate": 9.474824473420261e-05, + "loss": 1.1734, + "theoretical_loss": 3.356683327650851, + "tokens_seen": 2914043904 + }, + { + "epoch": 1.26, + "learning_rate": 9.473219658976931e-05, + "loss": 1.2419, + "theoretical_loss": 3.3566775386792673, + "tokens_seen": 2914109440 + }, + { + "epoch": 1.26, + "learning_rate": 9.471614844533601e-05, + "loss": 1.1339, + "theoretical_loss": 3.356671749874323, + "tokens_seen": 2914174976 + }, + { + "epoch": 1.26, + "learning_rate": 9.470010030090271e-05, + "loss": 1.0975, + "theoretical_loss": 3.3566659612360104, + "tokens_seen": 2914240512 + }, + { + "epoch": 1.26, + "learning_rate": 9.468405215646941e-05, + "loss": 1.2599, + "theoretical_loss": 3.3566601727643204, + "tokens_seen": 2914306048 + }, + { + "epoch": 1.26, + "learning_rate": 9.466800401203611e-05, + "loss": 1.3322, + "theoretical_loss": 3.3566543844592447, + "tokens_seen": 2914371584 + }, + { + "epoch": 1.26, + "learning_rate": 9.465195586760282e-05, + "loss": 1.3123, + "theoretical_loss": 3.3566485963207753, + "tokens_seen": 2914437120 + }, + { + "epoch": 1.26, + "learning_rate": 9.463590772316952e-05, + "loss": 1.19, + "theoretical_loss": 3.356642808348903, + "tokens_seen": 2914502656 + }, + { + "epoch": 1.26, + "learning_rate": 9.461985957873622e-05, + "loss": 1.2045, + "theoretical_loss": 3.3566370205436193, + "tokens_seen": 2914568192 + }, + { + "epoch": 1.26, + "learning_rate": 9.460381143430292e-05, + "loss": 1.1947, + "theoretical_loss": 3.356631232904916, + "tokens_seen": 2914633728 + }, + { + "epoch": 1.26, + "learning_rate": 9.458776328986962e-05, + "loss": 1.2571, + "theoretical_loss": 3.3566254454327837, + "tokens_seen": 2914699264 + }, + { + "epoch": 1.26, + "learning_rate": 9.457171514543632e-05, + "loss": 1.2038, + "theoretical_loss": 3.356619658127215, + "tokens_seen": 2914764800 + }, + { + "epoch": 1.26, + "learning_rate": 9.455566700100302e-05, + "loss": 1.2002, + "theoretical_loss": 3.356613870988201, + "tokens_seen": 2914830336 + }, + { + "epoch": 1.26, + "learning_rate": 9.45396188565697e-05, + "loss": 1.2642, + "theoretical_loss": 3.3566080840157326, + "tokens_seen": 2914895872 + }, + { + "epoch": 1.26, + "learning_rate": 9.45235707121364e-05, + "loss": 1.2401, + "theoretical_loss": 3.356602297209802, + "tokens_seen": 2914961408 + }, + { + "epoch": 1.26, + "learning_rate": 9.450752256770311e-05, + "loss": 1.0662, + "theoretical_loss": 3.3565965105704003, + "tokens_seen": 2915026944 + }, + { + "epoch": 1.26, + "learning_rate": 9.449147442326981e-05, + "loss": 1.2075, + "theoretical_loss": 3.3565907240975195, + "tokens_seen": 2915092480 + }, + { + "epoch": 1.26, + "learning_rate": 9.447542627883651e-05, + "loss": 1.2243, + "theoretical_loss": 3.35658493779115, + "tokens_seen": 2915158016 + }, + { + "epoch": 1.26, + "learning_rate": 9.445937813440321e-05, + "loss": 1.1087, + "theoretical_loss": 3.356579151651284, + "tokens_seen": 2915223552 + }, + { + "epoch": 1.26, + "learning_rate": 9.444332998996992e-05, + "loss": 1.2269, + "theoretical_loss": 3.3565733656779124, + "tokens_seen": 2915289088 + }, + { + "epoch": 1.26, + "learning_rate": 9.442728184553662e-05, + "loss": 1.3166, + "theoretical_loss": 3.3565675798710277, + "tokens_seen": 2915354624 + }, + { + "epoch": 1.26, + "learning_rate": 9.441123370110332e-05, + "loss": 1.1379, + "theoretical_loss": 3.3565617942306205, + "tokens_seen": 2915420160 + }, + { + "epoch": 1.26, + "learning_rate": 9.439518555667001e-05, + "loss": 1.149, + "theoretical_loss": 3.3565560087566824, + "tokens_seen": 2915485696 + }, + { + "epoch": 1.26, + "learning_rate": 9.437913741223671e-05, + "loss": 1.2862, + "theoretical_loss": 3.3565502234492053, + "tokens_seen": 2915551232 + }, + { + "epoch": 1.26, + "learning_rate": 9.436308926780341e-05, + "loss": 1.1999, + "theoretical_loss": 3.35654443830818, + "tokens_seen": 2915616768 + }, + { + "epoch": 1.26, + "learning_rate": 9.434704112337011e-05, + "loss": 1.1323, + "theoretical_loss": 3.3565386533335984, + "tokens_seen": 2915682304 + }, + { + "epoch": 1.26, + "learning_rate": 9.433099297893681e-05, + "loss": 1.1926, + "theoretical_loss": 3.356532868525452, + "tokens_seen": 2915747840 + }, + { + "epoch": 1.26, + "learning_rate": 9.431494483450351e-05, + "loss": 1.2504, + "theoretical_loss": 3.356527083883732, + "tokens_seen": 2915813376 + }, + { + "epoch": 1.26, + "learning_rate": 9.429889669007022e-05, + "loss": 1.1946, + "theoretical_loss": 3.3565212994084304, + "tokens_seen": 2915878912 + }, + { + "epoch": 1.26, + "learning_rate": 9.428284854563692e-05, + "loss": 1.1809, + "theoretical_loss": 3.356515515099538, + "tokens_seen": 2915944448 + }, + { + "epoch": 1.26, + "learning_rate": 9.426680040120362e-05, + "loss": 1.2863, + "theoretical_loss": 3.356509730957047, + "tokens_seen": 2916009984 + }, + { + "epoch": 1.26, + "learning_rate": 9.425075225677032e-05, + "loss": 1.2103, + "theoretical_loss": 3.356503946980948, + "tokens_seen": 2916075520 + }, + { + "epoch": 1.26, + "learning_rate": 9.423470411233702e-05, + "loss": 1.1898, + "theoretical_loss": 3.356498163171233, + "tokens_seen": 2916141056 + }, + { + "epoch": 1.26, + "learning_rate": 9.421865596790372e-05, + "loss": 1.1173, + "theoretical_loss": 3.3564923795278934, + "tokens_seen": 2916206592 + }, + { + "epoch": 1.26, + "learning_rate": 9.42026078234704e-05, + "loss": 1.2656, + "theoretical_loss": 3.356486596050921, + "tokens_seen": 2916272128 + }, + { + "epoch": 1.26, + "objective/train/docs_used": 1076441, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2835360765457153, + "objective/train/theoretical_loss": 3.3564808127403065, + "objective/train/tokens_used": 2936797664, + "theoretical_loss": 3.3564808127403065, + "tokens_seen": 2916337664 + }, + { + "epoch": 1.26, + "learning_rate": 9.41865596790371e-05, + "loss": 1.2835, + "theoretical_loss": 3.3564808127403065, + "tokens_seen": 2916337664 + }, + { + "epoch": 1.26, + "learning_rate": 9.417051153460381e-05, + "loss": 1.206, + "theoretical_loss": 3.356475029596042, + "tokens_seen": 2916403200 + }, + { + "epoch": 1.26, + "learning_rate": 9.415446339017051e-05, + "loss": 1.1534, + "theoretical_loss": 3.356469246618119, + "tokens_seen": 2916468736 + }, + { + "epoch": 1.26, + "learning_rate": 9.413841524573722e-05, + "loss": 1.1595, + "theoretical_loss": 3.3564634638065285, + "tokens_seen": 2916534272 + }, + { + "epoch": 1.26, + "learning_rate": 9.412236710130392e-05, + "loss": 1.1191, + "theoretical_loss": 3.3564576811612623, + "tokens_seen": 2916599808 + }, + { + "epoch": 1.26, + "learning_rate": 9.410631895687062e-05, + "loss": 1.1794, + "theoretical_loss": 3.356451898682312, + "tokens_seen": 2916665344 + }, + { + "epoch": 1.26, + "learning_rate": 9.409027081243732e-05, + "loss": 1.1015, + "theoretical_loss": 3.356446116369669, + "tokens_seen": 2916730880 + }, + { + "epoch": 1.26, + "learning_rate": 9.407422266800402e-05, + "loss": 1.1609, + "theoretical_loss": 3.3564403342233247, + "tokens_seen": 2916796416 + }, + { + "epoch": 1.26, + "learning_rate": 9.405817452357073e-05, + "loss": 1.2617, + "theoretical_loss": 3.3564345522432704, + "tokens_seen": 2916861952 + }, + { + "epoch": 1.26, + "learning_rate": 9.404212637913741e-05, + "loss": 1.2099, + "theoretical_loss": 3.3564287704294977, + "tokens_seen": 2916927488 + }, + { + "epoch": 1.26, + "learning_rate": 9.402607823470411e-05, + "loss": 1.2193, + "theoretical_loss": 3.3564229887819983, + "tokens_seen": 2916993024 + }, + { + "epoch": 1.26, + "learning_rate": 9.401003009027081e-05, + "loss": 1.1888, + "theoretical_loss": 3.3564172073007636, + "tokens_seen": 2917058560 + }, + { + "epoch": 1.26, + "learning_rate": 9.399398194583751e-05, + "loss": 1.1951, + "theoretical_loss": 3.356411425985785, + "tokens_seen": 2917124096 + }, + { + "epoch": 1.26, + "learning_rate": 9.397793380140421e-05, + "loss": 1.2251, + "theoretical_loss": 3.3564056448370536, + "tokens_seen": 2917189632 + }, + { + "epoch": 1.26, + "learning_rate": 9.396188565697092e-05, + "loss": 1.284, + "theoretical_loss": 3.356399863854562, + "tokens_seen": 2917255168 + }, + { + "epoch": 1.26, + "learning_rate": 9.394583751253762e-05, + "loss": 1.2504, + "theoretical_loss": 3.3563940830383, + "tokens_seen": 2917320704 + }, + { + "epoch": 1.26, + "learning_rate": 9.392978936810432e-05, + "loss": 1.2069, + "theoretical_loss": 3.356388302388261, + "tokens_seen": 2917386240 + }, + { + "epoch": 1.26, + "learning_rate": 9.391374122367102e-05, + "loss": 1.1338, + "theoretical_loss": 3.356382521904435, + "tokens_seen": 2917451776 + }, + { + "epoch": 1.26, + "learning_rate": 9.389769307923772e-05, + "loss": 1.1788, + "theoretical_loss": 3.356376741586814, + "tokens_seen": 2917517312 + }, + { + "epoch": 1.26, + "learning_rate": 9.388164493480442e-05, + "loss": 1.1511, + "theoretical_loss": 3.35637096143539, + "tokens_seen": 2917582848 + }, + { + "epoch": 1.26, + "learning_rate": 9.386559679037112e-05, + "loss": 1.2088, + "theoretical_loss": 3.356365181450154, + "tokens_seen": 2917648384 + }, + { + "epoch": 1.26, + "learning_rate": 9.38495486459378e-05, + "loss": 1.2531, + "theoretical_loss": 3.3563594016310967, + "tokens_seen": 2917713920 + }, + { + "epoch": 1.26, + "learning_rate": 9.383350050150452e-05, + "loss": 1.2333, + "theoretical_loss": 3.356353621978211, + "tokens_seen": 2917779456 + }, + { + "epoch": 1.26, + "learning_rate": 9.381745235707122e-05, + "loss": 1.1747, + "theoretical_loss": 3.3563478424914877, + "tokens_seen": 2917844992 + }, + { + "epoch": 1.26, + "learning_rate": 9.380140421263792e-05, + "loss": 1.1261, + "theoretical_loss": 3.3563420631709184, + "tokens_seen": 2917910528 + }, + { + "epoch": 1.26, + "learning_rate": 9.378535606820462e-05, + "loss": 1.2853, + "theoretical_loss": 3.3563362840164945, + "tokens_seen": 2917976064 + }, + { + "epoch": 1.26, + "learning_rate": 9.376930792377132e-05, + "loss": 1.1557, + "theoretical_loss": 3.3563305050282075, + "tokens_seen": 2918041600 + }, + { + "epoch": 1.26, + "learning_rate": 9.375325977933802e-05, + "loss": 1.199, + "theoretical_loss": 3.356324726206049, + "tokens_seen": 2918107136 + }, + { + "epoch": 1.26, + "learning_rate": 9.373721163490472e-05, + "loss": 1.0766, + "theoretical_loss": 3.3563189475500104, + "tokens_seen": 2918172672 + }, + { + "epoch": 1.26, + "learning_rate": 9.372116349047142e-05, + "loss": 1.1925, + "theoretical_loss": 3.3563131690600834, + "tokens_seen": 2918238208 + }, + { + "epoch": 1.27, + "learning_rate": 9.370511534603813e-05, + "loss": 1.1125, + "theoretical_loss": 3.356307390736259, + "tokens_seen": 2918303744 + }, + { + "epoch": 1.27, + "learning_rate": 9.368906720160481e-05, + "loss": 1.1716, + "theoretical_loss": 3.3563016125785294, + "tokens_seen": 2918369280 + }, + { + "epoch": 1.27, + "learning_rate": 9.367301905717151e-05, + "loss": 1.1327, + "theoretical_loss": 3.356295834586885, + "tokens_seen": 2918434816 + }, + { + "epoch": 1.27, + "learning_rate": 9.365697091273821e-05, + "loss": 1.2214, + "theoretical_loss": 3.3562900567613188, + "tokens_seen": 2918500352 + }, + { + "epoch": 1.27, + "learning_rate": 9.364092276830491e-05, + "loss": 1.1549, + "theoretical_loss": 3.3562842791018213, + "tokens_seen": 2918565888 + }, + { + "epoch": 1.27, + "learning_rate": 9.362487462387162e-05, + "loss": 1.0862, + "theoretical_loss": 3.3562785016083843, + "tokens_seen": 2918631424 + }, + { + "epoch": 1.27, + "learning_rate": 9.360882647943832e-05, + "loss": 1.1685, + "theoretical_loss": 3.356272724280999, + "tokens_seen": 2918696960 + }, + { + "epoch": 1.27, + "learning_rate": 9.359277833500502e-05, + "loss": 1.2025, + "theoretical_loss": 3.356266947119657, + "tokens_seen": 2918762496 + }, + { + "epoch": 1.27, + "learning_rate": 9.357673019057172e-05, + "loss": 1.1647, + "theoretical_loss": 3.35626117012435, + "tokens_seen": 2918828032 + }, + { + "epoch": 1.27, + "learning_rate": 9.356068204613842e-05, + "loss": 1.1674, + "theoretical_loss": 3.3562553932950694, + "tokens_seen": 2918893568 + }, + { + "epoch": 1.27, + "learning_rate": 9.354463390170512e-05, + "loss": 1.1438, + "theoretical_loss": 3.356249616631807, + "tokens_seen": 2918959104 + }, + { + "epoch": 1.27, + "learning_rate": 9.352858575727182e-05, + "loss": 1.1125, + "theoretical_loss": 3.3562438401345536, + "tokens_seen": 2919024640 + }, + { + "epoch": 1.27, + "learning_rate": 9.351253761283852e-05, + "loss": 1.1816, + "theoretical_loss": 3.3562380638033016, + "tokens_seen": 2919090176 + }, + { + "epoch": 1.27, + "learning_rate": 9.349648946840522e-05, + "loss": 1.2185, + "theoretical_loss": 3.3562322876380417, + "tokens_seen": 2919155712 + }, + { + "epoch": 1.27, + "learning_rate": 9.348044132397192e-05, + "loss": 1.2129, + "theoretical_loss": 3.3562265116387655, + "tokens_seen": 2919221248 + }, + { + "epoch": 1.27, + "learning_rate": 9.346439317953862e-05, + "loss": 1.2357, + "theoretical_loss": 3.356220735805465, + "tokens_seen": 2919286784 + }, + { + "epoch": 1.27, + "learning_rate": 9.344834503510532e-05, + "loss": 1.1268, + "theoretical_loss": 3.3562149601381313, + "tokens_seen": 2919352320 + }, + { + "epoch": 1.27, + "learning_rate": 9.343229689067202e-05, + "loss": 1.1876, + "theoretical_loss": 3.3562091846367563, + "tokens_seen": 2919417856 + }, + { + "epoch": 1.27, + "learning_rate": 9.341624874623872e-05, + "loss": 1.1788, + "theoretical_loss": 3.356203409301331, + "tokens_seen": 2919483392 + }, + { + "epoch": 1.27, + "learning_rate": 9.340020060180542e-05, + "loss": 1.1839, + "theoretical_loss": 3.3561976341318474, + "tokens_seen": 2919548928 + }, + { + "epoch": 1.27, + "learning_rate": 9.338415245737212e-05, + "loss": 1.1444, + "theoretical_loss": 3.3561918591282964, + "tokens_seen": 2919614464 + }, + { + "epoch": 1.27, + "learning_rate": 9.336810431293883e-05, + "loss": 1.1931, + "theoretical_loss": 3.3561860842906697, + "tokens_seen": 2919680000 + }, + { + "epoch": 1.27, + "learning_rate": 9.335205616850553e-05, + "loss": 1.0722, + "theoretical_loss": 3.356180309618959, + "tokens_seen": 2919745536 + }, + { + "epoch": 1.27, + "learning_rate": 9.333600802407221e-05, + "loss": 1.2992, + "theoretical_loss": 3.3561745351131567, + "tokens_seen": 2919811072 + }, + { + "epoch": 1.27, + "learning_rate": 9.331995987963891e-05, + "loss": 1.1744, + "theoretical_loss": 3.3561687607732527, + "tokens_seen": 2919876608 + }, + { + "epoch": 1.27, + "learning_rate": 9.330391173520561e-05, + "loss": 1.0462, + "theoretical_loss": 3.356162986599239, + "tokens_seen": 2919942144 + }, + { + "epoch": 1.27, + "learning_rate": 9.328786359077231e-05, + "loss": 1.2857, + "theoretical_loss": 3.3561572125911074, + "tokens_seen": 2920007680 + }, + { + "epoch": 1.27, + "learning_rate": 9.327181544633902e-05, + "loss": 1.1915, + "theoretical_loss": 3.3561514387488494, + "tokens_seen": 2920073216 + }, + { + "epoch": 1.27, + "learning_rate": 9.325576730190572e-05, + "loss": 1.2838, + "theoretical_loss": 3.3561456650724564, + "tokens_seen": 2920138752 + }, + { + "epoch": 1.27, + "learning_rate": 9.323971915747243e-05, + "loss": 1.28, + "theoretical_loss": 3.35613989156192, + "tokens_seen": 2920204288 + }, + { + "epoch": 1.27, + "learning_rate": 9.322367101303913e-05, + "loss": 1.2324, + "theoretical_loss": 3.356134118217232, + "tokens_seen": 2920269824 + }, + { + "epoch": 1.27, + "learning_rate": 9.320762286860583e-05, + "loss": 1.2706, + "theoretical_loss": 3.356128345038383, + "tokens_seen": 2920335360 + }, + { + "epoch": 1.27, + "learning_rate": 9.319157472417253e-05, + "loss": 1.1968, + "theoretical_loss": 3.356122572025366, + "tokens_seen": 2920400896 + }, + { + "epoch": 1.27, + "learning_rate": 9.317552657973922e-05, + "loss": 1.1917, + "theoretical_loss": 3.3561167991781704, + "tokens_seen": 2920466432 + }, + { + "epoch": 1.27, + "learning_rate": 9.315947843530592e-05, + "loss": 1.1815, + "theoretical_loss": 3.3561110264967895, + "tokens_seen": 2920531968 + }, + { + "epoch": 1.27, + "learning_rate": 9.314343029087262e-05, + "loss": 1.127, + "theoretical_loss": 3.356105253981214, + "tokens_seen": 2920597504 + }, + { + "epoch": 1.27, + "learning_rate": 9.312738214643932e-05, + "loss": 1.1672, + "theoretical_loss": 3.356099481631436, + "tokens_seen": 2920663040 + }, + { + "epoch": 1.27, + "learning_rate": 9.311133400200602e-05, + "loss": 1.1957, + "theoretical_loss": 3.3560937094474466, + "tokens_seen": 2920728576 + }, + { + "epoch": 1.27, + "learning_rate": 9.309528585757272e-05, + "loss": 1.2238, + "theoretical_loss": 3.3560879374292374, + "tokens_seen": 2920794112 + }, + { + "epoch": 1.27, + "learning_rate": 9.307923771313942e-05, + "loss": 1.1814, + "theoretical_loss": 3.3560821655767996, + "tokens_seen": 2920859648 + }, + { + "epoch": 1.27, + "learning_rate": 9.306318956870612e-05, + "loss": 1.2348, + "theoretical_loss": 3.3560763938901252, + "tokens_seen": 2920925184 + }, + { + "epoch": 1.27, + "learning_rate": 9.304714142427282e-05, + "loss": 1.1863, + "theoretical_loss": 3.356070622369206, + "tokens_seen": 2920990720 + }, + { + "epoch": 1.27, + "learning_rate": 9.303109327983952e-05, + "loss": 1.1636, + "theoretical_loss": 3.3560648510140325, + "tokens_seen": 2921056256 + }, + { + "epoch": 1.27, + "learning_rate": 9.301504513540623e-05, + "loss": 1.1929, + "theoretical_loss": 3.356059079824597, + "tokens_seen": 2921121792 + }, + { + "epoch": 1.27, + "learning_rate": 9.299899699097293e-05, + "loss": 1.1604, + "theoretical_loss": 3.356053308800891, + "tokens_seen": 2921187328 + }, + { + "epoch": 1.27, + "learning_rate": 9.298294884653961e-05, + "loss": 1.1412, + "theoretical_loss": 3.356047537942905, + "tokens_seen": 2921252864 + }, + { + "epoch": 1.27, + "learning_rate": 9.296690070210631e-05, + "loss": 1.2494, + "theoretical_loss": 3.356041767250632, + "tokens_seen": 2921318400 + }, + { + "epoch": 1.27, + "learning_rate": 9.295085255767301e-05, + "loss": 1.2364, + "theoretical_loss": 3.356035996724063, + "tokens_seen": 2921383936 + }, + { + "epoch": 1.27, + "learning_rate": 9.293480441323973e-05, + "loss": 1.2043, + "theoretical_loss": 3.356030226363189, + "tokens_seen": 2921449472 + }, + { + "epoch": 1.27, + "learning_rate": 9.291875626880643e-05, + "loss": 1.2065, + "theoretical_loss": 3.3560244561680026, + "tokens_seen": 2921515008 + }, + { + "epoch": 1.27, + "learning_rate": 9.290270812437313e-05, + "loss": 1.1172, + "theoretical_loss": 3.356018686138494, + "tokens_seen": 2921580544 + }, + { + "epoch": 1.27, + "learning_rate": 9.288665997993983e-05, + "loss": 1.2854, + "theoretical_loss": 3.356012916274656, + "tokens_seen": 2921646080 + }, + { + "epoch": 1.27, + "learning_rate": 9.287061183550653e-05, + "loss": 1.0814, + "theoretical_loss": 3.356007146576479, + "tokens_seen": 2921711616 + }, + { + "epoch": 1.27, + "learning_rate": 9.285456369107323e-05, + "loss": 1.1573, + "theoretical_loss": 3.356001377043955, + "tokens_seen": 2921777152 + }, + { + "epoch": 1.27, + "learning_rate": 9.283851554663993e-05, + "loss": 1.1153, + "theoretical_loss": 3.355995607677076, + "tokens_seen": 2921842688 + }, + { + "epoch": 1.27, + "learning_rate": 9.282246740220662e-05, + "loss": 1.2305, + "theoretical_loss": 3.355989838475833, + "tokens_seen": 2921908224 + }, + { + "epoch": 1.27, + "learning_rate": 9.280641925777332e-05, + "loss": 1.2318, + "theoretical_loss": 3.355984069440217, + "tokens_seen": 2921973760 + }, + { + "epoch": 1.27, + "learning_rate": 9.279037111334002e-05, + "loss": 1.2285, + "theoretical_loss": 3.3559783005702206, + "tokens_seen": 2922039296 + }, + { + "epoch": 1.27, + "learning_rate": 9.277432296890672e-05, + "loss": 1.2014, + "theoretical_loss": 3.3559725318658353, + "tokens_seen": 2922104832 + }, + { + "epoch": 1.27, + "learning_rate": 9.275827482447342e-05, + "loss": 1.2476, + "theoretical_loss": 3.355966763327052, + "tokens_seen": 2922170368 + }, + { + "epoch": 1.27, + "learning_rate": 9.274222668004012e-05, + "loss": 1.1672, + "theoretical_loss": 3.355960994953862, + "tokens_seen": 2922235904 + }, + { + "epoch": 1.27, + "learning_rate": 9.272617853560682e-05, + "loss": 1.2698, + "theoretical_loss": 3.3559552267462576, + "tokens_seen": 2922301440 + }, + { + "epoch": 1.27, + "learning_rate": 9.271013039117352e-05, + "loss": 1.1266, + "theoretical_loss": 3.35594945870423, + "tokens_seen": 2922366976 + }, + { + "epoch": 1.27, + "learning_rate": 9.269408224674022e-05, + "loss": 1.1005, + "theoretical_loss": 3.355943690827771, + "tokens_seen": 2922432512 + }, + { + "epoch": 1.27, + "learning_rate": 9.267803410230693e-05, + "loss": 1.2341, + "theoretical_loss": 3.3559379231168713, + "tokens_seen": 2922498048 + }, + { + "epoch": 1.27, + "learning_rate": 9.266198595787363e-05, + "loss": 1.2685, + "theoretical_loss": 3.3559321555715234, + "tokens_seen": 2922563584 + }, + { + "epoch": 1.27, + "learning_rate": 9.264593781344033e-05, + "loss": 1.0936, + "theoretical_loss": 3.3559263881917185, + "tokens_seen": 2922629120 + }, + { + "epoch": 1.27, + "learning_rate": 9.262988966900703e-05, + "loss": 1.1501, + "theoretical_loss": 3.355920620977448, + "tokens_seen": 2922694656 + }, + { + "epoch": 1.27, + "learning_rate": 9.261384152457373e-05, + "loss": 1.2301, + "theoretical_loss": 3.3559148539287036, + "tokens_seen": 2922760192 + }, + { + "epoch": 1.27, + "learning_rate": 9.259779338014043e-05, + "loss": 1.2121, + "theoretical_loss": 3.3559090870454766, + "tokens_seen": 2922825728 + }, + { + "epoch": 1.27, + "objective/train/docs_used": 1079191, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1836445331573486, + "objective/train/theoretical_loss": 3.355903320327759, + "objective/train/tokens_used": 2943351264, + "theoretical_loss": 3.355903320327759, + "tokens_seen": 2922891264 + }, + { + "epoch": 1.27, + "learning_rate": 9.258174523570713e-05, + "loss": 1.1836, + "theoretical_loss": 3.355903320327759, + "tokens_seen": 2922891264 + }, + { + "epoch": 1.27, + "learning_rate": 9.256569709127383e-05, + "loss": 1.1779, + "theoretical_loss": 3.355897553775542, + "tokens_seen": 2922956800 + }, + { + "epoch": 1.27, + "learning_rate": 9.254964894684053e-05, + "loss": 1.1857, + "theoretical_loss": 3.355891787388817, + "tokens_seen": 2923022336 + }, + { + "epoch": 1.27, + "learning_rate": 9.253360080240723e-05, + "loss": 1.1252, + "theoretical_loss": 3.355886021167576, + "tokens_seen": 2923087872 + }, + { + "epoch": 1.27, + "learning_rate": 9.251755265797393e-05, + "loss": 1.2106, + "theoretical_loss": 3.35588025511181, + "tokens_seen": 2923153408 + }, + { + "epoch": 1.27, + "learning_rate": 9.250150451354063e-05, + "loss": 1.1373, + "theoretical_loss": 3.355874489221511, + "tokens_seen": 2923218944 + }, + { + "epoch": 1.27, + "learning_rate": 9.248545636910732e-05, + "loss": 1.1691, + "theoretical_loss": 3.35586872349667, + "tokens_seen": 2923284480 + }, + { + "epoch": 1.27, + "learning_rate": 9.246940822467402e-05, + "loss": 1.1712, + "theoretical_loss": 3.3558629579372794, + "tokens_seen": 2923350016 + }, + { + "epoch": 1.27, + "learning_rate": 9.245336008024072e-05, + "loss": 1.3198, + "theoretical_loss": 3.35585719254333, + "tokens_seen": 2923415552 + }, + { + "epoch": 1.27, + "learning_rate": 9.243731193580742e-05, + "loss": 1.1832, + "theoretical_loss": 3.3558514273148132, + "tokens_seen": 2923481088 + }, + { + "epoch": 1.27, + "learning_rate": 9.242126379137412e-05, + "loss": 1.2838, + "theoretical_loss": 3.3558456622517214, + "tokens_seen": 2923546624 + }, + { + "epoch": 1.27, + "learning_rate": 9.240521564694082e-05, + "loss": 1.1393, + "theoretical_loss": 3.3558398973540458, + "tokens_seen": 2923612160 + }, + { + "epoch": 1.27, + "learning_rate": 9.238916750250752e-05, + "loss": 1.2024, + "theoretical_loss": 3.3558341326217773, + "tokens_seen": 2923677696 + }, + { + "epoch": 1.27, + "learning_rate": 9.237311935807422e-05, + "loss": 1.1977, + "theoretical_loss": 3.3558283680549086, + "tokens_seen": 2923743232 + }, + { + "epoch": 1.27, + "learning_rate": 9.235707121364092e-05, + "loss": 1.1813, + "theoretical_loss": 3.35582260365343, + "tokens_seen": 2923808768 + }, + { + "epoch": 1.27, + "learning_rate": 9.234102306920764e-05, + "loss": 1.1533, + "theoretical_loss": 3.355816839417334, + "tokens_seen": 2923874304 + }, + { + "epoch": 1.27, + "learning_rate": 9.232497492477433e-05, + "loss": 1.3611, + "theoretical_loss": 3.3558110753466117, + "tokens_seen": 2923939840 + }, + { + "epoch": 1.27, + "learning_rate": 9.230892678034103e-05, + "loss": 1.0685, + "theoretical_loss": 3.355805311441255, + "tokens_seen": 2924005376 + }, + { + "epoch": 1.27, + "learning_rate": 9.229287863590773e-05, + "loss": 1.1806, + "theoretical_loss": 3.355799547701255, + "tokens_seen": 2924070912 + }, + { + "epoch": 1.27, + "learning_rate": 9.227683049147443e-05, + "loss": 1.0763, + "theoretical_loss": 3.3557937841266035, + "tokens_seen": 2924136448 + }, + { + "epoch": 1.27, + "learning_rate": 9.226078234704113e-05, + "loss": 1.2147, + "theoretical_loss": 3.355788020717292, + "tokens_seen": 2924201984 + }, + { + "epoch": 1.27, + "learning_rate": 9.224473420260783e-05, + "loss": 1.1584, + "theoretical_loss": 3.355782257473312, + "tokens_seen": 2924267520 + }, + { + "epoch": 1.27, + "learning_rate": 9.222868605817453e-05, + "loss": 1.2562, + "theoretical_loss": 3.355776494394655, + "tokens_seen": 2924333056 + }, + { + "epoch": 1.27, + "learning_rate": 9.221263791374123e-05, + "loss": 1.1066, + "theoretical_loss": 3.355770731481313, + "tokens_seen": 2924398592 + }, + { + "epoch": 1.27, + "learning_rate": 9.219658976930793e-05, + "loss": 1.2105, + "theoretical_loss": 3.355764968733277, + "tokens_seen": 2924464128 + }, + { + "epoch": 1.27, + "learning_rate": 9.218054162487463e-05, + "loss": 1.2525, + "theoretical_loss": 3.355759206150539, + "tokens_seen": 2924529664 + }, + { + "epoch": 1.27, + "learning_rate": 9.216449348044133e-05, + "loss": 1.1911, + "theoretical_loss": 3.3557534437330903, + "tokens_seen": 2924595200 + }, + { + "epoch": 1.27, + "learning_rate": 9.214844533600803e-05, + "loss": 1.1984, + "theoretical_loss": 3.355747681480922, + "tokens_seen": 2924660736 + }, + { + "epoch": 1.27, + "learning_rate": 9.213239719157472e-05, + "loss": 1.1304, + "theoretical_loss": 3.3557419193940268, + "tokens_seen": 2924726272 + }, + { + "epoch": 1.27, + "learning_rate": 9.211634904714142e-05, + "loss": 1.301, + "theoretical_loss": 3.355736157472395, + "tokens_seen": 2924791808 + }, + { + "epoch": 1.27, + "learning_rate": 9.210030090270812e-05, + "loss": 1.1272, + "theoretical_loss": 3.355730395716019, + "tokens_seen": 2924857344 + }, + { + "epoch": 1.27, + "learning_rate": 9.208425275827482e-05, + "loss": 1.1704, + "theoretical_loss": 3.3557246341248903, + "tokens_seen": 2924922880 + }, + { + "epoch": 1.27, + "learning_rate": 9.206820461384152e-05, + "loss": 1.1236, + "theoretical_loss": 3.3557188726989997, + "tokens_seen": 2924988416 + }, + { + "epoch": 1.27, + "learning_rate": 9.205215646940822e-05, + "loss": 1.1213, + "theoretical_loss": 3.35571311143834, + "tokens_seen": 2925053952 + }, + { + "epoch": 1.27, + "learning_rate": 9.203610832497494e-05, + "loss": 1.1665, + "theoretical_loss": 3.355707350342902, + "tokens_seen": 2925119488 + }, + { + "epoch": 1.27, + "learning_rate": 9.202006018054164e-05, + "loss": 1.1152, + "theoretical_loss": 3.3557015894126767, + "tokens_seen": 2925185024 + }, + { + "epoch": 1.27, + "learning_rate": 9.200401203610834e-05, + "loss": 1.2313, + "theoretical_loss": 3.355695828647657, + "tokens_seen": 2925250560 + }, + { + "epoch": 1.27, + "learning_rate": 9.198796389167504e-05, + "loss": 1.1608, + "theoretical_loss": 3.3556900680478337, + "tokens_seen": 2925316096 + }, + { + "epoch": 1.27, + "learning_rate": 9.197191574724173e-05, + "loss": 1.1344, + "theoretical_loss": 3.355684307613198, + "tokens_seen": 2925381632 + }, + { + "epoch": 1.27, + "learning_rate": 9.195586760280843e-05, + "loss": 1.1594, + "theoretical_loss": 3.3556785473437425, + "tokens_seen": 2925447168 + }, + { + "epoch": 1.27, + "learning_rate": 9.193981945837513e-05, + "loss": 1.1394, + "theoretical_loss": 3.355672787239458, + "tokens_seen": 2925512704 + }, + { + "epoch": 1.27, + "learning_rate": 9.192377131394183e-05, + "loss": 1.0994, + "theoretical_loss": 3.3556670273003357, + "tokens_seen": 2925578240 + }, + { + "epoch": 1.27, + "learning_rate": 9.190772316950853e-05, + "loss": 1.1914, + "theoretical_loss": 3.355661267526368, + "tokens_seen": 2925643776 + }, + { + "epoch": 1.27, + "learning_rate": 9.189167502507523e-05, + "loss": 1.1406, + "theoretical_loss": 3.3556555079175463, + "tokens_seen": 2925709312 + }, + { + "epoch": 1.27, + "learning_rate": 9.187562688064193e-05, + "loss": 1.263, + "theoretical_loss": 3.355649748473862, + "tokens_seen": 2925774848 + }, + { + "epoch": 1.27, + "learning_rate": 9.185957873620863e-05, + "loss": 1.1957, + "theoretical_loss": 3.3556439891953067, + "tokens_seen": 2925840384 + }, + { + "epoch": 1.27, + "learning_rate": 9.184353059177533e-05, + "loss": 1.119, + "theoretical_loss": 3.3556382300818717, + "tokens_seen": 2925905920 + }, + { + "epoch": 1.27, + "learning_rate": 9.182748244734203e-05, + "loss": 1.1775, + "theoretical_loss": 3.355632471133549, + "tokens_seen": 2925971456 + }, + { + "epoch": 1.27, + "learning_rate": 9.181143430290873e-05, + "loss": 1.0969, + "theoretical_loss": 3.35562671235033, + "tokens_seen": 2926036992 + }, + { + "epoch": 1.27, + "learning_rate": 9.179538615847543e-05, + "loss": 1.1875, + "theoretical_loss": 3.355620953732206, + "tokens_seen": 2926102528 + }, + { + "epoch": 1.27, + "learning_rate": 9.177933801404212e-05, + "loss": 1.0429, + "theoretical_loss": 3.355615195279169, + "tokens_seen": 2926168064 + }, + { + "epoch": 1.27, + "learning_rate": 9.176328986960882e-05, + "loss": 1.196, + "theoretical_loss": 3.3556094369912106, + "tokens_seen": 2926233600 + }, + { + "epoch": 1.27, + "learning_rate": 9.174724172517552e-05, + "loss": 1.1012, + "theoretical_loss": 3.355603678868322, + "tokens_seen": 2926299136 + }, + { + "epoch": 1.27, + "learning_rate": 9.173119358074224e-05, + "loss": 1.0958, + "theoretical_loss": 3.355597920910495, + "tokens_seen": 2926364672 + }, + { + "epoch": 1.27, + "learning_rate": 9.171514543630894e-05, + "loss": 1.2158, + "theoretical_loss": 3.355592163117721, + "tokens_seen": 2926430208 + }, + { + "epoch": 1.27, + "learning_rate": 9.169909729187564e-05, + "loss": 1.1472, + "theoretical_loss": 3.3555864054899915, + "tokens_seen": 2926495744 + }, + { + "epoch": 1.27, + "learning_rate": 9.168304914744234e-05, + "loss": 1.2316, + "theoretical_loss": 3.3555806480272987, + "tokens_seen": 2926561280 + }, + { + "epoch": 1.27, + "learning_rate": 9.166700100300904e-05, + "loss": 1.1762, + "theoretical_loss": 3.3555748907296334, + "tokens_seen": 2926626816 + }, + { + "epoch": 1.27, + "learning_rate": 9.165095285857574e-05, + "loss": 1.2584, + "theoretical_loss": 3.3555691335969877, + "tokens_seen": 2926692352 + }, + { + "epoch": 1.27, + "learning_rate": 9.163490471414244e-05, + "loss": 1.1704, + "theoretical_loss": 3.3555633766293527, + "tokens_seen": 2926757888 + }, + { + "epoch": 1.27, + "learning_rate": 9.161885656970913e-05, + "loss": 1.2441, + "theoretical_loss": 3.3555576198267207, + "tokens_seen": 2926823424 + }, + { + "epoch": 1.27, + "learning_rate": 9.160280842527583e-05, + "loss": 1.1976, + "theoretical_loss": 3.355551863189082, + "tokens_seen": 2926888960 + }, + { + "epoch": 1.27, + "learning_rate": 9.158676028084253e-05, + "loss": 1.1473, + "theoretical_loss": 3.35554610671643, + "tokens_seen": 2926954496 + }, + { + "epoch": 1.27, + "learning_rate": 9.157071213640923e-05, + "loss": 1.1918, + "theoretical_loss": 3.355540350408755, + "tokens_seen": 2927020032 + }, + { + "epoch": 1.27, + "learning_rate": 9.155466399197593e-05, + "loss": 1.2459, + "theoretical_loss": 3.355534594266049, + "tokens_seen": 2927085568 + }, + { + "epoch": 1.27, + "learning_rate": 9.153861584754263e-05, + "loss": 1.2846, + "theoretical_loss": 3.355528838288303, + "tokens_seen": 2927151104 + }, + { + "epoch": 1.27, + "learning_rate": 9.152256770310933e-05, + "loss": 1.1535, + "theoretical_loss": 3.355523082475509, + "tokens_seen": 2927216640 + }, + { + "epoch": 1.27, + "learning_rate": 9.150651955867603e-05, + "loss": 1.0826, + "theoretical_loss": 3.355517326827659, + "tokens_seen": 2927282176 + }, + { + "epoch": 1.27, + "learning_rate": 9.149047141424273e-05, + "loss": 1.1354, + "theoretical_loss": 3.355511571344744, + "tokens_seen": 2927347712 + }, + { + "epoch": 1.27, + "learning_rate": 9.147442326980943e-05, + "loss": 1.1369, + "theoretical_loss": 3.3555058160267555, + "tokens_seen": 2927413248 + }, + { + "epoch": 1.27, + "learning_rate": 9.145837512537613e-05, + "loss": 1.1714, + "theoretical_loss": 3.355500060873686, + "tokens_seen": 2927478784 + }, + { + "epoch": 1.27, + "learning_rate": 9.144232698094283e-05, + "loss": 1.2554, + "theoretical_loss": 3.3554943058855256, + "tokens_seen": 2927544320 + }, + { + "epoch": 1.27, + "learning_rate": 9.142627883650953e-05, + "loss": 1.2209, + "theoretical_loss": 3.355488551062267, + "tokens_seen": 2927609856 + }, + { + "epoch": 1.27, + "learning_rate": 9.141023069207623e-05, + "loss": 1.1382, + "theoretical_loss": 3.355482796403902, + "tokens_seen": 2927675392 + }, + { + "epoch": 1.27, + "learning_rate": 9.139418254764294e-05, + "loss": 1.1997, + "theoretical_loss": 3.355477041910421, + "tokens_seen": 2927740928 + }, + { + "epoch": 1.27, + "learning_rate": 9.137813440320964e-05, + "loss": 1.167, + "theoretical_loss": 3.3554712875818167, + "tokens_seen": 2927806464 + }, + { + "epoch": 1.27, + "learning_rate": 9.136208625877634e-05, + "loss": 1.1973, + "theoretical_loss": 3.35546553341808, + "tokens_seen": 2927872000 + }, + { + "epoch": 1.27, + "learning_rate": 9.134603811434304e-05, + "loss": 1.221, + "theoretical_loss": 3.3554597794192027, + "tokens_seen": 2927937536 + }, + { + "epoch": 1.27, + "learning_rate": 9.132998996990974e-05, + "loss": 1.3759, + "theoretical_loss": 3.3554540255851766, + "tokens_seen": 2928003072 + }, + { + "epoch": 1.27, + "learning_rate": 9.131394182547644e-05, + "loss": 1.2074, + "theoretical_loss": 3.3554482719159933, + "tokens_seen": 2928068608 + }, + { + "epoch": 1.27, + "learning_rate": 9.129789368104314e-05, + "loss": 1.1692, + "theoretical_loss": 3.355442518411644, + "tokens_seen": 2928134144 + }, + { + "epoch": 1.27, + "learning_rate": 9.128184553660984e-05, + "loss": 1.1902, + "theoretical_loss": 3.3554367650721204, + "tokens_seen": 2928199680 + }, + { + "epoch": 1.27, + "learning_rate": 9.126579739217653e-05, + "loss": 1.1323, + "theoretical_loss": 3.355431011897414, + "tokens_seen": 2928265216 + }, + { + "epoch": 1.27, + "learning_rate": 9.124974924774323e-05, + "loss": 1.2256, + "theoretical_loss": 3.355425258887517, + "tokens_seen": 2928330752 + }, + { + "epoch": 1.27, + "learning_rate": 9.123370110330993e-05, + "loss": 1.2091, + "theoretical_loss": 3.3554195060424203, + "tokens_seen": 2928396288 + }, + { + "epoch": 1.27, + "learning_rate": 9.121765295887663e-05, + "loss": 1.145, + "theoretical_loss": 3.355413753362116, + "tokens_seen": 2928461824 + }, + { + "epoch": 1.27, + "learning_rate": 9.120160481444333e-05, + "loss": 1.1288, + "theoretical_loss": 3.355408000846595, + "tokens_seen": 2928527360 + }, + { + "epoch": 1.27, + "learning_rate": 9.118555667001003e-05, + "loss": 1.2939, + "theoretical_loss": 3.3554022484958494, + "tokens_seen": 2928592896 + }, + { + "epoch": 1.27, + "learning_rate": 9.116950852557673e-05, + "loss": 1.2095, + "theoretical_loss": 3.3553964963098712, + "tokens_seen": 2928658432 + }, + { + "epoch": 1.27, + "learning_rate": 9.115346038114343e-05, + "loss": 1.0776, + "theoretical_loss": 3.355390744288651, + "tokens_seen": 2928723968 + }, + { + "epoch": 1.27, + "learning_rate": 9.113741223671015e-05, + "loss": 1.2547, + "theoretical_loss": 3.3553849924321812, + "tokens_seen": 2928789504 + }, + { + "epoch": 1.27, + "learning_rate": 9.112136409227685e-05, + "loss": 1.2, + "theoretical_loss": 3.355379240740453, + "tokens_seen": 2928855040 + }, + { + "epoch": 1.27, + "learning_rate": 9.110531594784353e-05, + "loss": 1.1097, + "theoretical_loss": 3.3553734892134583, + "tokens_seen": 2928920576 + }, + { + "epoch": 1.27, + "learning_rate": 9.108926780341023e-05, + "loss": 1.2953, + "theoretical_loss": 3.3553677378511884, + "tokens_seen": 2928986112 + }, + { + "epoch": 1.27, + "learning_rate": 9.107321965897693e-05, + "loss": 1.2197, + "theoretical_loss": 3.3553619866536346, + "tokens_seen": 2929051648 + }, + { + "epoch": 1.27, + "learning_rate": 9.105717151454364e-05, + "loss": 1.2723, + "theoretical_loss": 3.3553562356207896, + "tokens_seen": 2929117184 + }, + { + "epoch": 1.27, + "learning_rate": 9.104112337011034e-05, + "loss": 1.2462, + "theoretical_loss": 3.355350484752644, + "tokens_seen": 2929182720 + }, + { + "epoch": 1.27, + "learning_rate": 9.102507522567704e-05, + "loss": 1.194, + "theoretical_loss": 3.355344734049189, + "tokens_seen": 2929248256 + }, + { + "epoch": 1.27, + "learning_rate": 9.100902708124374e-05, + "loss": 1.1561, + "theoretical_loss": 3.3553389835104177, + "tokens_seen": 2929313792 + }, + { + "epoch": 1.27, + "learning_rate": 9.099297893681044e-05, + "loss": 1.2059, + "theoretical_loss": 3.355333233136321, + "tokens_seen": 2929379328 + }, + { + "epoch": 1.27, + "objective/train/docs_used": 1081316, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1302776336669922, + "objective/train/theoretical_loss": 3.35532748292689, + "objective/train/tokens_used": 2949904864, + "theoretical_loss": 3.35532748292689, + "tokens_seen": 2929444864 + }, + { + "epoch": 1.27, + "learning_rate": 9.097693079237714e-05, + "loss": 1.1303, + "theoretical_loss": 3.35532748292689, + "tokens_seen": 2929444864 + }, + { + "epoch": 1.27, + "learning_rate": 9.096088264794384e-05, + "loss": 1.2039, + "theoretical_loss": 3.3553217328821168, + "tokens_seen": 2929510400 + }, + { + "epoch": 1.27, + "learning_rate": 9.094483450351054e-05, + "loss": 1.2342, + "theoretical_loss": 3.3553159830019927, + "tokens_seen": 2929575936 + }, + { + "epoch": 1.27, + "learning_rate": 9.092878635907724e-05, + "loss": 1.2986, + "theoretical_loss": 3.3553102332865095, + "tokens_seen": 2929641472 + }, + { + "epoch": 1.27, + "learning_rate": 9.091273821464393e-05, + "loss": 1.1961, + "theoretical_loss": 3.355304483735659, + "tokens_seen": 2929707008 + }, + { + "epoch": 1.27, + "learning_rate": 9.089669007021063e-05, + "loss": 1.148, + "theoretical_loss": 3.3552987343494327, + "tokens_seen": 2929772544 + }, + { + "epoch": 1.27, + "learning_rate": 9.088064192577733e-05, + "loss": 1.2822, + "theoretical_loss": 3.355292985127822, + "tokens_seen": 2929838080 + }, + { + "epoch": 1.27, + "learning_rate": 9.086459378134403e-05, + "loss": 1.196, + "theoretical_loss": 3.3552872360708186, + "tokens_seen": 2929903616 + }, + { + "epoch": 1.27, + "learning_rate": 9.084854563691073e-05, + "loss": 1.1127, + "theoretical_loss": 3.355281487178414, + "tokens_seen": 2929969152 + }, + { + "epoch": 1.27, + "learning_rate": 9.083249749247744e-05, + "loss": 1.2736, + "theoretical_loss": 3.3552757384506, + "tokens_seen": 2930034688 + }, + { + "epoch": 1.27, + "learning_rate": 9.081644934804414e-05, + "loss": 1.2043, + "theoretical_loss": 3.355269989887368, + "tokens_seen": 2930100224 + }, + { + "epoch": 1.27, + "learning_rate": 9.080040120361085e-05, + "loss": 1.1476, + "theoretical_loss": 3.35526424148871, + "tokens_seen": 2930165760 + }, + { + "epoch": 1.27, + "learning_rate": 9.078435305917755e-05, + "loss": 1.2358, + "theoretical_loss": 3.355258493254617, + "tokens_seen": 2930231296 + }, + { + "epoch": 1.27, + "learning_rate": 9.076830491474425e-05, + "loss": 1.2355, + "theoretical_loss": 3.355252745185081, + "tokens_seen": 2930296832 + }, + { + "epoch": 1.27, + "learning_rate": 9.075225677031093e-05, + "loss": 1.2966, + "theoretical_loss": 3.355246997280094, + "tokens_seen": 2930362368 + }, + { + "epoch": 1.27, + "learning_rate": 9.073620862587763e-05, + "loss": 1.2255, + "theoretical_loss": 3.3552412495396466, + "tokens_seen": 2930427904 + }, + { + "epoch": 1.27, + "learning_rate": 9.072016048144434e-05, + "loss": 1.3236, + "theoretical_loss": 3.3552355019637314, + "tokens_seen": 2930493440 + }, + { + "epoch": 1.27, + "learning_rate": 9.070411233701104e-05, + "loss": 1.3804, + "theoretical_loss": 3.3552297545523393, + "tokens_seen": 2930558976 + }, + { + "epoch": 1.27, + "learning_rate": 9.068806419257774e-05, + "loss": 1.1997, + "theoretical_loss": 3.355224007305462, + "tokens_seen": 2930624512 + }, + { + "epoch": 1.27, + "learning_rate": 9.067201604814444e-05, + "loss": 1.2816, + "theoretical_loss": 3.3552182602230918, + "tokens_seen": 2930690048 + }, + { + "epoch": 1.27, + "learning_rate": 9.065596790371114e-05, + "loss": 1.0796, + "theoretical_loss": 3.3552125133052195, + "tokens_seen": 2930755584 + }, + { + "epoch": 1.27, + "learning_rate": 9.063991975927784e-05, + "loss": 1.144, + "theoretical_loss": 3.3552067665518375, + "tokens_seen": 2930821120 + }, + { + "epoch": 1.27, + "learning_rate": 9.062387161484454e-05, + "loss": 1.3099, + "theoretical_loss": 3.3552010199629367, + "tokens_seen": 2930886656 + }, + { + "epoch": 1.27, + "learning_rate": 9.060782347041124e-05, + "loss": 1.1626, + "theoretical_loss": 3.355195273538509, + "tokens_seen": 2930952192 + }, + { + "epoch": 1.27, + "learning_rate": 9.059177532597794e-05, + "loss": 1.1455, + "theoretical_loss": 3.3551895272785455, + "tokens_seen": 2931017728 + }, + { + "epoch": 1.27, + "learning_rate": 9.057572718154464e-05, + "loss": 1.1601, + "theoretical_loss": 3.355183781183039, + "tokens_seen": 2931083264 + }, + { + "epoch": 1.27, + "learning_rate": 9.055967903711133e-05, + "loss": 1.1492, + "theoretical_loss": 3.35517803525198, + "tokens_seen": 2931148800 + }, + { + "epoch": 1.27, + "learning_rate": 9.054363089267803e-05, + "loss": 1.1451, + "theoretical_loss": 3.3551722894853606, + "tokens_seen": 2931214336 + }, + { + "epoch": 1.27, + "learning_rate": 9.052758274824474e-05, + "loss": 1.1021, + "theoretical_loss": 3.355166543883173, + "tokens_seen": 2931279872 + }, + { + "epoch": 1.27, + "learning_rate": 9.051153460381144e-05, + "loss": 1.1702, + "theoretical_loss": 3.3551607984454073, + "tokens_seen": 2931345408 + }, + { + "epoch": 1.27, + "learning_rate": 9.049548645937814e-05, + "loss": 1.2108, + "theoretical_loss": 3.355155053172056, + "tokens_seen": 2931410944 + }, + { + "epoch": 1.27, + "learning_rate": 9.047943831494484e-05, + "loss": 1.251, + "theoretical_loss": 3.355149308063111, + "tokens_seen": 2931476480 + }, + { + "epoch": 1.27, + "learning_rate": 9.046339017051155e-05, + "loss": 1.2404, + "theoretical_loss": 3.3551435631185633, + "tokens_seen": 2931542016 + }, + { + "epoch": 1.27, + "learning_rate": 9.044734202607825e-05, + "loss": 1.1002, + "theoretical_loss": 3.355137818338405, + "tokens_seen": 2931607552 + }, + { + "epoch": 1.27, + "learning_rate": 9.043129388164495e-05, + "loss": 1.17, + "theoretical_loss": 3.3551320737226282, + "tokens_seen": 2931673088 + }, + { + "epoch": 1.27, + "learning_rate": 9.041524573721163e-05, + "loss": 1.2805, + "theoretical_loss": 3.355126329271223, + "tokens_seen": 2931738624 + }, + { + "epoch": 1.27, + "learning_rate": 9.039919759277833e-05, + "loss": 1.2259, + "theoretical_loss": 3.3551205849841823, + "tokens_seen": 2931804160 + }, + { + "epoch": 1.27, + "learning_rate": 9.038314944834503e-05, + "loss": 1.1142, + "theoretical_loss": 3.3551148408614977, + "tokens_seen": 2931869696 + }, + { + "epoch": 1.27, + "learning_rate": 9.036710130391174e-05, + "loss": 1.2842, + "theoretical_loss": 3.35510909690316, + "tokens_seen": 2931935232 + }, + { + "epoch": 1.27, + "learning_rate": 9.035105315947844e-05, + "loss": 1.2821, + "theoretical_loss": 3.355103353109161, + "tokens_seen": 2932000768 + }, + { + "epoch": 1.27, + "learning_rate": 9.033500501504514e-05, + "loss": 1.1737, + "theoretical_loss": 3.3550976094794933, + "tokens_seen": 2932066304 + }, + { + "epoch": 1.27, + "learning_rate": 9.031895687061184e-05, + "loss": 1.1378, + "theoretical_loss": 3.3550918660141473, + "tokens_seen": 2932131840 + }, + { + "epoch": 1.27, + "learning_rate": 9.030290872617854e-05, + "loss": 1.3521, + "theoretical_loss": 3.3550861227131157, + "tokens_seen": 2932197376 + }, + { + "epoch": 1.27, + "learning_rate": 9.028686058174524e-05, + "loss": 1.2334, + "theoretical_loss": 3.355080379576389, + "tokens_seen": 2932262912 + }, + { + "epoch": 1.27, + "learning_rate": 9.027081243731194e-05, + "loss": 1.119, + "theoretical_loss": 3.35507463660396, + "tokens_seen": 2932328448 + }, + { + "epoch": 1.27, + "learning_rate": 9.025476429287864e-05, + "loss": 1.2064, + "theoretical_loss": 3.3550688937958193, + "tokens_seen": 2932393984 + }, + { + "epoch": 1.27, + "learning_rate": 9.023871614844534e-05, + "loss": 1.2428, + "theoretical_loss": 3.355063151151959, + "tokens_seen": 2932459520 + }, + { + "epoch": 1.27, + "learning_rate": 9.022266800401204e-05, + "loss": 1.203, + "theoretical_loss": 3.355057408672371, + "tokens_seen": 2932525056 + }, + { + "epoch": 1.27, + "learning_rate": 9.020661985957874e-05, + "loss": 1.1332, + "theoretical_loss": 3.3550516663570464, + "tokens_seen": 2932590592 + }, + { + "epoch": 1.27, + "learning_rate": 9.019057171514544e-05, + "loss": 1.2479, + "theoretical_loss": 3.355045924205977, + "tokens_seen": 2932656128 + }, + { + "epoch": 1.27, + "learning_rate": 9.017452357071214e-05, + "loss": 1.2865, + "theoretical_loss": 3.3550401822191547, + "tokens_seen": 2932721664 + }, + { + "epoch": 1.27, + "learning_rate": 9.015847542627884e-05, + "loss": 1.333, + "theoretical_loss": 3.3550344403965706, + "tokens_seen": 2932787200 + }, + { + "epoch": 1.27, + "learning_rate": 9.014242728184554e-05, + "loss": 1.1908, + "theoretical_loss": 3.355028698738217, + "tokens_seen": 2932852736 + }, + { + "epoch": 1.27, + "learning_rate": 9.012637913741224e-05, + "loss": 1.2617, + "theoretical_loss": 3.355022957244085, + "tokens_seen": 2932918272 + }, + { + "epoch": 1.27, + "learning_rate": 9.011033099297895e-05, + "loss": 1.2143, + "theoretical_loss": 3.3550172159141667, + "tokens_seen": 2932983808 + }, + { + "epoch": 1.27, + "learning_rate": 9.009428284854565e-05, + "loss": 1.2479, + "theoretical_loss": 3.355011474748453, + "tokens_seen": 2933049344 + }, + { + "epoch": 1.27, + "learning_rate": 9.007823470411235e-05, + "loss": 1.0956, + "theoretical_loss": 3.3550057337469363, + "tokens_seen": 2933114880 + }, + { + "epoch": 1.27, + "learning_rate": 9.006218655967903e-05, + "loss": 1.18, + "theoretical_loss": 3.3549999929096077, + "tokens_seen": 2933180416 + }, + { + "epoch": 1.27, + "learning_rate": 9.004613841524573e-05, + "loss": 1.395, + "theoretical_loss": 3.3549942522364593, + "tokens_seen": 2933245952 + }, + { + "epoch": 1.27, + "learning_rate": 9.003009027081244e-05, + "loss": 1.1578, + "theoretical_loss": 3.3549885117274822, + "tokens_seen": 2933311488 + }, + { + "epoch": 1.27, + "learning_rate": 9.001404212637914e-05, + "loss": 1.2252, + "theoretical_loss": 3.3549827713826685, + "tokens_seen": 2933377024 + }, + { + "epoch": 1.27, + "learning_rate": 8.999799398194584e-05, + "loss": 1.4127, + "theoretical_loss": 3.3549770312020097, + "tokens_seen": 2933442560 + }, + { + "epoch": 1.27, + "learning_rate": 8.998194583751254e-05, + "loss": 1.2165, + "theoretical_loss": 3.354971291185498, + "tokens_seen": 2933508096 + }, + { + "epoch": 1.27, + "learning_rate": 8.996589769307924e-05, + "loss": 1.1826, + "theoretical_loss": 3.3549655513331236, + "tokens_seen": 2933573632 + }, + { + "epoch": 1.27, + "learning_rate": 8.994984954864594e-05, + "loss": 1.1054, + "theoretical_loss": 3.3549598116448793, + "tokens_seen": 2933639168 + }, + { + "epoch": 1.27, + "learning_rate": 8.993380140421265e-05, + "loss": 1.101, + "theoretical_loss": 3.3549540721207567, + "tokens_seen": 2933704704 + }, + { + "epoch": 1.27, + "learning_rate": 8.991775325977935e-05, + "loss": 1.1952, + "theoretical_loss": 3.3549483327607463, + "tokens_seen": 2933770240 + }, + { + "epoch": 1.27, + "learning_rate": 8.990170511534604e-05, + "loss": 1.218, + "theoretical_loss": 3.3549425935648416, + "tokens_seen": 2933835776 + }, + { + "epoch": 1.27, + "learning_rate": 8.988565697091274e-05, + "loss": 1.1993, + "theoretical_loss": 3.3549368545330327, + "tokens_seen": 2933901312 + }, + { + "epoch": 1.27, + "learning_rate": 8.986960882647944e-05, + "loss": 1.1773, + "theoretical_loss": 3.354931115665312, + "tokens_seen": 2933966848 + }, + { + "epoch": 1.27, + "learning_rate": 8.985356068204614e-05, + "loss": 1.1632, + "theoretical_loss": 3.354925376961671, + "tokens_seen": 2934032384 + }, + { + "epoch": 1.27, + "learning_rate": 8.983751253761284e-05, + "loss": 1.2485, + "theoretical_loss": 3.354919638422101, + "tokens_seen": 2934097920 + }, + { + "epoch": 1.27, + "learning_rate": 8.982146439317954e-05, + "loss": 1.1905, + "theoretical_loss": 3.3549139000465944, + "tokens_seen": 2934163456 + }, + { + "epoch": 1.27, + "learning_rate": 8.980541624874624e-05, + "loss": 1.2703, + "theoretical_loss": 3.3549081618351417, + "tokens_seen": 2934228992 + }, + { + "epoch": 1.27, + "learning_rate": 8.978936810431294e-05, + "loss": 1.1497, + "theoretical_loss": 3.3549024237877356, + "tokens_seen": 2934294528 + }, + { + "epoch": 1.27, + "learning_rate": 8.977331995987965e-05, + "loss": 1.2598, + "theoretical_loss": 3.3548966859043676, + "tokens_seen": 2934360064 + }, + { + "epoch": 1.27, + "learning_rate": 8.975727181544635e-05, + "loss": 1.2868, + "theoretical_loss": 3.354890948185029, + "tokens_seen": 2934425600 + }, + { + "epoch": 1.27, + "learning_rate": 8.974122367101305e-05, + "loss": 1.2102, + "theoretical_loss": 3.3548852106297113, + "tokens_seen": 2934491136 + }, + { + "epoch": 1.27, + "learning_rate": 8.972517552657975e-05, + "loss": 1.1445, + "theoretical_loss": 3.3548794732384066, + "tokens_seen": 2934556672 + }, + { + "epoch": 1.27, + "learning_rate": 8.970912738214643e-05, + "loss": 1.2507, + "theoretical_loss": 3.3548737360111063, + "tokens_seen": 2934622208 + }, + { + "epoch": 1.27, + "learning_rate": 8.969307923771313e-05, + "loss": 1.131, + "theoretical_loss": 3.3548679989478023, + "tokens_seen": 2934687744 + }, + { + "epoch": 1.27, + "learning_rate": 8.967703109327984e-05, + "loss": 1.1206, + "theoretical_loss": 3.354862262048486, + "tokens_seen": 2934753280 + }, + { + "epoch": 1.27, + "learning_rate": 8.966098294884654e-05, + "loss": 1.2144, + "theoretical_loss": 3.354856525313149, + "tokens_seen": 2934818816 + }, + { + "epoch": 1.27, + "learning_rate": 8.964493480441324e-05, + "loss": 1.0963, + "theoretical_loss": 3.354850788741783, + "tokens_seen": 2934884352 + }, + { + "epoch": 1.27, + "learning_rate": 8.962888665997995e-05, + "loss": 1.206, + "theoretical_loss": 3.35484505233438, + "tokens_seen": 2934949888 + }, + { + "epoch": 1.27, + "learning_rate": 8.961283851554665e-05, + "loss": 1.3295, + "theoretical_loss": 3.3548393160909313, + "tokens_seen": 2935015424 + }, + { + "epoch": 1.27, + "learning_rate": 8.959679037111335e-05, + "loss": 1.1345, + "theoretical_loss": 3.3548335800114284, + "tokens_seen": 2935080960 + }, + { + "epoch": 1.27, + "learning_rate": 8.958074222668005e-05, + "loss": 1.1945, + "theoretical_loss": 3.354827844095863, + "tokens_seen": 2935146496 + }, + { + "epoch": 1.27, + "learning_rate": 8.956469408224675e-05, + "loss": 1.2437, + "theoretical_loss": 3.3548221083442273, + "tokens_seen": 2935212032 + }, + { + "epoch": 1.27, + "learning_rate": 8.954864593781344e-05, + "loss": 1.27, + "theoretical_loss": 3.3548163727565123, + "tokens_seen": 2935277568 + }, + { + "epoch": 1.27, + "learning_rate": 8.953259779338014e-05, + "loss": 1.224, + "theoretical_loss": 3.3548106373327107, + "tokens_seen": 2935343104 + }, + { + "epoch": 1.27, + "learning_rate": 8.951654964894684e-05, + "loss": 1.1385, + "theoretical_loss": 3.3548049020728126, + "tokens_seen": 2935408640 + }, + { + "epoch": 1.27, + "learning_rate": 8.950050150451354e-05, + "loss": 1.118, + "theoretical_loss": 3.3547991669768105, + "tokens_seen": 2935474176 + }, + { + "epoch": 1.27, + "learning_rate": 8.948445336008024e-05, + "loss": 1.2264, + "theoretical_loss": 3.3547934320446964, + "tokens_seen": 2935539712 + }, + { + "epoch": 1.27, + "learning_rate": 8.946840521564694e-05, + "loss": 1.2857, + "theoretical_loss": 3.3547876972764614, + "tokens_seen": 2935605248 + }, + { + "epoch": 1.27, + "learning_rate": 8.945235707121364e-05, + "loss": 1.2231, + "theoretical_loss": 3.354781962672097, + "tokens_seen": 2935670784 + }, + { + "epoch": 1.27, + "learning_rate": 8.943630892678035e-05, + "loss": 1.1718, + "theoretical_loss": 3.3547762282315956, + "tokens_seen": 2935736320 + }, + { + "epoch": 1.27, + "learning_rate": 8.942026078234705e-05, + "loss": 1.1876, + "theoretical_loss": 3.3547704939549483, + "tokens_seen": 2935801856 + }, + { + "epoch": 1.27, + "learning_rate": 8.940421263791375e-05, + "loss": 1.1702, + "theoretical_loss": 3.354764759842147, + "tokens_seen": 2935867392 + }, + { + "epoch": 1.27, + "learning_rate": 8.938816449348045e-05, + "loss": 1.0864, + "theoretical_loss": 3.354759025893183, + "tokens_seen": 2935932928 + }, + { + "epoch": 1.27, + "objective/train/docs_used": 1083688, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2070889472961426, + "objective/train/theoretical_loss": 3.3547532921080485, + "objective/train/tokens_used": 2956458464, + "theoretical_loss": 3.3547532921080485, + "tokens_seen": 2935998464 + }, + { + "epoch": 1.27, + "learning_rate": 8.937211634904715e-05, + "loss": 1.2071, + "theoretical_loss": 3.3547532921080485, + "tokens_seen": 2935998464 + }, + { + "epoch": 1.27, + "learning_rate": 8.935606820461383e-05, + "loss": 1.1344, + "theoretical_loss": 3.354747558486735, + "tokens_seen": 2936064000 + }, + { + "epoch": 1.27, + "learning_rate": 8.934002006018054e-05, + "loss": 1.1763, + "theoretical_loss": 3.3547418250292336, + "tokens_seen": 2936129536 + }, + { + "epoch": 1.27, + "learning_rate": 8.932397191574725e-05, + "loss": 1.1648, + "theoretical_loss": 3.354736091735537, + "tokens_seen": 2936195072 + }, + { + "epoch": 1.27, + "learning_rate": 8.930792377131395e-05, + "loss": 1.195, + "theoretical_loss": 3.354730358605636, + "tokens_seen": 2936260608 + }, + { + "epoch": 1.27, + "learning_rate": 8.929187562688065e-05, + "loss": 1.2173, + "theoretical_loss": 3.3547246256395225, + "tokens_seen": 2936326144 + }, + { + "epoch": 1.27, + "learning_rate": 8.927582748244735e-05, + "loss": 1.2324, + "theoretical_loss": 3.354718892837188, + "tokens_seen": 2936391680 + }, + { + "epoch": 1.27, + "learning_rate": 8.925977933801405e-05, + "loss": 1.1483, + "theoretical_loss": 3.3547131601986244, + "tokens_seen": 2936457216 + }, + { + "epoch": 1.27, + "learning_rate": 8.924373119358075e-05, + "loss": 1.2639, + "theoretical_loss": 3.354707427723824, + "tokens_seen": 2936522752 + }, + { + "epoch": 1.27, + "learning_rate": 8.922768304914745e-05, + "loss": 1.1674, + "theoretical_loss": 3.354701695412777, + "tokens_seen": 2936588288 + }, + { + "epoch": 1.27, + "learning_rate": 8.921163490471415e-05, + "loss": 1.1818, + "theoretical_loss": 3.3546959632654763, + "tokens_seen": 2936653824 + }, + { + "epoch": 1.27, + "learning_rate": 8.919558676028084e-05, + "loss": 1.2001, + "theoretical_loss": 3.3546902312819133, + "tokens_seen": 2936719360 + }, + { + "epoch": 1.27, + "learning_rate": 8.917953861584754e-05, + "loss": 1.1477, + "theoretical_loss": 3.3546844994620795, + "tokens_seen": 2936784896 + }, + { + "epoch": 1.27, + "learning_rate": 8.916349047141424e-05, + "loss": 1.1201, + "theoretical_loss": 3.354678767805966, + "tokens_seen": 2936850432 + }, + { + "epoch": 1.27, + "learning_rate": 8.914744232698094e-05, + "loss": 1.1592, + "theoretical_loss": 3.3546730363135655, + "tokens_seen": 2936915968 + }, + { + "epoch": 1.27, + "learning_rate": 8.913139418254764e-05, + "loss": 1.1729, + "theoretical_loss": 3.3546673049848694, + "tokens_seen": 2936981504 + }, + { + "epoch": 1.27, + "learning_rate": 8.911534603811434e-05, + "loss": 1.1491, + "theoretical_loss": 3.354661573819869, + "tokens_seen": 2937047040 + }, + { + "epoch": 1.27, + "learning_rate": 8.909929789368104e-05, + "loss": 1.2338, + "theoretical_loss": 3.354655842818556, + "tokens_seen": 2937112576 + }, + { + "epoch": 1.27, + "learning_rate": 8.908324974924775e-05, + "loss": 1.1939, + "theoretical_loss": 3.3546501119809227, + "tokens_seen": 2937178112 + }, + { + "epoch": 1.27, + "learning_rate": 8.906720160481445e-05, + "loss": 1.1371, + "theoretical_loss": 3.3546443813069597, + "tokens_seen": 2937243648 + }, + { + "epoch": 1.27, + "learning_rate": 8.905115346038115e-05, + "loss": 1.2038, + "theoretical_loss": 3.3546386507966597, + "tokens_seen": 2937309184 + }, + { + "epoch": 1.27, + "learning_rate": 8.903510531594785e-05, + "loss": 1.228, + "theoretical_loss": 3.354632920450014, + "tokens_seen": 2937374720 + }, + { + "epoch": 1.27, + "learning_rate": 8.901905717151455e-05, + "loss": 1.1832, + "theoretical_loss": 3.354627190267014, + "tokens_seen": 2937440256 + }, + { + "epoch": 1.27, + "learning_rate": 8.900300902708125e-05, + "loss": 1.2807, + "theoretical_loss": 3.354621460247652, + "tokens_seen": 2937505792 + }, + { + "epoch": 1.27, + "learning_rate": 8.898696088264795e-05, + "loss": 1.2316, + "theoretical_loss": 3.354615730391919, + "tokens_seen": 2937571328 + }, + { + "epoch": 1.27, + "learning_rate": 8.897091273821465e-05, + "loss": 1.2815, + "theoretical_loss": 3.3546100006998065, + "tokens_seen": 2937636864 + }, + { + "epoch": 1.27, + "learning_rate": 8.895486459378135e-05, + "loss": 1.192, + "theoretical_loss": 3.3546042711713073, + "tokens_seen": 2937702400 + }, + { + "epoch": 1.27, + "learning_rate": 8.893881644934805e-05, + "loss": 1.1593, + "theoretical_loss": 3.3545985418064124, + "tokens_seen": 2937767936 + }, + { + "epoch": 1.27, + "learning_rate": 8.892276830491475e-05, + "loss": 1.1394, + "theoretical_loss": 3.354592812605113, + "tokens_seen": 2937833472 + }, + { + "epoch": 1.27, + "learning_rate": 8.890672016048145e-05, + "loss": 1.1473, + "theoretical_loss": 3.3545870835674023, + "tokens_seen": 2937899008 + }, + { + "epoch": 1.27, + "learning_rate": 8.889067201604815e-05, + "loss": 1.2385, + "theoretical_loss": 3.3545813546932703, + "tokens_seen": 2937964544 + }, + { + "epoch": 1.27, + "learning_rate": 8.887462387161485e-05, + "loss": 1.1769, + "theoretical_loss": 3.3545756259827093, + "tokens_seen": 2938030080 + }, + { + "epoch": 1.27, + "learning_rate": 8.885857572718155e-05, + "loss": 1.191, + "theoretical_loss": 3.354569897435711, + "tokens_seen": 2938095616 + }, + { + "epoch": 1.27, + "learning_rate": 8.884252758274824e-05, + "loss": 1.1718, + "theoretical_loss": 3.354564169052267, + "tokens_seen": 2938161152 + }, + { + "epoch": 1.27, + "learning_rate": 8.882647943831494e-05, + "loss": 1.2288, + "theoretical_loss": 3.3545584408323696, + "tokens_seen": 2938226688 + }, + { + "epoch": 1.27, + "learning_rate": 8.881043129388164e-05, + "loss": 1.228, + "theoretical_loss": 3.3545527127760097, + "tokens_seen": 2938292224 + }, + { + "epoch": 1.27, + "learning_rate": 8.879438314944834e-05, + "loss": 1.2075, + "theoretical_loss": 3.3545469848831795, + "tokens_seen": 2938357760 + }, + { + "epoch": 1.27, + "learning_rate": 8.877833500501504e-05, + "loss": 1.1034, + "theoretical_loss": 3.35454125715387, + "tokens_seen": 2938423296 + }, + { + "epoch": 1.27, + "learning_rate": 8.876228686058174e-05, + "loss": 1.1821, + "theoretical_loss": 3.354535529588073, + "tokens_seen": 2938488832 + }, + { + "epoch": 1.27, + "learning_rate": 8.874623871614845e-05, + "loss": 1.0922, + "theoretical_loss": 3.3545298021857812, + "tokens_seen": 2938554368 + }, + { + "epoch": 1.27, + "learning_rate": 8.873019057171516e-05, + "loss": 1.1965, + "theoretical_loss": 3.354524074946986, + "tokens_seen": 2938619904 + }, + { + "epoch": 1.27, + "learning_rate": 8.871414242728186e-05, + "loss": 1.2711, + "theoretical_loss": 3.354518347871678, + "tokens_seen": 2938685440 + }, + { + "epoch": 1.27, + "learning_rate": 8.869809428284855e-05, + "loss": 1.1753, + "theoretical_loss": 3.3545126209598495, + "tokens_seen": 2938750976 + }, + { + "epoch": 1.27, + "learning_rate": 8.868204613841525e-05, + "loss": 1.2875, + "theoretical_loss": 3.3545068942114926, + "tokens_seen": 2938816512 + }, + { + "epoch": 1.27, + "learning_rate": 8.866599799398195e-05, + "loss": 1.2679, + "theoretical_loss": 3.3545011676265983, + "tokens_seen": 2938882048 + }, + { + "epoch": 1.27, + "learning_rate": 8.864994984954865e-05, + "loss": 1.2457, + "theoretical_loss": 3.354495441205159, + "tokens_seen": 2938947584 + }, + { + "epoch": 1.27, + "learning_rate": 8.863390170511535e-05, + "loss": 1.1841, + "theoretical_loss": 3.354489714947166, + "tokens_seen": 2939013120 + }, + { + "epoch": 1.27, + "learning_rate": 8.861785356068205e-05, + "loss": 1.2946, + "theoretical_loss": 3.3544839888526106, + "tokens_seen": 2939078656 + }, + { + "epoch": 1.27, + "learning_rate": 8.860180541624875e-05, + "loss": 1.151, + "theoretical_loss": 3.3544782629214853, + "tokens_seen": 2939144192 + }, + { + "epoch": 1.27, + "learning_rate": 8.858575727181545e-05, + "loss": 1.1698, + "theoretical_loss": 3.3544725371537814, + "tokens_seen": 2939209728 + }, + { + "epoch": 1.27, + "learning_rate": 8.856970912738215e-05, + "loss": 1.2389, + "theoretical_loss": 3.3544668115494902, + "tokens_seen": 2939275264 + }, + { + "epoch": 1.27, + "learning_rate": 8.855366098294885e-05, + "loss": 1.3249, + "theoretical_loss": 3.3544610861086044, + "tokens_seen": 2939340800 + }, + { + "epoch": 1.27, + "learning_rate": 8.853761283851555e-05, + "loss": 1.2793, + "theoretical_loss": 3.3544553608311145, + "tokens_seen": 2939406336 + }, + { + "epoch": 1.27, + "learning_rate": 8.852156469408225e-05, + "loss": 1.0976, + "theoretical_loss": 3.354449635717013, + "tokens_seen": 2939471872 + }, + { + "epoch": 1.27, + "learning_rate": 8.850551654964894e-05, + "loss": 1.2782, + "theoretical_loss": 3.3544439107662916, + "tokens_seen": 2939537408 + }, + { + "epoch": 1.27, + "learning_rate": 8.848946840521564e-05, + "loss": 1.0713, + "theoretical_loss": 3.3544381859789416, + "tokens_seen": 2939602944 + }, + { + "epoch": 1.27, + "learning_rate": 8.847342026078234e-05, + "loss": 1.1461, + "theoretical_loss": 3.3544324613549548, + "tokens_seen": 2939668480 + }, + { + "epoch": 1.27, + "learning_rate": 8.845737211634904e-05, + "loss": 1.2477, + "theoretical_loss": 3.354426736894323, + "tokens_seen": 2939734016 + }, + { + "epoch": 1.27, + "learning_rate": 8.844132397191574e-05, + "loss": 1.0743, + "theoretical_loss": 3.3544210125970375, + "tokens_seen": 2939799552 + }, + { + "epoch": 1.27, + "learning_rate": 8.842527582748246e-05, + "loss": 1.1829, + "theoretical_loss": 3.354415288463091, + "tokens_seen": 2939865088 + }, + { + "epoch": 1.27, + "learning_rate": 8.840922768304916e-05, + "loss": 1.1876, + "theoretical_loss": 3.354409564492474, + "tokens_seen": 2939930624 + }, + { + "epoch": 1.27, + "learning_rate": 8.839317953861586e-05, + "loss": 1.2993, + "theoretical_loss": 3.3544038406851793, + "tokens_seen": 2939996160 + }, + { + "epoch": 1.27, + "learning_rate": 8.837713139418256e-05, + "loss": 1.1418, + "theoretical_loss": 3.354398117041198, + "tokens_seen": 2940061696 + }, + { + "epoch": 1.27, + "learning_rate": 8.836108324974926e-05, + "loss": 1.1919, + "theoretical_loss": 3.354392393560521, + "tokens_seen": 2940127232 + }, + { + "epoch": 1.27, + "learning_rate": 8.834503510531595e-05, + "loss": 1.2075, + "theoretical_loss": 3.3543866702431417, + "tokens_seen": 2940192768 + }, + { + "epoch": 1.27, + "learning_rate": 8.832898696088265e-05, + "loss": 1.2289, + "theoretical_loss": 3.3543809470890507, + "tokens_seen": 2940258304 + }, + { + "epoch": 1.27, + "learning_rate": 8.831293881644935e-05, + "loss": 1.2707, + "theoretical_loss": 3.3543752240982396, + "tokens_seen": 2940323840 + }, + { + "epoch": 1.27, + "learning_rate": 8.829689067201605e-05, + "loss": 1.263, + "theoretical_loss": 3.3543695012707007, + "tokens_seen": 2940389376 + }, + { + "epoch": 1.27, + "learning_rate": 8.828084252758275e-05, + "loss": 1.3109, + "theoretical_loss": 3.3543637786064258, + "tokens_seen": 2940454912 + }, + { + "epoch": 1.27, + "learning_rate": 8.826479438314945e-05, + "loss": 1.223, + "theoretical_loss": 3.354358056105406, + "tokens_seen": 2940520448 + }, + { + "epoch": 1.27, + "learning_rate": 8.824874623871615e-05, + "loss": 1.3385, + "theoretical_loss": 3.354352333767633, + "tokens_seen": 2940585984 + }, + { + "epoch": 1.27, + "learning_rate": 8.823269809428285e-05, + "loss": 1.3637, + "theoretical_loss": 3.3543466115930993, + "tokens_seen": 2940651520 + }, + { + "epoch": 1.27, + "learning_rate": 8.821664994984955e-05, + "loss": 1.2281, + "theoretical_loss": 3.3543408895817954, + "tokens_seen": 2940717056 + }, + { + "epoch": 1.27, + "learning_rate": 8.820060180541625e-05, + "loss": 1.1118, + "theoretical_loss": 3.3543351677337143, + "tokens_seen": 2940782592 + }, + { + "epoch": 1.27, + "learning_rate": 8.818455366098295e-05, + "loss": 1.1391, + "theoretical_loss": 3.3543294460488466, + "tokens_seen": 2940848128 + }, + { + "epoch": 1.27, + "learning_rate": 8.816850551654965e-05, + "loss": 1.2235, + "theoretical_loss": 3.354323724527185, + "tokens_seen": 2940913664 + }, + { + "epoch": 1.27, + "learning_rate": 8.815245737211634e-05, + "loss": 1.2066, + "theoretical_loss": 3.3543180031687205, + "tokens_seen": 2940979200 + }, + { + "epoch": 1.27, + "learning_rate": 8.813640922768306e-05, + "loss": 1.1408, + "theoretical_loss": 3.3543122819734448, + "tokens_seen": 2941044736 + }, + { + "epoch": 1.27, + "learning_rate": 8.812036108324976e-05, + "loss": 1.2348, + "theoretical_loss": 3.35430656094135, + "tokens_seen": 2941110272 + }, + { + "epoch": 1.27, + "learning_rate": 8.810431293881646e-05, + "loss": 1.2037, + "theoretical_loss": 3.354300840072428, + "tokens_seen": 2941175808 + }, + { + "epoch": 1.27, + "learning_rate": 8.808826479438316e-05, + "loss": 1.2346, + "theoretical_loss": 3.3542951193666695, + "tokens_seen": 2941241344 + }, + { + "epoch": 1.27, + "learning_rate": 8.807221664994986e-05, + "loss": 1.3037, + "theoretical_loss": 3.354289398824067, + "tokens_seen": 2941306880 + }, + { + "epoch": 1.27, + "learning_rate": 8.805616850551656e-05, + "loss": 1.2134, + "theoretical_loss": 3.354283678444612, + "tokens_seen": 2941372416 + }, + { + "epoch": 1.27, + "learning_rate": 8.804012036108326e-05, + "loss": 1.0822, + "theoretical_loss": 3.354277958228297, + "tokens_seen": 2941437952 + }, + { + "epoch": 1.27, + "learning_rate": 8.802407221664996e-05, + "loss": 1.1727, + "theoretical_loss": 3.3542722381751124, + "tokens_seen": 2941503488 + }, + { + "epoch": 1.27, + "learning_rate": 8.800802407221666e-05, + "loss": 1.124, + "theoretical_loss": 3.3542665182850504, + "tokens_seen": 2941569024 + }, + { + "epoch": 1.27, + "learning_rate": 8.799197592778335e-05, + "loss": 1.1547, + "theoretical_loss": 3.354260798558103, + "tokens_seen": 2941634560 + }, + { + "epoch": 1.27, + "learning_rate": 8.797592778335005e-05, + "loss": 1.2302, + "theoretical_loss": 3.3542550789942616, + "tokens_seen": 2941700096 + }, + { + "epoch": 1.27, + "learning_rate": 8.795987963891675e-05, + "loss": 1.1653, + "theoretical_loss": 3.3542493595935183, + "tokens_seen": 2941765632 + }, + { + "epoch": 1.27, + "learning_rate": 8.794383149448345e-05, + "loss": 1.2306, + "theoretical_loss": 3.354243640355864, + "tokens_seen": 2941831168 + }, + { + "epoch": 1.27, + "learning_rate": 8.792778335005015e-05, + "loss": 1.2425, + "theoretical_loss": 3.3542379212812916, + "tokens_seen": 2941896704 + }, + { + "epoch": 1.27, + "learning_rate": 8.791173520561685e-05, + "loss": 1.1725, + "theoretical_loss": 3.354232202369792, + "tokens_seen": 2941962240 + }, + { + "epoch": 1.27, + "learning_rate": 8.789568706118355e-05, + "loss": 1.1929, + "theoretical_loss": 3.354226483621357, + "tokens_seen": 2942027776 + }, + { + "epoch": 1.27, + "learning_rate": 8.787963891675025e-05, + "loss": 1.2373, + "theoretical_loss": 3.3542207650359788, + "tokens_seen": 2942093312 + }, + { + "epoch": 1.27, + "learning_rate": 8.786359077231695e-05, + "loss": 1.1814, + "theoretical_loss": 3.3542150466136484, + "tokens_seen": 2942158848 + }, + { + "epoch": 1.27, + "learning_rate": 8.784754262788365e-05, + "loss": 1.2033, + "theoretical_loss": 3.354209328354358, + "tokens_seen": 2942224384 + }, + { + "epoch": 1.27, + "learning_rate": 8.783149448345035e-05, + "loss": 1.2131, + "theoretical_loss": 3.354203610258099, + "tokens_seen": 2942289920 + }, + { + "epoch": 1.27, + "learning_rate": 8.781544633901706e-05, + "loss": 1.273, + "theoretical_loss": 3.3541978923248634, + "tokens_seen": 2942355456 + }, + { + "epoch": 1.27, + "learning_rate": 8.779939819458376e-05, + "loss": 1.2305, + "theoretical_loss": 3.354192174554643, + "tokens_seen": 2942420992 + }, + { + "epoch": 1.27, + "learning_rate": 8.778335005015046e-05, + "loss": 1.2596, + "theoretical_loss": 3.354186456947429, + "tokens_seen": 2942486528 + }, + { + "epoch": 1.27, + "objective/train/docs_used": 1086060, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1779783964157104, + "objective/train/theoretical_loss": 3.3541807395032137, + "objective/train/tokens_used": 2963012064, + "theoretical_loss": 3.3541807395032137, + "tokens_seen": 2942552064 + }, + { + "epoch": 1.27, + "learning_rate": 8.776730190571716e-05, + "loss": 1.178, + "theoretical_loss": 3.3541807395032137, + "tokens_seen": 2942552064 + }, + { + "epoch": 1.27, + "learning_rate": 8.775125376128386e-05, + "loss": 1.3041, + "theoretical_loss": 3.354175022221989, + "tokens_seen": 2942617600 + }, + { + "epoch": 1.27, + "learning_rate": 8.773520561685056e-05, + "loss": 1.1906, + "theoretical_loss": 3.3541693051037456, + "tokens_seen": 2942683136 + }, + { + "epoch": 1.27, + "learning_rate": 8.771915747241726e-05, + "loss": 1.1857, + "theoretical_loss": 3.3541635881484764, + "tokens_seen": 2942748672 + }, + { + "epoch": 1.27, + "learning_rate": 8.770310932798396e-05, + "loss": 1.2243, + "theoretical_loss": 3.354157871356172, + "tokens_seen": 2942814208 + }, + { + "epoch": 1.27, + "learning_rate": 8.768706118355066e-05, + "loss": 1.1178, + "theoretical_loss": 3.354152154726825, + "tokens_seen": 2942879744 + }, + { + "epoch": 1.27, + "learning_rate": 8.767101303911736e-05, + "loss": 1.2196, + "theoretical_loss": 3.3541464382604267, + "tokens_seen": 2942945280 + }, + { + "epoch": 1.27, + "learning_rate": 8.765496489468406e-05, + "loss": 1.2838, + "theoretical_loss": 3.354140721956969, + "tokens_seen": 2943010816 + }, + { + "epoch": 1.27, + "learning_rate": 8.763891675025075e-05, + "loss": 1.2786, + "theoretical_loss": 3.3541350058164436, + "tokens_seen": 2943076352 + }, + { + "epoch": 1.27, + "learning_rate": 8.762286860581745e-05, + "loss": 1.2057, + "theoretical_loss": 3.354129289838842, + "tokens_seen": 2943141888 + }, + { + "epoch": 1.27, + "learning_rate": 8.760682046138415e-05, + "loss": 1.21, + "theoretical_loss": 3.3541235740241566, + "tokens_seen": 2943207424 + }, + { + "epoch": 1.27, + "learning_rate": 8.759077231695085e-05, + "loss": 1.1476, + "theoretical_loss": 3.354117858372378, + "tokens_seen": 2943272960 + }, + { + "epoch": 1.27, + "learning_rate": 8.757472417251755e-05, + "loss": 1.1128, + "theoretical_loss": 3.3541121428834995, + "tokens_seen": 2943338496 + }, + { + "epoch": 1.27, + "learning_rate": 8.755867602808425e-05, + "loss": 1.3006, + "theoretical_loss": 3.354106427557511, + "tokens_seen": 2943404032 + }, + { + "epoch": 1.27, + "learning_rate": 8.754262788365095e-05, + "loss": 1.1665, + "theoretical_loss": 3.3541007123944055, + "tokens_seen": 2943469568 + }, + { + "epoch": 1.27, + "learning_rate": 8.752657973921767e-05, + "loss": 1.1213, + "theoretical_loss": 3.3540949973941747, + "tokens_seen": 2943535104 + }, + { + "epoch": 1.27, + "learning_rate": 8.751053159478437e-05, + "loss": 1.1516, + "theoretical_loss": 3.3540892825568096, + "tokens_seen": 2943600640 + }, + { + "epoch": 1.27, + "learning_rate": 8.749448345035107e-05, + "loss": 1.233, + "theoretical_loss": 3.3540835678823027, + "tokens_seen": 2943666176 + }, + { + "epoch": 1.27, + "learning_rate": 8.747843530591775e-05, + "loss": 1.1438, + "theoretical_loss": 3.354077853370645, + "tokens_seen": 2943731712 + }, + { + "epoch": 1.27, + "learning_rate": 8.746238716148446e-05, + "loss": 1.2611, + "theoretical_loss": 3.354072139021829, + "tokens_seen": 2943797248 + }, + { + "epoch": 1.27, + "learning_rate": 8.744633901705116e-05, + "loss": 1.1291, + "theoretical_loss": 3.354066424835846, + "tokens_seen": 2943862784 + }, + { + "epoch": 1.27, + "learning_rate": 8.743029087261786e-05, + "loss": 1.1615, + "theoretical_loss": 3.3540607108126874, + "tokens_seen": 2943928320 + }, + { + "epoch": 1.27, + "learning_rate": 8.741424272818456e-05, + "loss": 1.3221, + "theoretical_loss": 3.3540549969523457, + "tokens_seen": 2943993856 + }, + { + "epoch": 1.27, + "learning_rate": 8.739819458375126e-05, + "loss": 1.2337, + "theoretical_loss": 3.3540492832548123, + "tokens_seen": 2944059392 + }, + { + "epoch": 1.27, + "learning_rate": 8.738214643931796e-05, + "loss": 1.1565, + "theoretical_loss": 3.3540435697200786, + "tokens_seen": 2944124928 + }, + { + "epoch": 1.27, + "learning_rate": 8.736609829488466e-05, + "loss": 1.1629, + "theoretical_loss": 3.354037856348137, + "tokens_seen": 2944190464 + }, + { + "epoch": 1.27, + "learning_rate": 8.735005015045136e-05, + "loss": 1.2374, + "theoretical_loss": 3.354032143138978, + "tokens_seen": 2944256000 + }, + { + "epoch": 1.27, + "learning_rate": 8.733400200601806e-05, + "loss": 1.1913, + "theoretical_loss": 3.3540264300925955, + "tokens_seen": 2944321536 + }, + { + "epoch": 1.27, + "learning_rate": 8.731795386158476e-05, + "loss": 1.2789, + "theoretical_loss": 3.354020717208979, + "tokens_seen": 2944387072 + }, + { + "epoch": 1.27, + "learning_rate": 8.730190571715146e-05, + "loss": 1.1287, + "theoretical_loss": 3.3540150044881214, + "tokens_seen": 2944452608 + }, + { + "epoch": 1.27, + "learning_rate": 8.728585757271815e-05, + "loss": 1.1465, + "theoretical_loss": 3.354009291930015, + "tokens_seen": 2944518144 + }, + { + "epoch": 1.27, + "learning_rate": 8.726980942828485e-05, + "loss": 1.265, + "theoretical_loss": 3.3540035795346497, + "tokens_seen": 2944583680 + }, + { + "epoch": 1.27, + "learning_rate": 8.725376128385155e-05, + "loss": 1.0332, + "theoretical_loss": 3.3539978673020188, + "tokens_seen": 2944649216 + }, + { + "epoch": 1.27, + "learning_rate": 8.723771313941825e-05, + "loss": 1.2594, + "theoretical_loss": 3.3539921552321132, + "tokens_seen": 2944714752 + }, + { + "epoch": 1.27, + "learning_rate": 8.722166499498496e-05, + "loss": 1.1695, + "theoretical_loss": 3.3539864433249256, + "tokens_seen": 2944780288 + }, + { + "epoch": 1.27, + "learning_rate": 8.720561685055167e-05, + "loss": 1.1605, + "theoretical_loss": 3.353980731580447, + "tokens_seen": 2944845824 + }, + { + "epoch": 1.27, + "learning_rate": 8.718956870611837e-05, + "loss": 1.1548, + "theoretical_loss": 3.3539750199986686, + "tokens_seen": 2944911360 + }, + { + "epoch": 1.27, + "learning_rate": 8.717352056168507e-05, + "loss": 1.2948, + "theoretical_loss": 3.3539693085795834, + "tokens_seen": 2944976896 + }, + { + "epoch": 1.27, + "learning_rate": 8.715747241725177e-05, + "loss": 1.1319, + "theoretical_loss": 3.3539635973231827, + "tokens_seen": 2945042432 + }, + { + "epoch": 1.27, + "learning_rate": 8.714142427281847e-05, + "loss": 1.2708, + "theoretical_loss": 3.353957886229458, + "tokens_seen": 2945107968 + }, + { + "epoch": 1.27, + "learning_rate": 8.712537612838516e-05, + "loss": 1.2488, + "theoretical_loss": 3.353952175298401, + "tokens_seen": 2945173504 + }, + { + "epoch": 1.27, + "learning_rate": 8.710932798395186e-05, + "loss": 1.1393, + "theoretical_loss": 3.353946464530004, + "tokens_seen": 2945239040 + }, + { + "epoch": 1.27, + "learning_rate": 8.709327983951856e-05, + "loss": 1.2422, + "theoretical_loss": 3.353940753924258, + "tokens_seen": 2945304576 + }, + { + "epoch": 1.27, + "learning_rate": 8.707723169508526e-05, + "loss": 1.2811, + "theoretical_loss": 3.353935043481155, + "tokens_seen": 2945370112 + }, + { + "epoch": 1.27, + "learning_rate": 8.706118355065196e-05, + "loss": 1.2654, + "theoretical_loss": 3.353929333200687, + "tokens_seen": 2945435648 + }, + { + "epoch": 1.27, + "learning_rate": 8.704513540621866e-05, + "loss": 1.067, + "theoretical_loss": 3.3539236230828458, + "tokens_seen": 2945501184 + }, + { + "epoch": 1.27, + "learning_rate": 8.702908726178536e-05, + "loss": 1.2383, + "theoretical_loss": 3.353917913127623, + "tokens_seen": 2945566720 + }, + { + "epoch": 1.27, + "learning_rate": 8.701303911735206e-05, + "loss": 1.1771, + "theoretical_loss": 3.35391220333501, + "tokens_seen": 2945632256 + }, + { + "epoch": 1.27, + "learning_rate": 8.699699097291876e-05, + "loss": 1.0873, + "theoretical_loss": 3.353906493704999, + "tokens_seen": 2945697792 + }, + { + "epoch": 1.27, + "learning_rate": 8.698094282848546e-05, + "loss": 1.0908, + "theoretical_loss": 3.353900784237582, + "tokens_seen": 2945763328 + }, + { + "epoch": 1.27, + "learning_rate": 8.696489468405216e-05, + "loss": 1.1831, + "theoretical_loss": 3.35389507493275, + "tokens_seen": 2945828864 + }, + { + "epoch": 1.27, + "learning_rate": 8.694884653961886e-05, + "loss": 1.1564, + "theoretical_loss": 3.353889365790495, + "tokens_seen": 2945894400 + }, + { + "epoch": 1.27, + "learning_rate": 8.693279839518556e-05, + "loss": 1.2198, + "theoretical_loss": 3.353883656810809, + "tokens_seen": 2945959936 + }, + { + "epoch": 1.27, + "learning_rate": 8.691675025075226e-05, + "loss": 1.1338, + "theoretical_loss": 3.3538779479936833, + "tokens_seen": 2946025472 + }, + { + "epoch": 1.27, + "learning_rate": 8.690070210631896e-05, + "loss": 1.1995, + "theoretical_loss": 3.3538722393391107, + "tokens_seen": 2946091008 + }, + { + "epoch": 1.27, + "learning_rate": 8.688465396188566e-05, + "loss": 1.3403, + "theoretical_loss": 3.353866530847082, + "tokens_seen": 2946156544 + }, + { + "epoch": 1.27, + "learning_rate": 8.686860581745237e-05, + "loss": 1.1259, + "theoretical_loss": 3.3538608225175888, + "tokens_seen": 2946222080 + }, + { + "epoch": 1.27, + "learning_rate": 8.685255767301907e-05, + "loss": 1.1795, + "theoretical_loss": 3.3538551143506234, + "tokens_seen": 2946287616 + }, + { + "epoch": 1.27, + "learning_rate": 8.683650952858577e-05, + "loss": 1.1048, + "theoretical_loss": 3.353849406346178, + "tokens_seen": 2946353152 + }, + { + "epoch": 1.27, + "learning_rate": 8.682046138415247e-05, + "loss": 1.1261, + "theoretical_loss": 3.3538436985042432, + "tokens_seen": 2946418688 + }, + { + "epoch": 1.27, + "learning_rate": 8.680441323971917e-05, + "loss": 1.151, + "theoretical_loss": 3.3538379908248115, + "tokens_seen": 2946484224 + }, + { + "epoch": 1.27, + "learning_rate": 8.678836509528587e-05, + "loss": 1.1292, + "theoretical_loss": 3.353832283307874, + "tokens_seen": 2946549760 + }, + { + "epoch": 1.27, + "learning_rate": 8.677231695085256e-05, + "loss": 1.1461, + "theoretical_loss": 3.3538265759534234, + "tokens_seen": 2946615296 + }, + { + "epoch": 1.27, + "learning_rate": 8.675626880641926e-05, + "loss": 1.1174, + "theoretical_loss": 3.3538208687614515, + "tokens_seen": 2946680832 + }, + { + "epoch": 1.27, + "learning_rate": 8.674022066198596e-05, + "loss": 1.1493, + "theoretical_loss": 3.3538151617319487, + "tokens_seen": 2946746368 + }, + { + "epoch": 1.27, + "learning_rate": 8.672417251755266e-05, + "loss": 1.1877, + "theoretical_loss": 3.353809454864908, + "tokens_seen": 2946811904 + }, + { + "epoch": 1.27, + "learning_rate": 8.670812437311936e-05, + "loss": 1.1568, + "theoretical_loss": 3.353803748160321, + "tokens_seen": 2946877440 + }, + { + "epoch": 1.27, + "learning_rate": 8.669207622868606e-05, + "loss": 1.2297, + "theoretical_loss": 3.353798041618179, + "tokens_seen": 2946942976 + }, + { + "epoch": 1.27, + "learning_rate": 8.667602808425276e-05, + "loss": 1.1665, + "theoretical_loss": 3.353792335238474, + "tokens_seen": 2947008512 + }, + { + "epoch": 1.27, + "learning_rate": 8.665997993981946e-05, + "loss": 1.1765, + "theoretical_loss": 3.353786629021198, + "tokens_seen": 2947074048 + }, + { + "epoch": 1.27, + "learning_rate": 8.664393179538616e-05, + "loss": 1.2277, + "theoretical_loss": 3.3537809229663424, + "tokens_seen": 2947139584 + }, + { + "epoch": 1.27, + "learning_rate": 8.662788365095286e-05, + "loss": 1.1974, + "theoretical_loss": 3.353775217073899, + "tokens_seen": 2947205120 + }, + { + "epoch": 1.27, + "learning_rate": 8.661183550651956e-05, + "loss": 1.2095, + "theoretical_loss": 3.35376951134386, + "tokens_seen": 2947270656 + }, + { + "epoch": 1.27, + "learning_rate": 8.659578736208626e-05, + "loss": 1.1339, + "theoretical_loss": 3.3537638057762167, + "tokens_seen": 2947336192 + }, + { + "epoch": 1.27, + "learning_rate": 8.657973921765296e-05, + "loss": 1.1745, + "theoretical_loss": 3.353758100370961, + "tokens_seen": 2947401728 + }, + { + "epoch": 1.27, + "learning_rate": 8.656369107321966e-05, + "loss": 1.1139, + "theoretical_loss": 3.3537523951280845, + "tokens_seen": 2947467264 + }, + { + "epoch": 1.27, + "learning_rate": 8.654764292878636e-05, + "loss": 1.1569, + "theoretical_loss": 3.353746690047579, + "tokens_seen": 2947532800 + }, + { + "epoch": 1.27, + "learning_rate": 8.653159478435307e-05, + "loss": 1.274, + "theoretical_loss": 3.353740985129437, + "tokens_seen": 2947598336 + }, + { + "epoch": 1.27, + "learning_rate": 8.651554663991977e-05, + "loss": 1.1395, + "theoretical_loss": 3.3537352803736495, + "tokens_seen": 2947663872 + }, + { + "epoch": 1.27, + "learning_rate": 8.649949849548647e-05, + "loss": 1.1751, + "theoretical_loss": 3.3537295757802084, + "tokens_seen": 2947729408 + }, + { + "epoch": 1.27, + "learning_rate": 8.648345035105317e-05, + "loss": 1.2033, + "theoretical_loss": 3.3537238713491053, + "tokens_seen": 2947794944 + }, + { + "epoch": 1.27, + "learning_rate": 8.646740220661987e-05, + "loss": 1.1149, + "theoretical_loss": 3.3537181670803324, + "tokens_seen": 2947860480 + }, + { + "epoch": 1.27, + "learning_rate": 8.645135406218657e-05, + "loss": 1.1244, + "theoretical_loss": 3.3537124629738813, + "tokens_seen": 2947926016 + }, + { + "epoch": 1.27, + "learning_rate": 8.643530591775326e-05, + "loss": 1.1807, + "theoretical_loss": 3.353706759029744, + "tokens_seen": 2947991552 + }, + { + "epoch": 1.27, + "learning_rate": 8.641925777331996e-05, + "loss": 1.1021, + "theoretical_loss": 3.353701055247911, + "tokens_seen": 2948057088 + }, + { + "epoch": 1.27, + "learning_rate": 8.640320962888666e-05, + "loss": 1.1306, + "theoretical_loss": 3.3536953516283763, + "tokens_seen": 2948122624 + }, + { + "epoch": 1.27, + "learning_rate": 8.638716148445336e-05, + "loss": 1.2255, + "theoretical_loss": 3.35368964817113, + "tokens_seen": 2948188160 + }, + { + "epoch": 1.27, + "learning_rate": 8.637111334002006e-05, + "loss": 1.2727, + "theoretical_loss": 3.353683944876164, + "tokens_seen": 2948253696 + }, + { + "epoch": 1.27, + "learning_rate": 8.635506519558676e-05, + "loss": 1.1788, + "theoretical_loss": 3.353678241743471, + "tokens_seen": 2948319232 + }, + { + "epoch": 1.27, + "learning_rate": 8.633901705115346e-05, + "loss": 1.1764, + "theoretical_loss": 3.3536725387730417, + "tokens_seen": 2948384768 + }, + { + "epoch": 1.27, + "learning_rate": 8.632296890672017e-05, + "loss": 1.098, + "theoretical_loss": 3.353666835964869, + "tokens_seen": 2948450304 + }, + { + "epoch": 1.27, + "learning_rate": 8.630692076228687e-05, + "loss": 1.1646, + "theoretical_loss": 3.353661133318943, + "tokens_seen": 2948515840 + }, + { + "epoch": 1.27, + "learning_rate": 8.629087261785357e-05, + "loss": 1.217, + "theoretical_loss": 3.353655430835258, + "tokens_seen": 2948581376 + }, + { + "epoch": 1.27, + "learning_rate": 8.627482447342026e-05, + "loss": 1.1375, + "theoretical_loss": 3.353649728513803, + "tokens_seen": 2948646912 + }, + { + "epoch": 1.27, + "learning_rate": 8.625877632898696e-05, + "loss": 1.1091, + "theoretical_loss": 3.353644026354572, + "tokens_seen": 2948712448 + }, + { + "epoch": 1.27, + "learning_rate": 8.624272818455366e-05, + "loss": 1.2002, + "theoretical_loss": 3.353638324357555, + "tokens_seen": 2948777984 + }, + { + "epoch": 1.27, + "learning_rate": 8.622668004012036e-05, + "loss": 1.2914, + "theoretical_loss": 3.3536326225227455, + "tokens_seen": 2948843520 + }, + { + "epoch": 1.27, + "learning_rate": 8.621063189568706e-05, + "loss": 1.2619, + "theoretical_loss": 3.3536269208501337, + "tokens_seen": 2948909056 + }, + { + "epoch": 1.27, + "learning_rate": 8.619458375125376e-05, + "loss": 1.2403, + "theoretical_loss": 3.3536212193397126, + "tokens_seen": 2948974592 + }, + { + "epoch": 1.27, + "learning_rate": 8.617853560682047e-05, + "loss": 1.0753, + "theoretical_loss": 3.3536155179914733, + "tokens_seen": 2949040128 + }, + { + "epoch": 1.27, + "objective/train/docs_used": 1088585, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1807368993759155, + "objective/train/theoretical_loss": 3.3536098168054074, + "objective/train/tokens_used": 2969565664, + "theoretical_loss": 3.3536098168054074, + "tokens_seen": 2949105664 + }, + { + "epoch": 1.27, + "learning_rate": 8.616248746238717e-05, + "loss": 1.1807, + "theoretical_loss": 3.3536098168054074, + "tokens_seen": 2949105664 + }, + { + "epoch": 1.27, + "learning_rate": 8.614643931795387e-05, + "loss": 1.213, + "theoretical_loss": 3.3536041157815077, + "tokens_seen": 2949171200 + }, + { + "epoch": 1.27, + "learning_rate": 8.613039117352057e-05, + "loss": 1.0945, + "theoretical_loss": 3.353598414919765, + "tokens_seen": 2949236736 + }, + { + "epoch": 1.27, + "learning_rate": 8.611434302908727e-05, + "loss": 1.1764, + "theoretical_loss": 3.3535927142201714, + "tokens_seen": 2949302272 + }, + { + "epoch": 1.27, + "learning_rate": 8.609829488465397e-05, + "loss": 1.1949, + "theoretical_loss": 3.3535870136827186, + "tokens_seen": 2949367808 + }, + { + "epoch": 1.27, + "learning_rate": 8.608224674022066e-05, + "loss": 1.1497, + "theoretical_loss": 3.353581313307399, + "tokens_seen": 2949433344 + }, + { + "epoch": 1.27, + "learning_rate": 8.606619859578736e-05, + "loss": 1.2185, + "theoretical_loss": 3.353575613094203, + "tokens_seen": 2949498880 + }, + { + "epoch": 1.27, + "learning_rate": 8.605015045135406e-05, + "loss": 1.1358, + "theoretical_loss": 3.353569913043124, + "tokens_seen": 2949564416 + }, + { + "epoch": 1.27, + "learning_rate": 8.603410230692077e-05, + "loss": 1.2233, + "theoretical_loss": 3.353564213154153, + "tokens_seen": 2949629952 + }, + { + "epoch": 1.27, + "learning_rate": 8.601805416248747e-05, + "loss": 1.1572, + "theoretical_loss": 3.3535585134272816, + "tokens_seen": 2949695488 + }, + { + "epoch": 1.27, + "learning_rate": 8.600200601805417e-05, + "loss": 1.2262, + "theoretical_loss": 3.353552813862502, + "tokens_seen": 2949761024 + }, + { + "epoch": 1.27, + "learning_rate": 8.598595787362087e-05, + "loss": 1.1801, + "theoretical_loss": 3.353547114459806, + "tokens_seen": 2949826560 + }, + { + "epoch": 1.27, + "learning_rate": 8.596990972918757e-05, + "loss": 1.1407, + "theoretical_loss": 3.3535414152191847, + "tokens_seen": 2949892096 + }, + { + "epoch": 1.27, + "learning_rate": 8.595386158475427e-05, + "loss": 1.226, + "theoretical_loss": 3.3535357161406307, + "tokens_seen": 2949957632 + }, + { + "epoch": 1.27, + "learning_rate": 8.593781344032098e-05, + "loss": 1.2095, + "theoretical_loss": 3.353530017224135, + "tokens_seen": 2950023168 + }, + { + "epoch": 1.27, + "learning_rate": 8.592176529588766e-05, + "loss": 1.2047, + "theoretical_loss": 3.353524318469691, + "tokens_seen": 2950088704 + }, + { + "epoch": 1.27, + "learning_rate": 8.590571715145436e-05, + "loss": 1.2483, + "theoretical_loss": 3.3535186198772884, + "tokens_seen": 2950154240 + }, + { + "epoch": 1.27, + "learning_rate": 8.588966900702106e-05, + "loss": 1.2403, + "theoretical_loss": 3.35351292144692, + "tokens_seen": 2950219776 + }, + { + "epoch": 1.27, + "learning_rate": 8.587362086258776e-05, + "loss": 1.1635, + "theoretical_loss": 3.353507223178578, + "tokens_seen": 2950285312 + }, + { + "epoch": 1.27, + "learning_rate": 8.585757271815446e-05, + "loss": 1.2234, + "theoretical_loss": 3.3535015250722537, + "tokens_seen": 2950350848 + }, + { + "epoch": 1.27, + "learning_rate": 8.584152457372117e-05, + "loss": 1.2236, + "theoretical_loss": 3.3534958271279387, + "tokens_seen": 2950416384 + }, + { + "epoch": 1.27, + "learning_rate": 8.582547642928787e-05, + "loss": 1.2507, + "theoretical_loss": 3.353490129345625, + "tokens_seen": 2950481920 + }, + { + "epoch": 1.27, + "learning_rate": 8.580942828485457e-05, + "loss": 1.1802, + "theoretical_loss": 3.3534844317253047, + "tokens_seen": 2950547456 + }, + { + "epoch": 1.27, + "learning_rate": 8.579338014042127e-05, + "loss": 1.1771, + "theoretical_loss": 3.3534787342669694, + "tokens_seen": 2950612992 + }, + { + "epoch": 1.27, + "learning_rate": 8.577733199598797e-05, + "loss": 1.1643, + "theoretical_loss": 3.3534730369706107, + "tokens_seen": 2950678528 + }, + { + "epoch": 1.27, + "learning_rate": 8.576128385155467e-05, + "loss": 1.252, + "theoretical_loss": 3.3534673398362203, + "tokens_seen": 2950744064 + }, + { + "epoch": 1.27, + "learning_rate": 8.574523570712137e-05, + "loss": 1.2167, + "theoretical_loss": 3.3534616428637904, + "tokens_seen": 2950809600 + }, + { + "epoch": 1.27, + "learning_rate": 8.572918756268807e-05, + "loss": 1.1528, + "theoretical_loss": 3.353455946053313, + "tokens_seen": 2950875136 + }, + { + "epoch": 1.27, + "learning_rate": 8.571313941825477e-05, + "loss": 1.1528, + "theoretical_loss": 3.353450249404779, + "tokens_seen": 2950940672 + }, + { + "epoch": 1.27, + "learning_rate": 8.569709127382147e-05, + "loss": 1.0751, + "theoretical_loss": 3.3534445529181807, + "tokens_seen": 2951006208 + }, + { + "epoch": 1.27, + "learning_rate": 8.568104312938817e-05, + "loss": 1.1298, + "theoretical_loss": 3.35343885659351, + "tokens_seen": 2951071744 + }, + { + "epoch": 1.27, + "learning_rate": 8.566499498495487e-05, + "loss": 1.4131, + "theoretical_loss": 3.353433160430759, + "tokens_seen": 2951137280 + }, + { + "epoch": 1.27, + "learning_rate": 8.564894684052157e-05, + "loss": 1.2205, + "theoretical_loss": 3.353427464429919, + "tokens_seen": 2951202816 + }, + { + "epoch": 1.27, + "learning_rate": 8.563289869608827e-05, + "loss": 1.1745, + "theoretical_loss": 3.353421768590982, + "tokens_seen": 2951268352 + }, + { + "epoch": 1.28, + "learning_rate": 8.561685055165497e-05, + "loss": 1.2199, + "theoretical_loss": 3.353416072913939, + "tokens_seen": 2951333888 + }, + { + "epoch": 1.28, + "learning_rate": 8.560080240722167e-05, + "loss": 1.1886, + "theoretical_loss": 3.3534103773987836, + "tokens_seen": 2951399424 + }, + { + "epoch": 1.28, + "learning_rate": 8.558475426278838e-05, + "loss": 1.1588, + "theoretical_loss": 3.3534046820455057, + "tokens_seen": 2951464960 + }, + { + "epoch": 1.28, + "learning_rate": 8.556870611835506e-05, + "loss": 1.2246, + "theoretical_loss": 3.3533989868540983, + "tokens_seen": 2951530496 + }, + { + "epoch": 1.28, + "learning_rate": 8.555265797392176e-05, + "loss": 1.2501, + "theoretical_loss": 3.3533932918245526, + "tokens_seen": 2951596032 + }, + { + "epoch": 1.28, + "learning_rate": 8.553660982948846e-05, + "loss": 1.1863, + "theoretical_loss": 3.3533875969568614, + "tokens_seen": 2951661568 + }, + { + "epoch": 1.28, + "learning_rate": 8.552056168505516e-05, + "loss": 1.2831, + "theoretical_loss": 3.353381902251015, + "tokens_seen": 2951727104 + }, + { + "epoch": 1.28, + "learning_rate": 8.550451354062187e-05, + "loss": 1.1679, + "theoretical_loss": 3.353376207707006, + "tokens_seen": 2951792640 + }, + { + "epoch": 1.28, + "learning_rate": 8.548846539618857e-05, + "loss": 1.1154, + "theoretical_loss": 3.353370513324827, + "tokens_seen": 2951858176 + }, + { + "epoch": 1.28, + "learning_rate": 8.547241725175527e-05, + "loss": 1.1947, + "theoretical_loss": 3.3533648191044683, + "tokens_seen": 2951923712 + }, + { + "epoch": 1.28, + "learning_rate": 8.545636910732197e-05, + "loss": 1.1914, + "theoretical_loss": 3.3533591250459223, + "tokens_seen": 2951989248 + }, + { + "epoch": 1.28, + "learning_rate": 8.544032096288867e-05, + "loss": 1.2041, + "theoretical_loss": 3.3533534311491815, + "tokens_seen": 2952054784 + }, + { + "epoch": 1.28, + "learning_rate": 8.542427281845538e-05, + "loss": 1.1798, + "theoretical_loss": 3.3533477374142366, + "tokens_seen": 2952120320 + }, + { + "epoch": 1.28, + "learning_rate": 8.540822467402207e-05, + "loss": 1.2213, + "theoretical_loss": 3.3533420438410797, + "tokens_seen": 2952185856 + }, + { + "epoch": 1.28, + "learning_rate": 8.539217652958877e-05, + "loss": 1.1731, + "theoretical_loss": 3.3533363504297036, + "tokens_seen": 2952251392 + }, + { + "epoch": 1.28, + "learning_rate": 8.537612838515547e-05, + "loss": 1.2176, + "theoretical_loss": 3.353330657180099, + "tokens_seen": 2952316928 + }, + { + "epoch": 1.28, + "learning_rate": 8.536008024072217e-05, + "loss": 1.2681, + "theoretical_loss": 3.3533249640922578, + "tokens_seen": 2952382464 + }, + { + "epoch": 1.28, + "learning_rate": 8.534403209628887e-05, + "loss": 1.1593, + "theoretical_loss": 3.353319271166172, + "tokens_seen": 2952448000 + }, + { + "epoch": 1.28, + "learning_rate": 8.532798395185557e-05, + "loss": 1.2578, + "theoretical_loss": 3.353313578401834, + "tokens_seen": 2952513536 + }, + { + "epoch": 1.28, + "learning_rate": 8.531193580742227e-05, + "loss": 1.1387, + "theoretical_loss": 3.353307885799235, + "tokens_seen": 2952579072 + }, + { + "epoch": 1.28, + "learning_rate": 8.529588766298897e-05, + "loss": 1.2113, + "theoretical_loss": 3.353302193358367, + "tokens_seen": 2952644608 + }, + { + "epoch": 1.28, + "learning_rate": 8.527983951855567e-05, + "loss": 1.1581, + "theoretical_loss": 3.3532965010792215, + "tokens_seen": 2952710144 + }, + { + "epoch": 1.28, + "learning_rate": 8.526379137412237e-05, + "loss": 1.2276, + "theoretical_loss": 3.3532908089617908, + "tokens_seen": 2952775680 + }, + { + "epoch": 1.28, + "learning_rate": 8.524774322968908e-05, + "loss": 1.2148, + "theoretical_loss": 3.353285117006066, + "tokens_seen": 2952841216 + }, + { + "epoch": 1.28, + "learning_rate": 8.523169508525578e-05, + "loss": 1.1728, + "theoretical_loss": 3.35327942521204, + "tokens_seen": 2952906752 + }, + { + "epoch": 1.28, + "learning_rate": 8.521564694082246e-05, + "loss": 1.1936, + "theoretical_loss": 3.3532737335797034, + "tokens_seen": 2952972288 + }, + { + "epoch": 1.28, + "learning_rate": 8.519959879638916e-05, + "loss": 1.1767, + "theoretical_loss": 3.353268042109049, + "tokens_seen": 2953037824 + }, + { + "epoch": 1.28, + "learning_rate": 8.518355065195586e-05, + "loss": 1.2054, + "theoretical_loss": 3.353262350800068, + "tokens_seen": 2953103360 + }, + { + "epoch": 1.28, + "learning_rate": 8.516750250752256e-05, + "loss": 1.2304, + "theoretical_loss": 3.353256659652753, + "tokens_seen": 2953168896 + }, + { + "epoch": 1.28, + "learning_rate": 8.515145436308927e-05, + "loss": 1.1571, + "theoretical_loss": 3.3532509686670946, + "tokens_seen": 2953234432 + }, + { + "epoch": 1.28, + "learning_rate": 8.513540621865597e-05, + "loss": 1.2667, + "theoretical_loss": 3.3532452778430857, + "tokens_seen": 2953299968 + }, + { + "epoch": 1.28, + "learning_rate": 8.511935807422268e-05, + "loss": 1.2201, + "theoretical_loss": 3.3532395871807177, + "tokens_seen": 2953365504 + }, + { + "epoch": 1.28, + "learning_rate": 8.510330992978938e-05, + "loss": 1.2014, + "theoretical_loss": 3.353233896679982, + "tokens_seen": 2953431040 + }, + { + "epoch": 1.28, + "learning_rate": 8.508726178535608e-05, + "loss": 1.1467, + "theoretical_loss": 3.353228206340871, + "tokens_seen": 2953496576 + }, + { + "epoch": 1.28, + "learning_rate": 8.507121364092278e-05, + "loss": 1.148, + "theoretical_loss": 3.353222516163377, + "tokens_seen": 2953562112 + }, + { + "epoch": 1.28, + "learning_rate": 8.505516549648947e-05, + "loss": 1.1479, + "theoretical_loss": 3.3532168261474906, + "tokens_seen": 2953627648 + }, + { + "epoch": 1.28, + "learning_rate": 8.503911735205617e-05, + "loss": 1.255, + "theoretical_loss": 3.3532111362932047, + "tokens_seen": 2953693184 + }, + { + "epoch": 1.28, + "learning_rate": 8.502306920762287e-05, + "loss": 1.2442, + "theoretical_loss": 3.3532054466005103, + "tokens_seen": 2953758720 + }, + { + "epoch": 1.28, + "learning_rate": 8.500702106318957e-05, + "loss": 1.2661, + "theoretical_loss": 3.3531997570694, + "tokens_seen": 2953824256 + }, + { + "epoch": 1.28, + "learning_rate": 8.499097291875627e-05, + "loss": 1.1142, + "theoretical_loss": 3.3531940676998646, + "tokens_seen": 2953889792 + }, + { + "epoch": 1.28, + "learning_rate": 8.497492477432297e-05, + "loss": 1.1555, + "theoretical_loss": 3.353188378491897, + "tokens_seen": 2953955328 + }, + { + "epoch": 1.28, + "learning_rate": 8.495887662988967e-05, + "loss": 1.1007, + "theoretical_loss": 3.3531826894454886, + "tokens_seen": 2954020864 + }, + { + "epoch": 1.28, + "learning_rate": 8.494282848545637e-05, + "loss": 1.2354, + "theoretical_loss": 3.353177000560631, + "tokens_seen": 2954086400 + }, + { + "epoch": 1.28, + "learning_rate": 8.492678034102307e-05, + "loss": 1.2437, + "theoretical_loss": 3.3531713118373165, + "tokens_seen": 2954151936 + }, + { + "epoch": 1.28, + "learning_rate": 8.491073219658977e-05, + "loss": 1.0921, + "theoretical_loss": 3.3531656232755362, + "tokens_seen": 2954217472 + }, + { + "epoch": 1.28, + "learning_rate": 8.489468405215648e-05, + "loss": 1.238, + "theoretical_loss": 3.3531599348752827, + "tokens_seen": 2954283008 + }, + { + "epoch": 1.28, + "learning_rate": 8.487863590772318e-05, + "loss": 1.2358, + "theoretical_loss": 3.3531542466365476, + "tokens_seen": 2954348544 + }, + { + "epoch": 1.28, + "learning_rate": 8.486258776328986e-05, + "loss": 1.0795, + "theoretical_loss": 3.3531485585593224, + "tokens_seen": 2954414080 + }, + { + "epoch": 1.28, + "learning_rate": 8.484653961885656e-05, + "loss": 1.2361, + "theoretical_loss": 3.3531428706435995, + "tokens_seen": 2954479616 + }, + { + "epoch": 1.28, + "learning_rate": 8.483049147442328e-05, + "loss": 1.2757, + "theoretical_loss": 3.35313718288937, + "tokens_seen": 2954545152 + }, + { + "epoch": 1.28, + "learning_rate": 8.481444332998998e-05, + "loss": 1.2059, + "theoretical_loss": 3.3531314952966267, + "tokens_seen": 2954610688 + }, + { + "epoch": 1.28, + "learning_rate": 8.479839518555668e-05, + "loss": 1.1462, + "theoretical_loss": 3.3531258078653603, + "tokens_seen": 2954676224 + }, + { + "epoch": 1.28, + "learning_rate": 8.478234704112338e-05, + "loss": 1.2036, + "theoretical_loss": 3.3531201205955634, + "tokens_seen": 2954741760 + }, + { + "epoch": 1.28, + "learning_rate": 8.476629889669008e-05, + "loss": 1.1387, + "theoretical_loss": 3.353114433487228, + "tokens_seen": 2954807296 + }, + { + "epoch": 1.28, + "learning_rate": 8.475025075225678e-05, + "loss": 1.2633, + "theoretical_loss": 3.3531087465403453, + "tokens_seen": 2954872832 + }, + { + "epoch": 1.28, + "learning_rate": 8.473420260782348e-05, + "loss": 1.1357, + "theoretical_loss": 3.353103059754907, + "tokens_seen": 2954938368 + }, + { + "epoch": 1.28, + "learning_rate": 8.471815446339017e-05, + "loss": 1.1968, + "theoretical_loss": 3.353097373130906, + "tokens_seen": 2955003904 + }, + { + "epoch": 1.28, + "learning_rate": 8.470210631895687e-05, + "loss": 1.0911, + "theoretical_loss": 3.3530916866683333, + "tokens_seen": 2955069440 + }, + { + "epoch": 1.28, + "learning_rate": 8.468605817452357e-05, + "loss": 1.2096, + "theoretical_loss": 3.353086000367181, + "tokens_seen": 2955134976 + }, + { + "epoch": 1.28, + "learning_rate": 8.467001003009027e-05, + "loss": 1.1983, + "theoretical_loss": 3.353080314227441, + "tokens_seen": 2955200512 + }, + { + "epoch": 1.28, + "learning_rate": 8.465396188565697e-05, + "loss": 1.1885, + "theoretical_loss": 3.3530746282491046, + "tokens_seen": 2955266048 + }, + { + "epoch": 1.28, + "learning_rate": 8.463791374122367e-05, + "loss": 1.1371, + "theoretical_loss": 3.353068942432164, + "tokens_seen": 2955331584 + }, + { + "epoch": 1.28, + "learning_rate": 8.462186559679037e-05, + "loss": 1.2269, + "theoretical_loss": 3.353063256776611, + "tokens_seen": 2955397120 + }, + { + "epoch": 1.28, + "learning_rate": 8.460581745235707e-05, + "loss": 1.1713, + "theoretical_loss": 3.353057571282438, + "tokens_seen": 2955462656 + }, + { + "epoch": 1.28, + "learning_rate": 8.458976930792377e-05, + "loss": 1.0992, + "theoretical_loss": 3.353051885949636, + "tokens_seen": 2955528192 + }, + { + "epoch": 1.28, + "learning_rate": 8.457372116349047e-05, + "loss": 1.3104, + "theoretical_loss": 3.3530462007781976, + "tokens_seen": 2955593728 + }, + { + "epoch": 1.28, + "objective/train/docs_used": 1091085, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2002942562103271, + "objective/train/theoretical_loss": 3.3530405157681145, + "objective/train/tokens_used": 2976119264, + "theoretical_loss": 3.3530405157681145, + "tokens_seen": 2955659264 + }, + { + "epoch": 1.28, + "learning_rate": 8.455767301905718e-05, + "loss": 1.2003, + "theoretical_loss": 3.3530405157681145, + "tokens_seen": 2955659264 + }, + { + "epoch": 1.28, + "learning_rate": 8.454162487462388e-05, + "loss": 1.1453, + "theoretical_loss": 3.353034830919378, + "tokens_seen": 2955724800 + }, + { + "epoch": 1.28, + "learning_rate": 8.452557673019058e-05, + "loss": 1.2345, + "theoretical_loss": 3.3530291462319797, + "tokens_seen": 2955790336 + }, + { + "epoch": 1.28, + "learning_rate": 8.450952858575728e-05, + "loss": 1.1656, + "theoretical_loss": 3.353023461705912, + "tokens_seen": 2955855872 + }, + { + "epoch": 1.28, + "learning_rate": 8.449348044132398e-05, + "loss": 1.1135, + "theoretical_loss": 3.3530177773411673, + "tokens_seen": 2955921408 + }, + { + "epoch": 1.28, + "learning_rate": 8.447743229689068e-05, + "loss": 1.1962, + "theoretical_loss": 3.3530120931377367, + "tokens_seen": 2955986944 + }, + { + "epoch": 1.28, + "learning_rate": 8.446138415245738e-05, + "loss": 1.3245, + "theoretical_loss": 3.353006409095612, + "tokens_seen": 2956052480 + }, + { + "epoch": 1.28, + "learning_rate": 8.444533600802408e-05, + "loss": 1.1327, + "theoretical_loss": 3.3530007252147858, + "tokens_seen": 2956118016 + }, + { + "epoch": 1.28, + "learning_rate": 8.442928786359078e-05, + "loss": 1.1433, + "theoretical_loss": 3.352995041495249, + "tokens_seen": 2956183552 + }, + { + "epoch": 1.28, + "learning_rate": 8.441323971915748e-05, + "loss": 1.3082, + "theoretical_loss": 3.352989357936994, + "tokens_seen": 2956249088 + }, + { + "epoch": 1.28, + "learning_rate": 8.439719157472418e-05, + "loss": 1.2019, + "theoretical_loss": 3.3529836745400123, + "tokens_seen": 2956314624 + }, + { + "epoch": 1.28, + "learning_rate": 8.438114343029088e-05, + "loss": 1.2336, + "theoretical_loss": 3.352977991304296, + "tokens_seen": 2956380160 + }, + { + "epoch": 1.28, + "learning_rate": 8.436509528585757e-05, + "loss": 1.2863, + "theoretical_loss": 3.352972308229837, + "tokens_seen": 2956445696 + }, + { + "epoch": 1.28, + "learning_rate": 8.434904714142427e-05, + "loss": 1.249, + "theoretical_loss": 3.3529666253166273, + "tokens_seen": 2956511232 + }, + { + "epoch": 1.28, + "learning_rate": 8.433299899699097e-05, + "loss": 1.1596, + "theoretical_loss": 3.352960942564658, + "tokens_seen": 2956576768 + }, + { + "epoch": 1.28, + "learning_rate": 8.431695085255767e-05, + "loss": 1.141, + "theoretical_loss": 3.3529552599739216, + "tokens_seen": 2956642304 + }, + { + "epoch": 1.28, + "learning_rate": 8.430090270812437e-05, + "loss": 1.0919, + "theoretical_loss": 3.35294957754441, + "tokens_seen": 2956707840 + }, + { + "epoch": 1.28, + "learning_rate": 8.428485456369107e-05, + "loss": 1.2524, + "theoretical_loss": 3.352943895276115, + "tokens_seen": 2956773376 + }, + { + "epoch": 1.28, + "learning_rate": 8.426880641925777e-05, + "loss": 1.207, + "theoretical_loss": 3.3529382131690277, + "tokens_seen": 2956838912 + }, + { + "epoch": 1.28, + "learning_rate": 8.425275827482447e-05, + "loss": 1.1997, + "theoretical_loss": 3.352932531223141, + "tokens_seen": 2956904448 + }, + { + "epoch": 1.28, + "learning_rate": 8.423671013039117e-05, + "loss": 1.188, + "theoretical_loss": 3.3529268494384463, + "tokens_seen": 2956969984 + }, + { + "epoch": 1.28, + "learning_rate": 8.422066198595789e-05, + "loss": 1.3023, + "theoretical_loss": 3.3529211678149355, + "tokens_seen": 2957035520 + }, + { + "epoch": 1.28, + "learning_rate": 8.420461384152458e-05, + "loss": 1.2477, + "theoretical_loss": 3.3529154863526003, + "tokens_seen": 2957101056 + }, + { + "epoch": 1.28, + "learning_rate": 8.418856569709128e-05, + "loss": 1.1759, + "theoretical_loss": 3.352909805051433, + "tokens_seen": 2957166592 + }, + { + "epoch": 1.28, + "learning_rate": 8.417251755265798e-05, + "loss": 1.1622, + "theoretical_loss": 3.352904123911425, + "tokens_seen": 2957232128 + }, + { + "epoch": 1.28, + "learning_rate": 8.415646940822468e-05, + "loss": 1.3214, + "theoretical_loss": 3.352898442932568, + "tokens_seen": 2957297664 + }, + { + "epoch": 1.28, + "learning_rate": 8.414042126379138e-05, + "loss": 1.1329, + "theoretical_loss": 3.3528927621148545, + "tokens_seen": 2957363200 + }, + { + "epoch": 1.28, + "learning_rate": 8.412437311935808e-05, + "loss": 1.1757, + "theoretical_loss": 3.3528870814582756, + "tokens_seen": 2957428736 + }, + { + "epoch": 1.28, + "learning_rate": 8.410832497492478e-05, + "loss": 1.2115, + "theoretical_loss": 3.352881400962824, + "tokens_seen": 2957494272 + }, + { + "epoch": 1.28, + "learning_rate": 8.409227683049148e-05, + "loss": 1.1896, + "theoretical_loss": 3.352875720628491, + "tokens_seen": 2957559808 + }, + { + "epoch": 1.28, + "learning_rate": 8.407622868605818e-05, + "loss": 1.2296, + "theoretical_loss": 3.3528700404552687, + "tokens_seen": 2957625344 + }, + { + "epoch": 1.28, + "learning_rate": 8.406018054162488e-05, + "loss": 1.1856, + "theoretical_loss": 3.3528643604431485, + "tokens_seen": 2957690880 + }, + { + "epoch": 1.28, + "learning_rate": 8.404413239719158e-05, + "loss": 1.1343, + "theoretical_loss": 3.352858680592123, + "tokens_seen": 2957756416 + }, + { + "epoch": 1.28, + "learning_rate": 8.402808425275828e-05, + "loss": 1.146, + "theoretical_loss": 3.3528530009021837, + "tokens_seen": 2957821952 + }, + { + "epoch": 1.28, + "learning_rate": 8.401203610832497e-05, + "loss": 1.1483, + "theoretical_loss": 3.3528473213733223, + "tokens_seen": 2957887488 + }, + { + "epoch": 1.28, + "learning_rate": 8.399598796389167e-05, + "loss": 1.1199, + "theoretical_loss": 3.3528416420055307, + "tokens_seen": 2957953024 + }, + { + "epoch": 1.28, + "learning_rate": 8.397993981945837e-05, + "loss": 1.2092, + "theoretical_loss": 3.352835962798801, + "tokens_seen": 2958018560 + }, + { + "epoch": 1.28, + "learning_rate": 8.396389167502507e-05, + "loss": 1.0768, + "theoretical_loss": 3.352830283753125, + "tokens_seen": 2958084096 + }, + { + "epoch": 1.28, + "learning_rate": 8.394784353059177e-05, + "loss": 1.177, + "theoretical_loss": 3.3528246048684944, + "tokens_seen": 2958149632 + }, + { + "epoch": 1.28, + "learning_rate": 8.393179538615849e-05, + "loss": 1.1732, + "theoretical_loss": 3.3528189261449013, + "tokens_seen": 2958215168 + }, + { + "epoch": 1.28, + "learning_rate": 8.391574724172519e-05, + "loss": 1.1446, + "theoretical_loss": 3.352813247582337, + "tokens_seen": 2958280704 + }, + { + "epoch": 1.28, + "learning_rate": 8.389969909729189e-05, + "loss": 1.0335, + "theoretical_loss": 3.3528075691807944, + "tokens_seen": 2958346240 + }, + { + "epoch": 1.28, + "learning_rate": 8.388365095285859e-05, + "loss": 1.2491, + "theoretical_loss": 3.352801890940264, + "tokens_seen": 2958411776 + }, + { + "epoch": 1.28, + "learning_rate": 8.386760280842529e-05, + "loss": 1.1979, + "theoretical_loss": 3.352796212860739, + "tokens_seen": 2958477312 + }, + { + "epoch": 1.28, + "learning_rate": 8.385155466399198e-05, + "loss": 1.2118, + "theoretical_loss": 3.3527905349422107, + "tokens_seen": 2958542848 + }, + { + "epoch": 1.28, + "learning_rate": 8.383550651955868e-05, + "loss": 1.1419, + "theoretical_loss": 3.3527848571846706, + "tokens_seen": 2958608384 + }, + { + "epoch": 1.28, + "learning_rate": 8.381945837512538e-05, + "loss": 1.2068, + "theoretical_loss": 3.3527791795881114, + "tokens_seen": 2958673920 + }, + { + "epoch": 1.28, + "learning_rate": 8.380341023069208e-05, + "loss": 1.1378, + "theoretical_loss": 3.3527735021525245, + "tokens_seen": 2958739456 + }, + { + "epoch": 1.28, + "learning_rate": 8.378736208625878e-05, + "loss": 1.1852, + "theoretical_loss": 3.352767824877901, + "tokens_seen": 2958804992 + }, + { + "epoch": 1.28, + "learning_rate": 8.377131394182548e-05, + "loss": 1.1735, + "theoretical_loss": 3.3527621477642344, + "tokens_seen": 2958870528 + }, + { + "epoch": 1.28, + "learning_rate": 8.375526579739218e-05, + "loss": 1.0806, + "theoretical_loss": 3.3527564708115154, + "tokens_seen": 2958936064 + }, + { + "epoch": 1.28, + "learning_rate": 8.373921765295888e-05, + "loss": 1.2662, + "theoretical_loss": 3.3527507940197365, + "tokens_seen": 2959001600 + }, + { + "epoch": 1.28, + "learning_rate": 8.372316950852558e-05, + "loss": 1.2448, + "theoretical_loss": 3.352745117388889, + "tokens_seen": 2959067136 + }, + { + "epoch": 1.28, + "learning_rate": 8.370712136409228e-05, + "loss": 1.25, + "theoretical_loss": 3.352739440918965, + "tokens_seen": 2959132672 + }, + { + "epoch": 1.28, + "learning_rate": 8.369107321965898e-05, + "loss": 1.2317, + "theoretical_loss": 3.3527337646099564, + "tokens_seen": 2959198208 + }, + { + "epoch": 1.28, + "learning_rate": 8.367502507522568e-05, + "loss": 1.1401, + "theoretical_loss": 3.352728088461855, + "tokens_seen": 2959263744 + }, + { + "epoch": 1.28, + "learning_rate": 8.365897693079237e-05, + "loss": 1.0944, + "theoretical_loss": 3.352722412474653, + "tokens_seen": 2959329280 + }, + { + "epoch": 1.28, + "learning_rate": 8.364292878635907e-05, + "loss": 1.1485, + "theoretical_loss": 3.352716736648342, + "tokens_seen": 2959394816 + }, + { + "epoch": 1.28, + "learning_rate": 8.362688064192579e-05, + "loss": 1.2258, + "theoretical_loss": 3.352711060982914, + "tokens_seen": 2959460352 + }, + { + "epoch": 1.28, + "learning_rate": 8.361083249749249e-05, + "loss": 1.1857, + "theoretical_loss": 3.3527053854783606, + "tokens_seen": 2959525888 + }, + { + "epoch": 1.28, + "learning_rate": 8.359478435305919e-05, + "loss": 1.1981, + "theoretical_loss": 3.352699710134674, + "tokens_seen": 2959591424 + }, + { + "epoch": 1.28, + "learning_rate": 8.357873620862589e-05, + "loss": 1.11, + "theoretical_loss": 3.352694034951846, + "tokens_seen": 2959656960 + }, + { + "epoch": 1.28, + "learning_rate": 8.356268806419259e-05, + "loss": 1.2276, + "theoretical_loss": 3.352688359929868, + "tokens_seen": 2959722496 + }, + { + "epoch": 1.28, + "learning_rate": 8.354663991975929e-05, + "loss": 1.2085, + "theoretical_loss": 3.3526826850687326, + "tokens_seen": 2959788032 + }, + { + "epoch": 1.28, + "learning_rate": 8.353059177532599e-05, + "loss": 1.1017, + "theoretical_loss": 3.3526770103684314, + "tokens_seen": 2959853568 + }, + { + "epoch": 1.28, + "learning_rate": 8.351454363089269e-05, + "loss": 1.2258, + "theoretical_loss": 3.352671335828956, + "tokens_seen": 2959919104 + }, + { + "epoch": 1.28, + "learning_rate": 8.349849548645938e-05, + "loss": 1.1934, + "theoretical_loss": 3.352665661450299, + "tokens_seen": 2959984640 + }, + { + "epoch": 1.28, + "learning_rate": 8.348244734202608e-05, + "loss": 1.2288, + "theoretical_loss": 3.352659987232452, + "tokens_seen": 2960050176 + }, + { + "epoch": 1.28, + "learning_rate": 8.346639919759278e-05, + "loss": 1.175, + "theoretical_loss": 3.3526543131754063, + "tokens_seen": 2960115712 + }, + { + "epoch": 1.28, + "learning_rate": 8.345035105315948e-05, + "loss": 1.2485, + "theoretical_loss": 3.3526486392791544, + "tokens_seen": 2960181248 + }, + { + "epoch": 1.28, + "learning_rate": 8.343430290872618e-05, + "loss": 1.1057, + "theoretical_loss": 3.3526429655436876, + "tokens_seen": 2960246784 + }, + { + "epoch": 1.28, + "learning_rate": 8.341825476429288e-05, + "loss": 1.1369, + "theoretical_loss": 3.3526372919689984, + "tokens_seen": 2960312320 + }, + { + "epoch": 1.28, + "learning_rate": 8.340220661985958e-05, + "loss": 1.1725, + "theoretical_loss": 3.352631618555079, + "tokens_seen": 2960377856 + }, + { + "epoch": 1.28, + "learning_rate": 8.338615847542628e-05, + "loss": 1.1715, + "theoretical_loss": 3.35262594530192, + "tokens_seen": 2960443392 + }, + { + "epoch": 1.28, + "learning_rate": 8.337011033099298e-05, + "loss": 1.2715, + "theoretical_loss": 3.352620272209514, + "tokens_seen": 2960508928 + }, + { + "epoch": 1.28, + "learning_rate": 8.335406218655968e-05, + "loss": 1.1974, + "theoretical_loss": 3.352614599277853, + "tokens_seen": 2960574464 + }, + { + "epoch": 1.28, + "learning_rate": 8.333801404212638e-05, + "loss": 1.2106, + "theoretical_loss": 3.352608926506929, + "tokens_seen": 2960640000 + }, + { + "epoch": 1.28, + "learning_rate": 8.332196589769308e-05, + "loss": 1.1643, + "theoretical_loss": 3.352603253896734, + "tokens_seen": 2960705536 + }, + { + "epoch": 1.28, + "learning_rate": 8.330591775325978e-05, + "loss": 1.2016, + "theoretical_loss": 3.352597581447259, + "tokens_seen": 2960771072 + }, + { + "epoch": 1.28, + "learning_rate": 8.328986960882648e-05, + "loss": 1.1459, + "theoretical_loss": 3.352591909158497, + "tokens_seen": 2960836608 + }, + { + "epoch": 1.28, + "learning_rate": 8.327382146439319e-05, + "loss": 1.0896, + "theoretical_loss": 3.352586237030439, + "tokens_seen": 2960902144 + }, + { + "epoch": 1.28, + "learning_rate": 8.325777331995989e-05, + "loss": 1.1709, + "theoretical_loss": 3.352580565063077, + "tokens_seen": 2960967680 + }, + { + "epoch": 1.28, + "learning_rate": 8.324172517552659e-05, + "loss": 1.1331, + "theoretical_loss": 3.3525748932564037, + "tokens_seen": 2961033216 + }, + { + "epoch": 1.28, + "learning_rate": 8.322567703109329e-05, + "loss": 1.1422, + "theoretical_loss": 3.35256922161041, + "tokens_seen": 2961098752 + }, + { + "epoch": 1.28, + "learning_rate": 8.320962888665999e-05, + "loss": 1.1014, + "theoretical_loss": 3.3525635501250886, + "tokens_seen": 2961164288 + }, + { + "epoch": 1.28, + "learning_rate": 8.319358074222669e-05, + "loss": 1.2245, + "theoretical_loss": 3.3525578788004307, + "tokens_seen": 2961229824 + }, + { + "epoch": 1.28, + "learning_rate": 8.317753259779339e-05, + "loss": 1.2767, + "theoretical_loss": 3.3525522076364282, + "tokens_seen": 2961295360 + }, + { + "epoch": 1.28, + "learning_rate": 8.316148445336009e-05, + "loss": 1.233, + "theoretical_loss": 3.352546536633074, + "tokens_seen": 2961360896 + }, + { + "epoch": 1.28, + "learning_rate": 8.314543630892678e-05, + "loss": 1.223, + "theoretical_loss": 3.352540865790359, + "tokens_seen": 2961426432 + }, + { + "epoch": 1.28, + "learning_rate": 8.312938816449348e-05, + "loss": 1.2235, + "theoretical_loss": 3.352535195108275, + "tokens_seen": 2961491968 + }, + { + "epoch": 1.28, + "learning_rate": 8.311334002006018e-05, + "loss": 1.2444, + "theoretical_loss": 3.3525295245868145, + "tokens_seen": 2961557504 + }, + { + "epoch": 1.28, + "learning_rate": 8.309729187562688e-05, + "loss": 1.1619, + "theoretical_loss": 3.352523854225969, + "tokens_seen": 2961623040 + }, + { + "epoch": 1.28, + "learning_rate": 8.308124373119358e-05, + "loss": 1.1496, + "theoretical_loss": 3.3525181840257314, + "tokens_seen": 2961688576 + }, + { + "epoch": 1.28, + "learning_rate": 8.306519558676028e-05, + "loss": 1.1803, + "theoretical_loss": 3.352512513986092, + "tokens_seen": 2961754112 + }, + { + "epoch": 1.28, + "learning_rate": 8.304914744232698e-05, + "loss": 1.1564, + "theoretical_loss": 3.352506844107044, + "tokens_seen": 2961819648 + }, + { + "epoch": 1.28, + "learning_rate": 8.303309929789368e-05, + "loss": 1.1345, + "theoretical_loss": 3.3525011743885784, + "tokens_seen": 2961885184 + }, + { + "epoch": 1.28, + "learning_rate": 8.30170511534604e-05, + "loss": 1.1635, + "theoretical_loss": 3.3524955048306873, + "tokens_seen": 2961950720 + }, + { + "epoch": 1.28, + "learning_rate": 8.30010030090271e-05, + "loss": 1.2045, + "theoretical_loss": 3.3524898354333628, + "tokens_seen": 2962016256 + }, + { + "epoch": 1.28, + "learning_rate": 8.298495486459378e-05, + "loss": 1.216, + "theoretical_loss": 3.3524841661965974, + "tokens_seen": 2962081792 + }, + { + "epoch": 1.28, + "learning_rate": 8.296890672016048e-05, + "loss": 1.1912, + "theoretical_loss": 3.352478497120382, + "tokens_seen": 2962147328 + }, + { + "epoch": 1.28, + "objective/train/docs_used": 1093205, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1135929822921753, + "objective/train/theoretical_loss": 3.3524728282047085, + "objective/train/tokens_used": 2982672864, + "theoretical_loss": 3.3524728282047085, + "tokens_seen": 2962212864 + }, + { + "epoch": 1.28, + "learning_rate": 8.295285857572718e-05, + "loss": 1.1136, + "theoretical_loss": 3.3524728282047085, + "tokens_seen": 2962212864 + }, + { + "epoch": 1.28, + "learning_rate": 8.293681043129389e-05, + "loss": 1.1994, + "theoretical_loss": 3.3524671594495694, + "tokens_seen": 2962278400 + }, + { + "epoch": 1.28, + "learning_rate": 8.292076228686059e-05, + "loss": 1.1319, + "theoretical_loss": 3.3524614908549566, + "tokens_seen": 2962343936 + }, + { + "epoch": 1.28, + "learning_rate": 8.290471414242729e-05, + "loss": 1.2593, + "theoretical_loss": 3.3524558224208616, + "tokens_seen": 2962409472 + }, + { + "epoch": 1.28, + "learning_rate": 8.288866599799399e-05, + "loss": 1.2147, + "theoretical_loss": 3.3524501541472764, + "tokens_seen": 2962475008 + }, + { + "epoch": 1.28, + "learning_rate": 8.287261785356069e-05, + "loss": 1.1428, + "theoretical_loss": 3.352444486034193, + "tokens_seen": 2962540544 + }, + { + "epoch": 1.28, + "learning_rate": 8.285656970912739e-05, + "loss": 1.1777, + "theoretical_loss": 3.3524388180816036, + "tokens_seen": 2962606080 + }, + { + "epoch": 1.28, + "learning_rate": 8.284052156469409e-05, + "loss": 1.1619, + "theoretical_loss": 3.3524331502894995, + "tokens_seen": 2962671616 + }, + { + "epoch": 1.28, + "learning_rate": 8.282447342026079e-05, + "loss": 1.0919, + "theoretical_loss": 3.352427482657873, + "tokens_seen": 2962737152 + }, + { + "epoch": 1.28, + "learning_rate": 8.280842527582749e-05, + "loss": 1.1064, + "theoretical_loss": 3.3524218151867156, + "tokens_seen": 2962802688 + }, + { + "epoch": 1.28, + "learning_rate": 8.279237713139418e-05, + "loss": 1.2159, + "theoretical_loss": 3.3524161478760197, + "tokens_seen": 2962868224 + }, + { + "epoch": 1.28, + "learning_rate": 8.277632898696088e-05, + "loss": 1.1944, + "theoretical_loss": 3.3524104807257773, + "tokens_seen": 2962933760 + }, + { + "epoch": 1.28, + "learning_rate": 8.276028084252758e-05, + "loss": 1.2317, + "theoretical_loss": 3.3524048137359794, + "tokens_seen": 2962999296 + }, + { + "epoch": 1.28, + "learning_rate": 8.274423269809428e-05, + "loss": 1.1834, + "theoretical_loss": 3.3523991469066194, + "tokens_seen": 2963064832 + }, + { + "epoch": 1.28, + "learning_rate": 8.2728184553661e-05, + "loss": 1.2949, + "theoretical_loss": 3.3523934802376876, + "tokens_seen": 2963130368 + }, + { + "epoch": 1.28, + "learning_rate": 8.27121364092277e-05, + "loss": 1.211, + "theoretical_loss": 3.352387813729177, + "tokens_seen": 2963195904 + }, + { + "epoch": 1.28, + "learning_rate": 8.26960882647944e-05, + "loss": 1.1881, + "theoretical_loss": 3.3523821473810793, + "tokens_seen": 2963261440 + }, + { + "epoch": 1.28, + "learning_rate": 8.26800401203611e-05, + "loss": 1.1483, + "theoretical_loss": 3.352376481193386, + "tokens_seen": 2963326976 + }, + { + "epoch": 1.28, + "learning_rate": 8.26639919759278e-05, + "loss": 1.1664, + "theoretical_loss": 3.3523708151660894, + "tokens_seen": 2963392512 + }, + { + "epoch": 1.28, + "learning_rate": 8.264794383149448e-05, + "loss": 1.2608, + "theoretical_loss": 3.3523651492991813, + "tokens_seen": 2963458048 + }, + { + "epoch": 1.28, + "learning_rate": 8.263189568706118e-05, + "loss": 1.0781, + "theoretical_loss": 3.352359483592654, + "tokens_seen": 2963523584 + }, + { + "epoch": 1.28, + "learning_rate": 8.261584754262788e-05, + "loss": 1.1236, + "theoretical_loss": 3.352353818046498, + "tokens_seen": 2963589120 + }, + { + "epoch": 1.28, + "learning_rate": 8.259979939819459e-05, + "loss": 1.1916, + "theoretical_loss": 3.3523481526607073, + "tokens_seen": 2963654656 + }, + { + "epoch": 1.28, + "learning_rate": 8.258375125376129e-05, + "loss": 1.2265, + "theoretical_loss": 3.352342487435272, + "tokens_seen": 2963720192 + }, + { + "epoch": 1.28, + "learning_rate": 8.256770310932799e-05, + "loss": 1.2097, + "theoretical_loss": 3.3523368223701855, + "tokens_seen": 2963785728 + }, + { + "epoch": 1.28, + "learning_rate": 8.255165496489469e-05, + "loss": 1.1094, + "theoretical_loss": 3.3523311574654384, + "tokens_seen": 2963851264 + }, + { + "epoch": 1.28, + "learning_rate": 8.253560682046139e-05, + "loss": 1.1993, + "theoretical_loss": 3.3523254927210235, + "tokens_seen": 2963916800 + }, + { + "epoch": 1.28, + "learning_rate": 8.251955867602809e-05, + "loss": 1.1977, + "theoretical_loss": 3.352319828136932, + "tokens_seen": 2963982336 + }, + { + "epoch": 1.28, + "learning_rate": 8.250351053159479e-05, + "loss": 1.1411, + "theoretical_loss": 3.3523141637131566, + "tokens_seen": 2964047872 + }, + { + "epoch": 1.28, + "learning_rate": 8.248746238716149e-05, + "loss": 1.2433, + "theoretical_loss": 3.352308499449689, + "tokens_seen": 2964113408 + }, + { + "epoch": 1.28, + "learning_rate": 8.247141424272819e-05, + "loss": 1.0582, + "theoretical_loss": 3.3523028353465207, + "tokens_seen": 2964178944 + }, + { + "epoch": 1.28, + "learning_rate": 8.245536609829488e-05, + "loss": 1.1673, + "theoretical_loss": 3.3522971714036442, + "tokens_seen": 2964244480 + }, + { + "epoch": 1.28, + "learning_rate": 8.243931795386158e-05, + "loss": 1.1026, + "theoretical_loss": 3.352291507621051, + "tokens_seen": 2964310016 + }, + { + "epoch": 1.28, + "learning_rate": 8.242326980942829e-05, + "loss": 1.2634, + "theoretical_loss": 3.3522858439987333, + "tokens_seen": 2964375552 + }, + { + "epoch": 1.28, + "learning_rate": 8.240722166499499e-05, + "loss": 1.1212, + "theoretical_loss": 3.3522801805366824, + "tokens_seen": 2964441088 + }, + { + "epoch": 1.28, + "learning_rate": 8.23911735205617e-05, + "loss": 1.2182, + "theoretical_loss": 3.352274517234891, + "tokens_seen": 2964506624 + }, + { + "epoch": 1.28, + "learning_rate": 8.23751253761284e-05, + "loss": 1.2173, + "theoretical_loss": 3.3522688540933507, + "tokens_seen": 2964572160 + }, + { + "epoch": 1.28, + "learning_rate": 8.23590772316951e-05, + "loss": 1.1764, + "theoretical_loss": 3.352263191112053, + "tokens_seen": 2964637696 + }, + { + "epoch": 1.28, + "learning_rate": 8.23430290872618e-05, + "loss": 1.2466, + "theoretical_loss": 3.352257528290991, + "tokens_seen": 2964703232 + }, + { + "epoch": 1.28, + "learning_rate": 8.23269809428285e-05, + "loss": 1.1566, + "theoretical_loss": 3.3522518656301554, + "tokens_seen": 2964768768 + }, + { + "epoch": 1.28, + "learning_rate": 8.23109327983952e-05, + "loss": 1.1765, + "theoretical_loss": 3.3522462031295386, + "tokens_seen": 2964834304 + }, + { + "epoch": 1.28, + "learning_rate": 8.229488465396188e-05, + "loss": 1.1475, + "theoretical_loss": 3.3522405407891327, + "tokens_seen": 2964899840 + }, + { + "epoch": 1.28, + "learning_rate": 8.227883650952858e-05, + "loss": 1.2734, + "theoretical_loss": 3.3522348786089293, + "tokens_seen": 2964965376 + }, + { + "epoch": 1.28, + "learning_rate": 8.226278836509528e-05, + "loss": 1.2202, + "theoretical_loss": 3.3522292165889205, + "tokens_seen": 2965030912 + }, + { + "epoch": 1.28, + "learning_rate": 8.224674022066199e-05, + "loss": 1.2006, + "theoretical_loss": 3.3522235547290986, + "tokens_seen": 2965096448 + }, + { + "epoch": 1.28, + "learning_rate": 8.223069207622869e-05, + "loss": 1.187, + "theoretical_loss": 3.352217893029455, + "tokens_seen": 2965161984 + }, + { + "epoch": 1.28, + "learning_rate": 8.221464393179539e-05, + "loss": 1.1165, + "theoretical_loss": 3.352212231489981, + "tokens_seen": 2965227520 + }, + { + "epoch": 1.28, + "learning_rate": 8.219859578736209e-05, + "loss": 1.2193, + "theoretical_loss": 3.35220657011067, + "tokens_seen": 2965293056 + }, + { + "epoch": 1.28, + "learning_rate": 8.218254764292879e-05, + "loss": 1.1161, + "theoretical_loss": 3.352200908891513, + "tokens_seen": 2965358592 + }, + { + "epoch": 1.28, + "learning_rate": 8.216649949849549e-05, + "loss": 1.2323, + "theoretical_loss": 3.3521952478325026, + "tokens_seen": 2965424128 + }, + { + "epoch": 1.28, + "learning_rate": 8.215045135406219e-05, + "loss": 1.0629, + "theoretical_loss": 3.35218958693363, + "tokens_seen": 2965489664 + }, + { + "epoch": 1.28, + "learning_rate": 8.213440320962889e-05, + "loss": 1.1631, + "theoretical_loss": 3.352183926194887, + "tokens_seen": 2965555200 + }, + { + "epoch": 1.28, + "learning_rate": 8.211835506519559e-05, + "loss": 1.1559, + "theoretical_loss": 3.352178265616266, + "tokens_seen": 2965620736 + }, + { + "epoch": 1.28, + "learning_rate": 8.210230692076229e-05, + "loss": 1.1968, + "theoretical_loss": 3.3521726051977594, + "tokens_seen": 2965686272 + }, + { + "epoch": 1.28, + "learning_rate": 8.208625877632899e-05, + "loss": 1.2597, + "theoretical_loss": 3.3521669449393583, + "tokens_seen": 2965751808 + }, + { + "epoch": 1.28, + "learning_rate": 8.207021063189569e-05, + "loss": 1.1635, + "theoretical_loss": 3.3521612848410554, + "tokens_seen": 2965817344 + }, + { + "epoch": 1.28, + "learning_rate": 8.205416248746239e-05, + "loss": 1.1558, + "theoretical_loss": 3.3521556249028417, + "tokens_seen": 2965882880 + }, + { + "epoch": 1.28, + "learning_rate": 8.20381143430291e-05, + "loss": 1.1828, + "theoretical_loss": 3.3521499651247098, + "tokens_seen": 2965948416 + }, + { + "epoch": 1.28, + "learning_rate": 8.20220661985958e-05, + "loss": 1.2286, + "theoretical_loss": 3.352144305506651, + "tokens_seen": 2966013952 + }, + { + "epoch": 1.28, + "learning_rate": 8.20060180541625e-05, + "loss": 1.1906, + "theoretical_loss": 3.352138646048658, + "tokens_seen": 2966079488 + }, + { + "epoch": 1.28, + "learning_rate": 8.19899699097292e-05, + "loss": 1.3157, + "theoretical_loss": 3.3521329867507226, + "tokens_seen": 2966145024 + }, + { + "epoch": 1.28, + "learning_rate": 8.19739217652959e-05, + "loss": 1.2425, + "theoretical_loss": 3.3521273276128367, + "tokens_seen": 2966210560 + }, + { + "epoch": 1.28, + "learning_rate": 8.19578736208626e-05, + "loss": 1.1843, + "theoretical_loss": 3.3521216686349913, + "tokens_seen": 2966276096 + }, + { + "epoch": 1.28, + "learning_rate": 8.194182547642928e-05, + "loss": 1.2101, + "theoretical_loss": 3.3521160098171796, + "tokens_seen": 2966341632 + }, + { + "epoch": 1.28, + "learning_rate": 8.192577733199598e-05, + "loss": 1.255, + "theoretical_loss": 3.3521103511593933, + "tokens_seen": 2966407168 + }, + { + "epoch": 1.28, + "learning_rate": 8.190972918756269e-05, + "loss": 1.216, + "theoretical_loss": 3.352104692661624, + "tokens_seen": 2966472704 + }, + { + "epoch": 1.28, + "learning_rate": 8.189368104312939e-05, + "loss": 1.1548, + "theoretical_loss": 3.3520990343238637, + "tokens_seen": 2966538240 + }, + { + "epoch": 1.28, + "learning_rate": 8.187763289869609e-05, + "loss": 1.1864, + "theoretical_loss": 3.3520933761461045, + "tokens_seen": 2966603776 + }, + { + "epoch": 1.28, + "learning_rate": 8.186158475426279e-05, + "loss": 1.2384, + "theoretical_loss": 3.352087718128338, + "tokens_seen": 2966669312 + }, + { + "epoch": 1.28, + "learning_rate": 8.184553660982949e-05, + "loss": 1.2189, + "theoretical_loss": 3.3520820602705563, + "tokens_seen": 2966734848 + }, + { + "epoch": 1.28, + "learning_rate": 8.18294884653962e-05, + "loss": 1.1558, + "theoretical_loss": 3.352076402572752, + "tokens_seen": 2966800384 + }, + { + "epoch": 1.28, + "learning_rate": 8.18134403209629e-05, + "loss": 1.2386, + "theoretical_loss": 3.352070745034916, + "tokens_seen": 2966865920 + }, + { + "epoch": 1.28, + "learning_rate": 8.17973921765296e-05, + "loss": 1.2127, + "theoretical_loss": 3.3520650876570404, + "tokens_seen": 2966931456 + }, + { + "epoch": 1.28, + "learning_rate": 8.178134403209629e-05, + "loss": 1.2351, + "theoretical_loss": 3.352059430439118, + "tokens_seen": 2966996992 + }, + { + "epoch": 1.28, + "learning_rate": 8.176529588766299e-05, + "loss": 1.1642, + "theoretical_loss": 3.35205377338114, + "tokens_seen": 2967062528 + }, + { + "epoch": 1.28, + "learning_rate": 8.174924774322969e-05, + "loss": 1.1957, + "theoretical_loss": 3.3520481164830986, + "tokens_seen": 2967128064 + }, + { + "epoch": 1.28, + "learning_rate": 8.173319959879639e-05, + "loss": 1.0859, + "theoretical_loss": 3.3520424597449856, + "tokens_seen": 2967193600 + }, + { + "epoch": 1.28, + "learning_rate": 8.171715145436309e-05, + "loss": 1.1612, + "theoretical_loss": 3.352036803166793, + "tokens_seen": 2967259136 + }, + { + "epoch": 1.28, + "learning_rate": 8.17011033099298e-05, + "loss": 1.1238, + "theoretical_loss": 3.3520311467485127, + "tokens_seen": 2967324672 + }, + { + "epoch": 1.28, + "learning_rate": 8.16850551654965e-05, + "loss": 1.2318, + "theoretical_loss": 3.3520254904901368, + "tokens_seen": 2967390208 + }, + { + "epoch": 1.28, + "learning_rate": 8.16690070210632e-05, + "loss": 1.2221, + "theoretical_loss": 3.3520198343916574, + "tokens_seen": 2967455744 + }, + { + "epoch": 1.28, + "learning_rate": 8.16529588766299e-05, + "loss": 1.2053, + "theoretical_loss": 3.352014178453066, + "tokens_seen": 2967521280 + }, + { + "epoch": 1.28, + "learning_rate": 8.16369107321966e-05, + "loss": 1.1472, + "theoretical_loss": 3.352008522674355, + "tokens_seen": 2967586816 + }, + { + "epoch": 1.28, + "learning_rate": 8.16208625877633e-05, + "loss": 1.2334, + "theoretical_loss": 3.352002867055516, + "tokens_seen": 2967652352 + }, + { + "epoch": 1.28, + "learning_rate": 8.160481444333e-05, + "loss": 1.2124, + "theoretical_loss": 3.351997211596541, + "tokens_seen": 2967717888 + }, + { + "epoch": 1.28, + "learning_rate": 8.158876629889668e-05, + "loss": 1.2619, + "theoretical_loss": 3.351991556297422, + "tokens_seen": 2967783424 + }, + { + "epoch": 1.28, + "learning_rate": 8.157271815446339e-05, + "loss": 1.1511, + "theoretical_loss": 3.3519859011581508, + "tokens_seen": 2967848960 + }, + { + "epoch": 1.28, + "learning_rate": 8.155667001003009e-05, + "loss": 1.221, + "theoretical_loss": 3.35198024617872, + "tokens_seen": 2967914496 + }, + { + "epoch": 1.28, + "learning_rate": 8.154062186559679e-05, + "loss": 1.2615, + "theoretical_loss": 3.351974591359121, + "tokens_seen": 2967980032 + }, + { + "epoch": 1.28, + "learning_rate": 8.15245737211635e-05, + "loss": 1.2662, + "theoretical_loss": 3.3519689366993455, + "tokens_seen": 2968045568 + }, + { + "epoch": 1.28, + "learning_rate": 8.15085255767302e-05, + "loss": 1.2232, + "theoretical_loss": 3.3519632821993857, + "tokens_seen": 2968111104 + }, + { + "epoch": 1.28, + "learning_rate": 8.14924774322969e-05, + "loss": 1.1631, + "theoretical_loss": 3.351957627859234, + "tokens_seen": 2968176640 + }, + { + "epoch": 1.28, + "learning_rate": 8.14764292878636e-05, + "loss": 1.2651, + "theoretical_loss": 3.351951973678882, + "tokens_seen": 2968242176 + }, + { + "epoch": 1.28, + "learning_rate": 8.14603811434303e-05, + "loss": 1.1591, + "theoretical_loss": 3.3519463196583215, + "tokens_seen": 2968307712 + }, + { + "epoch": 1.28, + "learning_rate": 8.1444332998997e-05, + "loss": 1.1313, + "theoretical_loss": 3.3519406657975446, + "tokens_seen": 2968373248 + }, + { + "epoch": 1.28, + "learning_rate": 8.142828485456369e-05, + "loss": 1.1459, + "theoretical_loss": 3.3519350120965434, + "tokens_seen": 2968438784 + }, + { + "epoch": 1.28, + "learning_rate": 8.141223671013039e-05, + "loss": 1.2505, + "theoretical_loss": 3.3519293585553096, + "tokens_seen": 2968504320 + }, + { + "epoch": 1.28, + "learning_rate": 8.139618856569709e-05, + "loss": 1.348, + "theoretical_loss": 3.351923705173835, + "tokens_seen": 2968569856 + }, + { + "epoch": 1.28, + "learning_rate": 8.138014042126379e-05, + "loss": 1.2521, + "theoretical_loss": 3.3519180519521123, + "tokens_seen": 2968635392 + }, + { + "epoch": 1.28, + "learning_rate": 8.13640922768305e-05, + "loss": 1.2369, + "theoretical_loss": 3.351912398890133, + "tokens_seen": 2968700928 + }, + { + "epoch": 1.28, + "objective/train/docs_used": 1095429, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2445905208587646, + "objective/train/theoretical_loss": 3.351906745987889, + "objective/train/tokens_used": 2989226464, + "theoretical_loss": 3.351906745987889, + "tokens_seen": 2968766464 + }, + { + "epoch": 1.28, + "learning_rate": 8.13480441323972e-05, + "loss": 1.2446, + "theoretical_loss": 3.351906745987889, + "tokens_seen": 2968766464 + }, + { + "epoch": 1.28, + "learning_rate": 8.13319959879639e-05, + "loss": 1.1154, + "theoretical_loss": 3.351901093245372, + "tokens_seen": 2968832000 + }, + { + "epoch": 1.28, + "learning_rate": 8.13159478435306e-05, + "loss": 1.2844, + "theoretical_loss": 3.3518954406625747, + "tokens_seen": 2968897536 + }, + { + "epoch": 1.28, + "learning_rate": 8.12998996990973e-05, + "loss": 1.2611, + "theoretical_loss": 3.3518897882394882, + "tokens_seen": 2968963072 + }, + { + "epoch": 1.28, + "learning_rate": 8.1283851554664e-05, + "loss": 1.1697, + "theoretical_loss": 3.351884135976105, + "tokens_seen": 2969028608 + }, + { + "epoch": 1.28, + "learning_rate": 8.12678034102307e-05, + "loss": 1.1805, + "theoretical_loss": 3.3518784838724174, + "tokens_seen": 2969094144 + }, + { + "epoch": 1.28, + "learning_rate": 8.12517552657974e-05, + "loss": 1.3415, + "theoretical_loss": 3.3518728319284166, + "tokens_seen": 2969159680 + }, + { + "epoch": 1.28, + "learning_rate": 8.123570712136408e-05, + "loss": 1.1812, + "theoretical_loss": 3.351867180144095, + "tokens_seen": 2969225216 + }, + { + "epoch": 1.28, + "learning_rate": 8.12196589769308e-05, + "loss": 1.1783, + "theoretical_loss": 3.3518615285194446, + "tokens_seen": 2969290752 + }, + { + "epoch": 1.28, + "learning_rate": 8.12036108324975e-05, + "loss": 1.2441, + "theoretical_loss": 3.351855877054457, + "tokens_seen": 2969356288 + }, + { + "epoch": 1.28, + "learning_rate": 8.11875626880642e-05, + "loss": 1.1579, + "theoretical_loss": 3.3518502257491245, + "tokens_seen": 2969421824 + }, + { + "epoch": 1.28, + "learning_rate": 8.11715145436309e-05, + "loss": 1.3006, + "theoretical_loss": 3.351844574603439, + "tokens_seen": 2969487360 + }, + { + "epoch": 1.28, + "learning_rate": 8.11554663991976e-05, + "loss": 1.1842, + "theoretical_loss": 3.351838923617392, + "tokens_seen": 2969552896 + }, + { + "epoch": 1.28, + "learning_rate": 8.11394182547643e-05, + "loss": 1.2384, + "theoretical_loss": 3.351833272790976, + "tokens_seen": 2969618432 + }, + { + "epoch": 1.28, + "learning_rate": 8.1123370110331e-05, + "loss": 1.2765, + "theoretical_loss": 3.3518276221241834, + "tokens_seen": 2969683968 + }, + { + "epoch": 1.28, + "learning_rate": 8.11073219658977e-05, + "loss": 1.2241, + "theoretical_loss": 3.3518219716170052, + "tokens_seen": 2969749504 + }, + { + "epoch": 1.28, + "learning_rate": 8.10912738214644e-05, + "loss": 1.1145, + "theoretical_loss": 3.3518163212694336, + "tokens_seen": 2969815040 + }, + { + "epoch": 1.28, + "learning_rate": 8.107522567703109e-05, + "loss": 1.258, + "theoretical_loss": 3.3518106710814615, + "tokens_seen": 2969880576 + }, + { + "epoch": 1.28, + "learning_rate": 8.105917753259779e-05, + "loss": 1.1951, + "theoretical_loss": 3.3518050210530794, + "tokens_seen": 2969946112 + }, + { + "epoch": 1.28, + "learning_rate": 8.104312938816449e-05, + "loss": 1.2021, + "theoretical_loss": 3.3517993711842804, + "tokens_seen": 2970011648 + }, + { + "epoch": 1.28, + "learning_rate": 8.102708124373119e-05, + "loss": 1.1384, + "theoretical_loss": 3.351793721475056, + "tokens_seen": 2970077184 + }, + { + "epoch": 1.28, + "learning_rate": 8.10110330992979e-05, + "loss": 1.2516, + "theoretical_loss": 3.351788071925398, + "tokens_seen": 2970142720 + }, + { + "epoch": 1.28, + "learning_rate": 8.09949849548646e-05, + "loss": 1.1332, + "theoretical_loss": 3.3517824225352992, + "tokens_seen": 2970208256 + }, + { + "epoch": 1.28, + "learning_rate": 8.09789368104313e-05, + "loss": 1.1375, + "theoretical_loss": 3.3517767733047505, + "tokens_seen": 2970273792 + }, + { + "epoch": 1.28, + "learning_rate": 8.0962888665998e-05, + "loss": 1.2697, + "theoretical_loss": 3.3517711242337445, + "tokens_seen": 2970339328 + }, + { + "epoch": 1.28, + "learning_rate": 8.09468405215647e-05, + "loss": 1.2129, + "theoretical_loss": 3.351765475322273, + "tokens_seen": 2970404864 + }, + { + "epoch": 1.28, + "learning_rate": 8.09307923771314e-05, + "loss": 1.1298, + "theoretical_loss": 3.351759826570328, + "tokens_seen": 2970470400 + }, + { + "epoch": 1.28, + "learning_rate": 8.09147442326981e-05, + "loss": 1.2025, + "theoretical_loss": 3.3517541779779014, + "tokens_seen": 2970535936 + }, + { + "epoch": 1.28, + "learning_rate": 8.08986960882648e-05, + "loss": 1.1036, + "theoretical_loss": 3.3517485295449854, + "tokens_seen": 2970601472 + }, + { + "epoch": 1.28, + "learning_rate": 8.08826479438315e-05, + "loss": 1.2103, + "theoretical_loss": 3.351742881271572, + "tokens_seen": 2970667008 + }, + { + "epoch": 1.28, + "learning_rate": 8.08665997993982e-05, + "loss": 1.1762, + "theoretical_loss": 3.351737233157653, + "tokens_seen": 2970732544 + }, + { + "epoch": 1.28, + "learning_rate": 8.08505516549649e-05, + "loss": 1.1729, + "theoretical_loss": 3.3517315852032206, + "tokens_seen": 2970798080 + }, + { + "epoch": 1.28, + "learning_rate": 8.08345035105316e-05, + "loss": 1.2584, + "theoretical_loss": 3.351725937408266, + "tokens_seen": 2970863616 + }, + { + "epoch": 1.28, + "learning_rate": 8.08184553660983e-05, + "loss": 1.1948, + "theoretical_loss": 3.351720289772782, + "tokens_seen": 2970929152 + }, + { + "epoch": 1.28, + "learning_rate": 8.0802407221665e-05, + "loss": 1.1971, + "theoretical_loss": 3.3517146422967605, + "tokens_seen": 2970994688 + }, + { + "epoch": 1.28, + "learning_rate": 8.07863590772317e-05, + "loss": 1.0887, + "theoretical_loss": 3.3517089949801933, + "tokens_seen": 2971060224 + }, + { + "epoch": 1.28, + "learning_rate": 8.07703109327984e-05, + "loss": 1.2006, + "theoretical_loss": 3.351703347823072, + "tokens_seen": 2971125760 + }, + { + "epoch": 1.28, + "learning_rate": 8.07542627883651e-05, + "loss": 1.2649, + "theoretical_loss": 3.3516977008253894, + "tokens_seen": 2971191296 + }, + { + "epoch": 1.28, + "learning_rate": 8.073821464393179e-05, + "loss": 1.1105, + "theoretical_loss": 3.3516920539871373, + "tokens_seen": 2971256832 + }, + { + "epoch": 1.28, + "learning_rate": 8.072216649949849e-05, + "loss": 1.2374, + "theoretical_loss": 3.351686407308307, + "tokens_seen": 2971322368 + }, + { + "epoch": 1.28, + "learning_rate": 8.070611835506519e-05, + "loss": 1.2247, + "theoretical_loss": 3.351680760788891, + "tokens_seen": 2971387904 + }, + { + "epoch": 1.28, + "learning_rate": 8.069007021063189e-05, + "loss": 1.1611, + "theoretical_loss": 3.351675114428881, + "tokens_seen": 2971453440 + }, + { + "epoch": 1.28, + "learning_rate": 8.06740220661986e-05, + "loss": 1.1741, + "theoretical_loss": 3.3516694682282697, + "tokens_seen": 2971518976 + }, + { + "epoch": 1.28, + "learning_rate": 8.06579739217653e-05, + "loss": 1.1825, + "theoretical_loss": 3.351663822187048, + "tokens_seen": 2971584512 + }, + { + "epoch": 1.28, + "learning_rate": 8.0641925777332e-05, + "loss": 1.2202, + "theoretical_loss": 3.351658176305209, + "tokens_seen": 2971650048 + }, + { + "epoch": 1.28, + "learning_rate": 8.062587763289871e-05, + "loss": 1.2291, + "theoretical_loss": 3.351652530582744, + "tokens_seen": 2971715584 + }, + { + "epoch": 1.28, + "learning_rate": 8.060982948846541e-05, + "loss": 1.1867, + "theoretical_loss": 3.351646885019645, + "tokens_seen": 2971781120 + }, + { + "epoch": 1.28, + "learning_rate": 8.059378134403211e-05, + "loss": 1.2581, + "theoretical_loss": 3.3516412396159043, + "tokens_seen": 2971846656 + }, + { + "epoch": 1.28, + "learning_rate": 8.05777331995988e-05, + "loss": 1.2464, + "theoretical_loss": 3.3516355943715137, + "tokens_seen": 2971912192 + }, + { + "epoch": 1.28, + "learning_rate": 8.05616850551655e-05, + "loss": 1.1519, + "theoretical_loss": 3.351629949286465, + "tokens_seen": 2971977728 + }, + { + "epoch": 1.28, + "learning_rate": 8.05456369107322e-05, + "loss": 1.1807, + "theoretical_loss": 3.3516243043607505, + "tokens_seen": 2972043264 + }, + { + "epoch": 1.28, + "learning_rate": 8.05295887662989e-05, + "loss": 1.1919, + "theoretical_loss": 3.351618659594362, + "tokens_seen": 2972108800 + }, + { + "epoch": 1.28, + "learning_rate": 8.05135406218656e-05, + "loss": 1.259, + "theoretical_loss": 3.351613014987292, + "tokens_seen": 2972174336 + }, + { + "epoch": 1.28, + "learning_rate": 8.04974924774323e-05, + "loss": 1.1602, + "theoretical_loss": 3.3516073705395315, + "tokens_seen": 2972239872 + }, + { + "epoch": 1.28, + "learning_rate": 8.0481444332999e-05, + "loss": 1.2324, + "theoretical_loss": 3.351601726251073, + "tokens_seen": 2972305408 + }, + { + "epoch": 1.28, + "learning_rate": 8.04653961885657e-05, + "loss": 1.079, + "theoretical_loss": 3.351596082121909, + "tokens_seen": 2972370944 + }, + { + "epoch": 1.28, + "learning_rate": 8.04493480441324e-05, + "loss": 1.1277, + "theoretical_loss": 3.351590438152031, + "tokens_seen": 2972436480 + }, + { + "epoch": 1.28, + "learning_rate": 8.04332998996991e-05, + "loss": 1.1539, + "theoretical_loss": 3.3515847943414308, + "tokens_seen": 2972502016 + }, + { + "epoch": 1.28, + "learning_rate": 8.04172517552658e-05, + "loss": 1.164, + "theoretical_loss": 3.351579150690101, + "tokens_seen": 2972567552 + }, + { + "epoch": 1.28, + "learning_rate": 8.04012036108325e-05, + "loss": 1.2533, + "theoretical_loss": 3.351573507198033, + "tokens_seen": 2972633088 + }, + { + "epoch": 1.28, + "learning_rate": 8.038515546639919e-05, + "loss": 1.1078, + "theoretical_loss": 3.3515678638652187, + "tokens_seen": 2972698624 + }, + { + "epoch": 1.28, + "learning_rate": 8.036910732196589e-05, + "loss": 1.2306, + "theoretical_loss": 3.3515622206916507, + "tokens_seen": 2972764160 + }, + { + "epoch": 1.28, + "learning_rate": 8.035305917753259e-05, + "loss": 1.2674, + "theoretical_loss": 3.3515565776773206, + "tokens_seen": 2972829696 + }, + { + "epoch": 1.28, + "learning_rate": 8.033701103309929e-05, + "loss": 1.1994, + "theoretical_loss": 3.3515509348222206, + "tokens_seen": 2972895232 + }, + { + "epoch": 1.28, + "learning_rate": 8.032096288866601e-05, + "loss": 1.258, + "theoretical_loss": 3.3515452921263424, + "tokens_seen": 2972960768 + }, + { + "epoch": 1.28, + "learning_rate": 8.030491474423271e-05, + "loss": 1.1575, + "theoretical_loss": 3.3515396495896788, + "tokens_seen": 2973026304 + }, + { + "epoch": 1.28, + "learning_rate": 8.028886659979941e-05, + "loss": 1.1139, + "theoretical_loss": 3.3515340072122206, + "tokens_seen": 2973091840 + }, + { + "epoch": 1.28, + "learning_rate": 8.027281845536611e-05, + "loss": 1.2127, + "theoretical_loss": 3.3515283649939605, + "tokens_seen": 2973157376 + }, + { + "epoch": 1.28, + "learning_rate": 8.025677031093281e-05, + "loss": 1.1425, + "theoretical_loss": 3.3515227229348907, + "tokens_seen": 2973222912 + }, + { + "epoch": 1.28, + "learning_rate": 8.024072216649951e-05, + "loss": 1.219, + "theoretical_loss": 3.3515170810350026, + "tokens_seen": 2973288448 + }, + { + "epoch": 1.28, + "learning_rate": 8.02246740220662e-05, + "loss": 1.047, + "theoretical_loss": 3.3515114392942884, + "tokens_seen": 2973353984 + }, + { + "epoch": 1.28, + "learning_rate": 8.02086258776329e-05, + "loss": 1.143, + "theoretical_loss": 3.3515057977127403, + "tokens_seen": 2973419520 + }, + { + "epoch": 1.28, + "learning_rate": 8.01925777331996e-05, + "loss": 1.196, + "theoretical_loss": 3.35150015629035, + "tokens_seen": 2973485056 + }, + { + "epoch": 1.28, + "learning_rate": 8.01765295887663e-05, + "loss": 1.3586, + "theoretical_loss": 3.35149451502711, + "tokens_seen": 2973550592 + }, + { + "epoch": 1.28, + "learning_rate": 8.0160481444333e-05, + "loss": 1.1603, + "theoretical_loss": 3.351488873923012, + "tokens_seen": 2973616128 + }, + { + "epoch": 1.28, + "learning_rate": 8.01444332998997e-05, + "loss": 1.2123, + "theoretical_loss": 3.351483232978048, + "tokens_seen": 2973681664 + }, + { + "epoch": 1.28, + "learning_rate": 8.01283851554664e-05, + "loss": 1.1666, + "theoretical_loss": 3.35147759219221, + "tokens_seen": 2973747200 + }, + { + "epoch": 1.28, + "learning_rate": 8.01123370110331e-05, + "loss": 1.193, + "theoretical_loss": 3.3514719515654896, + "tokens_seen": 2973812736 + }, + { + "epoch": 1.28, + "learning_rate": 8.00962888665998e-05, + "loss": 1.1783, + "theoretical_loss": 3.3514663110978797, + "tokens_seen": 2973878272 + }, + { + "epoch": 1.28, + "learning_rate": 8.00802407221665e-05, + "loss": 1.2448, + "theoretical_loss": 3.3514606707893715, + "tokens_seen": 2973943808 + }, + { + "epoch": 1.28, + "learning_rate": 8.00641925777332e-05, + "loss": 1.19, + "theoretical_loss": 3.3514550306399578, + "tokens_seen": 2974009344 + }, + { + "epoch": 1.28, + "learning_rate": 8.00481444332999e-05, + "loss": 1.2203, + "theoretical_loss": 3.3514493906496297, + "tokens_seen": 2974074880 + }, + { + "epoch": 1.28, + "learning_rate": 8.003209628886659e-05, + "loss": 1.15, + "theoretical_loss": 3.3514437508183796, + "tokens_seen": 2974140416 + }, + { + "epoch": 1.28, + "learning_rate": 8.00160481444333e-05, + "loss": 1.0725, + "theoretical_loss": 3.3514381111461997, + "tokens_seen": 2974205952 + }, + { + "epoch": 1.28, + "learning_rate": 8e-05, + "loss": 1.1934, + "theoretical_loss": 3.351432471633082, + "tokens_seen": 2974271488 + }, + { + "epoch": 1.28, + "learning_rate": 7.998395185556671e-05, + "loss": 1.136, + "theoretical_loss": 3.351426832279018, + "tokens_seen": 2974337024 + }, + { + "epoch": 1.28, + "learning_rate": 7.996790371113341e-05, + "loss": 1.2095, + "theoretical_loss": 3.3514211930840005, + "tokens_seen": 2974402560 + }, + { + "epoch": 1.28, + "learning_rate": 7.995185556670011e-05, + "loss": 1.1648, + "theoretical_loss": 3.351415554048021, + "tokens_seen": 2974468096 + }, + { + "epoch": 1.28, + "learning_rate": 7.993580742226681e-05, + "loss": 1.2463, + "theoretical_loss": 3.3514099151710712, + "tokens_seen": 2974533632 + }, + { + "epoch": 1.28, + "learning_rate": 7.991975927783351e-05, + "loss": 1.164, + "theoretical_loss": 3.351404276453144, + "tokens_seen": 2974599168 + }, + { + "epoch": 1.28, + "learning_rate": 7.990371113340021e-05, + "loss": 1.1555, + "theoretical_loss": 3.3513986378942304, + "tokens_seen": 2974664704 + }, + { + "epoch": 1.28, + "learning_rate": 7.988766298896691e-05, + "loss": 1.143, + "theoretical_loss": 3.3513929994943235, + "tokens_seen": 2974730240 + }, + { + "epoch": 1.28, + "learning_rate": 7.98716148445336e-05, + "loss": 1.1743, + "theoretical_loss": 3.3513873612534146, + "tokens_seen": 2974795776 + }, + { + "epoch": 1.28, + "learning_rate": 7.98555667001003e-05, + "loss": 1.185, + "theoretical_loss": 3.351381723171496, + "tokens_seen": 2974861312 + }, + { + "epoch": 1.28, + "learning_rate": 7.9839518555667e-05, + "loss": 1.1817, + "theoretical_loss": 3.351376085248559, + "tokens_seen": 2974926848 + }, + { + "epoch": 1.28, + "learning_rate": 7.98234704112337e-05, + "loss": 1.2186, + "theoretical_loss": 3.3513704474845967, + "tokens_seen": 2974992384 + }, + { + "epoch": 1.28, + "learning_rate": 7.98074222668004e-05, + "loss": 1.2396, + "theoretical_loss": 3.3513648098796005, + "tokens_seen": 2975057920 + }, + { + "epoch": 1.28, + "learning_rate": 7.97913741223671e-05, + "loss": 1.0855, + "theoretical_loss": 3.351359172433562, + "tokens_seen": 2975123456 + }, + { + "epoch": 1.28, + "learning_rate": 7.97753259779338e-05, + "loss": 1.2141, + "theoretical_loss": 3.3513535351464747, + "tokens_seen": 2975188992 + }, + { + "epoch": 1.28, + "learning_rate": 7.97592778335005e-05, + "loss": 1.1087, + "theoretical_loss": 3.351347898018329, + "tokens_seen": 2975254528 + }, + { + "epoch": 1.28, + "objective/train/docs_used": 1097793, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1768453121185303, + "objective/train/theoretical_loss": 3.3513422610491177, + "objective/train/tokens_used": 2995780064, + "theoretical_loss": 3.3513422610491177, + "tokens_seen": 2975320064 + }, + { + "epoch": 1.28, + "learning_rate": 7.97432296890672e-05, + "loss": 1.1768, + "theoretical_loss": 3.3513422610491177, + "tokens_seen": 2975320064 + }, + { + "epoch": 1.28, + "learning_rate": 7.972718154463392e-05, + "loss": 1.139, + "theoretical_loss": 3.351336624238833, + "tokens_seen": 2975385600 + }, + { + "epoch": 1.28, + "learning_rate": 7.97111334002006e-05, + "loss": 1.1547, + "theoretical_loss": 3.351330987587466, + "tokens_seen": 2975451136 + }, + { + "epoch": 1.28, + "learning_rate": 7.96950852557673e-05, + "loss": 1.2097, + "theoretical_loss": 3.3513253510950096, + "tokens_seen": 2975516672 + }, + { + "epoch": 1.28, + "learning_rate": 7.9679037111334e-05, + "loss": 1.2293, + "theoretical_loss": 3.351319714761456, + "tokens_seen": 2975582208 + }, + { + "epoch": 1.28, + "learning_rate": 7.96629889669007e-05, + "loss": 1.1497, + "theoretical_loss": 3.351314078586796, + "tokens_seen": 2975647744 + }, + { + "epoch": 1.28, + "learning_rate": 7.964694082246741e-05, + "loss": 1.2399, + "theoretical_loss": 3.351308442571023, + "tokens_seen": 2975713280 + }, + { + "epoch": 1.28, + "learning_rate": 7.963089267803411e-05, + "loss": 1.2524, + "theoretical_loss": 3.351302806714128, + "tokens_seen": 2975778816 + }, + { + "epoch": 1.28, + "learning_rate": 7.961484453360081e-05, + "loss": 1.2464, + "theoretical_loss": 3.3512971710161037, + "tokens_seen": 2975844352 + }, + { + "epoch": 1.28, + "learning_rate": 7.959879638916751e-05, + "loss": 1.2653, + "theoretical_loss": 3.3512915354769417, + "tokens_seen": 2975909888 + }, + { + "epoch": 1.28, + "learning_rate": 7.958274824473421e-05, + "loss": 1.2309, + "theoretical_loss": 3.3512859000966344, + "tokens_seen": 2975975424 + }, + { + "epoch": 1.28, + "learning_rate": 7.956670010030091e-05, + "loss": 1.1532, + "theoretical_loss": 3.3512802648751734, + "tokens_seen": 2976040960 + }, + { + "epoch": 1.28, + "learning_rate": 7.955065195586761e-05, + "loss": 1.2331, + "theoretical_loss": 3.351274629812551, + "tokens_seen": 2976106496 + }, + { + "epoch": 1.28, + "learning_rate": 7.953460381143431e-05, + "loss": 1.2262, + "theoretical_loss": 3.351268994908759, + "tokens_seen": 2976172032 + }, + { + "epoch": 1.28, + "learning_rate": 7.9518555667001e-05, + "loss": 1.2246, + "theoretical_loss": 3.35126336016379, + "tokens_seen": 2976237568 + }, + { + "epoch": 1.28, + "learning_rate": 7.95025075225677e-05, + "loss": 1.1943, + "theoretical_loss": 3.3512577255776352, + "tokens_seen": 2976303104 + }, + { + "epoch": 1.28, + "learning_rate": 7.94864593781344e-05, + "loss": 1.2011, + "theoretical_loss": 3.3512520911502874, + "tokens_seen": 2976368640 + }, + { + "epoch": 1.28, + "learning_rate": 7.94704112337011e-05, + "loss": 1.1827, + "theoretical_loss": 3.351246456881738, + "tokens_seen": 2976434176 + }, + { + "epoch": 1.28, + "learning_rate": 7.94543630892678e-05, + "loss": 1.1884, + "theoretical_loss": 3.351240822771979, + "tokens_seen": 2976499712 + }, + { + "epoch": 1.28, + "learning_rate": 7.94383149448345e-05, + "loss": 1.1525, + "theoretical_loss": 3.3512351888210032, + "tokens_seen": 2976565248 + }, + { + "epoch": 1.28, + "learning_rate": 7.942226680040122e-05, + "loss": 1.2977, + "theoretical_loss": 3.3512295550288025, + "tokens_seen": 2976630784 + }, + { + "epoch": 1.28, + "learning_rate": 7.940621865596792e-05, + "loss": 1.15, + "theoretical_loss": 3.351223921395368, + "tokens_seen": 2976696320 + }, + { + "epoch": 1.28, + "learning_rate": 7.939017051153462e-05, + "loss": 1.1776, + "theoretical_loss": 3.3512182879206924, + "tokens_seen": 2976761856 + }, + { + "epoch": 1.28, + "learning_rate": 7.937412236710132e-05, + "loss": 1.1228, + "theoretical_loss": 3.3512126546047676, + "tokens_seen": 2976827392 + }, + { + "epoch": 1.28, + "learning_rate": 7.9358074222668e-05, + "loss": 1.2427, + "theoretical_loss": 3.3512070214475855, + "tokens_seen": 2976892928 + }, + { + "epoch": 1.28, + "learning_rate": 7.93420260782347e-05, + "loss": 1.2501, + "theoretical_loss": 3.3512013884491387, + "tokens_seen": 2976958464 + }, + { + "epoch": 1.28, + "learning_rate": 7.93259779338014e-05, + "loss": 1.2283, + "theoretical_loss": 3.3511957556094187, + "tokens_seen": 2977024000 + }, + { + "epoch": 1.28, + "learning_rate": 7.93099297893681e-05, + "loss": 1.2225, + "theoretical_loss": 3.3511901229284176, + "tokens_seen": 2977089536 + }, + { + "epoch": 1.28, + "learning_rate": 7.929388164493481e-05, + "loss": 1.1373, + "theoretical_loss": 3.3511844904061276, + "tokens_seen": 2977155072 + }, + { + "epoch": 1.28, + "learning_rate": 7.927783350050151e-05, + "loss": 1.1329, + "theoretical_loss": 3.3511788580425406, + "tokens_seen": 2977220608 + }, + { + "epoch": 1.28, + "learning_rate": 7.926178535606821e-05, + "loss": 1.1769, + "theoretical_loss": 3.3511732258376488, + "tokens_seen": 2977286144 + }, + { + "epoch": 1.28, + "learning_rate": 7.924573721163491e-05, + "loss": 1.2101, + "theoretical_loss": 3.351167593791444, + "tokens_seen": 2977351680 + }, + { + "epoch": 1.28, + "learning_rate": 7.922968906720161e-05, + "loss": 1.2482, + "theoretical_loss": 3.351161961903918, + "tokens_seen": 2977417216 + }, + { + "epoch": 1.28, + "learning_rate": 7.921364092276831e-05, + "loss": 1.1159, + "theoretical_loss": 3.3511563301750638, + "tokens_seen": 2977482752 + }, + { + "epoch": 1.28, + "learning_rate": 7.919759277833501e-05, + "loss": 1.1825, + "theoretical_loss": 3.3511506986048722, + "tokens_seen": 2977548288 + }, + { + "epoch": 1.28, + "learning_rate": 7.918154463390171e-05, + "loss": 1.3102, + "theoretical_loss": 3.351145067193336, + "tokens_seen": 2977613824 + }, + { + "epoch": 1.28, + "learning_rate": 7.91654964894684e-05, + "loss": 1.1873, + "theoretical_loss": 3.3511394359404476, + "tokens_seen": 2977679360 + }, + { + "epoch": 1.28, + "learning_rate": 7.91494483450351e-05, + "loss": 1.185, + "theoretical_loss": 3.351133804846198, + "tokens_seen": 2977744896 + }, + { + "epoch": 1.28, + "learning_rate": 7.91334002006018e-05, + "loss": 1.2028, + "theoretical_loss": 3.3511281739105803, + "tokens_seen": 2977810432 + }, + { + "epoch": 1.28, + "learning_rate": 7.911735205616851e-05, + "loss": 1.1523, + "theoretical_loss": 3.3511225431335854, + "tokens_seen": 2977875968 + }, + { + "epoch": 1.28, + "learning_rate": 7.910130391173521e-05, + "loss": 1.22, + "theoretical_loss": 3.3511169125152063, + "tokens_seen": 2977941504 + }, + { + "epoch": 1.28, + "learning_rate": 7.908525576730192e-05, + "loss": 1.1591, + "theoretical_loss": 3.3511112820554345, + "tokens_seen": 2978007040 + }, + { + "epoch": 1.28, + "learning_rate": 7.906920762286862e-05, + "loss": 1.2377, + "theoretical_loss": 3.3511056517542626, + "tokens_seen": 2978072576 + }, + { + "epoch": 1.28, + "learning_rate": 7.905315947843532e-05, + "loss": 1.232, + "theoretical_loss": 3.351100021611682, + "tokens_seen": 2978138112 + }, + { + "epoch": 1.28, + "learning_rate": 7.903711133400202e-05, + "loss": 1.166, + "theoretical_loss": 3.351094391627685, + "tokens_seen": 2978203648 + }, + { + "epoch": 1.28, + "learning_rate": 7.902106318956872e-05, + "loss": 1.2255, + "theoretical_loss": 3.351088761802264, + "tokens_seen": 2978269184 + }, + { + "epoch": 1.28, + "learning_rate": 7.90050150451354e-05, + "loss": 1.0854, + "theoretical_loss": 3.35108313213541, + "tokens_seen": 2978334720 + }, + { + "epoch": 1.28, + "learning_rate": 7.89889669007021e-05, + "loss": 1.1927, + "theoretical_loss": 3.3510775026271165, + "tokens_seen": 2978400256 + }, + { + "epoch": 1.28, + "learning_rate": 7.89729187562688e-05, + "loss": 1.1954, + "theoretical_loss": 3.3510718732773745, + "tokens_seen": 2978465792 + }, + { + "epoch": 1.28, + "learning_rate": 7.895687061183551e-05, + "loss": 1.1456, + "theoretical_loss": 3.3510662440861765, + "tokens_seen": 2978531328 + }, + { + "epoch": 1.28, + "learning_rate": 7.894082246740221e-05, + "loss": 1.2008, + "theoretical_loss": 3.3510606150535143, + "tokens_seen": 2978596864 + }, + { + "epoch": 1.28, + "learning_rate": 7.892477432296891e-05, + "loss": 1.2338, + "theoretical_loss": 3.3510549861793804, + "tokens_seen": 2978662400 + }, + { + "epoch": 1.28, + "learning_rate": 7.890872617853561e-05, + "loss": 1.1929, + "theoretical_loss": 3.351049357463766, + "tokens_seen": 2978727936 + }, + { + "epoch": 1.28, + "learning_rate": 7.889267803410231e-05, + "loss": 1.293, + "theoretical_loss": 3.3510437289066637, + "tokens_seen": 2978793472 + }, + { + "epoch": 1.28, + "learning_rate": 7.887662988966901e-05, + "loss": 1.2396, + "theoretical_loss": 3.3510381005080654, + "tokens_seen": 2978859008 + }, + { + "epoch": 1.28, + "learning_rate": 7.886058174523571e-05, + "loss": 1.1676, + "theoretical_loss": 3.3510324722679634, + "tokens_seen": 2978924544 + }, + { + "epoch": 1.28, + "learning_rate": 7.884453360080241e-05, + "loss": 1.1566, + "theoretical_loss": 3.3510268441863498, + "tokens_seen": 2978990080 + }, + { + "epoch": 1.28, + "learning_rate": 7.882848545636911e-05, + "loss": 1.1382, + "theoretical_loss": 3.351021216263216, + "tokens_seen": 2979055616 + }, + { + "epoch": 1.28, + "learning_rate": 7.881243731193581e-05, + "loss": 1.1371, + "theoretical_loss": 3.351015588498555, + "tokens_seen": 2979121152 + }, + { + "epoch": 1.28, + "learning_rate": 7.879638916750251e-05, + "loss": 1.2341, + "theoretical_loss": 3.351009960892358, + "tokens_seen": 2979186688 + }, + { + "epoch": 1.28, + "learning_rate": 7.878034102306921e-05, + "loss": 1.1734, + "theoretical_loss": 3.3510043334446173, + "tokens_seen": 2979252224 + }, + { + "epoch": 1.28, + "learning_rate": 7.876429287863591e-05, + "loss": 1.0696, + "theoretical_loss": 3.350998706155325, + "tokens_seen": 2979317760 + }, + { + "epoch": 1.28, + "learning_rate": 7.874824473420262e-05, + "loss": 1.2275, + "theoretical_loss": 3.3509930790244735, + "tokens_seen": 2979383296 + }, + { + "epoch": 1.28, + "learning_rate": 7.873219658976932e-05, + "loss": 1.1641, + "theoretical_loss": 3.3509874520520544, + "tokens_seen": 2979448832 + }, + { + "epoch": 1.28, + "learning_rate": 7.871614844533602e-05, + "loss": 1.1391, + "theoretical_loss": 3.35098182523806, + "tokens_seen": 2979514368 + }, + { + "epoch": 1.28, + "learning_rate": 7.870010030090272e-05, + "loss": 1.1883, + "theoretical_loss": 3.350976198582482, + "tokens_seen": 2979579904 + }, + { + "epoch": 1.28, + "learning_rate": 7.868405215646942e-05, + "loss": 1.1896, + "theoretical_loss": 3.350970572085313, + "tokens_seen": 2979645440 + }, + { + "epoch": 1.28, + "learning_rate": 7.86680040120361e-05, + "loss": 1.196, + "theoretical_loss": 3.350964945746545, + "tokens_seen": 2979710976 + }, + { + "epoch": 1.28, + "learning_rate": 7.86519558676028e-05, + "loss": 1.168, + "theoretical_loss": 3.3509593195661695, + "tokens_seen": 2979776512 + }, + { + "epoch": 1.28, + "learning_rate": 7.86359077231695e-05, + "loss": 1.1875, + "theoretical_loss": 3.350953693544179, + "tokens_seen": 2979842048 + }, + { + "epoch": 1.28, + "learning_rate": 7.861985957873621e-05, + "loss": 1.2158, + "theoretical_loss": 3.3509480676805654, + "tokens_seen": 2979907584 + }, + { + "epoch": 1.28, + "learning_rate": 7.860381143430291e-05, + "loss": 1.223, + "theoretical_loss": 3.3509424419753207, + "tokens_seen": 2979973120 + }, + { + "epoch": 1.28, + "learning_rate": 7.858776328986961e-05, + "loss": 1.0492, + "theoretical_loss": 3.350936816428437, + "tokens_seen": 2980038656 + }, + { + "epoch": 1.28, + "learning_rate": 7.857171514543631e-05, + "loss": 1.1611, + "theoretical_loss": 3.3509311910399067, + "tokens_seen": 2980104192 + }, + { + "epoch": 1.28, + "learning_rate": 7.855566700100301e-05, + "loss": 1.1344, + "theoretical_loss": 3.3509255658097215, + "tokens_seen": 2980169728 + }, + { + "epoch": 1.28, + "learning_rate": 7.853961885656971e-05, + "loss": 1.238, + "theoretical_loss": 3.3509199407378736, + "tokens_seen": 2980235264 + }, + { + "epoch": 1.28, + "learning_rate": 7.852357071213642e-05, + "loss": 1.1637, + "theoretical_loss": 3.3509143158243546, + "tokens_seen": 2980300800 + }, + { + "epoch": 1.28, + "learning_rate": 7.850752256770311e-05, + "loss": 1.1134, + "theoretical_loss": 3.3509086910691575, + "tokens_seen": 2980366336 + }, + { + "epoch": 1.28, + "learning_rate": 7.849147442326981e-05, + "loss": 1.1474, + "theoretical_loss": 3.3509030664722737, + "tokens_seen": 2980431872 + }, + { + "epoch": 1.28, + "learning_rate": 7.847542627883651e-05, + "loss": 1.2144, + "theoretical_loss": 3.350897442033695, + "tokens_seen": 2980497408 + }, + { + "epoch": 1.28, + "learning_rate": 7.845937813440321e-05, + "loss": 1.2617, + "theoretical_loss": 3.350891817753414, + "tokens_seen": 2980562944 + }, + { + "epoch": 1.28, + "learning_rate": 7.844332998996991e-05, + "loss": 1.1707, + "theoretical_loss": 3.350886193631423, + "tokens_seen": 2980628480 + }, + { + "epoch": 1.28, + "learning_rate": 7.842728184553661e-05, + "loss": 1.2432, + "theoretical_loss": 3.3508805696677135, + "tokens_seen": 2980694016 + }, + { + "epoch": 1.28, + "learning_rate": 7.841123370110332e-05, + "loss": 1.2553, + "theoretical_loss": 3.350874945862278, + "tokens_seen": 2980759552 + }, + { + "epoch": 1.28, + "learning_rate": 7.839518555667002e-05, + "loss": 1.1456, + "theoretical_loss": 3.350869322215108, + "tokens_seen": 2980825088 + }, + { + "epoch": 1.28, + "learning_rate": 7.837913741223672e-05, + "loss": 1.2553, + "theoretical_loss": 3.350863698726196, + "tokens_seen": 2980890624 + }, + { + "epoch": 1.28, + "learning_rate": 7.836308926780342e-05, + "loss": 1.1981, + "theoretical_loss": 3.350858075395534, + "tokens_seen": 2980956160 + }, + { + "epoch": 1.28, + "learning_rate": 7.834704112337012e-05, + "loss": 1.259, + "theoretical_loss": 3.350852452223114, + "tokens_seen": 2981021696 + }, + { + "epoch": 1.28, + "learning_rate": 7.833099297893682e-05, + "loss": 1.2367, + "theoretical_loss": 3.3508468292089275, + "tokens_seen": 2981087232 + }, + { + "epoch": 1.28, + "learning_rate": 7.83149448345035e-05, + "loss": 1.207, + "theoretical_loss": 3.3508412063529676, + "tokens_seen": 2981152768 + }, + { + "epoch": 1.28, + "learning_rate": 7.82988966900702e-05, + "loss": 1.211, + "theoretical_loss": 3.3508355836552264, + "tokens_seen": 2981218304 + }, + { + "epoch": 1.28, + "learning_rate": 7.82828485456369e-05, + "loss": 1.1467, + "theoretical_loss": 3.350829961115695, + "tokens_seen": 2981283840 + }, + { + "epoch": 1.28, + "learning_rate": 7.826680040120361e-05, + "loss": 1.2345, + "theoretical_loss": 3.350824338734366, + "tokens_seen": 2981349376 + }, + { + "epoch": 1.28, + "learning_rate": 7.825075225677031e-05, + "loss": 1.2664, + "theoretical_loss": 3.3508187165112315, + "tokens_seen": 2981414912 + }, + { + "epoch": 1.28, + "learning_rate": 7.823470411233701e-05, + "loss": 1.1947, + "theoretical_loss": 3.3508130944462833, + "tokens_seen": 2981480448 + }, + { + "epoch": 1.28, + "learning_rate": 7.821865596790372e-05, + "loss": 1.145, + "theoretical_loss": 3.3508074725395143, + "tokens_seen": 2981545984 + }, + { + "epoch": 1.28, + "learning_rate": 7.820260782347042e-05, + "loss": 1.2202, + "theoretical_loss": 3.3508018507909156, + "tokens_seen": 2981611520 + }, + { + "epoch": 1.28, + "learning_rate": 7.818655967903712e-05, + "loss": 1.2195, + "theoretical_loss": 3.3507962292004794, + "tokens_seen": 2981677056 + }, + { + "epoch": 1.28, + "learning_rate": 7.817051153460382e-05, + "loss": 1.1815, + "theoretical_loss": 3.350790607768198, + "tokens_seen": 2981742592 + }, + { + "epoch": 1.28, + "learning_rate": 7.815446339017051e-05, + "loss": 1.1043, + "theoretical_loss": 3.3507849864940638, + "tokens_seen": 2981808128 + }, + { + "epoch": 1.28, + "objective/train/docs_used": 1100253, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1956806182861328, + "objective/train/theoretical_loss": 3.3507793653780684, + "objective/train/tokens_used": 3002333664, + "theoretical_loss": 3.3507793653780684, + "tokens_seen": 2981873664 + }, + { + "epoch": 1.28, + "learning_rate": 7.813841524573721e-05, + "loss": 1.1957, + "theoretical_loss": 3.3507793653780684, + "tokens_seen": 2981873664 + }, + { + "epoch": 1.28, + "learning_rate": 7.812236710130391e-05, + "loss": 1.2128, + "theoretical_loss": 3.350773744420204, + "tokens_seen": 2981939200 + }, + { + "epoch": 1.28, + "learning_rate": 7.810631895687061e-05, + "loss": 1.2269, + "theoretical_loss": 3.3507681236204627, + "tokens_seen": 2982004736 + }, + { + "epoch": 1.28, + "learning_rate": 7.809027081243731e-05, + "loss": 1.1527, + "theoretical_loss": 3.350762502978837, + "tokens_seen": 2982070272 + }, + { + "epoch": 1.28, + "learning_rate": 7.807422266800401e-05, + "loss": 1.1204, + "theoretical_loss": 3.350756882495318, + "tokens_seen": 2982135808 + }, + { + "epoch": 1.28, + "learning_rate": 7.805817452357072e-05, + "loss": 1.2334, + "theoretical_loss": 3.3507512621698985, + "tokens_seen": 2982201344 + }, + { + "epoch": 1.28, + "learning_rate": 7.804212637913742e-05, + "loss": 1.2509, + "theoretical_loss": 3.3507456420025705, + "tokens_seen": 2982266880 + }, + { + "epoch": 1.28, + "learning_rate": 7.802607823470412e-05, + "loss": 1.1421, + "theoretical_loss": 3.350740021993326, + "tokens_seen": 2982332416 + }, + { + "epoch": 1.28, + "learning_rate": 7.801003009027082e-05, + "loss": 1.2069, + "theoretical_loss": 3.350734402142157, + "tokens_seen": 2982397952 + }, + { + "epoch": 1.28, + "learning_rate": 7.799398194583752e-05, + "loss": 1.1967, + "theoretical_loss": 3.3507287824490555, + "tokens_seen": 2982463488 + }, + { + "epoch": 1.28, + "learning_rate": 7.797793380140422e-05, + "loss": 1.1277, + "theoretical_loss": 3.350723162914014, + "tokens_seen": 2982529024 + }, + { + "epoch": 1.28, + "learning_rate": 7.79618856569709e-05, + "loss": 1.1519, + "theoretical_loss": 3.3507175435370242, + "tokens_seen": 2982594560 + }, + { + "epoch": 1.28, + "learning_rate": 7.79458375125376e-05, + "loss": 1.2175, + "theoretical_loss": 3.3507119243180785, + "tokens_seen": 2982660096 + }, + { + "epoch": 1.28, + "learning_rate": 7.792978936810431e-05, + "loss": 1.1411, + "theoretical_loss": 3.3507063052571686, + "tokens_seen": 2982725632 + }, + { + "epoch": 1.28, + "learning_rate": 7.791374122367102e-05, + "loss": 1.0812, + "theoretical_loss": 3.350700686354287, + "tokens_seen": 2982791168 + }, + { + "epoch": 1.28, + "learning_rate": 7.789769307923772e-05, + "loss": 1.129, + "theoretical_loss": 3.350695067609425, + "tokens_seen": 2982856704 + }, + { + "epoch": 1.28, + "learning_rate": 7.788164493480442e-05, + "loss": 1.1347, + "theoretical_loss": 3.3506894490225756, + "tokens_seen": 2982922240 + }, + { + "epoch": 1.28, + "learning_rate": 7.786559679037112e-05, + "loss": 1.2093, + "theoretical_loss": 3.3506838305937308, + "tokens_seen": 2982987776 + }, + { + "epoch": 1.28, + "learning_rate": 7.784954864593782e-05, + "loss": 1.0837, + "theoretical_loss": 3.3506782123228818, + "tokens_seen": 2983053312 + }, + { + "epoch": 1.28, + "learning_rate": 7.783350050150452e-05, + "loss": 1.1246, + "theoretical_loss": 3.350672594210022, + "tokens_seen": 2983118848 + }, + { + "epoch": 1.28, + "learning_rate": 7.781745235707123e-05, + "loss": 1.3169, + "theoretical_loss": 3.350666976255142, + "tokens_seen": 2983184384 + }, + { + "epoch": 1.28, + "learning_rate": 7.780140421263791e-05, + "loss": 1.1508, + "theoretical_loss": 3.350661358458235, + "tokens_seen": 2983249920 + }, + { + "epoch": 1.28, + "learning_rate": 7.778535606820461e-05, + "loss": 1.2179, + "theoretical_loss": 3.3506557408192927, + "tokens_seen": 2983315456 + }, + { + "epoch": 1.28, + "learning_rate": 7.776930792377131e-05, + "loss": 1.1514, + "theoretical_loss": 3.3506501233383075, + "tokens_seen": 2983380992 + }, + { + "epoch": 1.28, + "learning_rate": 7.775325977933801e-05, + "loss": 1.2003, + "theoretical_loss": 3.350644506015271, + "tokens_seen": 2983446528 + }, + { + "epoch": 1.28, + "learning_rate": 7.773721163490471e-05, + "loss": 1.1265, + "theoretical_loss": 3.3506388888501757, + "tokens_seen": 2983512064 + }, + { + "epoch": 1.28, + "learning_rate": 7.772116349047142e-05, + "loss": 1.2076, + "theoretical_loss": 3.350633271843013, + "tokens_seen": 2983577600 + }, + { + "epoch": 1.28, + "learning_rate": 7.770511534603812e-05, + "loss": 1.1574, + "theoretical_loss": 3.3506276549937763, + "tokens_seen": 2983643136 + }, + { + "epoch": 1.28, + "learning_rate": 7.768906720160482e-05, + "loss": 1.1858, + "theoretical_loss": 3.3506220383024563, + "tokens_seen": 2983708672 + }, + { + "epoch": 1.28, + "learning_rate": 7.767301905717152e-05, + "loss": 1.1002, + "theoretical_loss": 3.350616421769046, + "tokens_seen": 2983774208 + }, + { + "epoch": 1.28, + "learning_rate": 7.765697091273822e-05, + "loss": 1.2133, + "theoretical_loss": 3.3506108053935373, + "tokens_seen": 2983839744 + }, + { + "epoch": 1.28, + "learning_rate": 7.764092276830492e-05, + "loss": 1.1681, + "theoretical_loss": 3.350605189175922, + "tokens_seen": 2983905280 + }, + { + "epoch": 1.28, + "learning_rate": 7.762487462387162e-05, + "loss": 1.2447, + "theoretical_loss": 3.3505995731161917, + "tokens_seen": 2983970816 + }, + { + "epoch": 1.28, + "learning_rate": 7.760882647943832e-05, + "loss": 1.1409, + "theoretical_loss": 3.3505939572143397, + "tokens_seen": 2984036352 + }, + { + "epoch": 1.28, + "learning_rate": 7.759277833500502e-05, + "loss": 1.1505, + "theoretical_loss": 3.350588341470358, + "tokens_seen": 2984101888 + }, + { + "epoch": 1.28, + "learning_rate": 7.757673019057172e-05, + "loss": 1.2056, + "theoretical_loss": 3.3505827258842373, + "tokens_seen": 2984167424 + }, + { + "epoch": 1.28, + "learning_rate": 7.756068204613842e-05, + "loss": 1.1612, + "theoretical_loss": 3.3505771104559714, + "tokens_seen": 2984232960 + }, + { + "epoch": 1.29, + "learning_rate": 7.754463390170512e-05, + "loss": 1.1559, + "theoretical_loss": 3.3505714951855516, + "tokens_seen": 2984298496 + }, + { + "epoch": 1.29, + "learning_rate": 7.752858575727182e-05, + "loss": 1.244, + "theoretical_loss": 3.3505658800729696, + "tokens_seen": 2984364032 + }, + { + "epoch": 1.29, + "learning_rate": 7.751253761283852e-05, + "loss": 1.2564, + "theoretical_loss": 3.3505602651182183, + "tokens_seen": 2984429568 + }, + { + "epoch": 1.29, + "learning_rate": 7.749648946840522e-05, + "loss": 1.1282, + "theoretical_loss": 3.350554650321289, + "tokens_seen": 2984495104 + }, + { + "epoch": 1.29, + "learning_rate": 7.748044132397192e-05, + "loss": 1.2128, + "theoretical_loss": 3.3505490356821745, + "tokens_seen": 2984560640 + }, + { + "epoch": 1.29, + "learning_rate": 7.746439317953863e-05, + "loss": 1.101, + "theoretical_loss": 3.350543421200867, + "tokens_seen": 2984626176 + }, + { + "epoch": 1.29, + "learning_rate": 7.744834503510531e-05, + "loss": 1.2208, + "theoretical_loss": 3.350537806877358, + "tokens_seen": 2984691712 + }, + { + "epoch": 1.29, + "learning_rate": 7.743229689067201e-05, + "loss": 1.1615, + "theoretical_loss": 3.3505321927116394, + "tokens_seen": 2984757248 + }, + { + "epoch": 1.29, + "learning_rate": 7.741624874623871e-05, + "loss": 1.0654, + "theoretical_loss": 3.350526578703704, + "tokens_seen": 2984822784 + }, + { + "epoch": 1.29, + "learning_rate": 7.740020060180541e-05, + "loss": 1.0572, + "theoretical_loss": 3.3505209648535437, + "tokens_seen": 2984888320 + }, + { + "epoch": 1.29, + "learning_rate": 7.738415245737212e-05, + "loss": 1.1432, + "theoretical_loss": 3.3505153511611505, + "tokens_seen": 2984953856 + }, + { + "epoch": 1.29, + "learning_rate": 7.736810431293882e-05, + "loss": 1.1392, + "theoretical_loss": 3.3505097376265165, + "tokens_seen": 2985019392 + }, + { + "epoch": 1.29, + "learning_rate": 7.735205616850552e-05, + "loss": 1.1966, + "theoretical_loss": 3.350504124249634, + "tokens_seen": 2985084928 + }, + { + "epoch": 1.29, + "learning_rate": 7.733600802407222e-05, + "loss": 1.0964, + "theoretical_loss": 3.350498511030495, + "tokens_seen": 2985150464 + }, + { + "epoch": 1.29, + "learning_rate": 7.731995987963893e-05, + "loss": 1.1593, + "theoretical_loss": 3.350492897969091, + "tokens_seen": 2985216000 + }, + { + "epoch": 1.29, + "learning_rate": 7.730391173520563e-05, + "loss": 1.1908, + "theoretical_loss": 3.350487285065415, + "tokens_seen": 2985281536 + }, + { + "epoch": 1.29, + "learning_rate": 7.728786359077232e-05, + "loss": 1.2649, + "theoretical_loss": 3.350481672319459, + "tokens_seen": 2985347072 + }, + { + "epoch": 1.29, + "learning_rate": 7.727181544633902e-05, + "loss": 1.179, + "theoretical_loss": 3.350476059731214, + "tokens_seen": 2985412608 + }, + { + "epoch": 1.29, + "learning_rate": 7.725576730190572e-05, + "loss": 1.1206, + "theoretical_loss": 3.350470447300674, + "tokens_seen": 2985478144 + }, + { + "epoch": 1.29, + "learning_rate": 7.723971915747242e-05, + "loss": 1.188, + "theoretical_loss": 3.3504648350278297, + "tokens_seen": 2985543680 + }, + { + "epoch": 1.29, + "learning_rate": 7.722367101303912e-05, + "loss": 1.2064, + "theoretical_loss": 3.3504592229126735, + "tokens_seen": 2985609216 + }, + { + "epoch": 1.29, + "learning_rate": 7.720762286860582e-05, + "loss": 1.2742, + "theoretical_loss": 3.3504536109551974, + "tokens_seen": 2985674752 + }, + { + "epoch": 1.29, + "learning_rate": 7.719157472417252e-05, + "loss": 1.125, + "theoretical_loss": 3.350447999155394, + "tokens_seen": 2985740288 + }, + { + "epoch": 1.29, + "learning_rate": 7.717552657973922e-05, + "loss": 1.1861, + "theoretical_loss": 3.350442387513255, + "tokens_seen": 2985805824 + }, + { + "epoch": 1.29, + "learning_rate": 7.715947843530592e-05, + "loss": 1.1701, + "theoretical_loss": 3.3504367760287725, + "tokens_seen": 2985871360 + }, + { + "epoch": 1.29, + "learning_rate": 7.714343029087262e-05, + "loss": 1.2177, + "theoretical_loss": 3.350431164701939, + "tokens_seen": 2985936896 + }, + { + "epoch": 1.29, + "learning_rate": 7.712738214643933e-05, + "loss": 1.2078, + "theoretical_loss": 3.3504255535327463, + "tokens_seen": 2986002432 + }, + { + "epoch": 1.29, + "learning_rate": 7.711133400200603e-05, + "loss": 1.1998, + "theoretical_loss": 3.3504199425211865, + "tokens_seen": 2986067968 + }, + { + "epoch": 1.29, + "learning_rate": 7.709528585757271e-05, + "loss": 1.1343, + "theoretical_loss": 3.3504143316672517, + "tokens_seen": 2986133504 + }, + { + "epoch": 1.29, + "learning_rate": 7.707923771313941e-05, + "loss": 1.2512, + "theoretical_loss": 3.350408720970934, + "tokens_seen": 2986199040 + }, + { + "epoch": 1.29, + "learning_rate": 7.706318956870611e-05, + "loss": 1.1724, + "theoretical_loss": 3.3504031104322256, + "tokens_seen": 2986264576 + }, + { + "epoch": 1.29, + "learning_rate": 7.704714142427281e-05, + "loss": 1.2433, + "theoretical_loss": 3.350397500051119, + "tokens_seen": 2986330112 + }, + { + "epoch": 1.29, + "learning_rate": 7.703109327983952e-05, + "loss": 1.1741, + "theoretical_loss": 3.3503918898276055, + "tokens_seen": 2986395648 + }, + { + "epoch": 1.29, + "learning_rate": 7.701504513540623e-05, + "loss": 1.2237, + "theoretical_loss": 3.3503862797616777, + "tokens_seen": 2986461184 + }, + { + "epoch": 1.29, + "learning_rate": 7.699899699097293e-05, + "loss": 1.1398, + "theoretical_loss": 3.350380669853328, + "tokens_seen": 2986526720 + }, + { + "epoch": 1.29, + "learning_rate": 7.698294884653963e-05, + "loss": 1.0962, + "theoretical_loss": 3.3503750601025475, + "tokens_seen": 2986592256 + }, + { + "epoch": 1.29, + "learning_rate": 7.696690070210633e-05, + "loss": 1.2486, + "theoretical_loss": 3.350369450509329, + "tokens_seen": 2986657792 + }, + { + "epoch": 1.29, + "learning_rate": 7.695085255767303e-05, + "loss": 1.2413, + "theoretical_loss": 3.350363841073665, + "tokens_seen": 2986723328 + }, + { + "epoch": 1.29, + "learning_rate": 7.693480441323972e-05, + "loss": 1.2569, + "theoretical_loss": 3.3503582317955476, + "tokens_seen": 2986788864 + }, + { + "epoch": 1.29, + "learning_rate": 7.691875626880642e-05, + "loss": 1.1433, + "theoretical_loss": 3.350352622674968, + "tokens_seen": 2986854400 + }, + { + "epoch": 1.29, + "learning_rate": 7.690270812437312e-05, + "loss": 1.2411, + "theoretical_loss": 3.350347013711919, + "tokens_seen": 2986919936 + }, + { + "epoch": 1.29, + "learning_rate": 7.688665997993982e-05, + "loss": 1.1676, + "theoretical_loss": 3.350341404906392, + "tokens_seen": 2986985472 + }, + { + "epoch": 1.29, + "learning_rate": 7.687061183550652e-05, + "loss": 1.1581, + "theoretical_loss": 3.3503357962583804, + "tokens_seen": 2987051008 + }, + { + "epoch": 1.29, + "learning_rate": 7.685456369107322e-05, + "loss": 1.22, + "theoretical_loss": 3.3503301877678755, + "tokens_seen": 2987116544 + }, + { + "epoch": 1.29, + "learning_rate": 7.683851554663992e-05, + "loss": 1.2177, + "theoretical_loss": 3.3503245794348695, + "tokens_seen": 2987182080 + }, + { + "epoch": 1.29, + "learning_rate": 7.682246740220662e-05, + "loss": 1.3257, + "theoretical_loss": 3.3503189712593544, + "tokens_seen": 2987247616 + }, + { + "epoch": 1.29, + "learning_rate": 7.680641925777332e-05, + "loss": 1.1389, + "theoretical_loss": 3.3503133632413227, + "tokens_seen": 2987313152 + }, + { + "epoch": 1.29, + "learning_rate": 7.679037111334003e-05, + "loss": 1.2413, + "theoretical_loss": 3.3503077553807663, + "tokens_seen": 2987378688 + }, + { + "epoch": 1.29, + "learning_rate": 7.677432296890673e-05, + "loss": 1.199, + "theoretical_loss": 3.3503021476776773, + "tokens_seen": 2987444224 + }, + { + "epoch": 1.29, + "learning_rate": 7.675827482447343e-05, + "loss": 1.2164, + "theoretical_loss": 3.3502965401320477, + "tokens_seen": 2987509760 + }, + { + "epoch": 1.29, + "learning_rate": 7.674222668004011e-05, + "loss": 1.1758, + "theoretical_loss": 3.35029093274387, + "tokens_seen": 2987575296 + }, + { + "epoch": 1.29, + "learning_rate": 7.672617853560683e-05, + "loss": 1.2, + "theoretical_loss": 3.350285325513136, + "tokens_seen": 2987640832 + }, + { + "epoch": 1.29, + "learning_rate": 7.671013039117353e-05, + "loss": 1.1595, + "theoretical_loss": 3.350279718439838, + "tokens_seen": 2987706368 + }, + { + "epoch": 1.29, + "learning_rate": 7.669408224674023e-05, + "loss": 1.1912, + "theoretical_loss": 3.3502741115239676, + "tokens_seen": 2987771904 + }, + { + "epoch": 1.29, + "learning_rate": 7.667803410230693e-05, + "loss": 1.1641, + "theoretical_loss": 3.3502685047655176, + "tokens_seen": 2987837440 + }, + { + "epoch": 1.29, + "learning_rate": 7.666198595787363e-05, + "loss": 1.1065, + "theoretical_loss": 3.3502628981644804, + "tokens_seen": 2987902976 + }, + { + "epoch": 1.29, + "learning_rate": 7.664593781344033e-05, + "loss": 1.2438, + "theoretical_loss": 3.350257291720847, + "tokens_seen": 2987968512 + }, + { + "epoch": 1.29, + "learning_rate": 7.662988966900703e-05, + "loss": 1.2325, + "theoretical_loss": 3.3502516854346105, + "tokens_seen": 2988034048 + }, + { + "epoch": 1.29, + "learning_rate": 7.661384152457373e-05, + "loss": 1.2077, + "theoretical_loss": 3.3502460793057627, + "tokens_seen": 2988099584 + }, + { + "epoch": 1.29, + "learning_rate": 7.659779338014042e-05, + "loss": 1.1354, + "theoretical_loss": 3.3502404733342956, + "tokens_seen": 2988165120 + }, + { + "epoch": 1.29, + "learning_rate": 7.658174523570712e-05, + "loss": 1.181, + "theoretical_loss": 3.3502348675202014, + "tokens_seen": 2988230656 + }, + { + "epoch": 1.29, + "learning_rate": 7.656569709127382e-05, + "loss": 1.1604, + "theoretical_loss": 3.3502292618634724, + "tokens_seen": 2988296192 + }, + { + "epoch": 1.29, + "learning_rate": 7.654964894684052e-05, + "loss": 1.1283, + "theoretical_loss": 3.3502236563641006, + "tokens_seen": 2988361728 + }, + { + "epoch": 1.29, + "objective/train/docs_used": 1102571, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2554247379302979, + "objective/train/theoretical_loss": 3.350218051022078, + "objective/train/tokens_used": 3008887264, + "theoretical_loss": 3.350218051022078, + "tokens_seen": 2988427264 + }, + { + "epoch": 1.29, + "learning_rate": 7.653360080240722e-05, + "loss": 1.2554, + "theoretical_loss": 3.350218051022078, + "tokens_seen": 2988427264 + }, + { + "epoch": 1.29, + "learning_rate": 7.651755265797392e-05, + "loss": 1.1969, + "theoretical_loss": 3.350212445837397, + "tokens_seen": 2988492800 + }, + { + "epoch": 1.29, + "learning_rate": 7.650150451354062e-05, + "loss": 1.2141, + "theoretical_loss": 3.35020684081005, + "tokens_seen": 2988558336 + }, + { + "epoch": 1.29, + "learning_rate": 7.648545636910732e-05, + "loss": 1.2206, + "theoretical_loss": 3.350201235940028, + "tokens_seen": 2988623872 + }, + { + "epoch": 1.29, + "learning_rate": 7.646940822467402e-05, + "loss": 1.1592, + "theoretical_loss": 3.3501956312273244, + "tokens_seen": 2988689408 + }, + { + "epoch": 1.29, + "learning_rate": 7.645336008024072e-05, + "loss": 1.1693, + "theoretical_loss": 3.350190026671931, + "tokens_seen": 2988754944 + }, + { + "epoch": 1.29, + "learning_rate": 7.643731193580743e-05, + "loss": 1.2046, + "theoretical_loss": 3.350184422273839, + "tokens_seen": 2988820480 + }, + { + "epoch": 1.29, + "learning_rate": 7.642126379137413e-05, + "loss": 1.2577, + "theoretical_loss": 3.350178818033042, + "tokens_seen": 2988886016 + }, + { + "epoch": 1.29, + "learning_rate": 7.640521564694083e-05, + "loss": 1.1948, + "theoretical_loss": 3.3501732139495313, + "tokens_seen": 2988951552 + }, + { + "epoch": 1.29, + "learning_rate": 7.638916750250753e-05, + "loss": 1.2323, + "theoretical_loss": 3.3501676100232984, + "tokens_seen": 2989017088 + }, + { + "epoch": 1.29, + "learning_rate": 7.637311935807423e-05, + "loss": 1.2298, + "theoretical_loss": 3.350162006254337, + "tokens_seen": 2989082624 + }, + { + "epoch": 1.29, + "learning_rate": 7.635707121364093e-05, + "loss": 1.1738, + "theoretical_loss": 3.3501564026426385, + "tokens_seen": 2989148160 + }, + { + "epoch": 1.29, + "learning_rate": 7.634102306920763e-05, + "loss": 1.2983, + "theoretical_loss": 3.350150799188195, + "tokens_seen": 2989213696 + }, + { + "epoch": 1.29, + "learning_rate": 7.632497492477433e-05, + "loss": 1.1301, + "theoretical_loss": 3.350145195890998, + "tokens_seen": 2989279232 + }, + { + "epoch": 1.29, + "learning_rate": 7.630892678034103e-05, + "loss": 1.2076, + "theoretical_loss": 3.3501395927510407, + "tokens_seen": 2989344768 + }, + { + "epoch": 1.29, + "learning_rate": 7.629287863590773e-05, + "loss": 1.2749, + "theoretical_loss": 3.3501339897683144, + "tokens_seen": 2989410304 + }, + { + "epoch": 1.29, + "learning_rate": 7.627683049147443e-05, + "loss": 1.1548, + "theoretical_loss": 3.3501283869428122, + "tokens_seen": 2989475840 + }, + { + "epoch": 1.29, + "learning_rate": 7.626078234704113e-05, + "loss": 1.1683, + "theoretical_loss": 3.350122784274525, + "tokens_seen": 2989541376 + }, + { + "epoch": 1.29, + "learning_rate": 7.624473420260782e-05, + "loss": 1.2405, + "theoretical_loss": 3.350117181763446, + "tokens_seen": 2989606912 + }, + { + "epoch": 1.29, + "learning_rate": 7.622868605817452e-05, + "loss": 1.2205, + "theoretical_loss": 3.3501115794095675, + "tokens_seen": 2989672448 + }, + { + "epoch": 1.29, + "learning_rate": 7.621263791374122e-05, + "loss": 1.2257, + "theoretical_loss": 3.3501059772128805, + "tokens_seen": 2989737984 + }, + { + "epoch": 1.29, + "learning_rate": 7.619658976930792e-05, + "loss": 1.1434, + "theoretical_loss": 3.3501003751733776, + "tokens_seen": 2989803520 + }, + { + "epoch": 1.29, + "learning_rate": 7.618054162487462e-05, + "loss": 1.1798, + "theoretical_loss": 3.3500947732910515, + "tokens_seen": 2989869056 + }, + { + "epoch": 1.29, + "learning_rate": 7.616449348044132e-05, + "loss": 1.1518, + "theoretical_loss": 3.3500891715658936, + "tokens_seen": 2989934592 + }, + { + "epoch": 1.29, + "learning_rate": 7.614844533600802e-05, + "loss": 1.2056, + "theoretical_loss": 3.3500835699978966, + "tokens_seen": 2990000128 + }, + { + "epoch": 1.29, + "learning_rate": 7.613239719157472e-05, + "loss": 1.1777, + "theoretical_loss": 3.3500779685870525, + "tokens_seen": 2990065664 + }, + { + "epoch": 1.29, + "learning_rate": 7.611634904714144e-05, + "loss": 1.1675, + "theoretical_loss": 3.3500723673333535, + "tokens_seen": 2990131200 + }, + { + "epoch": 1.29, + "learning_rate": 7.610030090270814e-05, + "loss": 1.1313, + "theoretical_loss": 3.3500667662367913, + "tokens_seen": 2990196736 + }, + { + "epoch": 1.29, + "learning_rate": 7.608425275827483e-05, + "loss": 1.233, + "theoretical_loss": 3.350061165297358, + "tokens_seen": 2990262272 + }, + { + "epoch": 1.29, + "learning_rate": 7.606820461384153e-05, + "loss": 1.2465, + "theoretical_loss": 3.350055564515047, + "tokens_seen": 2990327808 + }, + { + "epoch": 1.29, + "learning_rate": 7.605215646940823e-05, + "loss": 1.1752, + "theoretical_loss": 3.350049963889849, + "tokens_seen": 2990393344 + }, + { + "epoch": 1.29, + "learning_rate": 7.603610832497493e-05, + "loss": 1.241, + "theoretical_loss": 3.350044363421757, + "tokens_seen": 2990458880 + }, + { + "epoch": 1.29, + "learning_rate": 7.602006018054163e-05, + "loss": 1.0551, + "theoretical_loss": 3.3500387631107627, + "tokens_seen": 2990524416 + }, + { + "epoch": 1.29, + "learning_rate": 7.600401203610833e-05, + "loss": 1.0877, + "theoretical_loss": 3.350033162956859, + "tokens_seen": 2990589952 + }, + { + "epoch": 1.29, + "learning_rate": 7.598796389167503e-05, + "loss": 1.1587, + "theoretical_loss": 3.3500275629600367, + "tokens_seen": 2990655488 + }, + { + "epoch": 1.29, + "learning_rate": 7.597191574724173e-05, + "loss": 1.1667, + "theoretical_loss": 3.350021963120289, + "tokens_seen": 2990721024 + }, + { + "epoch": 1.29, + "learning_rate": 7.595586760280843e-05, + "loss": 1.0319, + "theoretical_loss": 3.3500163634376077, + "tokens_seen": 2990786560 + }, + { + "epoch": 1.29, + "learning_rate": 7.593981945837513e-05, + "loss": 1.1858, + "theoretical_loss": 3.3500107639119854, + "tokens_seen": 2990852096 + }, + { + "epoch": 1.29, + "learning_rate": 7.592377131394183e-05, + "loss": 1.1822, + "theoretical_loss": 3.3500051645434135, + "tokens_seen": 2990917632 + }, + { + "epoch": 1.29, + "learning_rate": 7.590772316950853e-05, + "loss": 1.1137, + "theoretical_loss": 3.3499995653318844, + "tokens_seen": 2990983168 + }, + { + "epoch": 1.29, + "learning_rate": 7.589167502507522e-05, + "loss": 1.1297, + "theoretical_loss": 3.3499939662773905, + "tokens_seen": 2991048704 + }, + { + "epoch": 1.29, + "learning_rate": 7.587562688064192e-05, + "loss": 1.1289, + "theoretical_loss": 3.3499883673799244, + "tokens_seen": 2991114240 + }, + { + "epoch": 1.29, + "learning_rate": 7.585957873620862e-05, + "loss": 1.2086, + "theoretical_loss": 3.349982768639477, + "tokens_seen": 2991179776 + }, + { + "epoch": 1.29, + "learning_rate": 7.584353059177532e-05, + "loss": 1.221, + "theoretical_loss": 3.3499771700560412, + "tokens_seen": 2991245312 + }, + { + "epoch": 1.29, + "learning_rate": 7.582748244734202e-05, + "loss": 1.1546, + "theoretical_loss": 3.3499715716296095, + "tokens_seen": 2991310848 + }, + { + "epoch": 1.29, + "learning_rate": 7.581143430290874e-05, + "loss": 1.1262, + "theoretical_loss": 3.349965973360174, + "tokens_seen": 2991376384 + }, + { + "epoch": 1.29, + "learning_rate": 7.579538615847544e-05, + "loss": 1.2579, + "theoretical_loss": 3.3499603752477256, + "tokens_seen": 2991441920 + }, + { + "epoch": 1.29, + "learning_rate": 7.577933801404214e-05, + "loss": 1.099, + "theoretical_loss": 3.349954777292258, + "tokens_seen": 2991507456 + }, + { + "epoch": 1.29, + "learning_rate": 7.576328986960884e-05, + "loss": 1.2272, + "theoretical_loss": 3.3499491794937626, + "tokens_seen": 2991572992 + }, + { + "epoch": 1.29, + "learning_rate": 7.574724172517554e-05, + "loss": 1.1466, + "theoretical_loss": 3.349943581852232, + "tokens_seen": 2991638528 + }, + { + "epoch": 1.29, + "learning_rate": 7.573119358074223e-05, + "loss": 1.1758, + "theoretical_loss": 3.3499379843676573, + "tokens_seen": 2991704064 + }, + { + "epoch": 1.29, + "learning_rate": 7.571514543630893e-05, + "loss": 1.175, + "theoretical_loss": 3.349932387040032, + "tokens_seen": 2991769600 + }, + { + "epoch": 1.29, + "learning_rate": 7.569909729187563e-05, + "loss": 1.1604, + "theoretical_loss": 3.3499267898693477, + "tokens_seen": 2991835136 + }, + { + "epoch": 1.29, + "learning_rate": 7.568304914744233e-05, + "loss": 1.13, + "theoretical_loss": 3.3499211928555965, + "tokens_seen": 2991900672 + }, + { + "epoch": 1.29, + "learning_rate": 7.566700100300903e-05, + "loss": 1.2119, + "theoretical_loss": 3.3499155959987705, + "tokens_seen": 2991966208 + }, + { + "epoch": 1.29, + "learning_rate": 7.565095285857573e-05, + "loss": 1.1987, + "theoretical_loss": 3.349909999298862, + "tokens_seen": 2992031744 + }, + { + "epoch": 1.29, + "learning_rate": 7.563490471414243e-05, + "loss": 1.0964, + "theoretical_loss": 3.3499044027558633, + "tokens_seen": 2992097280 + }, + { + "epoch": 1.29, + "learning_rate": 7.561885656970913e-05, + "loss": 1.1975, + "theoretical_loss": 3.349898806369766, + "tokens_seen": 2992162816 + }, + { + "epoch": 1.29, + "learning_rate": 7.560280842527583e-05, + "loss": 1.2101, + "theoretical_loss": 3.3498932101405634, + "tokens_seen": 2992228352 + }, + { + "epoch": 1.29, + "learning_rate": 7.558676028084253e-05, + "loss": 1.1846, + "theoretical_loss": 3.3498876140682463, + "tokens_seen": 2992293888 + }, + { + "epoch": 1.29, + "learning_rate": 7.557071213640923e-05, + "loss": 1.2401, + "theoretical_loss": 3.3498820181528077, + "tokens_seen": 2992359424 + }, + { + "epoch": 1.29, + "learning_rate": 7.555466399197593e-05, + "loss": 1.1366, + "theoretical_loss": 3.3498764223942397, + "tokens_seen": 2992424960 + }, + { + "epoch": 1.29, + "learning_rate": 7.553861584754262e-05, + "loss": 1.1492, + "theoretical_loss": 3.349870826792534, + "tokens_seen": 2992490496 + }, + { + "epoch": 1.29, + "learning_rate": 7.552256770310933e-05, + "loss": 1.2057, + "theoretical_loss": 3.3498652313476835, + "tokens_seen": 2992556032 + }, + { + "epoch": 1.29, + "learning_rate": 7.550651955867604e-05, + "loss": 1.2519, + "theoretical_loss": 3.34985963605968, + "tokens_seen": 2992621568 + }, + { + "epoch": 1.29, + "learning_rate": 7.549047141424274e-05, + "loss": 1.2243, + "theoretical_loss": 3.3498540409285154, + "tokens_seen": 2992687104 + }, + { + "epoch": 1.29, + "learning_rate": 7.547442326980944e-05, + "loss": 1.1811, + "theoretical_loss": 3.349848445954182, + "tokens_seen": 2992752640 + }, + { + "epoch": 1.29, + "learning_rate": 7.545837512537614e-05, + "loss": 1.0994, + "theoretical_loss": 3.349842851136672, + "tokens_seen": 2992818176 + }, + { + "epoch": 1.29, + "learning_rate": 7.544232698094284e-05, + "loss": 1.0841, + "theoretical_loss": 3.349837256475978, + "tokens_seen": 2992883712 + }, + { + "epoch": 1.29, + "learning_rate": 7.542627883650954e-05, + "loss": 1.23, + "theoretical_loss": 3.349831661972092, + "tokens_seen": 2992949248 + }, + { + "epoch": 1.29, + "learning_rate": 7.541023069207624e-05, + "loss": 1.1197, + "theoretical_loss": 3.3498260676250053, + "tokens_seen": 2993014784 + }, + { + "epoch": 1.29, + "learning_rate": 7.539418254764294e-05, + "loss": 1.1429, + "theoretical_loss": 3.3498204734347112, + "tokens_seen": 2993080320 + }, + { + "epoch": 1.29, + "learning_rate": 7.537813440320963e-05, + "loss": 1.1402, + "theoretical_loss": 3.349814879401202, + "tokens_seen": 2993145856 + }, + { + "epoch": 1.29, + "learning_rate": 7.536208625877633e-05, + "loss": 1.2175, + "theoretical_loss": 3.3498092855244685, + "tokens_seen": 2993211392 + }, + { + "epoch": 1.29, + "learning_rate": 7.534603811434303e-05, + "loss": 1.1829, + "theoretical_loss": 3.349803691804504, + "tokens_seen": 2993276928 + }, + { + "epoch": 1.29, + "learning_rate": 7.532998996990973e-05, + "loss": 1.2246, + "theoretical_loss": 3.3497980982413003, + "tokens_seen": 2993342464 + }, + { + "epoch": 1.29, + "learning_rate": 7.531394182547643e-05, + "loss": 1.2166, + "theoretical_loss": 3.3497925048348494, + "tokens_seen": 2993408000 + }, + { + "epoch": 1.29, + "learning_rate": 7.529789368104313e-05, + "loss": 1.2271, + "theoretical_loss": 3.3497869115851437, + "tokens_seen": 2993473536 + }, + { + "epoch": 1.29, + "learning_rate": 7.528184553660983e-05, + "loss": 1.2807, + "theoretical_loss": 3.349781318492176, + "tokens_seen": 2993539072 + }, + { + "epoch": 1.29, + "learning_rate": 7.526579739217653e-05, + "loss": 1.275, + "theoretical_loss": 3.3497757255559373, + "tokens_seen": 2993604608 + }, + { + "epoch": 1.29, + "learning_rate": 7.524974924774323e-05, + "loss": 1.1808, + "theoretical_loss": 3.3497701327764204, + "tokens_seen": 2993670144 + }, + { + "epoch": 1.29, + "learning_rate": 7.523370110330993e-05, + "loss": 1.1376, + "theoretical_loss": 3.3497645401536174, + "tokens_seen": 2993735680 + }, + { + "epoch": 1.29, + "learning_rate": 7.521765295887663e-05, + "loss": 1.1678, + "theoretical_loss": 3.349758947687521, + "tokens_seen": 2993801216 + }, + { + "epoch": 1.29, + "learning_rate": 7.520160481444333e-05, + "loss": 1.1719, + "theoretical_loss": 3.3497533553781222, + "tokens_seen": 2993866752 + }, + { + "epoch": 1.29, + "learning_rate": 7.518555667001003e-05, + "loss": 1.1923, + "theoretical_loss": 3.3497477632254142, + "tokens_seen": 2993932288 + }, + { + "epoch": 1.29, + "learning_rate": 7.516950852557673e-05, + "loss": 1.2179, + "theoretical_loss": 3.349742171229389, + "tokens_seen": 2993997824 + }, + { + "epoch": 1.29, + "learning_rate": 7.515346038114344e-05, + "loss": 1.2843, + "theoretical_loss": 3.349736579390038, + "tokens_seen": 2994063360 + }, + { + "epoch": 1.29, + "learning_rate": 7.513741223671014e-05, + "loss": 1.2248, + "theoretical_loss": 3.3497309877073547, + "tokens_seen": 2994128896 + }, + { + "epoch": 1.29, + "learning_rate": 7.512136409227684e-05, + "loss": 1.2278, + "theoretical_loss": 3.3497253961813303, + "tokens_seen": 2994194432 + }, + { + "epoch": 1.29, + "learning_rate": 7.510531594784354e-05, + "loss": 1.1715, + "theoretical_loss": 3.349719804811957, + "tokens_seen": 2994259968 + }, + { + "epoch": 1.29, + "learning_rate": 7.508926780341024e-05, + "loss": 1.158, + "theoretical_loss": 3.3497142135992277, + "tokens_seen": 2994325504 + }, + { + "epoch": 1.29, + "learning_rate": 7.507321965897694e-05, + "loss": 1.3223, + "theoretical_loss": 3.3497086225431336, + "tokens_seen": 2994391040 + }, + { + "epoch": 1.29, + "learning_rate": 7.505717151454364e-05, + "loss": 1.1736, + "theoretical_loss": 3.349703031643668, + "tokens_seen": 2994456576 + }, + { + "epoch": 1.29, + "learning_rate": 7.504112337011034e-05, + "loss": 1.2037, + "theoretical_loss": 3.349697440900822, + "tokens_seen": 2994522112 + }, + { + "epoch": 1.29, + "learning_rate": 7.502507522567703e-05, + "loss": 1.2805, + "theoretical_loss": 3.3496918503145885, + "tokens_seen": 2994587648 + }, + { + "epoch": 1.29, + "learning_rate": 7.500902708124373e-05, + "loss": 1.1259, + "theoretical_loss": 3.3496862598849595, + "tokens_seen": 2994653184 + }, + { + "epoch": 1.29, + "learning_rate": 7.499297893681043e-05, + "loss": 1.1608, + "theoretical_loss": 3.349680669611927, + "tokens_seen": 2994718720 + }, + { + "epoch": 1.29, + "learning_rate": 7.497693079237713e-05, + "loss": 1.2151, + "theoretical_loss": 3.3496750794954835, + "tokens_seen": 2994784256 + }, + { + "epoch": 1.29, + "learning_rate": 7.496088264794383e-05, + "loss": 1.0868, + "theoretical_loss": 3.349669489535621, + "tokens_seen": 2994849792 + }, + { + "epoch": 1.29, + "learning_rate": 7.494483450351053e-05, + "loss": 1.2266, + "theoretical_loss": 3.349663899732332, + "tokens_seen": 2994915328 + }, + { + "epoch": 1.29, + "objective/train/docs_used": 1105308, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2574517726898193, + "objective/train/theoretical_loss": 3.3496583100856077, + "objective/train/tokens_used": 3015440864, + "theoretical_loss": 3.3496583100856077, + "tokens_seen": 2994980864 + }, + { + "epoch": 1.29, + "learning_rate": 7.492878635907723e-05, + "loss": 1.2575, + "theoretical_loss": 3.3496583100856077, + "tokens_seen": 2994980864 + }, + { + "epoch": 1.29, + "learning_rate": 7.491273821464395e-05, + "loss": 1.3683, + "theoretical_loss": 3.3496527205954414, + "tokens_seen": 2995046400 + }, + { + "epoch": 1.29, + "learning_rate": 7.489669007021065e-05, + "loss": 1.2154, + "theoretical_loss": 3.349647131261825, + "tokens_seen": 2995111936 + }, + { + "epoch": 1.29, + "learning_rate": 7.488064192577733e-05, + "loss": 1.207, + "theoretical_loss": 3.3496415420847505, + "tokens_seen": 2995177472 + }, + { + "epoch": 1.29, + "learning_rate": 7.486459378134403e-05, + "loss": 1.2041, + "theoretical_loss": 3.34963595306421, + "tokens_seen": 2995243008 + }, + { + "epoch": 1.29, + "learning_rate": 7.484854563691073e-05, + "loss": 1.2095, + "theoretical_loss": 3.3496303642001957, + "tokens_seen": 2995308544 + }, + { + "epoch": 1.29, + "learning_rate": 7.483249749247743e-05, + "loss": 1.2322, + "theoretical_loss": 3.3496247754927, + "tokens_seen": 2995374080 + }, + { + "epoch": 1.29, + "learning_rate": 7.481644934804414e-05, + "loss": 1.1706, + "theoretical_loss": 3.3496191869417156, + "tokens_seen": 2995439616 + }, + { + "epoch": 1.29, + "learning_rate": 7.480040120361084e-05, + "loss": 1.1738, + "theoretical_loss": 3.3496135985472337, + "tokens_seen": 2995505152 + }, + { + "epoch": 1.29, + "learning_rate": 7.478435305917754e-05, + "loss": 1.1906, + "theoretical_loss": 3.3496080103092467, + "tokens_seen": 2995570688 + }, + { + "epoch": 1.29, + "learning_rate": 7.476830491474424e-05, + "loss": 1.1799, + "theoretical_loss": 3.3496024222277474, + "tokens_seen": 2995636224 + }, + { + "epoch": 1.29, + "learning_rate": 7.475225677031094e-05, + "loss": 1.1907, + "theoretical_loss": 3.3495968343027274, + "tokens_seen": 2995701760 + }, + { + "epoch": 1.29, + "learning_rate": 7.473620862587764e-05, + "loss": 1.1505, + "theoretical_loss": 3.349591246534179, + "tokens_seen": 2995767296 + }, + { + "epoch": 1.29, + "learning_rate": 7.472016048144434e-05, + "loss": 1.318, + "theoretical_loss": 3.349585658922095, + "tokens_seen": 2995832832 + }, + { + "epoch": 1.29, + "learning_rate": 7.470411233701104e-05, + "loss": 1.1672, + "theoretical_loss": 3.3495800714664665, + "tokens_seen": 2995898368 + }, + { + "epoch": 1.29, + "learning_rate": 7.468806419257773e-05, + "loss": 1.1855, + "theoretical_loss": 3.3495744841672868, + "tokens_seen": 2995963904 + }, + { + "epoch": 1.29, + "learning_rate": 7.467201604814443e-05, + "loss": 1.2799, + "theoretical_loss": 3.349568897024547, + "tokens_seen": 2996029440 + }, + { + "epoch": 1.29, + "learning_rate": 7.465596790371113e-05, + "loss": 1.0926, + "theoretical_loss": 3.3495633100382407, + "tokens_seen": 2996094976 + }, + { + "epoch": 1.29, + "learning_rate": 7.463991975927783e-05, + "loss": 1.1225, + "theoretical_loss": 3.349557723208359, + "tokens_seen": 2996160512 + }, + { + "epoch": 1.29, + "learning_rate": 7.462387161484454e-05, + "loss": 1.2635, + "theoretical_loss": 3.349552136534894, + "tokens_seen": 2996226048 + }, + { + "epoch": 1.29, + "learning_rate": 7.460782347041124e-05, + "loss": 1.1699, + "theoretical_loss": 3.3495465500178385, + "tokens_seen": 2996291584 + }, + { + "epoch": 1.29, + "learning_rate": 7.459177532597794e-05, + "loss": 1.2025, + "theoretical_loss": 3.3495409636571845, + "tokens_seen": 2996357120 + }, + { + "epoch": 1.29, + "learning_rate": 7.457572718154464e-05, + "loss": 1.1269, + "theoretical_loss": 3.3495353774529244, + "tokens_seen": 2996422656 + }, + { + "epoch": 1.29, + "learning_rate": 7.455967903711135e-05, + "loss": 1.2655, + "theoretical_loss": 3.3495297914050495, + "tokens_seen": 2996488192 + }, + { + "epoch": 1.29, + "learning_rate": 7.454363089267805e-05, + "loss": 1.0558, + "theoretical_loss": 3.3495242055135535, + "tokens_seen": 2996553728 + }, + { + "epoch": 1.29, + "learning_rate": 7.452758274824473e-05, + "loss": 1.201, + "theoretical_loss": 3.3495186197784275, + "tokens_seen": 2996619264 + }, + { + "epoch": 1.29, + "learning_rate": 7.451153460381143e-05, + "loss": 1.1127, + "theoretical_loss": 3.349513034199664, + "tokens_seen": 2996684800 + }, + { + "epoch": 1.29, + "learning_rate": 7.449548645937813e-05, + "loss": 1.0868, + "theoretical_loss": 3.349507448777255, + "tokens_seen": 2996750336 + }, + { + "epoch": 1.29, + "learning_rate": 7.447943831494484e-05, + "loss": 1.2921, + "theoretical_loss": 3.3495018635111933, + "tokens_seen": 2996815872 + }, + { + "epoch": 1.29, + "learning_rate": 7.446339017051154e-05, + "loss": 1.1873, + "theoretical_loss": 3.34949627840147, + "tokens_seen": 2996881408 + }, + { + "epoch": 1.29, + "learning_rate": 7.444734202607824e-05, + "loss": 1.2153, + "theoretical_loss": 3.349490693448079, + "tokens_seen": 2996946944 + }, + { + "epoch": 1.29, + "learning_rate": 7.443129388164494e-05, + "loss": 1.3177, + "theoretical_loss": 3.3494851086510105, + "tokens_seen": 2997012480 + }, + { + "epoch": 1.29, + "learning_rate": 7.441524573721164e-05, + "loss": 1.2157, + "theoretical_loss": 3.3494795240102584, + "tokens_seen": 2997078016 + }, + { + "epoch": 1.29, + "learning_rate": 7.439919759277834e-05, + "loss": 1.2401, + "theoretical_loss": 3.349473939525814, + "tokens_seen": 2997143552 + }, + { + "epoch": 1.29, + "learning_rate": 7.438314944834504e-05, + "loss": 1.1502, + "theoretical_loss": 3.34946835519767, + "tokens_seen": 2997209088 + }, + { + "epoch": 1.29, + "learning_rate": 7.436710130391174e-05, + "loss": 1.1887, + "theoretical_loss": 3.349462771025818, + "tokens_seen": 2997274624 + }, + { + "epoch": 1.29, + "learning_rate": 7.435105315947844e-05, + "loss": 1.0961, + "theoretical_loss": 3.3494571870102505, + "tokens_seen": 2997340160 + }, + { + "epoch": 1.29, + "learning_rate": 7.433500501504513e-05, + "loss": 1.2856, + "theoretical_loss": 3.34945160315096, + "tokens_seen": 2997405696 + }, + { + "epoch": 1.29, + "learning_rate": 7.431895687061184e-05, + "loss": 1.0814, + "theoretical_loss": 3.3494460194479387, + "tokens_seen": 2997471232 + }, + { + "epoch": 1.29, + "learning_rate": 7.430290872617854e-05, + "loss": 1.1675, + "theoretical_loss": 3.3494404359011782, + "tokens_seen": 2997536768 + }, + { + "epoch": 1.29, + "learning_rate": 7.428686058174524e-05, + "loss": 1.15, + "theoretical_loss": 3.349434852510671, + "tokens_seen": 2997602304 + }, + { + "epoch": 1.29, + "learning_rate": 7.427081243731194e-05, + "loss": 1.163, + "theoretical_loss": 3.34942926927641, + "tokens_seen": 2997667840 + }, + { + "epoch": 1.29, + "learning_rate": 7.425476429287864e-05, + "loss": 1.2043, + "theoretical_loss": 3.349423686198386, + "tokens_seen": 2997733376 + }, + { + "epoch": 1.29, + "learning_rate": 7.423871614844534e-05, + "loss": 1.1861, + "theoretical_loss": 3.3494181032765926, + "tokens_seen": 2997798912 + }, + { + "epoch": 1.29, + "learning_rate": 7.422266800401205e-05, + "loss": 1.0847, + "theoretical_loss": 3.3494125205110215, + "tokens_seen": 2997864448 + }, + { + "epoch": 1.29, + "learning_rate": 7.420661985957875e-05, + "loss": 1.2207, + "theoretical_loss": 3.3494069379016644, + "tokens_seen": 2997929984 + }, + { + "epoch": 1.29, + "learning_rate": 7.419057171514545e-05, + "loss": 1.1117, + "theoretical_loss": 3.3494013554485145, + "tokens_seen": 2997995520 + }, + { + "epoch": 1.29, + "learning_rate": 7.417452357071213e-05, + "loss": 1.2107, + "theoretical_loss": 3.349395773151563, + "tokens_seen": 2998061056 + }, + { + "epoch": 1.29, + "learning_rate": 7.415847542627883e-05, + "loss": 1.1922, + "theoretical_loss": 3.349390191010803, + "tokens_seen": 2998126592 + }, + { + "epoch": 1.29, + "learning_rate": 7.414242728184553e-05, + "loss": 1.1861, + "theoretical_loss": 3.349384609026226, + "tokens_seen": 2998192128 + }, + { + "epoch": 1.29, + "learning_rate": 7.412637913741224e-05, + "loss": 1.1868, + "theoretical_loss": 3.3493790271978248, + "tokens_seen": 2998257664 + }, + { + "epoch": 1.29, + "learning_rate": 7.411033099297894e-05, + "loss": 1.2202, + "theoretical_loss": 3.349373445525591, + "tokens_seen": 2998323200 + }, + { + "epoch": 1.29, + "learning_rate": 7.409428284854564e-05, + "loss": 1.1705, + "theoretical_loss": 3.3493678640095172, + "tokens_seen": 2998388736 + }, + { + "epoch": 1.29, + "learning_rate": 7.407823470411234e-05, + "loss": 1.1874, + "theoretical_loss": 3.349362282649596, + "tokens_seen": 2998454272 + }, + { + "epoch": 1.29, + "learning_rate": 7.406218655967904e-05, + "loss": 1.2046, + "theoretical_loss": 3.349356701445819, + "tokens_seen": 2998519808 + }, + { + "epoch": 1.29, + "learning_rate": 7.404613841524574e-05, + "loss": 1.1268, + "theoretical_loss": 3.3493511203981785, + "tokens_seen": 2998585344 + }, + { + "epoch": 1.29, + "learning_rate": 7.403009027081244e-05, + "loss": 1.2517, + "theoretical_loss": 3.349345539506667, + "tokens_seen": 2998650880 + }, + { + "epoch": 1.29, + "learning_rate": 7.401404212637914e-05, + "loss": 1.2374, + "theoretical_loss": 3.3493399587712767, + "tokens_seen": 2998716416 + }, + { + "epoch": 1.29, + "learning_rate": 7.399799398194584e-05, + "loss": 1.1815, + "theoretical_loss": 3.3493343781919993, + "tokens_seen": 2998781952 + }, + { + "epoch": 1.29, + "learning_rate": 7.398194583751254e-05, + "loss": 1.1623, + "theoretical_loss": 3.3493287977688277, + "tokens_seen": 2998847488 + }, + { + "epoch": 1.29, + "learning_rate": 7.396589769307924e-05, + "loss": 1.1555, + "theoretical_loss": 3.3493232175017535, + "tokens_seen": 2998913024 + }, + { + "epoch": 1.29, + "learning_rate": 7.394984954864594e-05, + "loss": 1.1668, + "theoretical_loss": 3.3493176373907696, + "tokens_seen": 2998978560 + }, + { + "epoch": 1.29, + "learning_rate": 7.393380140421264e-05, + "loss": 1.21, + "theoretical_loss": 3.349312057435868, + "tokens_seen": 2999044096 + }, + { + "epoch": 1.29, + "learning_rate": 7.391775325977934e-05, + "loss": 1.2229, + "theoretical_loss": 3.3493064776370405, + "tokens_seen": 2999109632 + }, + { + "epoch": 1.29, + "learning_rate": 7.390170511534604e-05, + "loss": 1.2323, + "theoretical_loss": 3.3493008979942793, + "tokens_seen": 2999175168 + }, + { + "epoch": 1.29, + "learning_rate": 7.388565697091275e-05, + "loss": 1.1314, + "theoretical_loss": 3.3492953185075773, + "tokens_seen": 2999240704 + }, + { + "epoch": 1.29, + "learning_rate": 7.386960882647945e-05, + "loss": 1.1457, + "theoretical_loss": 3.3492897391769265, + "tokens_seen": 2999306240 + }, + { + "epoch": 1.29, + "learning_rate": 7.385356068204615e-05, + "loss": 1.1893, + "theoretical_loss": 3.349284160002319, + "tokens_seen": 2999371776 + }, + { + "epoch": 1.29, + "learning_rate": 7.383751253761285e-05, + "loss": 1.2993, + "theoretical_loss": 3.349278580983747, + "tokens_seen": 2999437312 + }, + { + "epoch": 1.29, + "learning_rate": 7.382146439317953e-05, + "loss": 1.1103, + "theoretical_loss": 3.3492730021212025, + "tokens_seen": 2999502848 + }, + { + "epoch": 1.29, + "learning_rate": 7.380541624874623e-05, + "loss": 1.1749, + "theoretical_loss": 3.349267423414678, + "tokens_seen": 2999568384 + }, + { + "epoch": 1.29, + "learning_rate": 7.378936810431294e-05, + "loss": 1.1382, + "theoretical_loss": 3.349261844864166, + "tokens_seen": 2999633920 + }, + { + "epoch": 1.29, + "learning_rate": 7.377331995987964e-05, + "loss": 1.2017, + "theoretical_loss": 3.3492562664696583, + "tokens_seen": 2999699456 + }, + { + "epoch": 1.29, + "learning_rate": 7.375727181544634e-05, + "loss": 1.1269, + "theoretical_loss": 3.3492506882311472, + "tokens_seen": 2999764992 + }, + { + "epoch": 1.29, + "learning_rate": 7.374122367101304e-05, + "loss": 1.16, + "theoretical_loss": 3.349245110148625, + "tokens_seen": 2999830528 + }, + { + "epoch": 1.29, + "learning_rate": 7.372517552657974e-05, + "loss": 1.2235, + "theoretical_loss": 3.349239532222084, + "tokens_seen": 2999896064 + }, + { + "epoch": 1.29, + "learning_rate": 7.370912738214645e-05, + "loss": 1.1772, + "theoretical_loss": 3.349233954451516, + "tokens_seen": 2999961600 + }, + { + "epoch": 1.29, + "learning_rate": 7.369307923771315e-05, + "loss": 1.1184, + "theoretical_loss": 3.349228376836914, + "tokens_seen": 3000027136 + }, + { + "epoch": 1.29, + "learning_rate": 7.367703109327985e-05, + "loss": 1.3793, + "theoretical_loss": 3.34922279937827, + "tokens_seen": 3000092672 + }, + { + "epoch": 1.29, + "learning_rate": 7.366098294884654e-05, + "loss": 1.1942, + "theoretical_loss": 3.3492172220755756, + "tokens_seen": 3000158208 + }, + { + "epoch": 1.29, + "learning_rate": 7.364493480441324e-05, + "loss": 1.0579, + "theoretical_loss": 3.3492116449288236, + "tokens_seen": 3000223744 + }, + { + "epoch": 1.29, + "learning_rate": 7.362888665997994e-05, + "loss": 1.1743, + "theoretical_loss": 3.349206067938006, + "tokens_seen": 3000289280 + }, + { + "epoch": 1.29, + "learning_rate": 7.361283851554664e-05, + "loss": 1.1666, + "theoretical_loss": 3.3492004911031152, + "tokens_seen": 3000354816 + }, + { + "epoch": 1.29, + "learning_rate": 7.359679037111334e-05, + "loss": 1.1432, + "theoretical_loss": 3.3491949144241437, + "tokens_seen": 3000420352 + }, + { + "epoch": 1.29, + "learning_rate": 7.358074222668004e-05, + "loss": 1.2481, + "theoretical_loss": 3.349189337901083, + "tokens_seen": 3000485888 + }, + { + "epoch": 1.29, + "learning_rate": 7.356469408224674e-05, + "loss": 1.1907, + "theoretical_loss": 3.349183761533926, + "tokens_seen": 3000551424 + }, + { + "epoch": 1.29, + "learning_rate": 7.354864593781344e-05, + "loss": 1.1209, + "theoretical_loss": 3.349178185322665, + "tokens_seen": 3000616960 + }, + { + "epoch": 1.29, + "learning_rate": 7.353259779338015e-05, + "loss": 1.1887, + "theoretical_loss": 3.349172609267291, + "tokens_seen": 3000682496 + }, + { + "epoch": 1.29, + "learning_rate": 7.351654964894685e-05, + "loss": 1.2595, + "theoretical_loss": 3.349167033367798, + "tokens_seen": 3000748032 + }, + { + "epoch": 1.29, + "learning_rate": 7.350050150451355e-05, + "loss": 1.2065, + "theoretical_loss": 3.3491614576241773, + "tokens_seen": 3000813568 + }, + { + "epoch": 1.29, + "learning_rate": 7.348445336008025e-05, + "loss": 1.2372, + "theoretical_loss": 3.3491558820364213, + "tokens_seen": 3000879104 + }, + { + "epoch": 1.29, + "learning_rate": 7.346840521564693e-05, + "loss": 1.2796, + "theoretical_loss": 3.3491503066045216, + "tokens_seen": 3000944640 + }, + { + "epoch": 1.29, + "learning_rate": 7.345235707121364e-05, + "loss": 1.1713, + "theoretical_loss": 3.3491447313284715, + "tokens_seen": 3001010176 + }, + { + "epoch": 1.29, + "learning_rate": 7.343630892678034e-05, + "loss": 1.2137, + "theoretical_loss": 3.349139156208263, + "tokens_seen": 3001075712 + }, + { + "epoch": 1.29, + "learning_rate": 7.342026078234705e-05, + "loss": 1.2062, + "theoretical_loss": 3.3491335812438874, + "tokens_seen": 3001141248 + }, + { + "epoch": 1.29, + "learning_rate": 7.340421263791375e-05, + "loss": 1.2091, + "theoretical_loss": 3.349128006435338, + "tokens_seen": 3001206784 + }, + { + "epoch": 1.29, + "learning_rate": 7.338816449348045e-05, + "loss": 1.2933, + "theoretical_loss": 3.349122431782607, + "tokens_seen": 3001272320 + }, + { + "epoch": 1.29, + "learning_rate": 7.337211634904715e-05, + "loss": 1.2107, + "theoretical_loss": 3.3491168572856855, + "tokens_seen": 3001337856 + }, + { + "epoch": 1.29, + "learning_rate": 7.335606820461385e-05, + "loss": 1.0926, + "theoretical_loss": 3.3491112829445675, + "tokens_seen": 3001403392 + }, + { + "epoch": 1.29, + "learning_rate": 7.334002006018055e-05, + "loss": 1.2118, + "theoretical_loss": 3.349105708759244, + "tokens_seen": 3001468928 + }, + { + "epoch": 1.29, + "objective/train/docs_used": 1107721, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1941602230072021, + "objective/train/theoretical_loss": 3.3491001347297074, + "objective/train/tokens_used": 3021994464, + "theoretical_loss": 3.3491001347297074, + "tokens_seen": 3001534464 + }, + { + "epoch": 1.29, + "learning_rate": 7.332397191574725e-05, + "loss": 1.1942, + "theoretical_loss": 3.3491001347297074, + "tokens_seen": 3001534464 + }, + { + "epoch": 1.29, + "learning_rate": 7.330792377131394e-05, + "loss": 1.159, + "theoretical_loss": 3.34909456085595, + "tokens_seen": 3001600000 + }, + { + "epoch": 1.29, + "learning_rate": 7.329187562688064e-05, + "loss": 1.1534, + "theoretical_loss": 3.3490889871379643, + "tokens_seen": 3001665536 + }, + { + "epoch": 1.29, + "learning_rate": 7.327582748244734e-05, + "loss": 1.2117, + "theoretical_loss": 3.349083413575743, + "tokens_seen": 3001731072 + }, + { + "epoch": 1.29, + "learning_rate": 7.325977933801404e-05, + "loss": 1.0831, + "theoretical_loss": 3.3490778401692767, + "tokens_seen": 3001796608 + }, + { + "epoch": 1.29, + "learning_rate": 7.324373119358074e-05, + "loss": 1.2331, + "theoretical_loss": 3.3490722669185593, + "tokens_seen": 3001862144 + }, + { + "epoch": 1.29, + "learning_rate": 7.322768304914744e-05, + "loss": 1.2998, + "theoretical_loss": 3.3490666938235822, + "tokens_seen": 3001927680 + }, + { + "epoch": 1.29, + "learning_rate": 7.321163490471414e-05, + "loss": 1.2097, + "theoretical_loss": 3.349061120884338, + "tokens_seen": 3001993216 + }, + { + "epoch": 1.29, + "learning_rate": 7.319558676028085e-05, + "loss": 1.2388, + "theoretical_loss": 3.349055548100819, + "tokens_seen": 3002058752 + }, + { + "epoch": 1.29, + "learning_rate": 7.317953861584755e-05, + "loss": 1.06, + "theoretical_loss": 3.349049975473017, + "tokens_seen": 3002124288 + }, + { + "epoch": 1.29, + "learning_rate": 7.316349047141425e-05, + "loss": 1.1617, + "theoretical_loss": 3.3490444030009243, + "tokens_seen": 3002189824 + }, + { + "epoch": 1.29, + "learning_rate": 7.314744232698095e-05, + "loss": 1.1971, + "theoretical_loss": 3.3490388306845342, + "tokens_seen": 3002255360 + }, + { + "epoch": 1.29, + "learning_rate": 7.313139418254765e-05, + "loss": 1.2045, + "theoretical_loss": 3.3490332585238374, + "tokens_seen": 3002320896 + }, + { + "epoch": 1.29, + "learning_rate": 7.311534603811435e-05, + "loss": 1.1221, + "theoretical_loss": 3.349027686518827, + "tokens_seen": 3002386432 + }, + { + "epoch": 1.29, + "learning_rate": 7.309929789368105e-05, + "loss": 1.114, + "theoretical_loss": 3.349022114669496, + "tokens_seen": 3002451968 + }, + { + "epoch": 1.29, + "learning_rate": 7.308324974924775e-05, + "loss": 1.1697, + "theoretical_loss": 3.3490165429758347, + "tokens_seen": 3002517504 + }, + { + "epoch": 1.29, + "learning_rate": 7.306720160481445e-05, + "loss": 1.2416, + "theoretical_loss": 3.3490109714378367, + "tokens_seen": 3002583040 + }, + { + "epoch": 1.29, + "learning_rate": 7.305115346038115e-05, + "loss": 1.1836, + "theoretical_loss": 3.349005400055494, + "tokens_seen": 3002648576 + }, + { + "epoch": 1.29, + "learning_rate": 7.303510531594785e-05, + "loss": 1.1463, + "theoretical_loss": 3.348999828828799, + "tokens_seen": 3002714112 + }, + { + "epoch": 1.29, + "learning_rate": 7.301905717151455e-05, + "loss": 1.1797, + "theoretical_loss": 3.348994257757744, + "tokens_seen": 3002779648 + }, + { + "epoch": 1.29, + "learning_rate": 7.300300902708125e-05, + "loss": 1.1821, + "theoretical_loss": 3.3489886868423207, + "tokens_seen": 3002845184 + }, + { + "epoch": 1.29, + "learning_rate": 7.298696088264795e-05, + "loss": 1.1346, + "theoretical_loss": 3.348983116082522, + "tokens_seen": 3002910720 + }, + { + "epoch": 1.29, + "learning_rate": 7.297091273821465e-05, + "loss": 1.1904, + "theoretical_loss": 3.3489775454783395, + "tokens_seen": 3002976256 + }, + { + "epoch": 1.29, + "learning_rate": 7.295486459378134e-05, + "loss": 1.2007, + "theoretical_loss": 3.348971975029766, + "tokens_seen": 3003041792 + }, + { + "epoch": 1.29, + "learning_rate": 7.293881644934804e-05, + "loss": 1.1453, + "theoretical_loss": 3.3489664047367937, + "tokens_seen": 3003107328 + }, + { + "epoch": 1.29, + "learning_rate": 7.292276830491474e-05, + "loss": 1.2033, + "theoretical_loss": 3.3489608345994144, + "tokens_seen": 3003172864 + }, + { + "epoch": 1.29, + "learning_rate": 7.290672016048144e-05, + "loss": 1.188, + "theoretical_loss": 3.3489552646176213, + "tokens_seen": 3003238400 + }, + { + "epoch": 1.29, + "learning_rate": 7.289067201604814e-05, + "loss": 1.2014, + "theoretical_loss": 3.348949694791406, + "tokens_seen": 3003303936 + }, + { + "epoch": 1.29, + "learning_rate": 7.287462387161484e-05, + "loss": 1.1935, + "theoretical_loss": 3.3489441251207603, + "tokens_seen": 3003369472 + }, + { + "epoch": 1.29, + "learning_rate": 7.285857572718154e-05, + "loss": 1.1317, + "theoretical_loss": 3.3489385556056774, + "tokens_seen": 3003435008 + }, + { + "epoch": 1.29, + "learning_rate": 7.284252758274825e-05, + "loss": 1.1721, + "theoretical_loss": 3.3489329862461488, + "tokens_seen": 3003500544 + }, + { + "epoch": 1.29, + "learning_rate": 7.282647943831495e-05, + "loss": 1.28, + "theoretical_loss": 3.3489274170421672, + "tokens_seen": 3003566080 + }, + { + "epoch": 1.29, + "learning_rate": 7.281043129388165e-05, + "loss": 1.1236, + "theoretical_loss": 3.348921847993725, + "tokens_seen": 3003631616 + }, + { + "epoch": 1.29, + "learning_rate": 7.279438314944835e-05, + "loss": 1.1848, + "theoretical_loss": 3.348916279100814, + "tokens_seen": 3003697152 + }, + { + "epoch": 1.29, + "learning_rate": 7.277833500501505e-05, + "loss": 1.1792, + "theoretical_loss": 3.348910710363427, + "tokens_seen": 3003762688 + }, + { + "epoch": 1.29, + "learning_rate": 7.276228686058175e-05, + "loss": 1.1256, + "theoretical_loss": 3.3489051417815556, + "tokens_seen": 3003828224 + }, + { + "epoch": 1.29, + "learning_rate": 7.274623871614845e-05, + "loss": 1.2649, + "theoretical_loss": 3.3488995733551925, + "tokens_seen": 3003893760 + }, + { + "epoch": 1.29, + "learning_rate": 7.273019057171515e-05, + "loss": 1.1663, + "theoretical_loss": 3.34889400508433, + "tokens_seen": 3003959296 + }, + { + "epoch": 1.29, + "learning_rate": 7.271414242728185e-05, + "loss": 1.2029, + "theoretical_loss": 3.34888843696896, + "tokens_seen": 3004024832 + }, + { + "epoch": 1.29, + "learning_rate": 7.269809428284855e-05, + "loss": 1.2004, + "theoretical_loss": 3.3488828690090755, + "tokens_seen": 3004090368 + }, + { + "epoch": 1.29, + "learning_rate": 7.268204613841525e-05, + "loss": 1.1655, + "theoretical_loss": 3.348877301204668, + "tokens_seen": 3004155904 + }, + { + "epoch": 1.29, + "learning_rate": 7.266599799398195e-05, + "loss": 1.1723, + "theoretical_loss": 3.34887173355573, + "tokens_seen": 3004221440 + }, + { + "epoch": 1.29, + "learning_rate": 7.264994984954865e-05, + "loss": 1.2394, + "theoretical_loss": 3.348866166062254, + "tokens_seen": 3004286976 + }, + { + "epoch": 1.29, + "learning_rate": 7.263390170511535e-05, + "loss": 1.2537, + "theoretical_loss": 3.348860598724232, + "tokens_seen": 3004352512 + }, + { + "epoch": 1.29, + "learning_rate": 7.261785356068204e-05, + "loss": 1.3413, + "theoretical_loss": 3.3488550315416563, + "tokens_seen": 3004418048 + }, + { + "epoch": 1.29, + "learning_rate": 7.260180541624874e-05, + "loss": 1.2052, + "theoretical_loss": 3.3488494645145193, + "tokens_seen": 3004483584 + }, + { + "epoch": 1.29, + "learning_rate": 7.258575727181544e-05, + "loss": 1.1245, + "theoretical_loss": 3.348843897642813, + "tokens_seen": 3004549120 + }, + { + "epoch": 1.29, + "learning_rate": 7.256970912738214e-05, + "loss": 1.1335, + "theoretical_loss": 3.34883833092653, + "tokens_seen": 3004614656 + }, + { + "epoch": 1.29, + "learning_rate": 7.255366098294884e-05, + "loss": 1.1898, + "theoretical_loss": 3.348832764365662, + "tokens_seen": 3004680192 + }, + { + "epoch": 1.29, + "learning_rate": 7.253761283851554e-05, + "loss": 1.125, + "theoretical_loss": 3.3488271979602025, + "tokens_seen": 3004745728 + }, + { + "epoch": 1.29, + "learning_rate": 7.252156469408226e-05, + "loss": 1.2086, + "theoretical_loss": 3.3488216317101425, + "tokens_seen": 3004811264 + }, + { + "epoch": 1.29, + "learning_rate": 7.250551654964896e-05, + "loss": 1.1703, + "theoretical_loss": 3.348816065615475, + "tokens_seen": 3004876800 + }, + { + "epoch": 1.29, + "learning_rate": 7.248946840521566e-05, + "loss": 1.1704, + "theoretical_loss": 3.348810499676192, + "tokens_seen": 3004942336 + }, + { + "epoch": 1.29, + "learning_rate": 7.247342026078236e-05, + "loss": 1.258, + "theoretical_loss": 3.3488049338922856, + "tokens_seen": 3005007872 + }, + { + "epoch": 1.29, + "learning_rate": 7.245737211634905e-05, + "loss": 1.1578, + "theoretical_loss": 3.348799368263748, + "tokens_seen": 3005073408 + }, + { + "epoch": 1.29, + "learning_rate": 7.244132397191575e-05, + "loss": 1.2271, + "theoretical_loss": 3.3487938027905724, + "tokens_seen": 3005138944 + }, + { + "epoch": 1.29, + "learning_rate": 7.242527582748245e-05, + "loss": 1.1369, + "theoretical_loss": 3.34878823747275, + "tokens_seen": 3005204480 + }, + { + "epoch": 1.29, + "learning_rate": 7.240922768304915e-05, + "loss": 1.1078, + "theoretical_loss": 3.3487826723102736, + "tokens_seen": 3005270016 + }, + { + "epoch": 1.29, + "learning_rate": 7.239317953861585e-05, + "loss": 1.1711, + "theoretical_loss": 3.3487771073031354, + "tokens_seen": 3005335552 + }, + { + "epoch": 1.29, + "learning_rate": 7.237713139418255e-05, + "loss": 1.2087, + "theoretical_loss": 3.3487715424513276, + "tokens_seen": 3005401088 + }, + { + "epoch": 1.29, + "learning_rate": 7.236108324974925e-05, + "loss": 1.1398, + "theoretical_loss": 3.3487659777548426, + "tokens_seen": 3005466624 + }, + { + "epoch": 1.29, + "learning_rate": 7.234503510531595e-05, + "loss": 1.2224, + "theoretical_loss": 3.3487604132136726, + "tokens_seen": 3005532160 + }, + { + "epoch": 1.29, + "learning_rate": 7.232898696088265e-05, + "loss": 1.1282, + "theoretical_loss": 3.34875484882781, + "tokens_seen": 3005597696 + }, + { + "epoch": 1.29, + "learning_rate": 7.231293881644935e-05, + "loss": 1.1838, + "theoretical_loss": 3.3487492845972464, + "tokens_seen": 3005663232 + }, + { + "epoch": 1.29, + "learning_rate": 7.229689067201605e-05, + "loss": 1.2793, + "theoretical_loss": 3.348743720521975, + "tokens_seen": 3005728768 + }, + { + "epoch": 1.29, + "learning_rate": 7.228084252758275e-05, + "loss": 1.2524, + "theoretical_loss": 3.348738156601988, + "tokens_seen": 3005794304 + }, + { + "epoch": 1.29, + "learning_rate": 7.226479438314944e-05, + "loss": 1.193, + "theoretical_loss": 3.348732592837277, + "tokens_seen": 3005859840 + }, + { + "epoch": 1.29, + "learning_rate": 7.224874623871614e-05, + "loss": 1.2152, + "theoretical_loss": 3.3487270292278346, + "tokens_seen": 3005925376 + }, + { + "epoch": 1.29, + "learning_rate": 7.223269809428284e-05, + "loss": 1.2034, + "theoretical_loss": 3.3487214657736537, + "tokens_seen": 3005990912 + }, + { + "epoch": 1.29, + "learning_rate": 7.221664994984956e-05, + "loss": 1.2772, + "theoretical_loss": 3.3487159024747255, + "tokens_seen": 3006056448 + }, + { + "epoch": 1.29, + "learning_rate": 7.220060180541626e-05, + "loss": 1.1888, + "theoretical_loss": 3.3487103393310433, + "tokens_seen": 3006121984 + }, + { + "epoch": 1.29, + "learning_rate": 7.218455366098296e-05, + "loss": 1.1612, + "theoretical_loss": 3.3487047763425988, + "tokens_seen": 3006187520 + }, + { + "epoch": 1.29, + "learning_rate": 7.216850551654966e-05, + "loss": 1.2821, + "theoretical_loss": 3.3486992135093843, + "tokens_seen": 3006253056 + }, + { + "epoch": 1.29, + "learning_rate": 7.215245737211636e-05, + "loss": 1.0727, + "theoretical_loss": 3.3486936508313923, + "tokens_seen": 3006318592 + }, + { + "epoch": 1.29, + "learning_rate": 7.213640922768306e-05, + "loss": 1.182, + "theoretical_loss": 3.3486880883086148, + "tokens_seen": 3006384128 + }, + { + "epoch": 1.29, + "learning_rate": 7.212036108324976e-05, + "loss": 1.1283, + "theoretical_loss": 3.3486825259410438, + "tokens_seen": 3006449664 + }, + { + "epoch": 1.29, + "learning_rate": 7.210431293881645e-05, + "loss": 1.2517, + "theoretical_loss": 3.348676963728673, + "tokens_seen": 3006515200 + }, + { + "epoch": 1.29, + "learning_rate": 7.208826479438315e-05, + "loss": 1.1879, + "theoretical_loss": 3.348671401671493, + "tokens_seen": 3006580736 + }, + { + "epoch": 1.29, + "learning_rate": 7.207221664994985e-05, + "loss": 1.1782, + "theoretical_loss": 3.348665839769497, + "tokens_seen": 3006646272 + }, + { + "epoch": 1.29, + "learning_rate": 7.205616850551655e-05, + "loss": 1.2364, + "theoretical_loss": 3.3486602780226775, + "tokens_seen": 3006711808 + }, + { + "epoch": 1.29, + "learning_rate": 7.204012036108325e-05, + "loss": 1.1484, + "theoretical_loss": 3.348654716431026, + "tokens_seen": 3006777344 + }, + { + "epoch": 1.29, + "learning_rate": 7.202407221664995e-05, + "loss": 1.1602, + "theoretical_loss": 3.348649154994535, + "tokens_seen": 3006842880 + }, + { + "epoch": 1.29, + "learning_rate": 7.202407221664995e-05, + "loss": 1.2373, + "theoretical_loss": 3.3486435937131973, + "tokens_seen": 3006908416 + }, + { + "epoch": 1.29, + "learning_rate": 7.200802407221665e-05, + "loss": 1.0577, + "theoretical_loss": 3.348638032587005, + "tokens_seen": 3006973952 + }, + { + "epoch": 1.29, + "learning_rate": 7.199197592778335e-05, + "loss": 1.2599, + "theoretical_loss": 3.34863247161595, + "tokens_seen": 3007039488 + }, + { + "epoch": 1.29, + "learning_rate": 7.197592778335005e-05, + "loss": 1.1621, + "theoretical_loss": 3.3486269108000246, + "tokens_seen": 3007105024 + }, + { + "epoch": 1.29, + "learning_rate": 7.195987963891675e-05, + "loss": 1.2074, + "theoretical_loss": 3.3486213501392217, + "tokens_seen": 3007170560 + }, + { + "epoch": 1.29, + "learning_rate": 7.194383149448345e-05, + "loss": 1.0779, + "theoretical_loss": 3.3486157896335333, + "tokens_seen": 3007236096 + }, + { + "epoch": 1.29, + "learning_rate": 7.192778335005015e-05, + "loss": 1.2247, + "theoretical_loss": 3.348610229282951, + "tokens_seen": 3007301632 + }, + { + "epoch": 1.29, + "learning_rate": 7.191173520561686e-05, + "loss": 1.2369, + "theoretical_loss": 3.3486046690874685, + "tokens_seen": 3007367168 + }, + { + "epoch": 1.29, + "learning_rate": 7.189568706118356e-05, + "loss": 1.0819, + "theoretical_loss": 3.348599109047077, + "tokens_seen": 3007432704 + }, + { + "epoch": 1.29, + "learning_rate": 7.187963891675026e-05, + "loss": 1.2636, + "theoretical_loss": 3.3485935491617687, + "tokens_seen": 3007498240 + }, + { + "epoch": 1.29, + "learning_rate": 7.186359077231696e-05, + "loss": 1.2285, + "theoretical_loss": 3.348587989431537, + "tokens_seen": 3007563776 + }, + { + "epoch": 1.29, + "learning_rate": 7.184754262788366e-05, + "loss": 1.1858, + "theoretical_loss": 3.348582429856373, + "tokens_seen": 3007629312 + }, + { + "epoch": 1.29, + "learning_rate": 7.183149448345036e-05, + "loss": 1.2034, + "theoretical_loss": 3.3485768704362693, + "tokens_seen": 3007694848 + }, + { + "epoch": 1.29, + "learning_rate": 7.181544633901706e-05, + "loss": 1.1383, + "theoretical_loss": 3.348571311171219, + "tokens_seen": 3007760384 + }, + { + "epoch": 1.29, + "learning_rate": 7.179939819458376e-05, + "loss": 1.2209, + "theoretical_loss": 3.3485657520612135, + "tokens_seen": 3007825920 + }, + { + "epoch": 1.29, + "learning_rate": 7.178335005015046e-05, + "loss": 1.2476, + "theoretical_loss": 3.348560193106245, + "tokens_seen": 3007891456 + }, + { + "epoch": 1.29, + "learning_rate": 7.176730190571716e-05, + "loss": 1.2912, + "theoretical_loss": 3.348554634306307, + "tokens_seen": 3007956992 + }, + { + "epoch": 1.29, + "learning_rate": 7.175125376128385e-05, + "loss": 1.1177, + "theoretical_loss": 3.3485490756613903, + "tokens_seen": 3008022528 + }, + { + "epoch": 1.29, + "objective/train/docs_used": 1109976, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1271756887435913, + "objective/train/theoretical_loss": 3.348543517171488, + "objective/train/tokens_used": 3028548064, + "theoretical_loss": 3.348543517171488, + "tokens_seen": 3008088064 + }, + { + "epoch": 1.29, + "learning_rate": 7.173520561685055e-05, + "loss": 1.1272, + "theoretical_loss": 3.348543517171488, + "tokens_seen": 3008088064 + }, + { + "epoch": 1.29, + "learning_rate": 7.171915747241725e-05, + "loss": 1.198, + "theoretical_loss": 3.3485379588365927, + "tokens_seen": 3008153600 + }, + { + "epoch": 1.29, + "learning_rate": 7.170310932798395e-05, + "loss": 1.2294, + "theoretical_loss": 3.348532400656696, + "tokens_seen": 3008219136 + }, + { + "epoch": 1.29, + "learning_rate": 7.168706118355065e-05, + "loss": 1.1723, + "theoretical_loss": 3.3485268426317902, + "tokens_seen": 3008284672 + }, + { + "epoch": 1.29, + "learning_rate": 7.167101303911735e-05, + "loss": 1.1773, + "theoretical_loss": 3.348521284761868, + "tokens_seen": 3008350208 + }, + { + "epoch": 1.29, + "learning_rate": 7.165496489468405e-05, + "loss": 1.187, + "theoretical_loss": 3.3485157270469217, + "tokens_seen": 3008415744 + }, + { + "epoch": 1.29, + "learning_rate": 7.163891675025075e-05, + "loss": 1.2229, + "theoretical_loss": 3.3485101694869437, + "tokens_seen": 3008481280 + }, + { + "epoch": 1.29, + "learning_rate": 7.162286860581745e-05, + "loss": 1.2036, + "theoretical_loss": 3.3485046120819257, + "tokens_seen": 3008546816 + }, + { + "epoch": 1.29, + "learning_rate": 7.160682046138417e-05, + "loss": 1.2226, + "theoretical_loss": 3.3484990548318603, + "tokens_seen": 3008612352 + }, + { + "epoch": 1.29, + "learning_rate": 7.159077231695085e-05, + "loss": 1.2179, + "theoretical_loss": 3.34849349773674, + "tokens_seen": 3008677888 + }, + { + "epoch": 1.29, + "learning_rate": 7.157472417251756e-05, + "loss": 1.1455, + "theoretical_loss": 3.3484879407965575, + "tokens_seen": 3008743424 + }, + { + "epoch": 1.29, + "learning_rate": 7.155867602808426e-05, + "loss": 1.1893, + "theoretical_loss": 3.3484823840113043, + "tokens_seen": 3008808960 + }, + { + "epoch": 1.29, + "learning_rate": 7.154262788365096e-05, + "loss": 1.2194, + "theoretical_loss": 3.348476827380973, + "tokens_seen": 3008874496 + }, + { + "epoch": 1.29, + "learning_rate": 7.152657973921766e-05, + "loss": 1.252, + "theoretical_loss": 3.3484712709055557, + "tokens_seen": 3008940032 + }, + { + "epoch": 1.29, + "learning_rate": 7.151053159478436e-05, + "loss": 1.1836, + "theoretical_loss": 3.3484657145850454, + "tokens_seen": 3009005568 + }, + { + "epoch": 1.29, + "learning_rate": 7.149448345035106e-05, + "loss": 1.2407, + "theoretical_loss": 3.3484601584194333, + "tokens_seen": 3009071104 + }, + { + "epoch": 1.29, + "learning_rate": 7.147843530591776e-05, + "loss": 1.1977, + "theoretical_loss": 3.3484546024087125, + "tokens_seen": 3009136640 + }, + { + "epoch": 1.29, + "learning_rate": 7.146238716148446e-05, + "loss": 1.2049, + "theoretical_loss": 3.3484490465528753, + "tokens_seen": 3009202176 + }, + { + "epoch": 1.29, + "learning_rate": 7.144633901705116e-05, + "loss": 1.092, + "theoretical_loss": 3.3484434908519143, + "tokens_seen": 3009267712 + }, + { + "epoch": 1.29, + "learning_rate": 7.143029087261786e-05, + "loss": 1.1955, + "theoretical_loss": 3.348437935305821, + "tokens_seen": 3009333248 + }, + { + "epoch": 1.29, + "learning_rate": 7.141424272818456e-05, + "loss": 1.2205, + "theoretical_loss": 3.3484323799145876, + "tokens_seen": 3009398784 + }, + { + "epoch": 1.29, + "learning_rate": 7.139819458375125e-05, + "loss": 1.1199, + "theoretical_loss": 3.3484268246782074, + "tokens_seen": 3009464320 + }, + { + "epoch": 1.29, + "learning_rate": 7.138214643931795e-05, + "loss": 1.1326, + "theoretical_loss": 3.348421269596672, + "tokens_seen": 3009529856 + }, + { + "epoch": 1.29, + "learning_rate": 7.136609829488465e-05, + "loss": 1.125, + "theoretical_loss": 3.348415714669974, + "tokens_seen": 3009595392 + }, + { + "epoch": 1.29, + "learning_rate": 7.135005015045135e-05, + "loss": 1.1552, + "theoretical_loss": 3.3484101598981058, + "tokens_seen": 3009660928 + }, + { + "epoch": 1.29, + "learning_rate": 7.133400200601805e-05, + "loss": 1.2124, + "theoretical_loss": 3.3484046052810594, + "tokens_seen": 3009726464 + }, + { + "epoch": 1.29, + "learning_rate": 7.131795386158477e-05, + "loss": 1.1456, + "theoretical_loss": 3.348399050818827, + "tokens_seen": 3009792000 + }, + { + "epoch": 1.29, + "learning_rate": 7.130190571715147e-05, + "loss": 1.1408, + "theoretical_loss": 3.3483934965114015, + "tokens_seen": 3009857536 + }, + { + "epoch": 1.29, + "learning_rate": 7.128585757271817e-05, + "loss": 1.3048, + "theoretical_loss": 3.348387942358775, + "tokens_seen": 3009923072 + }, + { + "epoch": 1.29, + "learning_rate": 7.126980942828487e-05, + "loss": 1.1473, + "theoretical_loss": 3.3483823883609394, + "tokens_seen": 3009988608 + }, + { + "epoch": 1.29, + "learning_rate": 7.125376128385157e-05, + "loss": 1.1589, + "theoretical_loss": 3.3483768345178873, + "tokens_seen": 3010054144 + }, + { + "epoch": 1.29, + "learning_rate": 7.123771313941825e-05, + "loss": 1.0871, + "theoretical_loss": 3.3483712808296113, + "tokens_seen": 3010119680 + }, + { + "epoch": 1.29, + "learning_rate": 7.122166499498496e-05, + "loss": 1.1525, + "theoretical_loss": 3.348365727296103, + "tokens_seen": 3010185216 + }, + { + "epoch": 1.29, + "learning_rate": 7.120561685055166e-05, + "loss": 1.3391, + "theoretical_loss": 3.3483601739173556, + "tokens_seen": 3010250752 + }, + { + "epoch": 1.29, + "learning_rate": 7.118956870611836e-05, + "loss": 1.1212, + "theoretical_loss": 3.3483546206933608, + "tokens_seen": 3010316288 + }, + { + "epoch": 1.29, + "learning_rate": 7.117352056168506e-05, + "loss": 1.1005, + "theoretical_loss": 3.3483490676241114, + "tokens_seen": 3010381824 + }, + { + "epoch": 1.29, + "learning_rate": 7.115747241725176e-05, + "loss": 1.3025, + "theoretical_loss": 3.3483435147095992, + "tokens_seen": 3010447360 + }, + { + "epoch": 1.29, + "learning_rate": 7.114142427281846e-05, + "loss": 1.1184, + "theoretical_loss": 3.3483379619498166, + "tokens_seen": 3010512896 + }, + { + "epoch": 1.29, + "learning_rate": 7.112537612838516e-05, + "loss": 1.1488, + "theoretical_loss": 3.3483324093447564, + "tokens_seen": 3010578432 + }, + { + "epoch": 1.29, + "learning_rate": 7.110932798395186e-05, + "loss": 1.2484, + "theoretical_loss": 3.3483268568944102, + "tokens_seen": 3010643968 + }, + { + "epoch": 1.29, + "learning_rate": 7.109327983951856e-05, + "loss": 1.1858, + "theoretical_loss": 3.3483213045987705, + "tokens_seen": 3010709504 + }, + { + "epoch": 1.29, + "learning_rate": 7.107723169508526e-05, + "loss": 1.1752, + "theoretical_loss": 3.3483157524578306, + "tokens_seen": 3010775040 + }, + { + "epoch": 1.29, + "learning_rate": 7.106118355065196e-05, + "loss": 1.1746, + "theoretical_loss": 3.3483102004715812, + "tokens_seen": 3010840576 + }, + { + "epoch": 1.29, + "learning_rate": 7.104513540621865e-05, + "loss": 1.1358, + "theoretical_loss": 3.348304648640016, + "tokens_seen": 3010906112 + }, + { + "epoch": 1.29, + "learning_rate": 7.102908726178535e-05, + "loss": 1.2571, + "theoretical_loss": 3.3482990969631268, + "tokens_seen": 3010971648 + }, + { + "epoch": 1.29, + "learning_rate": 7.101303911735206e-05, + "loss": 1.2356, + "theoretical_loss": 3.3482935454409057, + "tokens_seen": 3011037184 + }, + { + "epoch": 1.29, + "learning_rate": 7.099699097291876e-05, + "loss": 1.2454, + "theoretical_loss": 3.3482879940733454, + "tokens_seen": 3011102720 + }, + { + "epoch": 1.29, + "learning_rate": 7.098094282848547e-05, + "loss": 1.3021, + "theoretical_loss": 3.3482824428604383, + "tokens_seen": 3011168256 + }, + { + "epoch": 1.29, + "learning_rate": 7.096489468405217e-05, + "loss": 1.2138, + "theoretical_loss": 3.348276891802176, + "tokens_seen": 3011233792 + }, + { + "epoch": 1.29, + "learning_rate": 7.094884653961887e-05, + "loss": 1.1143, + "theoretical_loss": 3.3482713408985516, + "tokens_seen": 3011299328 + }, + { + "epoch": 1.29, + "learning_rate": 7.093279839518557e-05, + "loss": 1.1075, + "theoretical_loss": 3.348265790149557, + "tokens_seen": 3011364864 + }, + { + "epoch": 1.29, + "learning_rate": 7.091675025075227e-05, + "loss": 1.2152, + "theoretical_loss": 3.348260239555185, + "tokens_seen": 3011430400 + }, + { + "epoch": 1.29, + "learning_rate": 7.090070210631895e-05, + "loss": 1.1794, + "theoretical_loss": 3.3482546891154272, + "tokens_seen": 3011495936 + }, + { + "epoch": 1.29, + "learning_rate": 7.088465396188566e-05, + "loss": 1.2012, + "theoretical_loss": 3.348249138830276, + "tokens_seen": 3011561472 + }, + { + "epoch": 1.29, + "learning_rate": 7.086860581745236e-05, + "loss": 1.121, + "theoretical_loss": 3.348243588699725, + "tokens_seen": 3011627008 + }, + { + "epoch": 1.29, + "learning_rate": 7.085255767301906e-05, + "loss": 1.1935, + "theoretical_loss": 3.3482380387237654, + "tokens_seen": 3011692544 + }, + { + "epoch": 1.29, + "learning_rate": 7.083650952858576e-05, + "loss": 1.2779, + "theoretical_loss": 3.348232488902389, + "tokens_seen": 3011758080 + }, + { + "epoch": 1.29, + "learning_rate": 7.082046138415246e-05, + "loss": 1.1187, + "theoretical_loss": 3.3482269392355897, + "tokens_seen": 3011823616 + }, + { + "epoch": 1.29, + "learning_rate": 7.080441323971916e-05, + "loss": 1.2505, + "theoretical_loss": 3.3482213897233586, + "tokens_seen": 3011889152 + }, + { + "epoch": 1.29, + "learning_rate": 7.078836509528586e-05, + "loss": 1.2244, + "theoretical_loss": 3.3482158403656883, + "tokens_seen": 3011954688 + }, + { + "epoch": 1.29, + "learning_rate": 7.077231695085256e-05, + "loss": 1.2197, + "theoretical_loss": 3.3482102911625713, + "tokens_seen": 3012020224 + }, + { + "epoch": 1.29, + "learning_rate": 7.075626880641926e-05, + "loss": 1.2252, + "theoretical_loss": 3.348204742114, + "tokens_seen": 3012085760 + }, + { + "epoch": 1.29, + "learning_rate": 7.074022066198596e-05, + "loss": 1.1428, + "theoretical_loss": 3.3481991932199664, + "tokens_seen": 3012151296 + }, + { + "epoch": 1.29, + "learning_rate": 7.072417251755266e-05, + "loss": 1.1925, + "theoretical_loss": 3.3481936444804634, + "tokens_seen": 3012216832 + }, + { + "epoch": 1.29, + "learning_rate": 7.070812437311936e-05, + "loss": 1.2262, + "theoretical_loss": 3.348188095895483, + "tokens_seen": 3012282368 + }, + { + "epoch": 1.29, + "learning_rate": 7.069207622868606e-05, + "loss": 1.1196, + "theoretical_loss": 3.348182547465017, + "tokens_seen": 3012347904 + }, + { + "epoch": 1.29, + "learning_rate": 7.067602808425276e-05, + "loss": 1.0982, + "theoretical_loss": 3.3481769991890586, + "tokens_seen": 3012413440 + }, + { + "epoch": 1.29, + "learning_rate": 7.065997993981946e-05, + "loss": 1.16, + "theoretical_loss": 3.3481714510676, + "tokens_seen": 3012478976 + }, + { + "epoch": 1.29, + "learning_rate": 7.064393179538616e-05, + "loss": 1.2261, + "theoretical_loss": 3.3481659031006332, + "tokens_seen": 3012544512 + }, + { + "epoch": 1.29, + "learning_rate": 7.062788365095287e-05, + "loss": 1.2202, + "theoretical_loss": 3.3481603552881505, + "tokens_seen": 3012610048 + }, + { + "epoch": 1.29, + "learning_rate": 7.061183550651957e-05, + "loss": 1.1733, + "theoretical_loss": 3.348154807630144, + "tokens_seen": 3012675584 + }, + { + "epoch": 1.29, + "learning_rate": 7.059578736208627e-05, + "loss": 1.1204, + "theoretical_loss": 3.348149260126607, + "tokens_seen": 3012741120 + }, + { + "epoch": 1.29, + "learning_rate": 7.057973921765297e-05, + "loss": 1.1699, + "theoretical_loss": 3.3481437127775315, + "tokens_seen": 3012806656 + }, + { + "epoch": 1.29, + "learning_rate": 7.056369107321967e-05, + "loss": 1.1499, + "theoretical_loss": 3.348138165582909, + "tokens_seen": 3012872192 + }, + { + "epoch": 1.29, + "learning_rate": 7.054764292878636e-05, + "loss": 1.1664, + "theoretical_loss": 3.3481326185427327, + "tokens_seen": 3012937728 + }, + { + "epoch": 1.29, + "learning_rate": 7.053159478435306e-05, + "loss": 1.1818, + "theoretical_loss": 3.348127071656995, + "tokens_seen": 3013003264 + }, + { + "epoch": 1.29, + "learning_rate": 7.051554663991976e-05, + "loss": 1.2524, + "theoretical_loss": 3.3481215249256877, + "tokens_seen": 3013068800 + }, + { + "epoch": 1.29, + "learning_rate": 7.049949849548646e-05, + "loss": 1.2734, + "theoretical_loss": 3.3481159783488033, + "tokens_seen": 3013134336 + }, + { + "epoch": 1.29, + "learning_rate": 7.048345035105316e-05, + "loss": 1.2477, + "theoretical_loss": 3.348110431926334, + "tokens_seen": 3013199872 + }, + { + "epoch": 1.29, + "learning_rate": 7.046740220661986e-05, + "loss": 1.1046, + "theoretical_loss": 3.348104885658273, + "tokens_seen": 3013265408 + }, + { + "epoch": 1.29, + "learning_rate": 7.045135406218656e-05, + "loss": 1.318, + "theoretical_loss": 3.3480993395446115, + "tokens_seen": 3013330944 + }, + { + "epoch": 1.29, + "learning_rate": 7.043530591775326e-05, + "loss": 1.15, + "theoretical_loss": 3.3480937935853428, + "tokens_seen": 3013396480 + }, + { + "epoch": 1.29, + "learning_rate": 7.041925777331997e-05, + "loss": 1.1635, + "theoretical_loss": 3.348088247780458, + "tokens_seen": 3013462016 + }, + { + "epoch": 1.29, + "learning_rate": 7.040320962888667e-05, + "loss": 1.1917, + "theoretical_loss": 3.3480827021299513, + "tokens_seen": 3013527552 + }, + { + "epoch": 1.29, + "learning_rate": 7.038716148445336e-05, + "loss": 1.223, + "theoretical_loss": 3.3480771566338134, + "tokens_seen": 3013593088 + }, + { + "epoch": 1.29, + "learning_rate": 7.037111334002006e-05, + "loss": 1.1139, + "theoretical_loss": 3.348071611292037, + "tokens_seen": 3013658624 + }, + { + "epoch": 1.29, + "learning_rate": 7.035506519558676e-05, + "loss": 1.2542, + "theoretical_loss": 3.3480660661046153, + "tokens_seen": 3013724160 + }, + { + "epoch": 1.29, + "learning_rate": 7.033901705115346e-05, + "loss": 1.3262, + "theoretical_loss": 3.3480605210715395, + "tokens_seen": 3013789696 + }, + { + "epoch": 1.29, + "learning_rate": 7.032296890672016e-05, + "loss": 1.2702, + "theoretical_loss": 3.3480549761928025, + "tokens_seen": 3013855232 + }, + { + "epoch": 1.29, + "learning_rate": 7.030692076228686e-05, + "loss": 1.2095, + "theoretical_loss": 3.348049431468397, + "tokens_seen": 3013920768 + }, + { + "epoch": 1.29, + "learning_rate": 7.029087261785357e-05, + "loss": 1.2009, + "theoretical_loss": 3.348043886898315, + "tokens_seen": 3013986304 + }, + { + "epoch": 1.29, + "learning_rate": 7.027482447342027e-05, + "loss": 1.2233, + "theoretical_loss": 3.3480383424825484, + "tokens_seen": 3014051840 + }, + { + "epoch": 1.29, + "learning_rate": 7.025877632898697e-05, + "loss": 1.1679, + "theoretical_loss": 3.34803279822109, + "tokens_seen": 3014117376 + }, + { + "epoch": 1.29, + "learning_rate": 7.024272818455367e-05, + "loss": 1.1678, + "theoretical_loss": 3.3480272541139326, + "tokens_seen": 3014182912 + }, + { + "epoch": 1.29, + "learning_rate": 7.022668004012037e-05, + "loss": 1.1981, + "theoretical_loss": 3.348021710161068, + "tokens_seen": 3014248448 + }, + { + "epoch": 1.29, + "learning_rate": 7.021063189568707e-05, + "loss": 1.1978, + "theoretical_loss": 3.348016166362488, + "tokens_seen": 3014313984 + }, + { + "epoch": 1.29, + "learning_rate": 7.019458375125376e-05, + "loss": 1.2042, + "theoretical_loss": 3.348010622718186, + "tokens_seen": 3014379520 + }, + { + "epoch": 1.29, + "learning_rate": 7.017853560682046e-05, + "loss": 1.1131, + "theoretical_loss": 3.348005079228154, + "tokens_seen": 3014445056 + }, + { + "epoch": 1.29, + "learning_rate": 7.016248746238716e-05, + "loss": 1.1666, + "theoretical_loss": 3.347999535892384, + "tokens_seen": 3014510592 + }, + { + "epoch": 1.29, + "learning_rate": 7.014643931795386e-05, + "loss": 1.1664, + "theoretical_loss": 3.347993992710869, + "tokens_seen": 3014576128 + }, + { + "epoch": 1.29, + "objective/train/docs_used": 1112428, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.205124020576477, + "objective/train/theoretical_loss": 3.3479884496836005, + "objective/train/tokens_used": 3035101664, + "theoretical_loss": 3.3479884496836005, + "tokens_seen": 3014641664 + }, + { + "epoch": 1.29, + "learning_rate": 7.013039117352056e-05, + "loss": 1.2051, + "theoretical_loss": 3.3479884496836005, + "tokens_seen": 3014641664 + }, + { + "epoch": 1.29, + "learning_rate": 7.011434302908727e-05, + "loss": 1.143, + "theoretical_loss": 3.347982906810572, + "tokens_seen": 3014707200 + }, + { + "epoch": 1.29, + "learning_rate": 7.009829488465397e-05, + "loss": 1.1816, + "theoretical_loss": 3.3479773640917747, + "tokens_seen": 3014772736 + }, + { + "epoch": 1.29, + "learning_rate": 7.008224674022067e-05, + "loss": 1.1029, + "theoretical_loss": 3.3479718215272016, + "tokens_seen": 3014838272 + }, + { + "epoch": 1.29, + "learning_rate": 7.006619859578737e-05, + "loss": 1.1901, + "theoretical_loss": 3.347966279116845, + "tokens_seen": 3014903808 + }, + { + "epoch": 1.29, + "learning_rate": 7.005015045135407e-05, + "loss": 1.1097, + "theoretical_loss": 3.3479607368606974, + "tokens_seen": 3014969344 + }, + { + "epoch": 1.29, + "learning_rate": 7.003410230692076e-05, + "loss": 1.1709, + "theoretical_loss": 3.3479551947587503, + "tokens_seen": 3015034880 + }, + { + "epoch": 1.29, + "learning_rate": 7.001805416248746e-05, + "loss": 1.1235, + "theoretical_loss": 3.347949652810997, + "tokens_seen": 3015100416 + }, + { + "epoch": 1.29, + "learning_rate": 7.000200601805416e-05, + "loss": 1.2256, + "theoretical_loss": 3.34794411101743, + "tokens_seen": 3015165952 + }, + { + "epoch": 1.29, + "learning_rate": 6.998595787362086e-05, + "loss": 1.1685, + "theoretical_loss": 3.3479385693780404, + "tokens_seen": 3015231488 + }, + { + "epoch": 1.29, + "learning_rate": 6.996990972918756e-05, + "loss": 1.1213, + "theoretical_loss": 3.3479330278928217, + "tokens_seen": 3015297024 + }, + { + "epoch": 1.29, + "learning_rate": 6.995386158475426e-05, + "loss": 1.1507, + "theoretical_loss": 3.3479274865617663, + "tokens_seen": 3015362560 + }, + { + "epoch": 1.29, + "learning_rate": 6.993781344032097e-05, + "loss": 1.2154, + "theoretical_loss": 3.347921945384866, + "tokens_seen": 3015428096 + }, + { + "epoch": 1.29, + "learning_rate": 6.992176529588767e-05, + "loss": 1.1524, + "theoretical_loss": 3.347916404362113, + "tokens_seen": 3015493632 + }, + { + "epoch": 1.29, + "learning_rate": 6.990571715145437e-05, + "loss": 1.151, + "theoretical_loss": 3.3479108634935, + "tokens_seen": 3015559168 + }, + { + "epoch": 1.29, + "learning_rate": 6.988966900702107e-05, + "loss": 1.2767, + "theoretical_loss": 3.3479053227790194, + "tokens_seen": 3015624704 + }, + { + "epoch": 1.29, + "learning_rate": 6.987362086258777e-05, + "loss": 1.0939, + "theoretical_loss": 3.347899782218664, + "tokens_seen": 3015690240 + }, + { + "epoch": 1.29, + "learning_rate": 6.985757271815447e-05, + "loss": 1.2246, + "theoretical_loss": 3.3478942418124253, + "tokens_seen": 3015755776 + }, + { + "epoch": 1.29, + "learning_rate": 6.984152457372116e-05, + "loss": 1.2575, + "theoretical_loss": 3.3478887015602963, + "tokens_seen": 3015821312 + }, + { + "epoch": 1.29, + "learning_rate": 6.982547642928786e-05, + "loss": 1.0793, + "theoretical_loss": 3.347883161462269, + "tokens_seen": 3015886848 + }, + { + "epoch": 1.29, + "learning_rate": 6.980942828485457e-05, + "loss": 1.2143, + "theoretical_loss": 3.347877621518336, + "tokens_seen": 3015952384 + }, + { + "epoch": 1.29, + "learning_rate": 6.979338014042127e-05, + "loss": 1.2736, + "theoretical_loss": 3.3478720817284895, + "tokens_seen": 3016017920 + }, + { + "epoch": 1.29, + "learning_rate": 6.977733199598797e-05, + "loss": 1.236, + "theoretical_loss": 3.3478665420927216, + "tokens_seen": 3016083456 + }, + { + "epoch": 1.29, + "learning_rate": 6.976128385155467e-05, + "loss": 1.0785, + "theoretical_loss": 3.347861002611025, + "tokens_seen": 3016148992 + }, + { + "epoch": 1.29, + "learning_rate": 6.974523570712137e-05, + "loss": 1.2035, + "theoretical_loss": 3.3478554632833926, + "tokens_seen": 3016214528 + }, + { + "epoch": 1.29, + "learning_rate": 6.972918756268807e-05, + "loss": 1.1493, + "theoretical_loss": 3.347849924109816, + "tokens_seen": 3016280064 + }, + { + "epoch": 1.29, + "learning_rate": 6.971313941825477e-05, + "loss": 1.1225, + "theoretical_loss": 3.3478443850902875, + "tokens_seen": 3016345600 + }, + { + "epoch": 1.29, + "learning_rate": 6.969709127382148e-05, + "loss": 1.1641, + "theoretical_loss": 3.3478388462248, + "tokens_seen": 3016411136 + }, + { + "epoch": 1.29, + "learning_rate": 6.968104312938816e-05, + "loss": 1.2073, + "theoretical_loss": 3.3478333075133455, + "tokens_seen": 3016476672 + }, + { + "epoch": 1.29, + "learning_rate": 6.966499498495486e-05, + "loss": 1.1726, + "theoretical_loss": 3.347827768955917, + "tokens_seen": 3016542208 + }, + { + "epoch": 1.29, + "learning_rate": 6.964894684052156e-05, + "loss": 1.1927, + "theoretical_loss": 3.347822230552506, + "tokens_seen": 3016607744 + }, + { + "epoch": 1.29, + "learning_rate": 6.963289869608826e-05, + "loss": 1.2666, + "theoretical_loss": 3.347816692303105, + "tokens_seen": 3016673280 + }, + { + "epoch": 1.29, + "learning_rate": 6.961685055165496e-05, + "loss": 1.191, + "theoretical_loss": 3.3478111542077067, + "tokens_seen": 3016738816 + }, + { + "epoch": 1.29, + "learning_rate": 6.960080240722167e-05, + "loss": 1.2042, + "theoretical_loss": 3.347805616266304, + "tokens_seen": 3016804352 + }, + { + "epoch": 1.29, + "learning_rate": 6.958475426278837e-05, + "loss": 1.1997, + "theoretical_loss": 3.3478000784788877, + "tokens_seen": 3016869888 + }, + { + "epoch": 1.29, + "learning_rate": 6.956870611835507e-05, + "loss": 1.1832, + "theoretical_loss": 3.347794540845452, + "tokens_seen": 3016935424 + }, + { + "epoch": 1.29, + "learning_rate": 6.955265797392177e-05, + "loss": 1.1809, + "theoretical_loss": 3.347789003365988, + "tokens_seen": 3017000960 + }, + { + "epoch": 1.29, + "learning_rate": 6.953660982948847e-05, + "loss": 1.2197, + "theoretical_loss": 3.3477834660404886, + "tokens_seen": 3017066496 + }, + { + "epoch": 1.29, + "learning_rate": 6.952056168505517e-05, + "loss": 1.1869, + "theoretical_loss": 3.3477779288689455, + "tokens_seen": 3017132032 + }, + { + "epoch": 1.29, + "learning_rate": 6.950451354062187e-05, + "loss": 1.1848, + "theoretical_loss": 3.3477723918513522, + "tokens_seen": 3017197568 + }, + { + "epoch": 1.29, + "learning_rate": 6.948846539618857e-05, + "loss": 1.2364, + "theoretical_loss": 3.3477668549877, + "tokens_seen": 3017263104 + }, + { + "epoch": 1.3, + "learning_rate": 6.947241725175527e-05, + "loss": 1.1041, + "theoretical_loss": 3.347761318277983, + "tokens_seen": 3017328640 + }, + { + "epoch": 1.3, + "learning_rate": 6.945636910732197e-05, + "loss": 1.1739, + "theoretical_loss": 3.3477557817221912, + "tokens_seen": 3017394176 + }, + { + "epoch": 1.3, + "learning_rate": 6.944032096288867e-05, + "loss": 1.1968, + "theoretical_loss": 3.3477502453203183, + "tokens_seen": 3017459712 + }, + { + "epoch": 1.3, + "learning_rate": 6.942427281845537e-05, + "loss": 1.2956, + "theoretical_loss": 3.347744709072357, + "tokens_seen": 3017525248 + }, + { + "epoch": 1.3, + "learning_rate": 6.940822467402207e-05, + "loss": 1.1241, + "theoretical_loss": 3.347739172978299, + "tokens_seen": 3017590784 + }, + { + "epoch": 1.3, + "learning_rate": 6.939217652958877e-05, + "loss": 1.1317, + "theoretical_loss": 3.3477336370381368, + "tokens_seen": 3017656320 + }, + { + "epoch": 1.3, + "learning_rate": 6.937612838515547e-05, + "loss": 1.1928, + "theoretical_loss": 3.347728101251863, + "tokens_seen": 3017721856 + }, + { + "epoch": 1.3, + "learning_rate": 6.936008024072217e-05, + "loss": 1.2104, + "theoretical_loss": 3.3477225656194696, + "tokens_seen": 3017787392 + }, + { + "epoch": 1.3, + "learning_rate": 6.934403209628888e-05, + "loss": 1.1824, + "theoretical_loss": 3.347717030140949, + "tokens_seen": 3017852928 + }, + { + "epoch": 1.3, + "learning_rate": 6.932798395185556e-05, + "loss": 1.2202, + "theoretical_loss": 3.347711494816294, + "tokens_seen": 3017918464 + }, + { + "epoch": 1.3, + "learning_rate": 6.931193580742226e-05, + "loss": 1.219, + "theoretical_loss": 3.3477059596454968, + "tokens_seen": 3017984000 + }, + { + "epoch": 1.3, + "learning_rate": 6.929588766298896e-05, + "loss": 1.243, + "theoretical_loss": 3.3477004246285498, + "tokens_seen": 3018049536 + }, + { + "epoch": 1.3, + "learning_rate": 6.927983951855566e-05, + "loss": 1.2693, + "theoretical_loss": 3.3476948897654455, + "tokens_seen": 3018115072 + }, + { + "epoch": 1.3, + "learning_rate": 6.926379137412237e-05, + "loss": 1.1987, + "theoretical_loss": 3.347689355056176, + "tokens_seen": 3018180608 + }, + { + "epoch": 1.3, + "learning_rate": 6.924774322968907e-05, + "loss": 1.1581, + "theoretical_loss": 3.347683820500734, + "tokens_seen": 3018246144 + }, + { + "epoch": 1.3, + "learning_rate": 6.923169508525577e-05, + "loss": 1.2339, + "theoretical_loss": 3.3476782860991117, + "tokens_seen": 3018311680 + }, + { + "epoch": 1.3, + "learning_rate": 6.921564694082248e-05, + "loss": 1.1644, + "theoretical_loss": 3.347672751851301, + "tokens_seen": 3018377216 + }, + { + "epoch": 1.3, + "learning_rate": 6.919959879638918e-05, + "loss": 1.2753, + "theoretical_loss": 3.3476672177572953, + "tokens_seen": 3018442752 + }, + { + "epoch": 1.3, + "learning_rate": 6.918355065195588e-05, + "loss": 1.144, + "theoretical_loss": 3.347661683817086, + "tokens_seen": 3018508288 + }, + { + "epoch": 1.3, + "learning_rate": 6.916750250752257e-05, + "loss": 1.2397, + "theoretical_loss": 3.3476561500306663, + "tokens_seen": 3018573824 + }, + { + "epoch": 1.3, + "learning_rate": 6.915145436308927e-05, + "loss": 1.1802, + "theoretical_loss": 3.347650616398028, + "tokens_seen": 3018639360 + }, + { + "epoch": 1.3, + "learning_rate": 6.913540621865597e-05, + "loss": 1.2574, + "theoretical_loss": 3.347645082919164, + "tokens_seen": 3018704896 + }, + { + "epoch": 1.3, + "learning_rate": 6.911935807422267e-05, + "loss": 1.2799, + "theoretical_loss": 3.347639549594066, + "tokens_seen": 3018770432 + }, + { + "epoch": 1.3, + "learning_rate": 6.910330992978937e-05, + "loss": 1.1281, + "theoretical_loss": 3.347634016422727, + "tokens_seen": 3018835968 + }, + { + "epoch": 1.3, + "learning_rate": 6.908726178535607e-05, + "loss": 1.1117, + "theoretical_loss": 3.3476284834051393, + "tokens_seen": 3018901504 + }, + { + "epoch": 1.3, + "learning_rate": 6.907121364092277e-05, + "loss": 1.0968, + "theoretical_loss": 3.347622950541295, + "tokens_seen": 3018967040 + }, + { + "epoch": 1.3, + "learning_rate": 6.905516549648947e-05, + "loss": 1.1564, + "theoretical_loss": 3.3476174178311866, + "tokens_seen": 3019032576 + }, + { + "epoch": 1.3, + "learning_rate": 6.903911735205617e-05, + "loss": 1.1859, + "theoretical_loss": 3.347611885274807, + "tokens_seen": 3019098112 + }, + { + "epoch": 1.3, + "learning_rate": 6.902306920762287e-05, + "loss": 1.2929, + "theoretical_loss": 3.3476063528721474, + "tokens_seen": 3019163648 + }, + { + "epoch": 1.3, + "learning_rate": 6.900702106318958e-05, + "loss": 1.1641, + "theoretical_loss": 3.3476008206232013, + "tokens_seen": 3019229184 + }, + { + "epoch": 1.3, + "learning_rate": 6.899097291875628e-05, + "loss": 1.13, + "theoretical_loss": 3.347595288527961, + "tokens_seen": 3019294720 + }, + { + "epoch": 1.3, + "learning_rate": 6.897492477432296e-05, + "loss": 1.2256, + "theoretical_loss": 3.3475897565864186, + "tokens_seen": 3019360256 + }, + { + "epoch": 1.3, + "learning_rate": 6.895887662988966e-05, + "loss": 1.1313, + "theoretical_loss": 3.3475842247985663, + "tokens_seen": 3019425792 + }, + { + "epoch": 1.3, + "learning_rate": 6.894282848545636e-05, + "loss": 1.2101, + "theoretical_loss": 3.3475786931643965, + "tokens_seen": 3019491328 + }, + { + "epoch": 1.3, + "learning_rate": 6.892678034102306e-05, + "loss": 1.2111, + "theoretical_loss": 3.3475731616839024, + "tokens_seen": 3019556864 + }, + { + "epoch": 1.3, + "learning_rate": 6.891073219658978e-05, + "loss": 1.153, + "theoretical_loss": 3.347567630357075, + "tokens_seen": 3019622400 + }, + { + "epoch": 1.3, + "learning_rate": 6.889468405215648e-05, + "loss": 1.1888, + "theoretical_loss": 3.347562099183908, + "tokens_seen": 3019687936 + }, + { + "epoch": 1.3, + "learning_rate": 6.887863590772318e-05, + "loss": 1.162, + "theoretical_loss": 3.3475565681643937, + "tokens_seen": 3019753472 + }, + { + "epoch": 1.3, + "learning_rate": 6.886258776328988e-05, + "loss": 1.2023, + "theoretical_loss": 3.3475510372985235, + "tokens_seen": 3019819008 + }, + { + "epoch": 1.3, + "learning_rate": 6.884653961885658e-05, + "loss": 1.2007, + "theoretical_loss": 3.3475455065862905, + "tokens_seen": 3019884544 + }, + { + "epoch": 1.3, + "learning_rate": 6.883049147442327e-05, + "loss": 1.2072, + "theoretical_loss": 3.3475399760276874, + "tokens_seen": 3019950080 + }, + { + "epoch": 1.3, + "learning_rate": 6.881444332998997e-05, + "loss": 1.2239, + "theoretical_loss": 3.3475344456227054, + "tokens_seen": 3020015616 + }, + { + "epoch": 1.3, + "learning_rate": 6.879839518555667e-05, + "loss": 1.1996, + "theoretical_loss": 3.3475289153713383, + "tokens_seen": 3020081152 + }, + { + "epoch": 1.3, + "learning_rate": 6.878234704112337e-05, + "loss": 1.1685, + "theoretical_loss": 3.3475233852735777, + "tokens_seen": 3020146688 + }, + { + "epoch": 1.3, + "learning_rate": 6.876629889669007e-05, + "loss": 1.2181, + "theoretical_loss": 3.3475178553294165, + "tokens_seen": 3020212224 + }, + { + "epoch": 1.3, + "learning_rate": 6.875025075225677e-05, + "loss": 1.1573, + "theoretical_loss": 3.3475123255388466, + "tokens_seen": 3020277760 + }, + { + "epoch": 1.3, + "learning_rate": 6.873420260782347e-05, + "loss": 1.1647, + "theoretical_loss": 3.34750679590186, + "tokens_seen": 3020343296 + }, + { + "epoch": 1.3, + "learning_rate": 6.871815446339017e-05, + "loss": 1.2367, + "theoretical_loss": 3.3475012664184502, + "tokens_seen": 3020408832 + }, + { + "epoch": 1.3, + "learning_rate": 6.870210631895687e-05, + "loss": 1.0765, + "theoretical_loss": 3.347495737088609, + "tokens_seen": 3020474368 + }, + { + "epoch": 1.3, + "learning_rate": 6.868605817452357e-05, + "loss": 1.1802, + "theoretical_loss": 3.347490207912329, + "tokens_seen": 3020539904 + }, + { + "epoch": 1.3, + "learning_rate": 6.867001003009028e-05, + "loss": 1.1536, + "theoretical_loss": 3.347484678889603, + "tokens_seen": 3020605440 + }, + { + "epoch": 1.3, + "learning_rate": 6.865396188565698e-05, + "loss": 1.3099, + "theoretical_loss": 3.347479150020422, + "tokens_seen": 3020670976 + }, + { + "epoch": 1.3, + "learning_rate": 6.863791374122366e-05, + "loss": 1.2077, + "theoretical_loss": 3.34747362130478, + "tokens_seen": 3020736512 + }, + { + "epoch": 1.3, + "learning_rate": 6.862186559679036e-05, + "loss": 1.1166, + "theoretical_loss": 3.3474680927426683, + "tokens_seen": 3020802048 + }, + { + "epoch": 1.3, + "learning_rate": 6.860581745235708e-05, + "loss": 1.1324, + "theoretical_loss": 3.34746256433408, + "tokens_seen": 3020867584 + }, + { + "epoch": 1.3, + "learning_rate": 6.858976930792378e-05, + "loss": 1.1853, + "theoretical_loss": 3.347457036079007, + "tokens_seen": 3020933120 + }, + { + "epoch": 1.3, + "learning_rate": 6.857372116349048e-05, + "loss": 1.2266, + "theoretical_loss": 3.347451507977442, + "tokens_seen": 3020998656 + }, + { + "epoch": 1.3, + "learning_rate": 6.855767301905718e-05, + "loss": 1.197, + "theoretical_loss": 3.347445980029377, + "tokens_seen": 3021064192 + }, + { + "epoch": 1.3, + "learning_rate": 6.854162487462388e-05, + "loss": 1.3027, + "theoretical_loss": 3.3474404522348054, + "tokens_seen": 3021129728 + }, + { + "epoch": 1.3, + "objective/train/docs_used": 1114759, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1078402996063232, + "objective/train/theoretical_loss": 3.3474349245937187, + "objective/train/tokens_used": 3041655264, + "theoretical_loss": 3.3474349245937187, + "tokens_seen": 3021195264 + }, + { + "epoch": 1.3, + "learning_rate": 6.852557673019058e-05, + "loss": 1.1078, + "theoretical_loss": 3.3474349245937187, + "tokens_seen": 3021195264 + }, + { + "epoch": 1.3, + "learning_rate": 6.850952858575728e-05, + "loss": 1.0517, + "theoretical_loss": 3.3474293971061098, + "tokens_seen": 3021260800 + }, + { + "epoch": 1.3, + "learning_rate": 6.849348044132398e-05, + "loss": 1.1746, + "theoretical_loss": 3.3474238697719705, + "tokens_seen": 3021326336 + }, + { + "epoch": 1.3, + "learning_rate": 6.847743229689067e-05, + "loss": 1.159, + "theoretical_loss": 3.3474183425912933, + "tokens_seen": 3021391872 + }, + { + "epoch": 1.3, + "learning_rate": 6.846138415245737e-05, + "loss": 1.1571, + "theoretical_loss": 3.3474128155640717, + "tokens_seen": 3021457408 + }, + { + "epoch": 1.3, + "learning_rate": 6.844533600802407e-05, + "loss": 1.1703, + "theoretical_loss": 3.347407288690297, + "tokens_seen": 3021522944 + }, + { + "epoch": 1.3, + "learning_rate": 6.842928786359077e-05, + "loss": 1.1841, + "theoretical_loss": 3.347401761969962, + "tokens_seen": 3021588480 + }, + { + "epoch": 1.3, + "learning_rate": 6.841323971915747e-05, + "loss": 1.2126, + "theoretical_loss": 3.3473962354030586, + "tokens_seen": 3021654016 + }, + { + "epoch": 1.3, + "learning_rate": 6.839719157472417e-05, + "loss": 1.1363, + "theoretical_loss": 3.34739070898958, + "tokens_seen": 3021719552 + }, + { + "epoch": 1.3, + "learning_rate": 6.838114343029087e-05, + "loss": 1.1826, + "theoretical_loss": 3.347385182729518, + "tokens_seen": 3021785088 + }, + { + "epoch": 1.3, + "learning_rate": 6.836509528585757e-05, + "loss": 1.1864, + "theoretical_loss": 3.3473796566228655, + "tokens_seen": 3021850624 + }, + { + "epoch": 1.3, + "learning_rate": 6.834904714142427e-05, + "loss": 1.1591, + "theoretical_loss": 3.3473741306696145, + "tokens_seen": 3021916160 + }, + { + "epoch": 1.3, + "learning_rate": 6.833299899699097e-05, + "loss": 1.3159, + "theoretical_loss": 3.3473686048697577, + "tokens_seen": 3021981696 + }, + { + "epoch": 1.3, + "learning_rate": 6.831695085255768e-05, + "loss": 1.2374, + "theoretical_loss": 3.3473630792232876, + "tokens_seen": 3022047232 + }, + { + "epoch": 1.3, + "learning_rate": 6.830090270812438e-05, + "loss": 1.1583, + "theoretical_loss": 3.3473575537301965, + "tokens_seen": 3022112768 + }, + { + "epoch": 1.3, + "learning_rate": 6.828485456369108e-05, + "loss": 1.3031, + "theoretical_loss": 3.3473520283904765, + "tokens_seen": 3022178304 + }, + { + "epoch": 1.3, + "learning_rate": 6.826880641925778e-05, + "loss": 1.2015, + "theoretical_loss": 3.34734650320412, + "tokens_seen": 3022243840 + }, + { + "epoch": 1.3, + "learning_rate": 6.825275827482448e-05, + "loss": 1.1838, + "theoretical_loss": 3.34734097817112, + "tokens_seen": 3022309376 + }, + { + "epoch": 1.3, + "learning_rate": 6.823671013039118e-05, + "loss": 1.2444, + "theoretical_loss": 3.347335453291469, + "tokens_seen": 3022374912 + }, + { + "epoch": 1.3, + "learning_rate": 6.822066198595788e-05, + "loss": 1.1607, + "theoretical_loss": 3.3473299285651588, + "tokens_seen": 3022440448 + }, + { + "epoch": 1.3, + "learning_rate": 6.820461384152458e-05, + "loss": 1.1236, + "theoretical_loss": 3.3473244039921815, + "tokens_seen": 3022505984 + }, + { + "epoch": 1.3, + "learning_rate": 6.818856569709128e-05, + "loss": 1.1592, + "theoretical_loss": 3.347318879572531, + "tokens_seen": 3022571520 + }, + { + "epoch": 1.3, + "learning_rate": 6.817251755265798e-05, + "loss": 1.2018, + "theoretical_loss": 3.347313355306198, + "tokens_seen": 3022637056 + }, + { + "epoch": 1.3, + "learning_rate": 6.815646940822468e-05, + "loss": 1.1825, + "theoretical_loss": 3.347307831193176, + "tokens_seen": 3022702592 + }, + { + "epoch": 1.3, + "learning_rate": 6.814042126379138e-05, + "loss": 1.126, + "theoretical_loss": 3.347302307233457, + "tokens_seen": 3022768128 + }, + { + "epoch": 1.3, + "learning_rate": 6.812437311935807e-05, + "loss": 1.171, + "theoretical_loss": 3.3472967834270335, + "tokens_seen": 3022833664 + }, + { + "epoch": 1.3, + "learning_rate": 6.810832497492477e-05, + "loss": 1.1577, + "theoretical_loss": 3.347291259773898, + "tokens_seen": 3022899200 + }, + { + "epoch": 1.3, + "learning_rate": 6.809227683049147e-05, + "loss": 1.1685, + "theoretical_loss": 3.347285736274043, + "tokens_seen": 3022964736 + }, + { + "epoch": 1.3, + "learning_rate": 6.807622868605817e-05, + "loss": 1.1851, + "theoretical_loss": 3.347280212927461, + "tokens_seen": 3023030272 + }, + { + "epoch": 1.3, + "learning_rate": 6.806018054162487e-05, + "loss": 1.1409, + "theoretical_loss": 3.347274689734144, + "tokens_seen": 3023095808 + }, + { + "epoch": 1.3, + "learning_rate": 6.804413239719157e-05, + "loss": 1.2555, + "theoretical_loss": 3.3472691666940846, + "tokens_seen": 3023161344 + }, + { + "epoch": 1.3, + "learning_rate": 6.802808425275827e-05, + "loss": 1.2681, + "theoretical_loss": 3.3472636438072754, + "tokens_seen": 3023226880 + }, + { + "epoch": 1.3, + "learning_rate": 6.801203610832499e-05, + "loss": 1.2287, + "theoretical_loss": 3.3472581210737085, + "tokens_seen": 3023292416 + }, + { + "epoch": 1.3, + "learning_rate": 6.799598796389169e-05, + "loss": 1.2459, + "theoretical_loss": 3.3472525984933768, + "tokens_seen": 3023357952 + }, + { + "epoch": 1.3, + "learning_rate": 6.797993981945839e-05, + "loss": 1.2629, + "theoretical_loss": 3.3472470760662727, + "tokens_seen": 3023423488 + }, + { + "epoch": 1.3, + "learning_rate": 6.796389167502508e-05, + "loss": 1.2219, + "theoretical_loss": 3.3472415537923883, + "tokens_seen": 3023489024 + }, + { + "epoch": 1.3, + "learning_rate": 6.794784353059178e-05, + "loss": 1.339, + "theoretical_loss": 3.3472360316717156, + "tokens_seen": 3023554560 + }, + { + "epoch": 1.3, + "learning_rate": 6.793179538615848e-05, + "loss": 1.2687, + "theoretical_loss": 3.3472305097042483, + "tokens_seen": 3023620096 + }, + { + "epoch": 1.3, + "learning_rate": 6.791574724172518e-05, + "loss": 1.2136, + "theoretical_loss": 3.3472249878899776, + "tokens_seen": 3023685632 + }, + { + "epoch": 1.3, + "learning_rate": 6.789969909729188e-05, + "loss": 1.1431, + "theoretical_loss": 3.3472194662288963, + "tokens_seen": 3023751168 + }, + { + "epoch": 1.3, + "learning_rate": 6.788365095285858e-05, + "loss": 1.2034, + "theoretical_loss": 3.347213944720997, + "tokens_seen": 3023816704 + }, + { + "epoch": 1.3, + "learning_rate": 6.786760280842528e-05, + "loss": 1.1502, + "theoretical_loss": 3.3472084233662724, + "tokens_seen": 3023882240 + }, + { + "epoch": 1.3, + "learning_rate": 6.785155466399198e-05, + "loss": 1.17, + "theoretical_loss": 3.3472029021647143, + "tokens_seen": 3023947776 + }, + { + "epoch": 1.3, + "learning_rate": 6.783550651955868e-05, + "loss": 1.2252, + "theoretical_loss": 3.347197381116316, + "tokens_seen": 3024013312 + }, + { + "epoch": 1.3, + "learning_rate": 6.781945837512538e-05, + "loss": 1.2169, + "theoretical_loss": 3.3471918602210686, + "tokens_seen": 3024078848 + }, + { + "epoch": 1.3, + "learning_rate": 6.780341023069208e-05, + "loss": 1.1241, + "theoretical_loss": 3.3471863394789656, + "tokens_seen": 3024144384 + }, + { + "epoch": 1.3, + "learning_rate": 6.778736208625878e-05, + "loss": 1.1606, + "theoretical_loss": 3.347180818889999, + "tokens_seen": 3024209920 + }, + { + "epoch": 1.3, + "learning_rate": 6.777131394182547e-05, + "loss": 1.2512, + "theoretical_loss": 3.347175298454162, + "tokens_seen": 3024275456 + }, + { + "epoch": 1.3, + "learning_rate": 6.775526579739217e-05, + "loss": 1.1433, + "theoretical_loss": 3.3471697781714456, + "tokens_seen": 3024340992 + }, + { + "epoch": 1.3, + "learning_rate": 6.773921765295887e-05, + "loss": 1.1473, + "theoretical_loss": 3.3471642580418433, + "tokens_seen": 3024406528 + }, + { + "epoch": 1.3, + "learning_rate": 6.772316950852557e-05, + "loss": 1.1651, + "theoretical_loss": 3.3471587380653474, + "tokens_seen": 3024472064 + }, + { + "epoch": 1.3, + "learning_rate": 6.770712136409229e-05, + "loss": 1.1788, + "theoretical_loss": 3.3471532182419503, + "tokens_seen": 3024537600 + }, + { + "epoch": 1.3, + "learning_rate": 6.769107321965899e-05, + "loss": 1.1385, + "theoretical_loss": 3.347147698571644, + "tokens_seen": 3024603136 + }, + { + "epoch": 1.3, + "learning_rate": 6.767502507522569e-05, + "loss": 1.211, + "theoretical_loss": 3.347142179054422, + "tokens_seen": 3024668672 + }, + { + "epoch": 1.3, + "learning_rate": 6.765897693079239e-05, + "loss": 1.197, + "theoretical_loss": 3.347136659690275, + "tokens_seen": 3024734208 + }, + { + "epoch": 1.3, + "learning_rate": 6.764292878635909e-05, + "loss": 1.1678, + "theoretical_loss": 3.3471311404791972, + "tokens_seen": 3024799744 + }, + { + "epoch": 1.3, + "learning_rate": 6.762688064192579e-05, + "loss": 1.1242, + "theoretical_loss": 3.34712562142118, + "tokens_seen": 3024865280 + }, + { + "epoch": 1.3, + "learning_rate": 6.761083249749248e-05, + "loss": 1.1961, + "theoretical_loss": 3.3471201025162163, + "tokens_seen": 3024930816 + }, + { + "epoch": 1.3, + "learning_rate": 6.759478435305918e-05, + "loss": 1.1214, + "theoretical_loss": 3.347114583764298, + "tokens_seen": 3024996352 + }, + { + "epoch": 1.3, + "learning_rate": 6.757873620862588e-05, + "loss": 1.0749, + "theoretical_loss": 3.3471090651654185, + "tokens_seen": 3025061888 + }, + { + "epoch": 1.3, + "learning_rate": 6.756268806419258e-05, + "loss": 1.1958, + "theoretical_loss": 3.347103546719569, + "tokens_seen": 3025127424 + }, + { + "epoch": 1.3, + "learning_rate": 6.754663991975928e-05, + "loss": 1.1661, + "theoretical_loss": 3.347098028426743, + "tokens_seen": 3025192960 + }, + { + "epoch": 1.3, + "learning_rate": 6.753059177532598e-05, + "loss": 1.2232, + "theoretical_loss": 3.3470925102869327, + "tokens_seen": 3025258496 + }, + { + "epoch": 1.3, + "learning_rate": 6.751454363089268e-05, + "loss": 1.1611, + "theoretical_loss": 3.34708699230013, + "tokens_seen": 3025324032 + }, + { + "epoch": 1.3, + "learning_rate": 6.749849548645938e-05, + "loss": 1.1032, + "theoretical_loss": 3.3470814744663278, + "tokens_seen": 3025389568 + }, + { + "epoch": 1.3, + "learning_rate": 6.748244734202608e-05, + "loss": 1.0942, + "theoretical_loss": 3.3470759567855186, + "tokens_seen": 3025455104 + }, + { + "epoch": 1.3, + "learning_rate": 6.746639919759278e-05, + "loss": 1.1568, + "theoretical_loss": 3.3470704392576947, + "tokens_seen": 3025520640 + }, + { + "epoch": 1.3, + "learning_rate": 6.745035105315948e-05, + "loss": 1.1705, + "theoretical_loss": 3.3470649218828488, + "tokens_seen": 3025586176 + }, + { + "epoch": 1.3, + "learning_rate": 6.743430290872618e-05, + "loss": 1.2748, + "theoretical_loss": 3.3470594046609725, + "tokens_seen": 3025651712 + }, + { + "epoch": 1.3, + "learning_rate": 6.741825476429287e-05, + "loss": 1.1728, + "theoretical_loss": 3.3470538875920592, + "tokens_seen": 3025717248 + }, + { + "epoch": 1.3, + "learning_rate": 6.740220661985958e-05, + "loss": 1.1582, + "theoretical_loss": 3.347048370676101, + "tokens_seen": 3025782784 + }, + { + "epoch": 1.3, + "learning_rate": 6.738615847542629e-05, + "loss": 1.0977, + "theoretical_loss": 3.3470428539130905, + "tokens_seen": 3025848320 + }, + { + "epoch": 1.3, + "learning_rate": 6.737011033099299e-05, + "loss": 1.2577, + "theoretical_loss": 3.3470373373030196, + "tokens_seen": 3025913856 + }, + { + "epoch": 1.3, + "learning_rate": 6.735406218655969e-05, + "loss": 1.1358, + "theoretical_loss": 3.3470318208458814, + "tokens_seen": 3025979392 + }, + { + "epoch": 1.3, + "learning_rate": 6.733801404212639e-05, + "loss": 1.1557, + "theoretical_loss": 3.347026304541668, + "tokens_seen": 3026044928 + }, + { + "epoch": 1.3, + "learning_rate": 6.732196589769309e-05, + "loss": 1.1839, + "theoretical_loss": 3.347020788390372, + "tokens_seen": 3026110464 + }, + { + "epoch": 1.3, + "learning_rate": 6.730591775325979e-05, + "loss": 1.1762, + "theoretical_loss": 3.3470152723919857, + "tokens_seen": 3026176000 + }, + { + "epoch": 1.3, + "learning_rate": 6.728986960882649e-05, + "loss": 1.1942, + "theoretical_loss": 3.347009756546502, + "tokens_seen": 3026241536 + }, + { + "epoch": 1.3, + "learning_rate": 6.727382146439319e-05, + "loss": 1.1693, + "theoretical_loss": 3.3470042408539125, + "tokens_seen": 3026307072 + }, + { + "epoch": 1.3, + "learning_rate": 6.725777331995988e-05, + "loss": 1.1328, + "theoretical_loss": 3.34699872531421, + "tokens_seen": 3026372608 + }, + { + "epoch": 1.3, + "learning_rate": 6.724172517552658e-05, + "loss": 1.2611, + "theoretical_loss": 3.346993209927388, + "tokens_seen": 3026438144 + }, + { + "epoch": 1.3, + "learning_rate": 6.722567703109328e-05, + "loss": 1.2701, + "theoretical_loss": 3.346987694693437, + "tokens_seen": 3026503680 + }, + { + "epoch": 1.3, + "learning_rate": 6.720962888665998e-05, + "loss": 1.1132, + "theoretical_loss": 3.346982179612351, + "tokens_seen": 3026569216 + }, + { + "epoch": 1.3, + "learning_rate": 6.719358074222668e-05, + "loss": 1.1098, + "theoretical_loss": 3.3469766646841217, + "tokens_seen": 3026634752 + }, + { + "epoch": 1.3, + "learning_rate": 6.717753259779338e-05, + "loss": 1.0857, + "theoretical_loss": 3.3469711499087422, + "tokens_seen": 3026700288 + }, + { + "epoch": 1.3, + "learning_rate": 6.716148445336008e-05, + "loss": 1.1615, + "theoretical_loss": 3.346965635286204, + "tokens_seen": 3026765824 + }, + { + "epoch": 1.3, + "learning_rate": 6.714543630892678e-05, + "loss": 1.1986, + "theoretical_loss": 3.3469601208165005, + "tokens_seen": 3026831360 + }, + { + "epoch": 1.3, + "learning_rate": 6.712938816449348e-05, + "loss": 1.1937, + "theoretical_loss": 3.3469546064996236, + "tokens_seen": 3026896896 + }, + { + "epoch": 1.3, + "learning_rate": 6.711334002006018e-05, + "loss": 1.205, + "theoretical_loss": 3.3469490923355663, + "tokens_seen": 3026962432 + }, + { + "epoch": 1.3, + "learning_rate": 6.709729187562688e-05, + "loss": 1.2199, + "theoretical_loss": 3.3469435783243204, + "tokens_seen": 3027027968 + }, + { + "epoch": 1.3, + "learning_rate": 6.708124373119358e-05, + "loss": 1.1956, + "theoretical_loss": 3.3469380644658786, + "tokens_seen": 3027093504 + }, + { + "epoch": 1.3, + "learning_rate": 6.706519558676028e-05, + "loss": 1.208, + "theoretical_loss": 3.346932550760233, + "tokens_seen": 3027159040 + }, + { + "epoch": 1.3, + "learning_rate": 6.704914744232698e-05, + "loss": 1.1118, + "theoretical_loss": 3.346927037207377, + "tokens_seen": 3027224576 + }, + { + "epoch": 1.3, + "learning_rate": 6.703309929789369e-05, + "loss": 1.1874, + "theoretical_loss": 3.3469215238073025, + "tokens_seen": 3027290112 + }, + { + "epoch": 1.3, + "learning_rate": 6.701705115346039e-05, + "loss": 1.1269, + "theoretical_loss": 3.346916010560002, + "tokens_seen": 3027355648 + }, + { + "epoch": 1.3, + "learning_rate": 6.700100300902709e-05, + "loss": 1.2463, + "theoretical_loss": 3.3469104974654673, + "tokens_seen": 3027421184 + }, + { + "epoch": 1.3, + "learning_rate": 6.698495486459379e-05, + "loss": 1.1673, + "theoretical_loss": 3.3469049845236922, + "tokens_seen": 3027486720 + }, + { + "epoch": 1.3, + "learning_rate": 6.696890672016049e-05, + "loss": 1.2841, + "theoretical_loss": 3.346899471734668, + "tokens_seen": 3027552256 + }, + { + "epoch": 1.3, + "learning_rate": 6.695285857572719e-05, + "loss": 1.0959, + "theoretical_loss": 3.346893959098388, + "tokens_seen": 3027617792 + }, + { + "epoch": 1.3, + "learning_rate": 6.693681043129389e-05, + "loss": 1.1245, + "theoretical_loss": 3.3468884466148436, + "tokens_seen": 3027683328 + }, + { + "epoch": 1.3, + "objective/train/docs_used": 1117494, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1437715291976929, + "objective/train/theoretical_loss": 3.3468829342840287, + "objective/train/tokens_used": 3048208864, + "theoretical_loss": 3.3468829342840287, + "tokens_seen": 3027748864 + }, + { + "epoch": 1.3, + "learning_rate": 6.692076228686058e-05, + "loss": 1.1438, + "theoretical_loss": 3.3468829342840287, + "tokens_seen": 3027748864 + }, + { + "epoch": 1.3, + "learning_rate": 6.690471414242728e-05, + "loss": 1.167, + "theoretical_loss": 3.3468774221059348, + "tokens_seen": 3027814400 + }, + { + "epoch": 1.3, + "learning_rate": 6.688866599799398e-05, + "loss": 1.2299, + "theoretical_loss": 3.346871910080554, + "tokens_seen": 3027879936 + }, + { + "epoch": 1.3, + "learning_rate": 6.687261785356068e-05, + "loss": 1.1222, + "theoretical_loss": 3.34686639820788, + "tokens_seen": 3027945472 + }, + { + "epoch": 1.3, + "learning_rate": 6.685656970912738e-05, + "loss": 1.1901, + "theoretical_loss": 3.3468608864879044, + "tokens_seen": 3028011008 + }, + { + "epoch": 1.3, + "learning_rate": 6.684052156469408e-05, + "loss": 1.2737, + "theoretical_loss": 3.3468553749206196, + "tokens_seen": 3028076544 + }, + { + "epoch": 1.3, + "learning_rate": 6.682447342026078e-05, + "loss": 1.1926, + "theoretical_loss": 3.3468498635060184, + "tokens_seen": 3028142080 + }, + { + "epoch": 1.3, + "learning_rate": 6.68084252758275e-05, + "loss": 1.15, + "theoretical_loss": 3.3468443522440934, + "tokens_seen": 3028207616 + }, + { + "epoch": 1.3, + "learning_rate": 6.67923771313942e-05, + "loss": 1.2405, + "theoretical_loss": 3.346838841134837, + "tokens_seen": 3028273152 + }, + { + "epoch": 1.3, + "learning_rate": 6.67763289869609e-05, + "loss": 1.2276, + "theoretical_loss": 3.346833330178241, + "tokens_seen": 3028338688 + }, + { + "epoch": 1.3, + "learning_rate": 6.676028084252758e-05, + "loss": 1.1847, + "theoretical_loss": 3.3468278193742984, + "tokens_seen": 3028404224 + }, + { + "epoch": 1.3, + "learning_rate": 6.674423269809428e-05, + "loss": 1.1963, + "theoretical_loss": 3.3468223087230022, + "tokens_seen": 3028469760 + }, + { + "epoch": 1.3, + "learning_rate": 6.672818455366098e-05, + "loss": 1.1728, + "theoretical_loss": 3.346816798224344, + "tokens_seen": 3028535296 + }, + { + "epoch": 1.3, + "learning_rate": 6.671213640922768e-05, + "loss": 1.1213, + "theoretical_loss": 3.3468112878783165, + "tokens_seen": 3028600832 + }, + { + "epoch": 1.3, + "learning_rate": 6.669608826479439e-05, + "loss": 1.2054, + "theoretical_loss": 3.3468057776849123, + "tokens_seen": 3028666368 + }, + { + "epoch": 1.3, + "learning_rate": 6.668004012036109e-05, + "loss": 1.0774, + "theoretical_loss": 3.3468002676441237, + "tokens_seen": 3028731904 + }, + { + "epoch": 1.3, + "learning_rate": 6.666399197592779e-05, + "loss": 1.1333, + "theoretical_loss": 3.3467947577559434, + "tokens_seen": 3028797440 + }, + { + "epoch": 1.3, + "learning_rate": 6.664794383149449e-05, + "loss": 1.1618, + "theoretical_loss": 3.346789248020364, + "tokens_seen": 3028862976 + }, + { + "epoch": 1.3, + "learning_rate": 6.663189568706119e-05, + "loss": 1.2571, + "theoretical_loss": 3.3467837384373773, + "tokens_seen": 3028928512 + }, + { + "epoch": 1.3, + "learning_rate": 6.661584754262789e-05, + "loss": 1.1973, + "theoretical_loss": 3.3467782290069765, + "tokens_seen": 3028994048 + }, + { + "epoch": 1.3, + "learning_rate": 6.659979939819459e-05, + "loss": 1.265, + "theoretical_loss": 3.346772719729154, + "tokens_seen": 3029059584 + }, + { + "epoch": 1.3, + "learning_rate": 6.658375125376129e-05, + "loss": 1.0326, + "theoretical_loss": 3.3467672106039017, + "tokens_seen": 3029125120 + }, + { + "epoch": 1.3, + "learning_rate": 6.656770310932798e-05, + "loss": 1.1677, + "theoretical_loss": 3.3467617016312126, + "tokens_seen": 3029190656 + }, + { + "epoch": 1.3, + "learning_rate": 6.655165496489468e-05, + "loss": 1.2128, + "theoretical_loss": 3.346756192811079, + "tokens_seen": 3029256192 + }, + { + "epoch": 1.3, + "learning_rate": 6.653560682046138e-05, + "loss": 1.0938, + "theoretical_loss": 3.346750684143493, + "tokens_seen": 3029321728 + }, + { + "epoch": 1.3, + "learning_rate": 6.651955867602808e-05, + "loss": 1.0941, + "theoretical_loss": 3.346745175628448, + "tokens_seen": 3029387264 + }, + { + "epoch": 1.3, + "learning_rate": 6.650351053159479e-05, + "loss": 1.125, + "theoretical_loss": 3.346739667265936, + "tokens_seen": 3029452800 + }, + { + "epoch": 1.3, + "learning_rate": 6.64874623871615e-05, + "loss": 1.283, + "theoretical_loss": 3.346734159055949, + "tokens_seen": 3029518336 + }, + { + "epoch": 1.3, + "learning_rate": 6.64714142427282e-05, + "loss": 1.1734, + "theoretical_loss": 3.34672865099848, + "tokens_seen": 3029583872 + }, + { + "epoch": 1.3, + "learning_rate": 6.64553660982949e-05, + "loss": 1.2053, + "theoretical_loss": 3.346723143093522, + "tokens_seen": 3029649408 + }, + { + "epoch": 1.3, + "learning_rate": 6.64393179538616e-05, + "loss": 1.1687, + "theoretical_loss": 3.346717635341066, + "tokens_seen": 3029714944 + }, + { + "epoch": 1.3, + "learning_rate": 6.64232698094283e-05, + "loss": 1.2633, + "theoretical_loss": 3.346712127741106, + "tokens_seen": 3029780480 + }, + { + "epoch": 1.3, + "learning_rate": 6.640722166499498e-05, + "loss": 1.164, + "theoretical_loss": 3.3467066202936326, + "tokens_seen": 3029846016 + }, + { + "epoch": 1.3, + "learning_rate": 6.639117352056168e-05, + "loss": 1.1709, + "theoretical_loss": 3.3467011129986406, + "tokens_seen": 3029911552 + }, + { + "epoch": 1.3, + "learning_rate": 6.637512537612838e-05, + "loss": 1.113, + "theoretical_loss": 3.3466956058561212, + "tokens_seen": 3029977088 + }, + { + "epoch": 1.3, + "learning_rate": 6.635907723169509e-05, + "loss": 1.1714, + "theoretical_loss": 3.3466900988660666, + "tokens_seen": 3030042624 + }, + { + "epoch": 1.3, + "learning_rate": 6.634302908726179e-05, + "loss": 1.2213, + "theoretical_loss": 3.3466845920284705, + "tokens_seen": 3030108160 + }, + { + "epoch": 1.3, + "learning_rate": 6.632698094282849e-05, + "loss": 1.1146, + "theoretical_loss": 3.346679085343324, + "tokens_seen": 3030173696 + }, + { + "epoch": 1.3, + "learning_rate": 6.631093279839519e-05, + "loss": 1.3139, + "theoretical_loss": 3.34667357881062, + "tokens_seen": 3030239232 + }, + { + "epoch": 1.3, + "learning_rate": 6.629488465396189e-05, + "loss": 1.2609, + "theoretical_loss": 3.3466680724303517, + "tokens_seen": 3030304768 + }, + { + "epoch": 1.3, + "learning_rate": 6.627883650952859e-05, + "loss": 1.2135, + "theoretical_loss": 3.346662566202511, + "tokens_seen": 3030370304 + }, + { + "epoch": 1.3, + "learning_rate": 6.626278836509529e-05, + "loss": 1.1204, + "theoretical_loss": 3.3466570601270904, + "tokens_seen": 3030435840 + }, + { + "epoch": 1.3, + "learning_rate": 6.624674022066199e-05, + "loss": 1.1771, + "theoretical_loss": 3.3466515542040822, + "tokens_seen": 3030501376 + }, + { + "epoch": 1.3, + "learning_rate": 6.623069207622869e-05, + "loss": 1.1415, + "theoretical_loss": 3.3466460484334792, + "tokens_seen": 3030566912 + }, + { + "epoch": 1.3, + "learning_rate": 6.621464393179539e-05, + "loss": 1.107, + "theoretical_loss": 3.346640542815274, + "tokens_seen": 3030632448 + }, + { + "epoch": 1.3, + "learning_rate": 6.619859578736209e-05, + "loss": 1.2562, + "theoretical_loss": 3.3466350373494587, + "tokens_seen": 3030697984 + }, + { + "epoch": 1.3, + "learning_rate": 6.618254764292879e-05, + "loss": 1.2719, + "theoretical_loss": 3.346629532036026, + "tokens_seen": 3030763520 + }, + { + "epoch": 1.3, + "learning_rate": 6.616649949849549e-05, + "loss": 1.2618, + "theoretical_loss": 3.3466240268749683, + "tokens_seen": 3030829056 + }, + { + "epoch": 1.3, + "learning_rate": 6.61504513540622e-05, + "loss": 1.1746, + "theoretical_loss": 3.346618521866278, + "tokens_seen": 3030894592 + }, + { + "epoch": 1.3, + "learning_rate": 6.61344032096289e-05, + "loss": 1.1637, + "theoretical_loss": 3.346613017009948, + "tokens_seen": 3030960128 + }, + { + "epoch": 1.3, + "learning_rate": 6.61183550651956e-05, + "loss": 1.2808, + "theoretical_loss": 3.3466075123059706, + "tokens_seen": 3031025664 + }, + { + "epoch": 1.3, + "learning_rate": 6.61023069207623e-05, + "loss": 1.1714, + "theoretical_loss": 3.3466020077543384, + "tokens_seen": 3031091200 + }, + { + "epoch": 1.3, + "learning_rate": 6.6086258776329e-05, + "loss": 1.1714, + "theoretical_loss": 3.346596503355043, + "tokens_seen": 3031156736 + }, + { + "epoch": 1.3, + "learning_rate": 6.60702106318957e-05, + "loss": 1.3013, + "theoretical_loss": 3.346590999108078, + "tokens_seen": 3031222272 + }, + { + "epoch": 1.3, + "learning_rate": 6.605416248746238e-05, + "loss": 1.1304, + "theoretical_loss": 3.3465854950134357, + "tokens_seen": 3031287808 + }, + { + "epoch": 1.3, + "learning_rate": 6.603811434302908e-05, + "loss": 1.1662, + "theoretical_loss": 3.346579991071108, + "tokens_seen": 3031353344 + }, + { + "epoch": 1.3, + "learning_rate": 6.602206619859578e-05, + "loss": 1.2129, + "theoretical_loss": 3.346574487281088, + "tokens_seen": 3031418880 + }, + { + "epoch": 1.3, + "learning_rate": 6.600601805416249e-05, + "loss": 1.1501, + "theoretical_loss": 3.346568983643368, + "tokens_seen": 3031484416 + }, + { + "epoch": 1.3, + "learning_rate": 6.598996990972919e-05, + "loss": 1.2313, + "theoretical_loss": 3.3465634801579403, + "tokens_seen": 3031549952 + }, + { + "epoch": 1.3, + "learning_rate": 6.597392176529589e-05, + "loss": 1.081, + "theoretical_loss": 3.3465579768247977, + "tokens_seen": 3031615488 + }, + { + "epoch": 1.3, + "learning_rate": 6.595787362086259e-05, + "loss": 1.2316, + "theoretical_loss": 3.346552473643932, + "tokens_seen": 3031681024 + }, + { + "epoch": 1.3, + "learning_rate": 6.594182547642929e-05, + "loss": 1.2402, + "theoretical_loss": 3.346546970615337, + "tokens_seen": 3031746560 + }, + { + "epoch": 1.3, + "learning_rate": 6.592577733199599e-05, + "loss": 1.1802, + "theoretical_loss": 3.346541467739004, + "tokens_seen": 3031812096 + }, + { + "epoch": 1.3, + "learning_rate": 6.59097291875627e-05, + "loss": 1.173, + "theoretical_loss": 3.346535965014926, + "tokens_seen": 3031877632 + }, + { + "epoch": 1.3, + "learning_rate": 6.589368104312939e-05, + "loss": 1.2022, + "theoretical_loss": 3.346530462443096, + "tokens_seen": 3031943168 + }, + { + "epoch": 1.3, + "learning_rate": 6.587763289869609e-05, + "loss": 1.1521, + "theoretical_loss": 3.3465249600235047, + "tokens_seen": 3032008704 + }, + { + "epoch": 1.3, + "learning_rate": 6.586158475426279e-05, + "loss": 1.1804, + "theoretical_loss": 3.3465194577561466, + "tokens_seen": 3032074240 + }, + { + "epoch": 1.3, + "learning_rate": 6.584553660982949e-05, + "loss": 1.2507, + "theoretical_loss": 3.3465139556410133, + "tokens_seen": 3032139776 + }, + { + "epoch": 1.3, + "learning_rate": 6.582948846539619e-05, + "loss": 1.2244, + "theoretical_loss": 3.346508453678098, + "tokens_seen": 3032205312 + }, + { + "epoch": 1.3, + "learning_rate": 6.581344032096289e-05, + "loss": 1.1497, + "theoretical_loss": 3.3465029518673917, + "tokens_seen": 3032270848 + }, + { + "epoch": 1.3, + "learning_rate": 6.57973921765296e-05, + "loss": 1.1836, + "theoretical_loss": 3.3464974502088882, + "tokens_seen": 3032336384 + }, + { + "epoch": 1.3, + "learning_rate": 6.57813440320963e-05, + "loss": 1.1048, + "theoretical_loss": 3.3464919487025795, + "tokens_seen": 3032401920 + }, + { + "epoch": 1.3, + "learning_rate": 6.5765295887663e-05, + "loss": 1.2308, + "theoretical_loss": 3.3464864473484583, + "tokens_seen": 3032467456 + }, + { + "epoch": 1.3, + "learning_rate": 6.57492477432297e-05, + "loss": 1.0965, + "theoretical_loss": 3.3464809461465173, + "tokens_seen": 3032532992 + }, + { + "epoch": 1.3, + "learning_rate": 6.57331995987964e-05, + "loss": 1.1651, + "theoretical_loss": 3.346475445096748, + "tokens_seen": 3032598528 + }, + { + "epoch": 1.3, + "learning_rate": 6.57171514543631e-05, + "loss": 1.2183, + "theoretical_loss": 3.3464699441991446, + "tokens_seen": 3032664064 + }, + { + "epoch": 1.3, + "learning_rate": 6.570110330992978e-05, + "loss": 1.2451, + "theoretical_loss": 3.346464443453698, + "tokens_seen": 3032729600 + }, + { + "epoch": 1.3, + "learning_rate": 6.568505516549648e-05, + "loss": 1.1598, + "theoretical_loss": 3.3464589428604015, + "tokens_seen": 3032795136 + }, + { + "epoch": 1.3, + "learning_rate": 6.566900702106319e-05, + "loss": 1.1686, + "theoretical_loss": 3.346453442419247, + "tokens_seen": 3032860672 + }, + { + "epoch": 1.3, + "learning_rate": 6.565295887662989e-05, + "loss": 1.1074, + "theoretical_loss": 3.346447942130228, + "tokens_seen": 3032926208 + }, + { + "epoch": 1.3, + "learning_rate": 6.563691073219659e-05, + "loss": 1.2605, + "theoretical_loss": 3.3464424419933363, + "tokens_seen": 3032991744 + }, + { + "epoch": 1.3, + "learning_rate": 6.562086258776329e-05, + "loss": 1.1893, + "theoretical_loss": 3.3464369420085642, + "tokens_seen": 3033057280 + }, + { + "epoch": 1.3, + "learning_rate": 6.560481444333e-05, + "loss": 1.2105, + "theoretical_loss": 3.3464314421759047, + "tokens_seen": 3033122816 + }, + { + "epoch": 1.3, + "learning_rate": 6.55887662988967e-05, + "loss": 1.2079, + "theoretical_loss": 3.3464259424953506, + "tokens_seen": 3033188352 + }, + { + "epoch": 1.3, + "learning_rate": 6.55727181544634e-05, + "loss": 1.1682, + "theoretical_loss": 3.3464204429668936, + "tokens_seen": 3033253888 + }, + { + "epoch": 1.3, + "learning_rate": 6.55566700100301e-05, + "loss": 1.2952, + "theoretical_loss": 3.346414943590527, + "tokens_seen": 3033319424 + }, + { + "epoch": 1.3, + "learning_rate": 6.554062186559679e-05, + "loss": 1.1359, + "theoretical_loss": 3.346409444366242, + "tokens_seen": 3033384960 + }, + { + "epoch": 1.3, + "learning_rate": 6.552457372116349e-05, + "loss": 1.2293, + "theoretical_loss": 3.346403945294033, + "tokens_seen": 3033450496 + }, + { + "epoch": 1.3, + "learning_rate": 6.550852557673019e-05, + "loss": 1.2281, + "theoretical_loss": 3.3463984463738905, + "tokens_seen": 3033516032 + }, + { + "epoch": 1.3, + "learning_rate": 6.549247743229689e-05, + "loss": 1.1347, + "theoretical_loss": 3.3463929476058087, + "tokens_seen": 3033581568 + }, + { + "epoch": 1.3, + "learning_rate": 6.547642928786359e-05, + "loss": 1.1787, + "theoretical_loss": 3.346387448989779, + "tokens_seen": 3033647104 + }, + { + "epoch": 1.3, + "learning_rate": 6.54603811434303e-05, + "loss": 1.2004, + "theoretical_loss": 3.346381950525795, + "tokens_seen": 3033712640 + }, + { + "epoch": 1.3, + "learning_rate": 6.5444332998997e-05, + "loss": 1.1717, + "theoretical_loss": 3.346376452213848, + "tokens_seen": 3033778176 + }, + { + "epoch": 1.3, + "learning_rate": 6.54282848545637e-05, + "loss": 1.1493, + "theoretical_loss": 3.3463709540539313, + "tokens_seen": 3033843712 + }, + { + "epoch": 1.3, + "learning_rate": 6.54122367101304e-05, + "loss": 1.2566, + "theoretical_loss": 3.346365456046037, + "tokens_seen": 3033909248 + }, + { + "epoch": 1.3, + "learning_rate": 6.53961885656971e-05, + "loss": 1.1899, + "theoretical_loss": 3.3463599581901575, + "tokens_seen": 3033974784 + }, + { + "epoch": 1.3, + "learning_rate": 6.53801404212638e-05, + "loss": 1.1237, + "theoretical_loss": 3.3463544604862863, + "tokens_seen": 3034040320 + }, + { + "epoch": 1.3, + "learning_rate": 6.53640922768305e-05, + "loss": 1.2157, + "theoretical_loss": 3.3463489629344148, + "tokens_seen": 3034105856 + }, + { + "epoch": 1.3, + "learning_rate": 6.534804413239718e-05, + "loss": 1.1894, + "theoretical_loss": 3.3463434655345354, + "tokens_seen": 3034171392 + }, + { + "epoch": 1.3, + "learning_rate": 6.533199598796389e-05, + "loss": 1.2553, + "theoretical_loss": 3.346337968286642, + "tokens_seen": 3034236928 + }, + { + "epoch": 1.3, + "objective/train/docs_used": 1119733, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2971986532211304, + "objective/train/theoretical_loss": 3.346332471190726, + "objective/train/tokens_used": 3054762464, + "theoretical_loss": 3.346332471190726, + "tokens_seen": 3034302464 + }, + { + "epoch": 1.3, + "learning_rate": 6.531594784353059e-05, + "loss": 1.2972, + "theoretical_loss": 3.346332471190726, + "tokens_seen": 3034302464 + }, + { + "epoch": 1.3, + "learning_rate": 6.52998996990973e-05, + "loss": 1.1893, + "theoretical_loss": 3.34632697424678, + "tokens_seen": 3034368000 + }, + { + "epoch": 1.3, + "learning_rate": 6.5283851554664e-05, + "loss": 1.2724, + "theoretical_loss": 3.3463214774547967, + "tokens_seen": 3034433536 + }, + { + "epoch": 1.3, + "learning_rate": 6.52678034102307e-05, + "loss": 1.2229, + "theoretical_loss": 3.3463159808147687, + "tokens_seen": 3034499072 + }, + { + "epoch": 1.3, + "learning_rate": 6.52517552657974e-05, + "loss": 1.0572, + "theoretical_loss": 3.3463104843266884, + "tokens_seen": 3034564608 + }, + { + "epoch": 1.3, + "learning_rate": 6.52357071213641e-05, + "loss": 1.1852, + "theoretical_loss": 3.3463049879905484, + "tokens_seen": 3034630144 + }, + { + "epoch": 1.3, + "learning_rate": 6.52196589769308e-05, + "loss": 1.0851, + "theoretical_loss": 3.3462994918063407, + "tokens_seen": 3034695680 + }, + { + "epoch": 1.3, + "learning_rate": 6.52036108324975e-05, + "loss": 1.1935, + "theoretical_loss": 3.346293995774059, + "tokens_seen": 3034761216 + }, + { + "epoch": 1.3, + "learning_rate": 6.518756268806419e-05, + "loss": 1.2397, + "theoretical_loss": 3.346288499893695, + "tokens_seen": 3034826752 + }, + { + "epoch": 1.3, + "learning_rate": 6.517151454363089e-05, + "loss": 1.2546, + "theoretical_loss": 3.3462830041652407, + "tokens_seen": 3034892288 + }, + { + "epoch": 1.3, + "learning_rate": 6.515546639919759e-05, + "loss": 1.1219, + "theoretical_loss": 3.3462775085886896, + "tokens_seen": 3034957824 + }, + { + "epoch": 1.3, + "learning_rate": 6.513941825476429e-05, + "loss": 1.1507, + "theoretical_loss": 3.346272013164034, + "tokens_seen": 3035023360 + }, + { + "epoch": 1.3, + "learning_rate": 6.5123370110331e-05, + "loss": 1.1731, + "theoretical_loss": 3.346266517891266, + "tokens_seen": 3035088896 + }, + { + "epoch": 1.3, + "learning_rate": 6.51073219658977e-05, + "loss": 1.1838, + "theoretical_loss": 3.3462610227703786, + "tokens_seen": 3035154432 + }, + { + "epoch": 1.3, + "learning_rate": 6.50912738214644e-05, + "loss": 1.1165, + "theoretical_loss": 3.346255527801364, + "tokens_seen": 3035219968 + }, + { + "epoch": 1.3, + "learning_rate": 6.50752256770311e-05, + "loss": 1.2423, + "theoretical_loss": 3.346250032984215, + "tokens_seen": 3035285504 + }, + { + "epoch": 1.3, + "learning_rate": 6.50591775325978e-05, + "loss": 1.2664, + "theoretical_loss": 3.3462445383189237, + "tokens_seen": 3035351040 + }, + { + "epoch": 1.3, + "learning_rate": 6.50431293881645e-05, + "loss": 1.1947, + "theoretical_loss": 3.3462390438054834, + "tokens_seen": 3035416576 + }, + { + "epoch": 1.3, + "learning_rate": 6.50270812437312e-05, + "loss": 1.1969, + "theoretical_loss": 3.3462335494438857, + "tokens_seen": 3035482112 + }, + { + "epoch": 1.3, + "learning_rate": 6.50110330992979e-05, + "loss": 1.1747, + "theoretical_loss": 3.3462280552341235, + "tokens_seen": 3035547648 + }, + { + "epoch": 1.3, + "learning_rate": 6.49949849548646e-05, + "loss": 1.2958, + "theoretical_loss": 3.3462225611761895, + "tokens_seen": 3035613184 + }, + { + "epoch": 1.3, + "learning_rate": 6.49789368104313e-05, + "loss": 1.0897, + "theoretical_loss": 3.3462170672700764, + "tokens_seen": 3035678720 + }, + { + "epoch": 1.3, + "learning_rate": 6.4962888665998e-05, + "loss": 1.1452, + "theoretical_loss": 3.346211573515776, + "tokens_seen": 3035744256 + }, + { + "epoch": 1.3, + "learning_rate": 6.49468405215647e-05, + "loss": 1.1565, + "theoretical_loss": 3.3462060799132813, + "tokens_seen": 3035809792 + }, + { + "epoch": 1.3, + "learning_rate": 6.49307923771314e-05, + "loss": 1.2165, + "theoretical_loss": 3.346200586462585, + "tokens_seen": 3035875328 + }, + { + "epoch": 1.3, + "learning_rate": 6.49147442326981e-05, + "loss": 1.2014, + "theoretical_loss": 3.3461950931636792, + "tokens_seen": 3035940864 + }, + { + "epoch": 1.3, + "learning_rate": 6.48986960882648e-05, + "loss": 1.099, + "theoretical_loss": 3.3461896000165567, + "tokens_seen": 3036006400 + }, + { + "epoch": 1.3, + "learning_rate": 6.48826479438315e-05, + "loss": 1.191, + "theoretical_loss": 3.34618410702121, + "tokens_seen": 3036071936 + }, + { + "epoch": 1.3, + "learning_rate": 6.48665997993982e-05, + "loss": 1.195, + "theoretical_loss": 3.346178614177632, + "tokens_seen": 3036137472 + }, + { + "epoch": 1.3, + "learning_rate": 6.485055165496489e-05, + "loss": 1.2453, + "theoretical_loss": 3.346173121485814, + "tokens_seen": 3036203008 + }, + { + "epoch": 1.3, + "learning_rate": 6.483450351053159e-05, + "loss": 1.2395, + "theoretical_loss": 3.34616762894575, + "tokens_seen": 3036268544 + }, + { + "epoch": 1.3, + "learning_rate": 6.481845536609829e-05, + "loss": 1.2072, + "theoretical_loss": 3.3461621365574317, + "tokens_seen": 3036334080 + }, + { + "epoch": 1.3, + "learning_rate": 6.480240722166499e-05, + "loss": 1.2477, + "theoretical_loss": 3.346156644320852, + "tokens_seen": 3036399616 + }, + { + "epoch": 1.3, + "learning_rate": 6.478635907723169e-05, + "loss": 1.2104, + "theoretical_loss": 3.3461511522360032, + "tokens_seen": 3036465152 + }, + { + "epoch": 1.3, + "learning_rate": 6.47703109327984e-05, + "loss": 1.1433, + "theoretical_loss": 3.3461456603028776, + "tokens_seen": 3036530688 + }, + { + "epoch": 1.3, + "learning_rate": 6.47542627883651e-05, + "loss": 1.1596, + "theoretical_loss": 3.346140168521468, + "tokens_seen": 3036596224 + }, + { + "epoch": 1.3, + "learning_rate": 6.47382146439318e-05, + "loss": 1.2389, + "theoretical_loss": 3.3461346768917677, + "tokens_seen": 3036661760 + }, + { + "epoch": 1.3, + "learning_rate": 6.47221664994985e-05, + "loss": 1.1819, + "theoretical_loss": 3.346129185413768, + "tokens_seen": 3036727296 + }, + { + "epoch": 1.3, + "learning_rate": 6.470611835506521e-05, + "loss": 1.2118, + "theoretical_loss": 3.3461236940874617, + "tokens_seen": 3036792832 + }, + { + "epoch": 1.3, + "learning_rate": 6.46900702106319e-05, + "loss": 1.226, + "theoretical_loss": 3.346118202912842, + "tokens_seen": 3036858368 + }, + { + "epoch": 1.3, + "learning_rate": 6.46740220661986e-05, + "loss": 1.0465, + "theoretical_loss": 3.3461127118899006, + "tokens_seen": 3036923904 + }, + { + "epoch": 1.3, + "learning_rate": 6.46579739217653e-05, + "loss": 1.0987, + "theoretical_loss": 3.3461072210186305, + "tokens_seen": 3036989440 + }, + { + "epoch": 1.3, + "learning_rate": 6.4641925777332e-05, + "loss": 1.1957, + "theoretical_loss": 3.3461017302990244, + "tokens_seen": 3037054976 + }, + { + "epoch": 1.3, + "learning_rate": 6.46258776328987e-05, + "loss": 1.1694, + "theoretical_loss": 3.3460962397310743, + "tokens_seen": 3037120512 + }, + { + "epoch": 1.3, + "learning_rate": 6.46098294884654e-05, + "loss": 1.1092, + "theoretical_loss": 3.3460907493147736, + "tokens_seen": 3037186048 + }, + { + "epoch": 1.3, + "learning_rate": 6.45937813440321e-05, + "loss": 1.2445, + "theoretical_loss": 3.346085259050114, + "tokens_seen": 3037251584 + }, + { + "epoch": 1.3, + "learning_rate": 6.45777331995988e-05, + "loss": 1.0759, + "theoretical_loss": 3.3460797689370883, + "tokens_seen": 3037317120 + }, + { + "epoch": 1.3, + "learning_rate": 6.45616850551655e-05, + "loss": 1.191, + "theoretical_loss": 3.3460742789756894, + "tokens_seen": 3037382656 + }, + { + "epoch": 1.3, + "learning_rate": 6.45456369107322e-05, + "loss": 1.2224, + "theoretical_loss": 3.346068789165909, + "tokens_seen": 3037448192 + }, + { + "epoch": 1.3, + "learning_rate": 6.45295887662989e-05, + "loss": 1.1853, + "theoretical_loss": 3.3460632995077404, + "tokens_seen": 3037513728 + }, + { + "epoch": 1.3, + "learning_rate": 6.45135406218656e-05, + "loss": 1.1864, + "theoretical_loss": 3.346057810001176, + "tokens_seen": 3037579264 + }, + { + "epoch": 1.3, + "learning_rate": 6.449749247743229e-05, + "loss": 1.1621, + "theoretical_loss": 3.346052320646208, + "tokens_seen": 3037644800 + }, + { + "epoch": 1.3, + "learning_rate": 6.448144433299899e-05, + "loss": 1.1499, + "theoretical_loss": 3.3460468314428295, + "tokens_seen": 3037710336 + }, + { + "epoch": 1.3, + "learning_rate": 6.446539618856569e-05, + "loss": 1.1599, + "theoretical_loss": 3.3460413423910325, + "tokens_seen": 3037775872 + }, + { + "epoch": 1.3, + "learning_rate": 6.444934804413239e-05, + "loss": 1.3157, + "theoretical_loss": 3.3460358534908097, + "tokens_seen": 3037841408 + }, + { + "epoch": 1.3, + "learning_rate": 6.44332998996991e-05, + "loss": 1.1702, + "theoretical_loss": 3.346030364742154, + "tokens_seen": 3037906944 + }, + { + "epoch": 1.3, + "learning_rate": 6.44172517552658e-05, + "loss": 1.2308, + "theoretical_loss": 3.3460248761450577, + "tokens_seen": 3037972480 + }, + { + "epoch": 1.3, + "learning_rate": 6.440120361083251e-05, + "loss": 1.2041, + "theoretical_loss": 3.346019387699513, + "tokens_seen": 3038038016 + }, + { + "epoch": 1.3, + "learning_rate": 6.438515546639921e-05, + "loss": 1.1391, + "theoretical_loss": 3.346013899405513, + "tokens_seen": 3038103552 + }, + { + "epoch": 1.3, + "learning_rate": 6.436910732196591e-05, + "loss": 1.1862, + "theoretical_loss": 3.3460084112630497, + "tokens_seen": 3038169088 + }, + { + "epoch": 1.3, + "learning_rate": 6.435305917753261e-05, + "loss": 1.1596, + "theoretical_loss": 3.3460029232721165, + "tokens_seen": 3038234624 + }, + { + "epoch": 1.3, + "learning_rate": 6.43370110330993e-05, + "loss": 1.178, + "theoretical_loss": 3.3459974354327047, + "tokens_seen": 3038300160 + }, + { + "epoch": 1.3, + "learning_rate": 6.4320962888666e-05, + "loss": 1.2173, + "theoretical_loss": 3.345991947744808, + "tokens_seen": 3038365696 + }, + { + "epoch": 1.3, + "learning_rate": 6.43049147442327e-05, + "loss": 1.1716, + "theoretical_loss": 3.3459864602084184, + "tokens_seen": 3038431232 + }, + { + "epoch": 1.3, + "learning_rate": 6.42888665997994e-05, + "loss": 1.1916, + "theoretical_loss": 3.345980972823529, + "tokens_seen": 3038496768 + }, + { + "epoch": 1.3, + "learning_rate": 6.42728184553661e-05, + "loss": 1.0448, + "theoretical_loss": 3.345975485590131, + "tokens_seen": 3038562304 + }, + { + "epoch": 1.3, + "learning_rate": 6.42567703109328e-05, + "loss": 1.1743, + "theoretical_loss": 3.3459699985082185, + "tokens_seen": 3038627840 + }, + { + "epoch": 1.3, + "learning_rate": 6.42407221664995e-05, + "loss": 1.1575, + "theoretical_loss": 3.345964511577783, + "tokens_seen": 3038693376 + }, + { + "epoch": 1.3, + "learning_rate": 6.42246740220662e-05, + "loss": 1.1247, + "theoretical_loss": 3.345959024798818, + "tokens_seen": 3038758912 + }, + { + "epoch": 1.3, + "learning_rate": 6.42086258776329e-05, + "loss": 1.2147, + "theoretical_loss": 3.345953538171315, + "tokens_seen": 3038824448 + }, + { + "epoch": 1.3, + "learning_rate": 6.41925777331996e-05, + "loss": 1.1392, + "theoretical_loss": 3.345948051695267, + "tokens_seen": 3038889984 + }, + { + "epoch": 1.3, + "learning_rate": 6.41765295887663e-05, + "loss": 1.1014, + "theoretical_loss": 3.345942565370667, + "tokens_seen": 3038955520 + }, + { + "epoch": 1.3, + "learning_rate": 6.4160481444333e-05, + "loss": 1.2249, + "theoretical_loss": 3.345937079197507, + "tokens_seen": 3039021056 + }, + { + "epoch": 1.3, + "learning_rate": 6.414443329989969e-05, + "loss": 1.2025, + "theoretical_loss": 3.34593159317578, + "tokens_seen": 3039086592 + }, + { + "epoch": 1.3, + "learning_rate": 6.412838515546639e-05, + "loss": 1.1269, + "theoretical_loss": 3.345926107305478, + "tokens_seen": 3039152128 + }, + { + "epoch": 1.3, + "learning_rate": 6.41123370110331e-05, + "loss": 1.2561, + "theoretical_loss": 3.3459206215865933, + "tokens_seen": 3039217664 + }, + { + "epoch": 1.3, + "learning_rate": 6.409628886659981e-05, + "loss": 1.0689, + "theoretical_loss": 3.3459151360191193, + "tokens_seen": 3039283200 + }, + { + "epoch": 1.3, + "learning_rate": 6.408024072216651e-05, + "loss": 1.1988, + "theoretical_loss": 3.3459096506030486, + "tokens_seen": 3039348736 + }, + { + "epoch": 1.3, + "learning_rate": 6.406419257773321e-05, + "loss": 1.1365, + "theoretical_loss": 3.345904165338373, + "tokens_seen": 3039414272 + }, + { + "epoch": 1.3, + "learning_rate": 6.406419257773321e-05, + "loss": 1.243, + "theoretical_loss": 3.345898680225086, + "tokens_seen": 3039479808 + }, + { + "epoch": 1.3, + "learning_rate": 6.404814443329991e-05, + "loss": 1.238, + "theoretical_loss": 3.345893195263179, + "tokens_seen": 3039545344 + }, + { + "epoch": 1.3, + "learning_rate": 6.403209628886661e-05, + "loss": 1.2093, + "theoretical_loss": 3.3458877104526454, + "tokens_seen": 3039610880 + }, + { + "epoch": 1.3, + "learning_rate": 6.401604814443331e-05, + "loss": 1.1692, + "theoretical_loss": 3.3458822257934777, + "tokens_seen": 3039676416 + }, + { + "epoch": 1.3, + "learning_rate": 6.400000000000001e-05, + "loss": 1.182, + "theoretical_loss": 3.345876741285668, + "tokens_seen": 3039741952 + }, + { + "epoch": 1.3, + "learning_rate": 6.39839518555667e-05, + "loss": 1.2274, + "theoretical_loss": 3.3458712569292093, + "tokens_seen": 3039807488 + }, + { + "epoch": 1.3, + "learning_rate": 6.39679037111334e-05, + "loss": 1.1208, + "theoretical_loss": 3.3458657727240944, + "tokens_seen": 3039873024 + }, + { + "epoch": 1.3, + "learning_rate": 6.39518555667001e-05, + "loss": 1.2228, + "theoretical_loss": 3.345860288670315, + "tokens_seen": 3039938560 + }, + { + "epoch": 1.3, + "learning_rate": 6.39358074222668e-05, + "loss": 1.1201, + "theoretical_loss": 3.345854804767864, + "tokens_seen": 3040004096 + }, + { + "epoch": 1.3, + "learning_rate": 6.39197592778335e-05, + "loss": 1.1435, + "theoretical_loss": 3.3458493210167344, + "tokens_seen": 3040069632 + }, + { + "epoch": 1.3, + "learning_rate": 6.39037111334002e-05, + "loss": 1.1507, + "theoretical_loss": 3.3458438374169184, + "tokens_seen": 3040135168 + }, + { + "epoch": 1.3, + "learning_rate": 6.38876629889669e-05, + "loss": 1.1101, + "theoretical_loss": 3.345838353968408, + "tokens_seen": 3040200704 + }, + { + "epoch": 1.3, + "learning_rate": 6.38716148445336e-05, + "loss": 1.1344, + "theoretical_loss": 3.345832870671197, + "tokens_seen": 3040266240 + }, + { + "epoch": 1.3, + "learning_rate": 6.38555667001003e-05, + "loss": 1.2116, + "theoretical_loss": 3.3458273875252775, + "tokens_seen": 3040331776 + }, + { + "epoch": 1.3, + "learning_rate": 6.3839518555667e-05, + "loss": 1.0915, + "theoretical_loss": 3.3458219045306414, + "tokens_seen": 3040397312 + }, + { + "epoch": 1.3, + "learning_rate": 6.38234704112337e-05, + "loss": 1.2032, + "theoretical_loss": 3.3458164216872825, + "tokens_seen": 3040462848 + }, + { + "epoch": 1.3, + "learning_rate": 6.38074222668004e-05, + "loss": 1.2022, + "theoretical_loss": 3.345810938995192, + "tokens_seen": 3040528384 + }, + { + "epoch": 1.3, + "learning_rate": 6.37913741223671e-05, + "loss": 1.119, + "theoretical_loss": 3.345805456454363, + "tokens_seen": 3040593920 + }, + { + "epoch": 1.3, + "learning_rate": 6.37753259779338e-05, + "loss": 1.2193, + "theoretical_loss": 3.345799974064789, + "tokens_seen": 3040659456 + }, + { + "epoch": 1.3, + "learning_rate": 6.37592778335005e-05, + "loss": 1.1466, + "theoretical_loss": 3.345794491826461, + "tokens_seen": 3040724992 + }, + { + "epoch": 1.3, + "learning_rate": 6.374322968906721e-05, + "loss": 1.1267, + "theoretical_loss": 3.3457890097393723, + "tokens_seen": 3040790528 + }, + { + "epoch": 1.3, + "objective/train/docs_used": 1122270, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.203256368637085, + "objective/train/theoretical_loss": 3.3457835278035155, + "objective/train/tokens_used": 3061316064, + "theoretical_loss": 3.3457835278035155, + "tokens_seen": 3040856064 + }, + { + "epoch": 1.3, + "learning_rate": 6.372718154463391e-05, + "loss": 1.2033, + "theoretical_loss": 3.3457835278035155, + "tokens_seen": 3040856064 + }, + { + "epoch": 1.3, + "learning_rate": 6.371113340020061e-05, + "loss": 1.1674, + "theoretical_loss": 3.3457780460188835, + "tokens_seen": 3040921600 + }, + { + "epoch": 1.3, + "learning_rate": 6.369508525576731e-05, + "loss": 1.2301, + "theoretical_loss": 3.3457725643854683, + "tokens_seen": 3040987136 + }, + { + "epoch": 1.3, + "learning_rate": 6.367903711133401e-05, + "loss": 1.1072, + "theoretical_loss": 3.345767082903263, + "tokens_seen": 3041052672 + }, + { + "epoch": 1.3, + "learning_rate": 6.366298896690071e-05, + "loss": 1.1373, + "theoretical_loss": 3.3457616015722595, + "tokens_seen": 3041118208 + }, + { + "epoch": 1.3, + "learning_rate": 6.364694082246741e-05, + "loss": 1.1081, + "theoretical_loss": 3.3457561203924504, + "tokens_seen": 3041183744 + }, + { + "epoch": 1.3, + "learning_rate": 6.36308926780341e-05, + "loss": 1.1559, + "theoretical_loss": 3.345750639363829, + "tokens_seen": 3041249280 + }, + { + "epoch": 1.3, + "learning_rate": 6.36148445336008e-05, + "loss": 1.1478, + "theoretical_loss": 3.3457451584863875, + "tokens_seen": 3041314816 + }, + { + "epoch": 1.3, + "learning_rate": 6.35987963891675e-05, + "loss": 1.1876, + "theoretical_loss": 3.3457396777601183, + "tokens_seen": 3041380352 + }, + { + "epoch": 1.3, + "learning_rate": 6.35827482447342e-05, + "loss": 1.1759, + "theoretical_loss": 3.3457341971850143, + "tokens_seen": 3041445888 + }, + { + "epoch": 1.3, + "learning_rate": 6.35667001003009e-05, + "loss": 1.1423, + "theoretical_loss": 3.345728716761068, + "tokens_seen": 3041511424 + }, + { + "epoch": 1.3, + "learning_rate": 6.35506519558676e-05, + "loss": 1.2417, + "theoretical_loss": 3.3457232364882716, + "tokens_seen": 3041576960 + }, + { + "epoch": 1.3, + "learning_rate": 6.35346038114343e-05, + "loss": 1.1872, + "theoretical_loss": 3.345717756366618, + "tokens_seen": 3041642496 + }, + { + "epoch": 1.3, + "learning_rate": 6.3518555667001e-05, + "loss": 1.1918, + "theoretical_loss": 3.3457122763960996, + "tokens_seen": 3041708032 + }, + { + "epoch": 1.3, + "learning_rate": 6.350250752256772e-05, + "loss": 1.1873, + "theoretical_loss": 3.3457067965767093, + "tokens_seen": 3041773568 + }, + { + "epoch": 1.3, + "learning_rate": 6.348645937813442e-05, + "loss": 1.1408, + "theoretical_loss": 3.345701316908439, + "tokens_seen": 3041839104 + }, + { + "epoch": 1.3, + "learning_rate": 6.34704112337011e-05, + "loss": 1.1898, + "theoretical_loss": 3.3456958373912826, + "tokens_seen": 3041904640 + }, + { + "epoch": 1.3, + "learning_rate": 6.34543630892678e-05, + "loss": 1.1101, + "theoretical_loss": 3.345690358025231, + "tokens_seen": 3041970176 + }, + { + "epoch": 1.3, + "learning_rate": 6.34383149448345e-05, + "loss": 1.1362, + "theoretical_loss": 3.345684878810278, + "tokens_seen": 3042035712 + }, + { + "epoch": 1.3, + "learning_rate": 6.34222668004012e-05, + "loss": 1.194, + "theoretical_loss": 3.345679399746416, + "tokens_seen": 3042101248 + }, + { + "epoch": 1.3, + "learning_rate": 6.340621865596791e-05, + "loss": 1.1247, + "theoretical_loss": 3.3456739208336366, + "tokens_seen": 3042166784 + }, + { + "epoch": 1.3, + "learning_rate": 6.339017051153461e-05, + "loss": 1.1988, + "theoretical_loss": 3.3456684420719336, + "tokens_seen": 3042232320 + }, + { + "epoch": 1.3, + "learning_rate": 6.337412236710131e-05, + "loss": 1.1679, + "theoretical_loss": 3.345662963461299, + "tokens_seen": 3042297856 + }, + { + "epoch": 1.3, + "learning_rate": 6.335807422266801e-05, + "loss": 1.1619, + "theoretical_loss": 3.345657485001726, + "tokens_seen": 3042363392 + }, + { + "epoch": 1.3, + "learning_rate": 6.334202607823471e-05, + "loss": 1.2242, + "theoretical_loss": 3.3456520066932063, + "tokens_seen": 3042428928 + }, + { + "epoch": 1.3, + "learning_rate": 6.332597793380141e-05, + "loss": 1.3195, + "theoretical_loss": 3.3456465285357324, + "tokens_seen": 3042494464 + }, + { + "epoch": 1.3, + "learning_rate": 6.330992978936811e-05, + "loss": 1.2, + "theoretical_loss": 3.3456410505292977, + "tokens_seen": 3042560000 + }, + { + "epoch": 1.3, + "learning_rate": 6.329388164493481e-05, + "loss": 1.0912, + "theoretical_loss": 3.3456355726738947, + "tokens_seen": 3042625536 + }, + { + "epoch": 1.3, + "learning_rate": 6.32778335005015e-05, + "loss": 1.1448, + "theoretical_loss": 3.3456300949695152, + "tokens_seen": 3042691072 + }, + { + "epoch": 1.3, + "learning_rate": 6.32617853560682e-05, + "loss": 1.1622, + "theoretical_loss": 3.345624617416153, + "tokens_seen": 3042756608 + }, + { + "epoch": 1.3, + "learning_rate": 6.32457372116349e-05, + "loss": 1.2076, + "theoretical_loss": 3.3456191400137993, + "tokens_seen": 3042822144 + }, + { + "epoch": 1.3, + "learning_rate": 6.32296890672016e-05, + "loss": 1.2773, + "theoretical_loss": 3.3456136627624473, + "tokens_seen": 3042887680 + }, + { + "epoch": 1.3, + "learning_rate": 6.32136409227683e-05, + "loss": 1.2276, + "theoretical_loss": 3.34560818566209, + "tokens_seen": 3042953216 + }, + { + "epoch": 1.3, + "learning_rate": 6.319759277833502e-05, + "loss": 1.1309, + "theoretical_loss": 3.3456027087127196, + "tokens_seen": 3043018752 + }, + { + "epoch": 1.3, + "learning_rate": 6.318154463390172e-05, + "loss": 1.1301, + "theoretical_loss": 3.3455972319143283, + "tokens_seen": 3043084288 + }, + { + "epoch": 1.3, + "learning_rate": 6.316549648946842e-05, + "loss": 1.1767, + "theoretical_loss": 3.3455917552669097, + "tokens_seen": 3043149824 + }, + { + "epoch": 1.3, + "learning_rate": 6.314944834503512e-05, + "loss": 1.1049, + "theoretical_loss": 3.3455862787704556, + "tokens_seen": 3043215360 + }, + { + "epoch": 1.3, + "learning_rate": 6.313340020060182e-05, + "loss": 1.2401, + "theoretical_loss": 3.3455808024249585, + "tokens_seen": 3043280896 + }, + { + "epoch": 1.3, + "learning_rate": 6.31173520561685e-05, + "loss": 1.2162, + "theoretical_loss": 3.3455753262304113, + "tokens_seen": 3043346432 + }, + { + "epoch": 1.3, + "learning_rate": 6.31013039117352e-05, + "loss": 1.1537, + "theoretical_loss": 3.345569850186807, + "tokens_seen": 3043411968 + }, + { + "epoch": 1.3, + "learning_rate": 6.30852557673019e-05, + "loss": 1.1583, + "theoretical_loss": 3.3455643742941374, + "tokens_seen": 3043477504 + }, + { + "epoch": 1.3, + "learning_rate": 6.306920762286861e-05, + "loss": 1.2549, + "theoretical_loss": 3.345558898552395, + "tokens_seen": 3043543040 + }, + { + "epoch": 1.3, + "learning_rate": 6.305315947843531e-05, + "loss": 1.1544, + "theoretical_loss": 3.3455534229615735, + "tokens_seen": 3043608576 + }, + { + "epoch": 1.3, + "learning_rate": 6.303711133400201e-05, + "loss": 1.2085, + "theoretical_loss": 3.3455479475216645, + "tokens_seen": 3043674112 + }, + { + "epoch": 1.3, + "learning_rate": 6.302106318956871e-05, + "loss": 1.085, + "theoretical_loss": 3.345542472232661, + "tokens_seen": 3043739648 + }, + { + "epoch": 1.3, + "learning_rate": 6.300501504513541e-05, + "loss": 1.1907, + "theoretical_loss": 3.3455369970945554, + "tokens_seen": 3043805184 + }, + { + "epoch": 1.3, + "learning_rate": 6.298896690070211e-05, + "loss": 1.1744, + "theoretical_loss": 3.3455315221073407, + "tokens_seen": 3043870720 + }, + { + "epoch": 1.3, + "learning_rate": 6.297291875626881e-05, + "loss": 1.2001, + "theoretical_loss": 3.345526047271009, + "tokens_seen": 3043936256 + }, + { + "epoch": 1.3, + "learning_rate": 6.295687061183551e-05, + "loss": 1.1168, + "theoretical_loss": 3.345520572585553, + "tokens_seen": 3044001792 + }, + { + "epoch": 1.3, + "learning_rate": 6.294082246740221e-05, + "loss": 1.1389, + "theoretical_loss": 3.3455150980509654, + "tokens_seen": 3044067328 + }, + { + "epoch": 1.3, + "learning_rate": 6.29247743229689e-05, + "loss": 1.0293, + "theoretical_loss": 3.3455096236672386, + "tokens_seen": 3044132864 + }, + { + "epoch": 1.3, + "learning_rate": 6.290872617853561e-05, + "loss": 1.2257, + "theoretical_loss": 3.3455041494343654, + "tokens_seen": 3044198400 + }, + { + "epoch": 1.3, + "learning_rate": 6.289267803410231e-05, + "loss": 1.2139, + "theoretical_loss": 3.3454986753523386, + "tokens_seen": 3044263936 + }, + { + "epoch": 1.3, + "learning_rate": 6.287662988966901e-05, + "loss": 1.0786, + "theoretical_loss": 3.3454932014211503, + "tokens_seen": 3044329472 + }, + { + "epoch": 1.3, + "learning_rate": 6.286058174523572e-05, + "loss": 1.1633, + "theoretical_loss": 3.3454877276407933, + "tokens_seen": 3044395008 + }, + { + "epoch": 1.3, + "learning_rate": 6.284453360080242e-05, + "loss": 1.1925, + "theoretical_loss": 3.3454822540112605, + "tokens_seen": 3044460544 + }, + { + "epoch": 1.3, + "learning_rate": 6.282848545636912e-05, + "loss": 1.1298, + "theoretical_loss": 3.345476780532544, + "tokens_seen": 3044526080 + }, + { + "epoch": 1.3, + "learning_rate": 6.281243731193582e-05, + "loss": 1.1716, + "theoretical_loss": 3.3454713072046367, + "tokens_seen": 3044591616 + }, + { + "epoch": 1.3, + "learning_rate": 6.279638916750252e-05, + "loss": 1.2754, + "theoretical_loss": 3.3454658340275314, + "tokens_seen": 3044657152 + }, + { + "epoch": 1.3, + "learning_rate": 6.27803410230692e-05, + "loss": 1.249, + "theoretical_loss": 3.34546036100122, + "tokens_seen": 3044722688 + }, + { + "epoch": 1.3, + "learning_rate": 6.27642928786359e-05, + "loss": 1.1649, + "theoretical_loss": 3.345454888125696, + "tokens_seen": 3044788224 + }, + { + "epoch": 1.3, + "learning_rate": 6.27482447342026e-05, + "loss": 1.2371, + "theoretical_loss": 3.3454494154009513, + "tokens_seen": 3044853760 + }, + { + "epoch": 1.3, + "learning_rate": 6.27321965897693e-05, + "loss": 1.1799, + "theoretical_loss": 3.3454439428269787, + "tokens_seen": 3044919296 + }, + { + "epoch": 1.3, + "learning_rate": 6.271614844533601e-05, + "loss": 1.1554, + "theoretical_loss": 3.3454384704037707, + "tokens_seen": 3044984832 + }, + { + "epoch": 1.3, + "learning_rate": 6.270010030090271e-05, + "loss": 1.2565, + "theoretical_loss": 3.34543299813132, + "tokens_seen": 3045050368 + }, + { + "epoch": 1.3, + "learning_rate": 6.268405215646941e-05, + "loss": 1.2257, + "theoretical_loss": 3.34542752600962, + "tokens_seen": 3045115904 + }, + { + "epoch": 1.3, + "learning_rate": 6.266800401203611e-05, + "loss": 1.1571, + "theoretical_loss": 3.3454220540386617, + "tokens_seen": 3045181440 + }, + { + "epoch": 1.3, + "learning_rate": 6.265195586760281e-05, + "loss": 1.2192, + "theoretical_loss": 3.3454165822184385, + "tokens_seen": 3045246976 + }, + { + "epoch": 1.3, + "learning_rate": 6.263590772316951e-05, + "loss": 1.1776, + "theoretical_loss": 3.3454111105489437, + "tokens_seen": 3045312512 + }, + { + "epoch": 1.3, + "learning_rate": 6.261985957873621e-05, + "loss": 1.2346, + "theoretical_loss": 3.3454056390301687, + "tokens_seen": 3045378048 + }, + { + "epoch": 1.3, + "learning_rate": 6.260381143430291e-05, + "loss": 1.2451, + "theoretical_loss": 3.345400167662107, + "tokens_seen": 3045443584 + }, + { + "epoch": 1.3, + "learning_rate": 6.258776328986961e-05, + "loss": 1.2632, + "theoretical_loss": 3.3453946964447505, + "tokens_seen": 3045509120 + }, + { + "epoch": 1.3, + "learning_rate": 6.257171514543631e-05, + "loss": 1.1557, + "theoretical_loss": 3.3453892253780926, + "tokens_seen": 3045574656 + }, + { + "epoch": 1.3, + "learning_rate": 6.255566700100301e-05, + "loss": 1.1544, + "theoretical_loss": 3.345383754462125, + "tokens_seen": 3045640192 + }, + { + "epoch": 1.3, + "learning_rate": 6.253961885656971e-05, + "loss": 1.1286, + "theoretical_loss": 3.3453782836968413, + "tokens_seen": 3045705728 + }, + { + "epoch": 1.3, + "learning_rate": 6.252357071213641e-05, + "loss": 1.2376, + "theoretical_loss": 3.3453728130822333, + "tokens_seen": 3045771264 + }, + { + "epoch": 1.3, + "learning_rate": 6.250752256770312e-05, + "loss": 1.1801, + "theoretical_loss": 3.345367342618294, + "tokens_seen": 3045836800 + }, + { + "epoch": 1.3, + "learning_rate": 6.249147442326982e-05, + "loss": 1.2363, + "theoretical_loss": 3.3453618723050162, + "tokens_seen": 3045902336 + }, + { + "epoch": 1.3, + "learning_rate": 6.247542627883652e-05, + "loss": 1.1933, + "theoretical_loss": 3.3453564021423916, + "tokens_seen": 3045967872 + }, + { + "epoch": 1.3, + "learning_rate": 6.245937813440322e-05, + "loss": 1.2054, + "theoretical_loss": 3.3453509321304136, + "tokens_seen": 3046033408 + }, + { + "epoch": 1.3, + "learning_rate": 6.244332998996992e-05, + "loss": 1.1324, + "theoretical_loss": 3.345345462269075, + "tokens_seen": 3046098944 + }, + { + "epoch": 1.3, + "learning_rate": 6.24272818455366e-05, + "loss": 1.208, + "theoretical_loss": 3.345339992558368, + "tokens_seen": 3046164480 + }, + { + "epoch": 1.3, + "learning_rate": 6.24112337011033e-05, + "loss": 1.1822, + "theoretical_loss": 3.345334522998285, + "tokens_seen": 3046230016 + }, + { + "epoch": 1.3, + "learning_rate": 6.239518555667e-05, + "loss": 1.1207, + "theoretical_loss": 3.3453290535888187, + "tokens_seen": 3046295552 + }, + { + "epoch": 1.3, + "learning_rate": 6.237913741223671e-05, + "loss": 1.1987, + "theoretical_loss": 3.3453235843299622, + "tokens_seen": 3046361088 + }, + { + "epoch": 1.3, + "learning_rate": 6.236308926780341e-05, + "loss": 1.2043, + "theoretical_loss": 3.345318115221708, + "tokens_seen": 3046426624 + }, + { + "epoch": 1.3, + "learning_rate": 6.234704112337011e-05, + "loss": 1.1242, + "theoretical_loss": 3.3453126462640483, + "tokens_seen": 3046492160 + }, + { + "epoch": 1.3, + "learning_rate": 6.233099297893681e-05, + "loss": 1.1755, + "theoretical_loss": 3.345307177456976, + "tokens_seen": 3046557696 + }, + { + "epoch": 1.3, + "learning_rate": 6.231494483450351e-05, + "loss": 1.2461, + "theoretical_loss": 3.3453017088004833, + "tokens_seen": 3046623232 + }, + { + "epoch": 1.3, + "learning_rate": 6.229889669007022e-05, + "loss": 1.233, + "theoretical_loss": 3.345296240294563, + "tokens_seen": 3046688768 + }, + { + "epoch": 1.3, + "learning_rate": 6.228284854563692e-05, + "loss": 1.192, + "theoretical_loss": 3.3452907719392084, + "tokens_seen": 3046754304 + }, + { + "epoch": 1.3, + "learning_rate": 6.226680040120361e-05, + "loss": 1.1012, + "theoretical_loss": 3.3452853037344115, + "tokens_seen": 3046819840 + }, + { + "epoch": 1.3, + "learning_rate": 6.225075225677031e-05, + "loss": 1.3046, + "theoretical_loss": 3.3452798356801647, + "tokens_seen": 3046885376 + }, + { + "epoch": 1.3, + "learning_rate": 6.223470411233701e-05, + "loss": 1.1903, + "theoretical_loss": 3.345274367776461, + "tokens_seen": 3046950912 + }, + { + "epoch": 1.3, + "learning_rate": 6.221865596790371e-05, + "loss": 1.1319, + "theoretical_loss": 3.3452689000232936, + "tokens_seen": 3047016448 + }, + { + "epoch": 1.3, + "learning_rate": 6.220260782347041e-05, + "loss": 1.1149, + "theoretical_loss": 3.345263432420654, + "tokens_seen": 3047081984 + }, + { + "epoch": 1.3, + "learning_rate": 6.218655967903711e-05, + "loss": 1.1806, + "theoretical_loss": 3.345257964968535, + "tokens_seen": 3047147520 + }, + { + "epoch": 1.3, + "learning_rate": 6.217051153460382e-05, + "loss": 1.2642, + "theoretical_loss": 3.34525249766693, + "tokens_seen": 3047213056 + }, + { + "epoch": 1.3, + "learning_rate": 6.215446339017052e-05, + "loss": 1.1333, + "theoretical_loss": 3.3452470305158304, + "tokens_seen": 3047278592 + }, + { + "epoch": 1.3, + "learning_rate": 6.213841524573722e-05, + "loss": 1.2006, + "theoretical_loss": 3.3452415635152297, + "tokens_seen": 3047344128 + }, + { + "epoch": 1.3, + "objective/train/docs_used": 1124661, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2000653743743896, + "objective/train/theoretical_loss": 3.3452360966651207, + "objective/train/tokens_used": 3067869664, + "theoretical_loss": 3.3452360966651207, + "tokens_seen": 3047409664 + }, + { + "epoch": 1.3, + "learning_rate": 6.212236710130392e-05, + "loss": 1.2001, + "theoretical_loss": 3.3452360966651207, + "tokens_seen": 3047409664 + }, + { + "epoch": 1.3, + "learning_rate": 6.210631895687062e-05, + "loss": 1.1517, + "theoretical_loss": 3.3452306299654957, + "tokens_seen": 3047475200 + }, + { + "epoch": 1.3, + "learning_rate": 6.209027081243732e-05, + "loss": 1.2681, + "theoretical_loss": 3.3452251634163472, + "tokens_seen": 3047540736 + }, + { + "epoch": 1.3, + "learning_rate": 6.2074222668004e-05, + "loss": 1.2038, + "theoretical_loss": 3.3452196970176677, + "tokens_seen": 3047606272 + }, + { + "epoch": 1.3, + "learning_rate": 6.20581745235707e-05, + "loss": 1.1363, + "theoretical_loss": 3.34521423076945, + "tokens_seen": 3047671808 + }, + { + "epoch": 1.3, + "learning_rate": 6.20421263791374e-05, + "loss": 1.2525, + "theoretical_loss": 3.3452087646716873, + "tokens_seen": 3047737344 + }, + { + "epoch": 1.3, + "learning_rate": 6.202607823470411e-05, + "loss": 1.1538, + "theoretical_loss": 3.345203298724371, + "tokens_seen": 3047802880 + }, + { + "epoch": 1.3, + "learning_rate": 6.201003009027082e-05, + "loss": 1.2398, + "theoretical_loss": 3.345197832927495, + "tokens_seen": 3047868416 + }, + { + "epoch": 1.3, + "learning_rate": 6.199398194583752e-05, + "loss": 1.1399, + "theoretical_loss": 3.3451923672810513, + "tokens_seen": 3047933952 + }, + { + "epoch": 1.3, + "learning_rate": 6.197793380140422e-05, + "loss": 1.1066, + "theoretical_loss": 3.3451869017850324, + "tokens_seen": 3047999488 + }, + { + "epoch": 1.3, + "learning_rate": 6.196188565697092e-05, + "loss": 1.2553, + "theoretical_loss": 3.345181436439431, + "tokens_seen": 3048065024 + }, + { + "epoch": 1.3, + "learning_rate": 6.194583751253762e-05, + "loss": 1.2063, + "theoretical_loss": 3.34517597124424, + "tokens_seen": 3048130560 + }, + { + "epoch": 1.3, + "learning_rate": 6.192978936810432e-05, + "loss": 1.1758, + "theoretical_loss": 3.3451705061994517, + "tokens_seen": 3048196096 + }, + { + "epoch": 1.3, + "learning_rate": 6.191374122367101e-05, + "loss": 1.1528, + "theoretical_loss": 3.3451650413050587, + "tokens_seen": 3048261632 + }, + { + "epoch": 1.3, + "learning_rate": 6.189769307923771e-05, + "loss": 1.131, + "theoretical_loss": 3.345159576561054, + "tokens_seen": 3048327168 + }, + { + "epoch": 1.3, + "learning_rate": 6.188164493480441e-05, + "loss": 1.1606, + "theoretical_loss": 3.34515411196743, + "tokens_seen": 3048392704 + }, + { + "epoch": 1.3, + "learning_rate": 6.186559679037111e-05, + "loss": 1.2068, + "theoretical_loss": 3.345148647524179, + "tokens_seen": 3048458240 + }, + { + "epoch": 1.3, + "learning_rate": 6.184954864593781e-05, + "loss": 1.2296, + "theoretical_loss": 3.345143183231295, + "tokens_seen": 3048523776 + }, + { + "epoch": 1.3, + "learning_rate": 6.183350050150452e-05, + "loss": 1.1532, + "theoretical_loss": 3.345137719088769, + "tokens_seen": 3048589312 + }, + { + "epoch": 1.3, + "learning_rate": 6.181745235707122e-05, + "loss": 1.1979, + "theoretical_loss": 3.345132255096594, + "tokens_seen": 3048654848 + }, + { + "epoch": 1.3, + "learning_rate": 6.180140421263792e-05, + "loss": 1.2655, + "theoretical_loss": 3.3451267912547626, + "tokens_seen": 3048720384 + }, + { + "epoch": 1.3, + "learning_rate": 6.178535606820462e-05, + "loss": 1.1665, + "theoretical_loss": 3.3451213275632683, + "tokens_seen": 3048785920 + }, + { + "epoch": 1.3, + "learning_rate": 6.176930792377132e-05, + "loss": 1.2239, + "theoretical_loss": 3.345115864022103, + "tokens_seen": 3048851456 + }, + { + "epoch": 1.3, + "learning_rate": 6.175325977933802e-05, + "loss": 1.1393, + "theoretical_loss": 3.3451104006312598, + "tokens_seen": 3048916992 + }, + { + "epoch": 1.3, + "learning_rate": 6.173721163490472e-05, + "loss": 1.1401, + "theoretical_loss": 3.3451049373907304, + "tokens_seen": 3048982528 + }, + { + "epoch": 1.3, + "learning_rate": 6.17211634904714e-05, + "loss": 1.2815, + "theoretical_loss": 3.3450994743005085, + "tokens_seen": 3049048064 + }, + { + "epoch": 1.3, + "learning_rate": 6.170511534603812e-05, + "loss": 1.1322, + "theoretical_loss": 3.345094011360586, + "tokens_seen": 3049113600 + }, + { + "epoch": 1.3, + "learning_rate": 6.168906720160482e-05, + "loss": 1.1959, + "theoretical_loss": 3.345088548570956, + "tokens_seen": 3049179136 + }, + { + "epoch": 1.3, + "learning_rate": 6.167301905717152e-05, + "loss": 1.1122, + "theoretical_loss": 3.3450830859316105, + "tokens_seen": 3049244672 + }, + { + "epoch": 1.3, + "learning_rate": 6.165697091273822e-05, + "loss": 1.1324, + "theoretical_loss": 3.345077623442543, + "tokens_seen": 3049310208 + }, + { + "epoch": 1.3, + "learning_rate": 6.164092276830492e-05, + "loss": 1.2, + "theoretical_loss": 3.3450721611037455, + "tokens_seen": 3049375744 + }, + { + "epoch": 1.3, + "learning_rate": 6.162487462387162e-05, + "loss": 1.063, + "theoretical_loss": 3.345066698915211, + "tokens_seen": 3049441280 + }, + { + "epoch": 1.3, + "learning_rate": 6.160882647943832e-05, + "loss": 1.1577, + "theoretical_loss": 3.345061236876932, + "tokens_seen": 3049506816 + }, + { + "epoch": 1.3, + "learning_rate": 6.159277833500502e-05, + "loss": 1.1386, + "theoretical_loss": 3.345055774988901, + "tokens_seen": 3049572352 + }, + { + "epoch": 1.3, + "learning_rate": 6.157673019057173e-05, + "loss": 1.2491, + "theoretical_loss": 3.345050313251111, + "tokens_seen": 3049637888 + }, + { + "epoch": 1.3, + "learning_rate": 6.156068204613841e-05, + "loss": 1.2013, + "theoretical_loss": 3.345044851663554, + "tokens_seen": 3049703424 + }, + { + "epoch": 1.3, + "learning_rate": 6.154463390170511e-05, + "loss": 1.1463, + "theoretical_loss": 3.3450393902262237, + "tokens_seen": 3049768960 + }, + { + "epoch": 1.3, + "learning_rate": 6.152858575727181e-05, + "loss": 1.1802, + "theoretical_loss": 3.3450339289391113, + "tokens_seen": 3049834496 + }, + { + "epoch": 1.3, + "learning_rate": 6.151253761283851e-05, + "loss": 1.0877, + "theoretical_loss": 3.3450284678022104, + "tokens_seen": 3049900032 + }, + { + "epoch": 1.3, + "learning_rate": 6.149648946840521e-05, + "loss": 1.2263, + "theoretical_loss": 3.345023006815514, + "tokens_seen": 3049965568 + }, + { + "epoch": 1.3, + "learning_rate": 6.148044132397192e-05, + "loss": 1.1168, + "theoretical_loss": 3.345017545979014, + "tokens_seen": 3050031104 + }, + { + "epoch": 1.3, + "learning_rate": 6.146439317953862e-05, + "loss": 1.1479, + "theoretical_loss": 3.3450120852927028, + "tokens_seen": 3050096640 + }, + { + "epoch": 1.3, + "learning_rate": 6.144834503510532e-05, + "loss": 1.1946, + "theoretical_loss": 3.3450066247565737, + "tokens_seen": 3050162176 + }, + { + "epoch": 1.3, + "learning_rate": 6.143229689067202e-05, + "loss": 1.2051, + "theoretical_loss": 3.3450011643706192, + "tokens_seen": 3050227712 + }, + { + "epoch": 1.31, + "learning_rate": 6.141624874623872e-05, + "loss": 1.1875, + "theoretical_loss": 3.344995704134832, + "tokens_seen": 3050293248 + }, + { + "epoch": 1.31, + "learning_rate": 6.140020060180542e-05, + "loss": 1.0933, + "theoretical_loss": 3.3449902440492045, + "tokens_seen": 3050358784 + }, + { + "epoch": 1.31, + "learning_rate": 6.138415245737212e-05, + "loss": 1.1643, + "theoretical_loss": 3.3449847841137297, + "tokens_seen": 3050424320 + }, + { + "epoch": 1.31, + "learning_rate": 6.136810431293882e-05, + "loss": 1.2011, + "theoretical_loss": 3.3449793243283996, + "tokens_seen": 3050489856 + }, + { + "epoch": 1.31, + "learning_rate": 6.135205616850552e-05, + "loss": 1.2386, + "theoretical_loss": 3.3449738646932072, + "tokens_seen": 3050555392 + }, + { + "epoch": 1.31, + "learning_rate": 6.133600802407222e-05, + "loss": 1.181, + "theoretical_loss": 3.3449684052081454, + "tokens_seen": 3050620928 + }, + { + "epoch": 1.31, + "learning_rate": 6.131995987963892e-05, + "loss": 1.3015, + "theoretical_loss": 3.344962945873207, + "tokens_seen": 3050686464 + }, + { + "epoch": 1.31, + "learning_rate": 6.130391173520562e-05, + "loss": 1.1417, + "theoretical_loss": 3.3449574866883838, + "tokens_seen": 3050752000 + }, + { + "epoch": 1.31, + "learning_rate": 6.128786359077232e-05, + "loss": 1.2275, + "theoretical_loss": 3.344952027653669, + "tokens_seen": 3050817536 + }, + { + "epoch": 1.31, + "learning_rate": 6.127181544633902e-05, + "loss": 1.1502, + "theoretical_loss": 3.344946568769055, + "tokens_seen": 3050883072 + }, + { + "epoch": 1.31, + "learning_rate": 6.125576730190572e-05, + "loss": 1.1754, + "theoretical_loss": 3.344941110034535, + "tokens_seen": 3050948608 + }, + { + "epoch": 1.31, + "learning_rate": 6.123971915747242e-05, + "loss": 1.2585, + "theoretical_loss": 3.3449356514501014, + "tokens_seen": 3051014144 + }, + { + "epoch": 1.31, + "learning_rate": 6.122367101303913e-05, + "loss": 1.2568, + "theoretical_loss": 3.3449301930157467, + "tokens_seen": 3051079680 + }, + { + "epoch": 1.31, + "learning_rate": 6.120762286860581e-05, + "loss": 1.2189, + "theoretical_loss": 3.3449247347314635, + "tokens_seen": 3051145216 + }, + { + "epoch": 1.31, + "learning_rate": 6.119157472417251e-05, + "loss": 1.1667, + "theoretical_loss": 3.344919276597244, + "tokens_seen": 3051210752 + }, + { + "epoch": 1.31, + "learning_rate": 6.117552657973921e-05, + "loss": 1.1023, + "theoretical_loss": 3.3449138186130822, + "tokens_seen": 3051276288 + }, + { + "epoch": 1.31, + "learning_rate": 6.115947843530591e-05, + "loss": 1.1223, + "theoretical_loss": 3.3449083607789696, + "tokens_seen": 3051341824 + }, + { + "epoch": 1.31, + "learning_rate": 6.114343029087262e-05, + "loss": 1.2957, + "theoretical_loss": 3.344902903094899, + "tokens_seen": 3051407360 + }, + { + "epoch": 1.31, + "learning_rate": 6.112738214643932e-05, + "loss": 1.2518, + "theoretical_loss": 3.3448974455608633, + "tokens_seen": 3051472896 + }, + { + "epoch": 1.31, + "learning_rate": 6.111133400200602e-05, + "loss": 1.2236, + "theoretical_loss": 3.3448919881768555, + "tokens_seen": 3051538432 + }, + { + "epoch": 1.31, + "learning_rate": 6.109528585757273e-05, + "loss": 1.2259, + "theoretical_loss": 3.3448865309428677, + "tokens_seen": 3051603968 + }, + { + "epoch": 1.31, + "learning_rate": 6.107923771313943e-05, + "loss": 1.1571, + "theoretical_loss": 3.3448810738588923, + "tokens_seen": 3051669504 + }, + { + "epoch": 1.31, + "learning_rate": 6.106318956870612e-05, + "loss": 1.1888, + "theoretical_loss": 3.3448756169249227, + "tokens_seen": 3051735040 + }, + { + "epoch": 1.31, + "learning_rate": 6.104714142427282e-05, + "loss": 1.1631, + "theoretical_loss": 3.3448701601409514, + "tokens_seen": 3051800576 + }, + { + "epoch": 1.31, + "learning_rate": 6.1031093279839526e-05, + "loss": 1.1234, + "theoretical_loss": 3.3448647035069703, + "tokens_seen": 3051866112 + }, + { + "epoch": 1.31, + "learning_rate": 6.101504513540622e-05, + "loss": 1.1156, + "theoretical_loss": 3.344859247022973, + "tokens_seen": 3051931648 + }, + { + "epoch": 1.31, + "learning_rate": 6.099899699097292e-05, + "loss": 1.1779, + "theoretical_loss": 3.3448537906889517, + "tokens_seen": 3051997184 + }, + { + "epoch": 1.31, + "learning_rate": 6.098294884653962e-05, + "loss": 1.1689, + "theoretical_loss": 3.344848334504899, + "tokens_seen": 3052062720 + }, + { + "epoch": 1.31, + "learning_rate": 6.096690070210632e-05, + "loss": 1.3703, + "theoretical_loss": 3.3448428784708084, + "tokens_seen": 3052128256 + }, + { + "epoch": 1.31, + "learning_rate": 6.095085255767302e-05, + "loss": 1.1571, + "theoretical_loss": 3.344837422586671, + "tokens_seen": 3052193792 + }, + { + "epoch": 1.31, + "learning_rate": 6.0934804413239723e-05, + "loss": 1.1362, + "theoretical_loss": 3.344831966852481, + "tokens_seen": 3052259328 + }, + { + "epoch": 1.31, + "learning_rate": 6.091875626880642e-05, + "loss": 1.1735, + "theoretical_loss": 3.34482651126823, + "tokens_seen": 3052324864 + }, + { + "epoch": 1.31, + "learning_rate": 6.090270812437312e-05, + "loss": 1.2177, + "theoretical_loss": 3.344821055833911, + "tokens_seen": 3052390400 + }, + { + "epoch": 1.31, + "learning_rate": 6.088665997993982e-05, + "loss": 1.1628, + "theoretical_loss": 3.344815600549517, + "tokens_seen": 3052455936 + }, + { + "epoch": 1.31, + "learning_rate": 6.087061183550652e-05, + "loss": 1.1997, + "theoretical_loss": 3.34481014541504, + "tokens_seen": 3052521472 + }, + { + "epoch": 1.31, + "learning_rate": 6.085456369107322e-05, + "loss": 1.163, + "theoretical_loss": 3.344804690430473, + "tokens_seen": 3052587008 + }, + { + "epoch": 1.31, + "learning_rate": 6.083851554663992e-05, + "loss": 1.1796, + "theoretical_loss": 3.3447992355958087, + "tokens_seen": 3052652544 + }, + { + "epoch": 1.31, + "learning_rate": 6.0822467402206614e-05, + "loss": 1.1575, + "theoretical_loss": 3.34479378091104, + "tokens_seen": 3052718080 + }, + { + "epoch": 1.31, + "learning_rate": 6.080641925777333e-05, + "loss": 1.182, + "theoretical_loss": 3.3447883263761593, + "tokens_seen": 3052783616 + }, + { + "epoch": 1.31, + "learning_rate": 6.079037111334003e-05, + "loss": 1.2255, + "theoretical_loss": 3.344782871991159, + "tokens_seen": 3052849152 + }, + { + "epoch": 1.31, + "learning_rate": 6.077432296890673e-05, + "loss": 1.1893, + "theoretical_loss": 3.3447774177560325, + "tokens_seen": 3052914688 + }, + { + "epoch": 1.31, + "learning_rate": 6.0758274824473424e-05, + "loss": 1.158, + "theoretical_loss": 3.344771963670772, + "tokens_seen": 3052980224 + }, + { + "epoch": 1.31, + "learning_rate": 6.0742226680040124e-05, + "loss": 1.259, + "theoretical_loss": 3.3447665097353694, + "tokens_seen": 3053045760 + }, + { + "epoch": 1.31, + "learning_rate": 6.0726178535606825e-05, + "loss": 1.2592, + "theoretical_loss": 3.3447610559498187, + "tokens_seen": 3053111296 + }, + { + "epoch": 1.31, + "learning_rate": 6.0710130391173526e-05, + "loss": 1.0245, + "theoretical_loss": 3.344755602314112, + "tokens_seen": 3053176832 + }, + { + "epoch": 1.31, + "learning_rate": 6.0694082246740226e-05, + "loss": 1.1179, + "theoretical_loss": 3.344750148828242, + "tokens_seen": 3053242368 + }, + { + "epoch": 1.31, + "learning_rate": 6.067803410230693e-05, + "loss": 1.1548, + "theoretical_loss": 3.344744695492201, + "tokens_seen": 3053307904 + }, + { + "epoch": 1.31, + "learning_rate": 6.066198595787362e-05, + "loss": 1.104, + "theoretical_loss": 3.344739242305982, + "tokens_seen": 3053373440 + }, + { + "epoch": 1.31, + "learning_rate": 6.064593781344032e-05, + "loss": 1.2406, + "theoretical_loss": 3.3447337892695783, + "tokens_seen": 3053438976 + }, + { + "epoch": 1.31, + "learning_rate": 6.062988966900702e-05, + "loss": 1.2567, + "theoretical_loss": 3.3447283363829814, + "tokens_seen": 3053504512 + }, + { + "epoch": 1.31, + "learning_rate": 6.061384152457372e-05, + "loss": 1.2673, + "theoretical_loss": 3.3447228836461846, + "tokens_seen": 3053570048 + }, + { + "epoch": 1.31, + "learning_rate": 6.059779338014042e-05, + "loss": 1.2013, + "theoretical_loss": 3.3447174310591805, + "tokens_seen": 3053635584 + }, + { + "epoch": 1.31, + "learning_rate": 6.0581745235707124e-05, + "loss": 1.0619, + "theoretical_loss": 3.344711978621962, + "tokens_seen": 3053701120 + }, + { + "epoch": 1.31, + "learning_rate": 6.056569709127382e-05, + "loss": 1.2141, + "theoretical_loss": 3.3447065263345213, + "tokens_seen": 3053766656 + }, + { + "epoch": 1.31, + "learning_rate": 6.054964894684052e-05, + "loss": 1.2836, + "theoretical_loss": 3.344701074196851, + "tokens_seen": 3053832192 + }, + { + "epoch": 1.31, + "learning_rate": 6.053360080240722e-05, + "loss": 1.2401, + "theoretical_loss": 3.3446956222089446, + "tokens_seen": 3053897728 + }, + { + "epoch": 1.31, + "objective/train/docs_used": 1126782, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2358450889587402, + "objective/train/theoretical_loss": 3.344690170370794, + "objective/train/tokens_used": 3074423264, + "theoretical_loss": 3.344690170370794, + "tokens_seen": 3053963264 + }, + { + "epoch": 1.31, + "learning_rate": 6.051755265797392e-05, + "loss": 1.2358, + "theoretical_loss": 3.344690170370794, + "tokens_seen": 3053963264 + }, + { + "epoch": 1.31, + "learning_rate": 6.050150451354063e-05, + "loss": 1.0993, + "theoretical_loss": 3.344684718682392, + "tokens_seen": 3054028800 + }, + { + "epoch": 1.31, + "learning_rate": 6.048545636910733e-05, + "loss": 1.155, + "theoretical_loss": 3.3446792671437313, + "tokens_seen": 3054094336 + }, + { + "epoch": 1.31, + "learning_rate": 6.046940822467403e-05, + "loss": 1.2358, + "theoretical_loss": 3.344673815754805, + "tokens_seen": 3054159872 + }, + { + "epoch": 1.31, + "learning_rate": 6.045336008024073e-05, + "loss": 1.1551, + "theoretical_loss": 3.344668364515605, + "tokens_seen": 3054225408 + }, + { + "epoch": 1.31, + "learning_rate": 6.043731193580743e-05, + "loss": 1.1571, + "theoretical_loss": 3.3446629134261245, + "tokens_seen": 3054290944 + }, + { + "epoch": 1.31, + "learning_rate": 6.042126379137413e-05, + "loss": 1.246, + "theoretical_loss": 3.3446574624863565, + "tokens_seen": 3054356480 + }, + { + "epoch": 1.31, + "learning_rate": 6.0405215646940824e-05, + "loss": 1.2441, + "theoretical_loss": 3.344652011696293, + "tokens_seen": 3054422016 + }, + { + "epoch": 1.31, + "learning_rate": 6.0389167502507525e-05, + "loss": 1.1709, + "theoretical_loss": 3.3446465610559266, + "tokens_seen": 3054487552 + }, + { + "epoch": 1.31, + "learning_rate": 6.0373119358074225e-05, + "loss": 1.1466, + "theoretical_loss": 3.344641110565251, + "tokens_seen": 3054553088 + }, + { + "epoch": 1.31, + "learning_rate": 6.0357071213640926e-05, + "loss": 1.1308, + "theoretical_loss": 3.3446356602242577, + "tokens_seen": 3054618624 + }, + { + "epoch": 1.31, + "learning_rate": 6.0341023069207627e-05, + "loss": 1.1075, + "theoretical_loss": 3.34463021003294, + "tokens_seen": 3054684160 + }, + { + "epoch": 1.31, + "learning_rate": 6.032497492477433e-05, + "loss": 1.2122, + "theoretical_loss": 3.3446247599912904, + "tokens_seen": 3054749696 + }, + { + "epoch": 1.31, + "learning_rate": 6.030892678034102e-05, + "loss": 1.1331, + "theoretical_loss": 3.3446193100993016, + "tokens_seen": 3054815232 + }, + { + "epoch": 1.31, + "learning_rate": 6.029287863590772e-05, + "loss": 1.0968, + "theoretical_loss": 3.344613860356966, + "tokens_seen": 3054880768 + }, + { + "epoch": 1.31, + "learning_rate": 6.027683049147442e-05, + "loss": 1.0429, + "theoretical_loss": 3.3446084107642773, + "tokens_seen": 3054946304 + }, + { + "epoch": 1.31, + "learning_rate": 6.026078234704112e-05, + "loss": 1.1477, + "theoretical_loss": 3.3446029613212267, + "tokens_seen": 3055011840 + }, + { + "epoch": 1.31, + "learning_rate": 6.0244734202607824e-05, + "loss": 1.1895, + "theoretical_loss": 3.344597512027808, + "tokens_seen": 3055077376 + }, + { + "epoch": 1.31, + "learning_rate": 6.0228686058174524e-05, + "loss": 1.2411, + "theoretical_loss": 3.344592062884014, + "tokens_seen": 3055142912 + }, + { + "epoch": 1.31, + "learning_rate": 6.021263791374122e-05, + "loss": 1.1022, + "theoretical_loss": 3.3445866138898364, + "tokens_seen": 3055208448 + }, + { + "epoch": 1.31, + "learning_rate": 6.019658976930793e-05, + "loss": 1.1013, + "theoretical_loss": 3.3445811650452684, + "tokens_seen": 3055273984 + }, + { + "epoch": 1.31, + "learning_rate": 6.018054162487463e-05, + "loss": 1.1691, + "theoretical_loss": 3.3445757163503025, + "tokens_seen": 3055339520 + }, + { + "epoch": 1.31, + "learning_rate": 6.0164493480441334e-05, + "loss": 1.2483, + "theoretical_loss": 3.344570267804932, + "tokens_seen": 3055405056 + }, + { + "epoch": 1.31, + "learning_rate": 6.014844533600803e-05, + "loss": 1.0931, + "theoretical_loss": 3.3445648194091486, + "tokens_seen": 3055470592 + }, + { + "epoch": 1.31, + "learning_rate": 6.013239719157473e-05, + "loss": 1.1934, + "theoretical_loss": 3.344559371162946, + "tokens_seen": 3055536128 + }, + { + "epoch": 1.31, + "learning_rate": 6.011634904714143e-05, + "loss": 1.1678, + "theoretical_loss": 3.344553923066316, + "tokens_seen": 3055601664 + }, + { + "epoch": 1.31, + "learning_rate": 6.010030090270813e-05, + "loss": 1.2084, + "theoretical_loss": 3.3445484751192525, + "tokens_seen": 3055667200 + }, + { + "epoch": 1.31, + "learning_rate": 6.008425275827483e-05, + "loss": 1.1926, + "theoretical_loss": 3.3445430273217465, + "tokens_seen": 3055732736 + }, + { + "epoch": 1.31, + "learning_rate": 6.006820461384153e-05, + "loss": 1.1252, + "theoretical_loss": 3.344537579673792, + "tokens_seen": 3055798272 + }, + { + "epoch": 1.31, + "learning_rate": 6.0052156469408224e-05, + "loss": 1.226, + "theoretical_loss": 3.344532132175381, + "tokens_seen": 3055863808 + }, + { + "epoch": 1.31, + "learning_rate": 6.0036108324974925e-05, + "loss": 1.1345, + "theoretical_loss": 3.3445266848265067, + "tokens_seen": 3055929344 + }, + { + "epoch": 1.31, + "learning_rate": 6.0020060180541626e-05, + "loss": 1.1897, + "theoretical_loss": 3.344521237627162, + "tokens_seen": 3055994880 + }, + { + "epoch": 1.31, + "learning_rate": 6.0004012036108326e-05, + "loss": 1.2009, + "theoretical_loss": 3.344515790577338, + "tokens_seen": 3056060416 + }, + { + "epoch": 1.31, + "learning_rate": 5.998796389167503e-05, + "loss": 1.2204, + "theoretical_loss": 3.344510343677029, + "tokens_seen": 3056125952 + }, + { + "epoch": 1.31, + "learning_rate": 5.997191574724173e-05, + "loss": 0.9985, + "theoretical_loss": 3.3445048969262277, + "tokens_seen": 3056191488 + }, + { + "epoch": 1.31, + "learning_rate": 5.995586760280842e-05, + "loss": 1.118, + "theoretical_loss": 3.3444994503249257, + "tokens_seen": 3056257024 + }, + { + "epoch": 1.31, + "learning_rate": 5.993981945837512e-05, + "loss": 1.091, + "theoretical_loss": 3.3444940038731166, + "tokens_seen": 3056322560 + }, + { + "epoch": 1.31, + "learning_rate": 5.992377131394182e-05, + "loss": 1.1184, + "theoretical_loss": 3.3444885575707923, + "tokens_seen": 3056388096 + }, + { + "epoch": 1.31, + "learning_rate": 5.990772316950854e-05, + "loss": 1.1552, + "theoretical_loss": 3.3444831114179467, + "tokens_seen": 3056453632 + }, + { + "epoch": 1.31, + "learning_rate": 5.989167502507523e-05, + "loss": 1.129, + "theoretical_loss": 3.3444776654145714, + "tokens_seen": 3056519168 + }, + { + "epoch": 1.31, + "learning_rate": 5.987562688064193e-05, + "loss": 1.1144, + "theoretical_loss": 3.344472219560659, + "tokens_seen": 3056584704 + }, + { + "epoch": 1.31, + "learning_rate": 5.985957873620863e-05, + "loss": 1.1503, + "theoretical_loss": 3.3444667738562033, + "tokens_seen": 3056650240 + }, + { + "epoch": 1.31, + "learning_rate": 5.984353059177533e-05, + "loss": 1.086, + "theoretical_loss": 3.3444613283011964, + "tokens_seen": 3056715776 + }, + { + "epoch": 1.31, + "learning_rate": 5.982748244734203e-05, + "loss": 1.1542, + "theoretical_loss": 3.3444558828956303, + "tokens_seen": 3056781312 + }, + { + "epoch": 1.31, + "learning_rate": 5.9811434302908734e-05, + "loss": 1.1443, + "theoretical_loss": 3.344450437639499, + "tokens_seen": 3056846848 + }, + { + "epoch": 1.31, + "learning_rate": 5.979538615847543e-05, + "loss": 1.1774, + "theoretical_loss": 3.344444992532794, + "tokens_seen": 3056912384 + }, + { + "epoch": 1.31, + "learning_rate": 5.977933801404213e-05, + "loss": 1.2164, + "theoretical_loss": 3.344439547575509, + "tokens_seen": 3056977920 + }, + { + "epoch": 1.31, + "learning_rate": 5.976328986960883e-05, + "loss": 1.1552, + "theoretical_loss": 3.344434102767636, + "tokens_seen": 3057043456 + }, + { + "epoch": 1.31, + "learning_rate": 5.974724172517553e-05, + "loss": 1.2726, + "theoretical_loss": 3.344428658109168, + "tokens_seen": 3057108992 + }, + { + "epoch": 1.31, + "learning_rate": 5.973119358074223e-05, + "loss": 1.1344, + "theoretical_loss": 3.344423213600097, + "tokens_seen": 3057174528 + }, + { + "epoch": 1.31, + "learning_rate": 5.971514543630893e-05, + "loss": 1.1274, + "theoretical_loss": 3.344417769240417, + "tokens_seen": 3057240064 + }, + { + "epoch": 1.31, + "learning_rate": 5.9699097291875625e-05, + "loss": 1.1646, + "theoretical_loss": 3.34441232503012, + "tokens_seen": 3057305600 + }, + { + "epoch": 1.31, + "learning_rate": 5.9683049147442326e-05, + "loss": 1.2697, + "theoretical_loss": 3.3444068809691982, + "tokens_seen": 3057371136 + }, + { + "epoch": 1.31, + "learning_rate": 5.9667001003009026e-05, + "loss": 1.207, + "theoretical_loss": 3.344401437057645, + "tokens_seen": 3057436672 + }, + { + "epoch": 1.31, + "learning_rate": 5.965095285857573e-05, + "loss": 1.2243, + "theoretical_loss": 3.3443959932954535, + "tokens_seen": 3057502208 + }, + { + "epoch": 1.31, + "learning_rate": 5.963490471414243e-05, + "loss": 1.1144, + "theoretical_loss": 3.344390549682615, + "tokens_seen": 3057567744 + }, + { + "epoch": 1.31, + "learning_rate": 5.961885656970913e-05, + "loss": 1.133, + "theoretical_loss": 3.3443851062191237, + "tokens_seen": 3057633280 + }, + { + "epoch": 1.31, + "learning_rate": 5.9602808425275835e-05, + "loss": 1.1494, + "theoretical_loss": 3.344379662904971, + "tokens_seen": 3057698816 + }, + { + "epoch": 1.31, + "learning_rate": 5.9586760280842536e-05, + "loss": 1.1195, + "theoretical_loss": 3.3443742197401507, + "tokens_seen": 3057764352 + }, + { + "epoch": 1.31, + "learning_rate": 5.957071213640924e-05, + "loss": 1.2049, + "theoretical_loss": 3.3443687767246546, + "tokens_seen": 3057829888 + }, + { + "epoch": 1.31, + "learning_rate": 5.955466399197594e-05, + "loss": 1.162, + "theoretical_loss": 3.3443633338584764, + "tokens_seen": 3057895424 + }, + { + "epoch": 1.31, + "learning_rate": 5.953861584754263e-05, + "loss": 1.1999, + "theoretical_loss": 3.3443578911416076, + "tokens_seen": 3057960960 + }, + { + "epoch": 1.31, + "learning_rate": 5.952256770310933e-05, + "loss": 1.1394, + "theoretical_loss": 3.3443524485740417, + "tokens_seen": 3058026496 + }, + { + "epoch": 1.31, + "learning_rate": 5.950651955867603e-05, + "loss": 1.1689, + "theoretical_loss": 3.3443470061557714, + "tokens_seen": 3058092032 + }, + { + "epoch": 1.31, + "learning_rate": 5.949047141424273e-05, + "loss": 1.1698, + "theoretical_loss": 3.3443415638867893, + "tokens_seen": 3058157568 + }, + { + "epoch": 1.31, + "learning_rate": 5.9474423269809434e-05, + "loss": 1.1438, + "theoretical_loss": 3.3443361217670877, + "tokens_seen": 3058223104 + }, + { + "epoch": 1.31, + "learning_rate": 5.9458375125376134e-05, + "loss": 1.1491, + "theoretical_loss": 3.34433067979666, + "tokens_seen": 3058288640 + }, + { + "epoch": 1.31, + "learning_rate": 5.944232698094283e-05, + "loss": 1.1436, + "theoretical_loss": 3.3443252379754984, + "tokens_seen": 3058354176 + }, + { + "epoch": 1.31, + "learning_rate": 5.942627883650953e-05, + "loss": 1.1437, + "theoretical_loss": 3.344319796303596, + "tokens_seen": 3058419712 + }, + { + "epoch": 1.31, + "learning_rate": 5.941023069207623e-05, + "loss": 1.3255, + "theoretical_loss": 3.344314354780945, + "tokens_seen": 3058485248 + }, + { + "epoch": 1.31, + "learning_rate": 5.939418254764293e-05, + "loss": 1.1324, + "theoretical_loss": 3.3443089134075383, + "tokens_seen": 3058550784 + }, + { + "epoch": 1.31, + "learning_rate": 5.937813440320963e-05, + "loss": 1.1812, + "theoretical_loss": 3.344303472183369, + "tokens_seen": 3058616320 + }, + { + "epoch": 1.31, + "learning_rate": 5.936208625877633e-05, + "loss": 1.1511, + "theoretical_loss": 3.3442980311084294, + "tokens_seen": 3058681856 + }, + { + "epoch": 1.31, + "learning_rate": 5.9346038114343025e-05, + "loss": 1.1815, + "theoretical_loss": 3.344292590182712, + "tokens_seen": 3058747392 + }, + { + "epoch": 1.31, + "learning_rate": 5.9329989969909726e-05, + "loss": 1.2619, + "theoretical_loss": 3.3442871494062105, + "tokens_seen": 3058812928 + }, + { + "epoch": 1.31, + "learning_rate": 5.9313941825476427e-05, + "loss": 1.1288, + "theoretical_loss": 3.344281708778916, + "tokens_seen": 3058878464 + }, + { + "epoch": 1.31, + "learning_rate": 5.929789368104314e-05, + "loss": 1.1776, + "theoretical_loss": 3.344276268300823, + "tokens_seen": 3058944000 + }, + { + "epoch": 1.31, + "learning_rate": 5.9281845536609835e-05, + "loss": 1.2232, + "theoretical_loss": 3.344270827971923, + "tokens_seen": 3059009536 + }, + { + "epoch": 1.31, + "learning_rate": 5.9265797392176535e-05, + "loss": 1.1271, + "theoretical_loss": 3.344265387792209, + "tokens_seen": 3059075072 + }, + { + "epoch": 1.31, + "learning_rate": 5.9249749247743236e-05, + "loss": 1.1787, + "theoretical_loss": 3.3442599477616737, + "tokens_seen": 3059140608 + }, + { + "epoch": 1.31, + "learning_rate": 5.9233701103309936e-05, + "loss": 1.2345, + "theoretical_loss": 3.34425450788031, + "tokens_seen": 3059206144 + }, + { + "epoch": 1.31, + "learning_rate": 5.921765295887664e-05, + "loss": 1.1577, + "theoretical_loss": 3.3442490681481107, + "tokens_seen": 3059271680 + }, + { + "epoch": 1.31, + "learning_rate": 5.920160481444334e-05, + "loss": 1.2137, + "theoretical_loss": 3.344243628565068, + "tokens_seen": 3059337216 + }, + { + "epoch": 1.31, + "learning_rate": 5.918555667001003e-05, + "loss": 1.2255, + "theoretical_loss": 3.3442381891311754, + "tokens_seen": 3059402752 + }, + { + "epoch": 1.31, + "learning_rate": 5.916950852557673e-05, + "loss": 1.1986, + "theoretical_loss": 3.344232749846425, + "tokens_seen": 3059468288 + }, + { + "epoch": 1.31, + "learning_rate": 5.915346038114343e-05, + "loss": 1.1739, + "theoretical_loss": 3.3442273107108096, + "tokens_seen": 3059533824 + }, + { + "epoch": 1.31, + "learning_rate": 5.9137412236710134e-05, + "loss": 1.2031, + "theoretical_loss": 3.3442218717243217, + "tokens_seen": 3059599360 + }, + { + "epoch": 1.31, + "learning_rate": 5.9121364092276834e-05, + "loss": 1.2515, + "theoretical_loss": 3.3442164328869546, + "tokens_seen": 3059664896 + }, + { + "epoch": 1.31, + "learning_rate": 5.9105315947843535e-05, + "loss": 1.1737, + "theoretical_loss": 3.3442109941987006, + "tokens_seen": 3059730432 + }, + { + "epoch": 1.31, + "learning_rate": 5.908926780341023e-05, + "loss": 1.2746, + "theoretical_loss": 3.3442055556595527, + "tokens_seen": 3059795968 + }, + { + "epoch": 1.31, + "learning_rate": 5.907321965897693e-05, + "loss": 1.3262, + "theoretical_loss": 3.3442001172695033, + "tokens_seen": 3059861504 + }, + { + "epoch": 1.31, + "learning_rate": 5.905717151454363e-05, + "loss": 1.1615, + "theoretical_loss": 3.3441946790285453, + "tokens_seen": 3059927040 + }, + { + "epoch": 1.31, + "learning_rate": 5.904112337011033e-05, + "loss": 1.2977, + "theoretical_loss": 3.3441892409366716, + "tokens_seen": 3059992576 + }, + { + "epoch": 1.31, + "learning_rate": 5.902507522567703e-05, + "loss": 1.1987, + "theoretical_loss": 3.3441838029938746, + "tokens_seen": 3060058112 + }, + { + "epoch": 1.31, + "learning_rate": 5.900902708124373e-05, + "loss": 1.2066, + "theoretical_loss": 3.344178365200147, + "tokens_seen": 3060123648 + }, + { + "epoch": 1.31, + "learning_rate": 5.899297893681044e-05, + "loss": 1.1527, + "theoretical_loss": 3.3441729275554817, + "tokens_seen": 3060189184 + }, + { + "epoch": 1.31, + "learning_rate": 5.897693079237714e-05, + "loss": 1.1577, + "theoretical_loss": 3.3441674900598715, + "tokens_seen": 3060254720 + }, + { + "epoch": 1.31, + "learning_rate": 5.896088264794384e-05, + "loss": 1.1106, + "theoretical_loss": 3.344162052713309, + "tokens_seen": 3060320256 + }, + { + "epoch": 1.31, + "learning_rate": 5.8944834503510534e-05, + "loss": 1.09, + "theoretical_loss": 3.344156615515787, + "tokens_seen": 3060385792 + }, + { + "epoch": 1.31, + "learning_rate": 5.8928786359077235e-05, + "loss": 1.1337, + "theoretical_loss": 3.344151178467298, + "tokens_seen": 3060451328 + }, + { + "epoch": 1.31, + "objective/train/docs_used": 1129376, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.184858798980713, + "objective/train/theoretical_loss": 3.344145741567835, + "objective/train/tokens_used": 3080976864, + "theoretical_loss": 3.344145741567835, + "tokens_seen": 3060516864 + }, + { + "epoch": 1.31, + "learning_rate": 5.8912738214643936e-05, + "loss": 1.1849, + "theoretical_loss": 3.344145741567835, + "tokens_seen": 3060516864 + }, + { + "epoch": 1.31, + "learning_rate": 5.8896690070210636e-05, + "loss": 1.2169, + "theoretical_loss": 3.3441403048173903, + "tokens_seen": 3060582400 + }, + { + "epoch": 1.31, + "learning_rate": 5.888064192577734e-05, + "loss": 1.2423, + "theoretical_loss": 3.3441348682159573, + "tokens_seen": 3060647936 + }, + { + "epoch": 1.31, + "learning_rate": 5.886459378134404e-05, + "loss": 1.246, + "theoretical_loss": 3.344129431763528, + "tokens_seen": 3060713472 + }, + { + "epoch": 1.31, + "learning_rate": 5.884854563691073e-05, + "loss": 1.2201, + "theoretical_loss": 3.344123995460096, + "tokens_seen": 3060779008 + }, + { + "epoch": 1.31, + "learning_rate": 5.883249749247743e-05, + "loss": 1.1889, + "theoretical_loss": 3.344118559305653, + "tokens_seen": 3060844544 + }, + { + "epoch": 1.31, + "learning_rate": 5.881644934804413e-05, + "loss": 1.2164, + "theoretical_loss": 3.3441131233001924, + "tokens_seen": 3060910080 + }, + { + "epoch": 1.31, + "learning_rate": 5.880040120361083e-05, + "loss": 1.1501, + "theoretical_loss": 3.3441076874437066, + "tokens_seen": 3060975616 + }, + { + "epoch": 1.31, + "learning_rate": 5.8784353059177534e-05, + "loss": 1.1643, + "theoretical_loss": 3.3441022517361887, + "tokens_seen": 3061041152 + }, + { + "epoch": 1.31, + "learning_rate": 5.8768304914744235e-05, + "loss": 1.1082, + "theoretical_loss": 3.344096816177631, + "tokens_seen": 3061106688 + }, + { + "epoch": 1.31, + "learning_rate": 5.875225677031093e-05, + "loss": 1.1876, + "theoretical_loss": 3.3440913807680266, + "tokens_seen": 3061172224 + }, + { + "epoch": 1.31, + "learning_rate": 5.873620862587763e-05, + "loss": 1.2506, + "theoretical_loss": 3.344085945507368, + "tokens_seen": 3061237760 + }, + { + "epoch": 1.31, + "learning_rate": 5.872016048144433e-05, + "loss": 1.1532, + "theoretical_loss": 3.3440805103956484, + "tokens_seen": 3061303296 + }, + { + "epoch": 1.31, + "learning_rate": 5.8704112337011044e-05, + "loss": 1.1654, + "theoretical_loss": 3.3440750754328596, + "tokens_seen": 3061368832 + }, + { + "epoch": 1.31, + "learning_rate": 5.868806419257774e-05, + "loss": 1.0702, + "theoretical_loss": 3.344069640618995, + "tokens_seen": 3061434368 + }, + { + "epoch": 1.31, + "learning_rate": 5.867201604814444e-05, + "loss": 1.2379, + "theoretical_loss": 3.344064205954047, + "tokens_seen": 3061499904 + }, + { + "epoch": 1.31, + "learning_rate": 5.865596790371114e-05, + "loss": 1.1231, + "theoretical_loss": 3.344058771438009, + "tokens_seen": 3061565440 + }, + { + "epoch": 1.31, + "learning_rate": 5.863991975927784e-05, + "loss": 1.1551, + "theoretical_loss": 3.344053337070873, + "tokens_seen": 3061630976 + }, + { + "epoch": 1.31, + "learning_rate": 5.862387161484454e-05, + "loss": 1.1479, + "theoretical_loss": 3.344047902852632, + "tokens_seen": 3061696512 + }, + { + "epoch": 1.31, + "learning_rate": 5.860782347041124e-05, + "loss": 1.1337, + "theoretical_loss": 3.344042468783279, + "tokens_seen": 3061762048 + }, + { + "epoch": 1.31, + "learning_rate": 5.8591775325977935e-05, + "loss": 1.3078, + "theoretical_loss": 3.3440370348628057, + "tokens_seen": 3061827584 + }, + { + "epoch": 1.31, + "learning_rate": 5.8575727181544635e-05, + "loss": 1.1481, + "theoretical_loss": 3.3440316010912063, + "tokens_seen": 3061893120 + }, + { + "epoch": 1.31, + "learning_rate": 5.8559679037111336e-05, + "loss": 1.2534, + "theoretical_loss": 3.3440261674684724, + "tokens_seen": 3061958656 + }, + { + "epoch": 1.31, + "learning_rate": 5.854363089267804e-05, + "loss": 1.2519, + "theoretical_loss": 3.3440207339945975, + "tokens_seen": 3062024192 + }, + { + "epoch": 1.31, + "learning_rate": 5.852758274824474e-05, + "loss": 1.3394, + "theoretical_loss": 3.344015300669574, + "tokens_seen": 3062089728 + }, + { + "epoch": 1.31, + "learning_rate": 5.851153460381144e-05, + "loss": 1.2627, + "theoretical_loss": 3.3440098674933942, + "tokens_seen": 3062155264 + }, + { + "epoch": 1.31, + "learning_rate": 5.849548645937813e-05, + "loss": 1.1145, + "theoretical_loss": 3.3440044344660516, + "tokens_seen": 3062220800 + }, + { + "epoch": 1.31, + "learning_rate": 5.847943831494483e-05, + "loss": 1.221, + "theoretical_loss": 3.3439990015875387, + "tokens_seen": 3062286336 + }, + { + "epoch": 1.31, + "learning_rate": 5.846339017051153e-05, + "loss": 1.2454, + "theoretical_loss": 3.343993568857848, + "tokens_seen": 3062351872 + }, + { + "epoch": 1.31, + "learning_rate": 5.8447342026078234e-05, + "loss": 1.1502, + "theoretical_loss": 3.3439881362769723, + "tokens_seen": 3062417408 + }, + { + "epoch": 1.31, + "learning_rate": 5.8431293881644934e-05, + "loss": 1.1222, + "theoretical_loss": 3.3439827038449046, + "tokens_seen": 3062482944 + }, + { + "epoch": 1.31, + "learning_rate": 5.8415245737211635e-05, + "loss": 1.1007, + "theoretical_loss": 3.3439772715616374, + "tokens_seen": 3062548480 + }, + { + "epoch": 1.31, + "learning_rate": 5.839919759277834e-05, + "loss": 1.1718, + "theoretical_loss": 3.343971839427163, + "tokens_seen": 3062614016 + }, + { + "epoch": 1.31, + "learning_rate": 5.838314944834504e-05, + "loss": 1.1087, + "theoretical_loss": 3.343966407441475, + "tokens_seen": 3062679552 + }, + { + "epoch": 1.31, + "learning_rate": 5.8367101303911744e-05, + "loss": 1.1592, + "theoretical_loss": 3.3439609756045665, + "tokens_seen": 3062745088 + }, + { + "epoch": 1.31, + "learning_rate": 5.8351053159478444e-05, + "loss": 1.1399, + "theoretical_loss": 3.3439555439164286, + "tokens_seen": 3062810624 + }, + { + "epoch": 1.31, + "learning_rate": 5.833500501504514e-05, + "loss": 1.1588, + "theoretical_loss": 3.3439501123770556, + "tokens_seen": 3062876160 + }, + { + "epoch": 1.31, + "learning_rate": 5.831895687061184e-05, + "loss": 1.1967, + "theoretical_loss": 3.3439446809864393, + "tokens_seen": 3062941696 + }, + { + "epoch": 1.31, + "learning_rate": 5.830290872617854e-05, + "loss": 1.1158, + "theoretical_loss": 3.343939249744573, + "tokens_seen": 3063007232 + }, + { + "epoch": 1.31, + "learning_rate": 5.828686058174524e-05, + "loss": 1.2204, + "theoretical_loss": 3.3439338186514487, + "tokens_seen": 3063072768 + }, + { + "epoch": 1.31, + "learning_rate": 5.827081243731194e-05, + "loss": 1.1144, + "theoretical_loss": 3.34392838770706, + "tokens_seen": 3063138304 + }, + { + "epoch": 1.31, + "learning_rate": 5.825476429287864e-05, + "loss": 1.3029, + "theoretical_loss": 3.343922956911399, + "tokens_seen": 3063203840 + }, + { + "epoch": 1.31, + "learning_rate": 5.8238716148445335e-05, + "loss": 1.2615, + "theoretical_loss": 3.343917526264459, + "tokens_seen": 3063269376 + }, + { + "epoch": 1.31, + "learning_rate": 5.8222668004012036e-05, + "loss": 1.098, + "theoretical_loss": 3.3439120957662327, + "tokens_seen": 3063334912 + }, + { + "epoch": 1.31, + "learning_rate": 5.8206619859578736e-05, + "loss": 1.1544, + "theoretical_loss": 3.3439066654167124, + "tokens_seen": 3063400448 + }, + { + "epoch": 1.31, + "learning_rate": 5.819057171514544e-05, + "loss": 1.1499, + "theoretical_loss": 3.343901235215891, + "tokens_seen": 3063465984 + }, + { + "epoch": 1.31, + "learning_rate": 5.817452357071214e-05, + "loss": 1.2314, + "theoretical_loss": 3.3438958051637613, + "tokens_seen": 3063531520 + }, + { + "epoch": 1.31, + "learning_rate": 5.815847542627884e-05, + "loss": 1.0675, + "theoretical_loss": 3.343890375260316, + "tokens_seen": 3063597056 + }, + { + "epoch": 1.31, + "learning_rate": 5.814242728184553e-05, + "loss": 1.1443, + "theoretical_loss": 3.343884945505548, + "tokens_seen": 3063662592 + }, + { + "epoch": 1.31, + "learning_rate": 5.812637913741223e-05, + "loss": 1.1314, + "theoretical_loss": 3.34387951589945, + "tokens_seen": 3063728128 + }, + { + "epoch": 1.31, + "learning_rate": 5.8110330992978933e-05, + "loss": 1.1137, + "theoretical_loss": 3.343874086442015, + "tokens_seen": 3063793664 + }, + { + "epoch": 1.31, + "learning_rate": 5.809428284854565e-05, + "loss": 1.2297, + "theoretical_loss": 3.343868657133235, + "tokens_seen": 3063859200 + }, + { + "epoch": 1.31, + "learning_rate": 5.807823470411234e-05, + "loss": 1.167, + "theoretical_loss": 3.3438632279731033, + "tokens_seen": 3063924736 + }, + { + "epoch": 1.31, + "learning_rate": 5.806218655967904e-05, + "loss": 1.1396, + "theoretical_loss": 3.3438577989616123, + "tokens_seen": 3063990272 + }, + { + "epoch": 1.31, + "learning_rate": 5.804613841524574e-05, + "loss": 1.1871, + "theoretical_loss": 3.3438523700987557, + "tokens_seen": 3064055808 + }, + { + "epoch": 1.31, + "learning_rate": 5.8030090270812443e-05, + "loss": 1.0695, + "theoretical_loss": 3.343846941384525, + "tokens_seen": 3064121344 + }, + { + "epoch": 1.31, + "learning_rate": 5.8014042126379144e-05, + "loss": 1.2308, + "theoretical_loss": 3.343841512818914, + "tokens_seen": 3064186880 + }, + { + "epoch": 1.31, + "learning_rate": 5.7997993981945845e-05, + "loss": 1.1437, + "theoretical_loss": 3.3438360844019144, + "tokens_seen": 3064252416 + }, + { + "epoch": 1.31, + "learning_rate": 5.798194583751254e-05, + "loss": 1.2108, + "theoretical_loss": 3.34383065613352, + "tokens_seen": 3064317952 + }, + { + "epoch": 1.31, + "learning_rate": 5.796589769307924e-05, + "loss": 1.1755, + "theoretical_loss": 3.343825228013723, + "tokens_seen": 3064383488 + }, + { + "epoch": 1.31, + "learning_rate": 5.794984954864594e-05, + "loss": 1.215, + "theoretical_loss": 3.343819800042516, + "tokens_seen": 3064449024 + }, + { + "epoch": 1.31, + "learning_rate": 5.793380140421264e-05, + "loss": 1.0744, + "theoretical_loss": 3.3438143722198923, + "tokens_seen": 3064514560 + }, + { + "epoch": 1.31, + "learning_rate": 5.791775325977934e-05, + "loss": 1.1757, + "theoretical_loss": 3.3438089445458443, + "tokens_seen": 3064580096 + }, + { + "epoch": 1.31, + "learning_rate": 5.790170511534604e-05, + "loss": 1.135, + "theoretical_loss": 3.3438035170203646, + "tokens_seen": 3064645632 + }, + { + "epoch": 1.31, + "learning_rate": 5.7885656970912736e-05, + "loss": 1.2141, + "theoretical_loss": 3.343798089643446, + "tokens_seen": 3064711168 + }, + { + "epoch": 1.31, + "learning_rate": 5.7869608826479436e-05, + "loss": 1.172, + "theoretical_loss": 3.3437926624150816, + "tokens_seen": 3064776704 + }, + { + "epoch": 1.31, + "learning_rate": 5.785356068204614e-05, + "loss": 1.2437, + "theoretical_loss": 3.3437872353352645, + "tokens_seen": 3064842240 + }, + { + "epoch": 1.31, + "learning_rate": 5.783751253761284e-05, + "loss": 1.2019, + "theoretical_loss": 3.3437818084039863, + "tokens_seen": 3064907776 + }, + { + "epoch": 1.31, + "learning_rate": 5.782146439317954e-05, + "loss": 1.1846, + "theoretical_loss": 3.3437763816212405, + "tokens_seen": 3064973312 + }, + { + "epoch": 1.31, + "learning_rate": 5.7805416248746246e-05, + "loss": 1.2501, + "theoretical_loss": 3.34377095498702, + "tokens_seen": 3065038848 + }, + { + "epoch": 1.31, + "learning_rate": 5.7789368104312946e-05, + "loss": 1.1504, + "theoretical_loss": 3.343765528501317, + "tokens_seen": 3065104384 + }, + { + "epoch": 1.31, + "learning_rate": 5.777331995987965e-05, + "loss": 1.1547, + "theoretical_loss": 3.343760102164125, + "tokens_seen": 3065169920 + }, + { + "epoch": 1.31, + "learning_rate": 5.775727181544635e-05, + "loss": 1.1964, + "theoretical_loss": 3.3437546759754353, + "tokens_seen": 3065235456 + }, + { + "epoch": 1.31, + "learning_rate": 5.774122367101305e-05, + "loss": 1.2003, + "theoretical_loss": 3.3437492499352426, + "tokens_seen": 3065300992 + }, + { + "epoch": 1.31, + "learning_rate": 5.772517552657974e-05, + "loss": 1.1487, + "theoretical_loss": 3.343743824043538, + "tokens_seen": 3065366528 + }, + { + "epoch": 1.31, + "learning_rate": 5.770912738214644e-05, + "loss": 1.1377, + "theoretical_loss": 3.343738398300316, + "tokens_seen": 3065432064 + }, + { + "epoch": 1.31, + "learning_rate": 5.769307923771314e-05, + "loss": 1.0892, + "theoretical_loss": 3.343732972705568, + "tokens_seen": 3065497600 + }, + { + "epoch": 1.31, + "learning_rate": 5.7677031093279844e-05, + "loss": 1.1325, + "theoretical_loss": 3.343727547259287, + "tokens_seen": 3065563136 + }, + { + "epoch": 1.31, + "learning_rate": 5.7660982948846544e-05, + "loss": 1.2257, + "theoretical_loss": 3.343722121961466, + "tokens_seen": 3065628672 + }, + { + "epoch": 1.31, + "learning_rate": 5.7644934804413245e-05, + "loss": 1.2225, + "theoretical_loss": 3.3437166968120975, + "tokens_seen": 3065694208 + }, + { + "epoch": 1.31, + "learning_rate": 5.762888665997994e-05, + "loss": 1.1887, + "theoretical_loss": 3.3437112718111743, + "tokens_seen": 3065759744 + }, + { + "epoch": 1.31, + "learning_rate": 5.761283851554664e-05, + "loss": 1.254, + "theoretical_loss": 3.3437058469586898, + "tokens_seen": 3065825280 + }, + { + "epoch": 1.31, + "learning_rate": 5.759679037111334e-05, + "loss": 1.2464, + "theoretical_loss": 3.343700422254636, + "tokens_seen": 3065890816 + }, + { + "epoch": 1.31, + "learning_rate": 5.758074222668004e-05, + "loss": 1.1276, + "theoretical_loss": 3.343694997699006, + "tokens_seen": 3065956352 + }, + { + "epoch": 1.31, + "learning_rate": 5.756469408224674e-05, + "loss": 1.1308, + "theoretical_loss": 3.343689573291792, + "tokens_seen": 3066021888 + }, + { + "epoch": 1.31, + "learning_rate": 5.754864593781344e-05, + "loss": 1.1925, + "theoretical_loss": 3.343684149032988, + "tokens_seen": 3066087424 + }, + { + "epoch": 1.31, + "learning_rate": 5.7532597793380136e-05, + "loss": 1.2842, + "theoretical_loss": 3.3436787249225857, + "tokens_seen": 3066152960 + }, + { + "epoch": 1.31, + "learning_rate": 5.751654964894684e-05, + "loss": 1.1633, + "theoretical_loss": 3.343673300960578, + "tokens_seen": 3066218496 + }, + { + "epoch": 1.31, + "learning_rate": 5.750050150451355e-05, + "loss": 1.1561, + "theoretical_loss": 3.343667877146958, + "tokens_seen": 3066284032 + }, + { + "epoch": 1.31, + "learning_rate": 5.748445336008025e-05, + "loss": 1.1083, + "theoretical_loss": 3.3436624534817185, + "tokens_seen": 3066349568 + }, + { + "epoch": 1.31, + "learning_rate": 5.7468405215646945e-05, + "loss": 1.2722, + "theoretical_loss": 3.343657029964852, + "tokens_seen": 3066415104 + }, + { + "epoch": 1.31, + "learning_rate": 5.7452357071213646e-05, + "loss": 1.1962, + "theoretical_loss": 3.3436516065963513, + "tokens_seen": 3066480640 + }, + { + "epoch": 1.31, + "learning_rate": 5.7436308926780347e-05, + "loss": 1.1257, + "theoretical_loss": 3.343646183376209, + "tokens_seen": 3066546176 + }, + { + "epoch": 1.31, + "learning_rate": 5.742026078234705e-05, + "loss": 1.1116, + "theoretical_loss": 3.3436407603044187, + "tokens_seen": 3066611712 + }, + { + "epoch": 1.31, + "learning_rate": 5.740421263791375e-05, + "loss": 1.1744, + "theoretical_loss": 3.343635337380972, + "tokens_seen": 3066677248 + }, + { + "epoch": 1.31, + "learning_rate": 5.738816449348045e-05, + "loss": 1.1676, + "theoretical_loss": 3.3436299146058626, + "tokens_seen": 3066742784 + }, + { + "epoch": 1.31, + "learning_rate": 5.737211634904714e-05, + "loss": 1.2061, + "theoretical_loss": 3.343624491979083, + "tokens_seen": 3066808320 + }, + { + "epoch": 1.31, + "learning_rate": 5.735606820461384e-05, + "loss": 1.1707, + "theoretical_loss": 3.3436190695006256, + "tokens_seen": 3066873856 + }, + { + "epoch": 1.31, + "learning_rate": 5.7340020060180544e-05, + "loss": 1.1787, + "theoretical_loss": 3.343613647170484, + "tokens_seen": 3066939392 + }, + { + "epoch": 1.31, + "learning_rate": 5.7323971915747244e-05, + "loss": 1.2094, + "theoretical_loss": 3.3436082249886496, + "tokens_seen": 3067004928 + }, + { + "epoch": 1.31, + "objective/train/docs_used": 1131735, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1784160137176514, + "objective/train/theoretical_loss": 3.343602802955117, + "objective/train/tokens_used": 3087530464, + "theoretical_loss": 3.343602802955117, + "tokens_seen": 3067070464 + }, + { + "epoch": 1.31, + "learning_rate": 5.7307923771313945e-05, + "loss": 1.1784, + "theoretical_loss": 3.343602802955117, + "tokens_seen": 3067070464 + }, + { + "epoch": 1.31, + "learning_rate": 5.7291875626880646e-05, + "loss": 1.1269, + "theoretical_loss": 3.3435973810698774, + "tokens_seen": 3067136000 + }, + { + "epoch": 1.31, + "learning_rate": 5.727582748244734e-05, + "loss": 1.1308, + "theoretical_loss": 3.343591959332924, + "tokens_seen": 3067201536 + }, + { + "epoch": 1.31, + "learning_rate": 5.725977933801404e-05, + "loss": 1.1629, + "theoretical_loss": 3.34358653774425, + "tokens_seen": 3067267072 + }, + { + "epoch": 1.31, + "learning_rate": 5.724373119358074e-05, + "loss": 1.1404, + "theoretical_loss": 3.3435811163038482, + "tokens_seen": 3067332608 + }, + { + "epoch": 1.31, + "learning_rate": 5.722768304914744e-05, + "loss": 1.2088, + "theoretical_loss": 3.343575695011711, + "tokens_seen": 3067398144 + }, + { + "epoch": 1.31, + "learning_rate": 5.721163490471414e-05, + "loss": 1.2173, + "theoretical_loss": 3.3435702738678312, + "tokens_seen": 3067463680 + }, + { + "epoch": 1.31, + "learning_rate": 5.719558676028085e-05, + "loss": 1.1367, + "theoretical_loss": 3.3435648528722015, + "tokens_seen": 3067529216 + }, + { + "epoch": 1.31, + "learning_rate": 5.717953861584755e-05, + "loss": 1.1216, + "theoretical_loss": 3.343559432024815, + "tokens_seen": 3067594752 + }, + { + "epoch": 1.31, + "learning_rate": 5.716349047141425e-05, + "loss": 1.1737, + "theoretical_loss": 3.343554011325664, + "tokens_seen": 3067660288 + }, + { + "epoch": 1.31, + "learning_rate": 5.714744232698095e-05, + "loss": 1.2129, + "theoretical_loss": 3.343548590774742, + "tokens_seen": 3067725824 + }, + { + "epoch": 1.31, + "learning_rate": 5.7131394182547645e-05, + "loss": 1.2517, + "theoretical_loss": 3.3435431703720413, + "tokens_seen": 3067791360 + }, + { + "epoch": 1.31, + "learning_rate": 5.7115346038114346e-05, + "loss": 1.1775, + "theoretical_loss": 3.3435377501175547, + "tokens_seen": 3067856896 + }, + { + "epoch": 1.31, + "learning_rate": 5.7099297893681046e-05, + "loss": 1.1672, + "theoretical_loss": 3.343532330011275, + "tokens_seen": 3067922432 + }, + { + "epoch": 1.31, + "learning_rate": 5.708324974924775e-05, + "loss": 1.2577, + "theoretical_loss": 3.3435269100531952, + "tokens_seen": 3067987968 + }, + { + "epoch": 1.31, + "learning_rate": 5.706720160481445e-05, + "loss": 1.2129, + "theoretical_loss": 3.3435214902433077, + "tokens_seen": 3068053504 + }, + { + "epoch": 1.31, + "learning_rate": 5.705115346038115e-05, + "loss": 1.1458, + "theoretical_loss": 3.3435160705816056, + "tokens_seen": 3068119040 + }, + { + "epoch": 1.31, + "learning_rate": 5.703510531594784e-05, + "loss": 1.1792, + "theoretical_loss": 3.3435106510680814, + "tokens_seen": 3068184576 + }, + { + "epoch": 1.31, + "learning_rate": 5.701905717151454e-05, + "loss": 1.1859, + "theoretical_loss": 3.343505231702728, + "tokens_seen": 3068250112 + }, + { + "epoch": 1.31, + "learning_rate": 5.700300902708124e-05, + "loss": 1.0868, + "theoretical_loss": 3.343499812485539, + "tokens_seen": 3068315648 + }, + { + "epoch": 1.31, + "learning_rate": 5.6986960882647944e-05, + "loss": 1.2024, + "theoretical_loss": 3.3434943934165053, + "tokens_seen": 3068381184 + }, + { + "epoch": 1.31, + "learning_rate": 5.6970912738214645e-05, + "loss": 1.1636, + "theoretical_loss": 3.3434889744956218, + "tokens_seen": 3068446720 + }, + { + "epoch": 1.31, + "learning_rate": 5.6954864593781345e-05, + "loss": 1.1273, + "theoretical_loss": 3.3434835557228797, + "tokens_seen": 3068512256 + }, + { + "epoch": 1.31, + "learning_rate": 5.693881644934804e-05, + "loss": 1.2004, + "theoretical_loss": 3.3434781370982725, + "tokens_seen": 3068577792 + }, + { + "epoch": 1.31, + "learning_rate": 5.692276830491474e-05, + "loss": 1.2308, + "theoretical_loss": 3.343472718621793, + "tokens_seen": 3068643328 + }, + { + "epoch": 1.31, + "learning_rate": 5.6906720160481454e-05, + "loss": 1.1432, + "theoretical_loss": 3.3434673002934336, + "tokens_seen": 3068708864 + }, + { + "epoch": 1.31, + "learning_rate": 5.6890672016048155e-05, + "loss": 1.2274, + "theoretical_loss": 3.3434618821131874, + "tokens_seen": 3068774400 + }, + { + "epoch": 1.31, + "learning_rate": 5.687462387161485e-05, + "loss": 1.1987, + "theoretical_loss": 3.343456464081047, + "tokens_seen": 3068839936 + }, + { + "epoch": 1.31, + "learning_rate": 5.685857572718155e-05, + "loss": 1.0499, + "theoretical_loss": 3.3434510461970057, + "tokens_seen": 3068905472 + }, + { + "epoch": 1.31, + "learning_rate": 5.684252758274825e-05, + "loss": 1.1666, + "theoretical_loss": 3.343445628461056, + "tokens_seen": 3068971008 + }, + { + "epoch": 1.31, + "learning_rate": 5.682647943831495e-05, + "loss": 1.2318, + "theoretical_loss": 3.34344021087319, + "tokens_seen": 3069036544 + }, + { + "epoch": 1.31, + "learning_rate": 5.681043129388165e-05, + "loss": 1.0337, + "theoretical_loss": 3.343434793433402, + "tokens_seen": 3069102080 + }, + { + "epoch": 1.31, + "learning_rate": 5.679438314944835e-05, + "loss": 1.0648, + "theoretical_loss": 3.343429376141683, + "tokens_seen": 3069167616 + }, + { + "epoch": 1.31, + "learning_rate": 5.6778335005015045e-05, + "loss": 1.0535, + "theoretical_loss": 3.3434239589980272, + "tokens_seen": 3069233152 + }, + { + "epoch": 1.31, + "learning_rate": 5.6762286860581746e-05, + "loss": 1.1416, + "theoretical_loss": 3.3434185420024267, + "tokens_seen": 3069298688 + }, + { + "epoch": 1.31, + "learning_rate": 5.674623871614845e-05, + "loss": 1.2068, + "theoretical_loss": 3.343413125154874, + "tokens_seen": 3069364224 + }, + { + "epoch": 1.31, + "learning_rate": 5.673019057171515e-05, + "loss": 1.205, + "theoretical_loss": 3.343407708455363, + "tokens_seen": 3069429760 + }, + { + "epoch": 1.31, + "learning_rate": 5.671414242728185e-05, + "loss": 1.218, + "theoretical_loss": 3.3434022919038857, + "tokens_seen": 3069495296 + }, + { + "epoch": 1.31, + "learning_rate": 5.669809428284855e-05, + "loss": 1.2135, + "theoretical_loss": 3.343396875500435, + "tokens_seen": 3069560832 + }, + { + "epoch": 1.31, + "learning_rate": 5.668204613841524e-05, + "loss": 1.1506, + "theoretical_loss": 3.3433914592450034, + "tokens_seen": 3069626368 + }, + { + "epoch": 1.31, + "learning_rate": 5.666599799398194e-05, + "loss": 1.2009, + "theoretical_loss": 3.343386043137585, + "tokens_seen": 3069691904 + }, + { + "epoch": 1.31, + "learning_rate": 5.6649949849548644e-05, + "loss": 1.1717, + "theoretical_loss": 3.3433806271781705, + "tokens_seen": 3069757440 + }, + { + "epoch": 1.31, + "learning_rate": 5.6633901705115344e-05, + "loss": 1.1958, + "theoretical_loss": 3.3433752113667543, + "tokens_seen": 3069822976 + }, + { + "epoch": 1.31, + "learning_rate": 5.6617853560682045e-05, + "loss": 1.1105, + "theoretical_loss": 3.3433697957033286, + "tokens_seen": 3069888512 + }, + { + "epoch": 1.31, + "learning_rate": 5.660180541624875e-05, + "loss": 1.1891, + "theoretical_loss": 3.3433643801878863, + "tokens_seen": 3069954048 + }, + { + "epoch": 1.31, + "learning_rate": 5.658575727181545e-05, + "loss": 1.1309, + "theoretical_loss": 3.3433589648204203, + "tokens_seen": 3070019584 + }, + { + "epoch": 1.31, + "learning_rate": 5.6569709127382154e-05, + "loss": 1.123, + "theoretical_loss": 3.343353549600923, + "tokens_seen": 3070085120 + }, + { + "epoch": 1.31, + "learning_rate": 5.6553660982948854e-05, + "loss": 1.2268, + "theoretical_loss": 3.343348134529388, + "tokens_seen": 3070150656 + }, + { + "epoch": 1.31, + "learning_rate": 5.6537612838515555e-05, + "loss": 1.2111, + "theoretical_loss": 3.3433427196058076, + "tokens_seen": 3070216192 + }, + { + "epoch": 1.31, + "learning_rate": 5.652156469408225e-05, + "loss": 1.299, + "theoretical_loss": 3.343337304830174, + "tokens_seen": 3070281728 + }, + { + "epoch": 1.31, + "learning_rate": 5.650551654964895e-05, + "loss": 1.2294, + "theoretical_loss": 3.343331890202481, + "tokens_seen": 3070347264 + }, + { + "epoch": 1.31, + "learning_rate": 5.648946840521565e-05, + "loss": 1.1998, + "theoretical_loss": 3.343326475722721, + "tokens_seen": 3070412800 + }, + { + "epoch": 1.31, + "learning_rate": 5.647342026078235e-05, + "loss": 1.1242, + "theoretical_loss": 3.343321061390887, + "tokens_seen": 3070478336 + }, + { + "epoch": 1.31, + "learning_rate": 5.645737211634905e-05, + "loss": 1.199, + "theoretical_loss": 3.343315647206971, + "tokens_seen": 3070543872 + }, + { + "epoch": 1.31, + "learning_rate": 5.644132397191575e-05, + "loss": 1.1521, + "theoretical_loss": 3.3433102331709668, + "tokens_seen": 3070609408 + }, + { + "epoch": 1.31, + "learning_rate": 5.6425275827482446e-05, + "loss": 1.1454, + "theoretical_loss": 3.343304819282867, + "tokens_seen": 3070674944 + }, + { + "epoch": 1.31, + "learning_rate": 5.6409227683049147e-05, + "loss": 1.1102, + "theoretical_loss": 3.3432994055426644, + "tokens_seen": 3070740480 + }, + { + "epoch": 1.31, + "learning_rate": 5.639317953861585e-05, + "loss": 1.131, + "theoretical_loss": 3.343293991950351, + "tokens_seen": 3070806016 + }, + { + "epoch": 1.31, + "learning_rate": 5.637713139418255e-05, + "loss": 1.1669, + "theoretical_loss": 3.3432885785059203, + "tokens_seen": 3070871552 + }, + { + "epoch": 1.31, + "learning_rate": 5.636108324974925e-05, + "loss": 1.2233, + "theoretical_loss": 3.3432831652093653, + "tokens_seen": 3070937088 + }, + { + "epoch": 1.31, + "learning_rate": 5.634503510531595e-05, + "loss": 1.1124, + "theoretical_loss": 3.3432777520606782, + "tokens_seen": 3071002624 + }, + { + "epoch": 1.31, + "learning_rate": 5.632898696088264e-05, + "loss": 1.1571, + "theoretical_loss": 3.3432723390598524, + "tokens_seen": 3071068160 + }, + { + "epoch": 1.31, + "learning_rate": 5.6312938816449344e-05, + "loss": 1.1467, + "theoretical_loss": 3.3432669262068804, + "tokens_seen": 3071133696 + }, + { + "epoch": 1.31, + "learning_rate": 5.629689067201606e-05, + "loss": 1.2129, + "theoretical_loss": 3.343261513501755, + "tokens_seen": 3071199232 + }, + { + "epoch": 1.31, + "learning_rate": 5.628084252758276e-05, + "loss": 1.1871, + "theoretical_loss": 3.343256100944469, + "tokens_seen": 3071264768 + }, + { + "epoch": 1.31, + "learning_rate": 5.626479438314945e-05, + "loss": 1.2065, + "theoretical_loss": 3.3432506885350155, + "tokens_seen": 3071330304 + }, + { + "epoch": 1.31, + "learning_rate": 5.624874623871615e-05, + "loss": 1.1821, + "theoretical_loss": 3.343245276273387, + "tokens_seen": 3071395840 + }, + { + "epoch": 1.31, + "learning_rate": 5.6232698094282854e-05, + "loss": 1.1567, + "theoretical_loss": 3.3432398641595764, + "tokens_seen": 3071461376 + }, + { + "epoch": 1.31, + "learning_rate": 5.6216649949849554e-05, + "loss": 1.1403, + "theoretical_loss": 3.343234452193576, + "tokens_seen": 3071526912 + }, + { + "epoch": 1.31, + "learning_rate": 5.6200601805416255e-05, + "loss": 1.1952, + "theoretical_loss": 3.3432290403753795, + "tokens_seen": 3071592448 + }, + { + "epoch": 1.31, + "learning_rate": 5.6184553660982955e-05, + "loss": 1.2125, + "theoretical_loss": 3.3432236287049792, + "tokens_seen": 3071657984 + }, + { + "epoch": 1.31, + "learning_rate": 5.616850551654965e-05, + "loss": 1.1492, + "theoretical_loss": 3.3432182171823683, + "tokens_seen": 3071723520 + }, + { + "epoch": 1.31, + "learning_rate": 5.615245737211635e-05, + "loss": 1.0999, + "theoretical_loss": 3.3432128058075388, + "tokens_seen": 3071789056 + }, + { + "epoch": 1.31, + "learning_rate": 5.613640922768305e-05, + "loss": 1.1264, + "theoretical_loss": 3.3432073945804843, + "tokens_seen": 3071854592 + }, + { + "epoch": 1.31, + "learning_rate": 5.612036108324975e-05, + "loss": 1.1382, + "theoretical_loss": 3.3432019835011975, + "tokens_seen": 3071920128 + }, + { + "epoch": 1.31, + "learning_rate": 5.610431293881645e-05, + "loss": 1.2387, + "theoretical_loss": 3.343196572569671, + "tokens_seen": 3071985664 + }, + { + "epoch": 1.31, + "learning_rate": 5.608826479438315e-05, + "loss": 1.2073, + "theoretical_loss": 3.3431911617858976, + "tokens_seen": 3072051200 + }, + { + "epoch": 1.31, + "learning_rate": 5.6072216649949846e-05, + "loss": 1.1491, + "theoretical_loss": 3.3431857511498704, + "tokens_seen": 3072116736 + }, + { + "epoch": 1.31, + "learning_rate": 5.605616850551655e-05, + "loss": 1.1695, + "theoretical_loss": 3.343180340661582, + "tokens_seen": 3072182272 + }, + { + "epoch": 1.31, + "learning_rate": 5.604012036108325e-05, + "loss": 1.1633, + "theoretical_loss": 3.343174930321025, + "tokens_seen": 3072247808 + }, + { + "epoch": 1.31, + "learning_rate": 5.602407221664995e-05, + "loss": 1.1489, + "theoretical_loss": 3.343169520128192, + "tokens_seen": 3072313344 + }, + { + "epoch": 1.31, + "learning_rate": 5.600802407221665e-05, + "loss": 1.1107, + "theoretical_loss": 3.3431641100830767, + "tokens_seen": 3072378880 + }, + { + "epoch": 1.31, + "learning_rate": 5.5991975927783356e-05, + "loss": 1.2705, + "theoretical_loss": 3.343158700185672, + "tokens_seen": 3072444416 + }, + { + "epoch": 1.31, + "learning_rate": 5.597592778335006e-05, + "loss": 1.0567, + "theoretical_loss": 3.3431532904359695, + "tokens_seen": 3072509952 + }, + { + "epoch": 1.31, + "learning_rate": 5.595987963891676e-05, + "loss": 1.1648, + "theoretical_loss": 3.3431478808339627, + "tokens_seen": 3072575488 + }, + { + "epoch": 1.31, + "learning_rate": 5.594383149448346e-05, + "loss": 1.2161, + "theoretical_loss": 3.3431424713796445, + "tokens_seen": 3072641024 + }, + { + "epoch": 1.31, + "learning_rate": 5.592778335005016e-05, + "loss": 1.1237, + "theoretical_loss": 3.3431370620730076, + "tokens_seen": 3072706560 + }, + { + "epoch": 1.31, + "learning_rate": 5.591173520561685e-05, + "loss": 1.1524, + "theoretical_loss": 3.343131652914045, + "tokens_seen": 3072772096 + }, + { + "epoch": 1.31, + "learning_rate": 5.589568706118355e-05, + "loss": 1.2059, + "theoretical_loss": 3.3431262439027494, + "tokens_seen": 3072837632 + }, + { + "epoch": 1.31, + "learning_rate": 5.5879638916750254e-05, + "loss": 1.0794, + "theoretical_loss": 3.3431208350391137, + "tokens_seen": 3072903168 + }, + { + "epoch": 1.31, + "learning_rate": 5.5863590772316955e-05, + "loss": 1.1476, + "theoretical_loss": 3.3431154263231306, + "tokens_seen": 3072968704 + }, + { + "epoch": 1.31, + "learning_rate": 5.5847542627883655e-05, + "loss": 1.1225, + "theoretical_loss": 3.3431100177547926, + "tokens_seen": 3073034240 + }, + { + "epoch": 1.31, + "learning_rate": 5.5831494483450356e-05, + "loss": 1.2457, + "theoretical_loss": 3.343104609334093, + "tokens_seen": 3073099776 + }, + { + "epoch": 1.31, + "learning_rate": 5.581544633901705e-05, + "loss": 1.1807, + "theoretical_loss": 3.3430992010610243, + "tokens_seen": 3073165312 + }, + { + "epoch": 1.31, + "learning_rate": 5.579939819458375e-05, + "loss": 1.2599, + "theoretical_loss": 3.34309379293558, + "tokens_seen": 3073230848 + }, + { + "epoch": 1.31, + "learning_rate": 5.578335005015045e-05, + "loss": 1.1871, + "theoretical_loss": 3.343088384957752, + "tokens_seen": 3073296384 + }, + { + "epoch": 1.31, + "learning_rate": 5.576730190571715e-05, + "loss": 1.1285, + "theoretical_loss": 3.3430829771275334, + "tokens_seen": 3073361920 + }, + { + "epoch": 1.31, + "learning_rate": 5.575125376128385e-05, + "loss": 1.1241, + "theoretical_loss": 3.343077569444918, + "tokens_seen": 3073427456 + }, + { + "epoch": 1.31, + "learning_rate": 5.573520561685055e-05, + "loss": 1.1659, + "theoretical_loss": 3.343072161909897, + "tokens_seen": 3073492992 + }, + { + "epoch": 1.31, + "learning_rate": 5.571915747241725e-05, + "loss": 1.2394, + "theoretical_loss": 3.343066754522464, + "tokens_seen": 3073558528 + }, + { + "epoch": 1.31, + "objective/train/docs_used": 1134040, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.010815978050232, + "objective/train/theoretical_loss": 3.343061347282612, + "objective/train/tokens_used": 3094084064, + "theoretical_loss": 3.343061347282612, + "tokens_seen": 3073624064 + }, + { + "epoch": 1.31, + "learning_rate": 5.570310932798396e-05, + "loss": 1.0108, + "theoretical_loss": 3.343061347282612, + "tokens_seen": 3073624064 + }, + { + "epoch": 1.31, + "learning_rate": 5.568706118355066e-05, + "loss": 1.0869, + "theoretical_loss": 3.343055940190334, + "tokens_seen": 3073689600 + }, + { + "epoch": 1.31, + "learning_rate": 5.567101303911736e-05, + "loss": 1.1983, + "theoretical_loss": 3.343050533245622, + "tokens_seen": 3073755136 + }, + { + "epoch": 1.31, + "learning_rate": 5.5654964894684056e-05, + "loss": 1.2011, + "theoretical_loss": 3.3430451264484695, + "tokens_seen": 3073820672 + }, + { + "epoch": 1.31, + "learning_rate": 5.563891675025076e-05, + "loss": 1.1577, + "theoretical_loss": 3.3430397197988695, + "tokens_seen": 3073886208 + }, + { + "epoch": 1.31, + "learning_rate": 5.562286860581746e-05, + "loss": 1.118, + "theoretical_loss": 3.343034313296814, + "tokens_seen": 3073951744 + }, + { + "epoch": 1.31, + "learning_rate": 5.560682046138416e-05, + "loss": 1.2263, + "theoretical_loss": 3.3430289069422963, + "tokens_seen": 3074017280 + }, + { + "epoch": 1.31, + "learning_rate": 5.559077231695086e-05, + "loss": 1.1049, + "theoretical_loss": 3.3430235007353093, + "tokens_seen": 3074082816 + }, + { + "epoch": 1.31, + "learning_rate": 5.557472417251756e-05, + "loss": 1.1246, + "theoretical_loss": 3.3430180946758457, + "tokens_seen": 3074148352 + }, + { + "epoch": 1.31, + "learning_rate": 5.555867602808425e-05, + "loss": 1.2154, + "theoretical_loss": 3.3430126887638987, + "tokens_seen": 3074213888 + }, + { + "epoch": 1.31, + "learning_rate": 5.5542627883650954e-05, + "loss": 1.2769, + "theoretical_loss": 3.343007282999461, + "tokens_seen": 3074279424 + }, + { + "epoch": 1.31, + "learning_rate": 5.5526579739217654e-05, + "loss": 1.2202, + "theoretical_loss": 3.3430018773825245, + "tokens_seen": 3074344960 + }, + { + "epoch": 1.31, + "learning_rate": 5.5510531594784355e-05, + "loss": 1.2386, + "theoretical_loss": 3.342996471913083, + "tokens_seen": 3074410496 + }, + { + "epoch": 1.31, + "learning_rate": 5.5494483450351056e-05, + "loss": 1.1957, + "theoretical_loss": 3.3429910665911295, + "tokens_seen": 3074476032 + }, + { + "epoch": 1.31, + "learning_rate": 5.5478435305917756e-05, + "loss": 1.1551, + "theoretical_loss": 3.342985661416656, + "tokens_seen": 3074541568 + }, + { + "epoch": 1.31, + "learning_rate": 5.546238716148445e-05, + "loss": 1.1568, + "theoretical_loss": 3.342980256389656, + "tokens_seen": 3074607104 + }, + { + "epoch": 1.31, + "learning_rate": 5.544633901705115e-05, + "loss": 1.1256, + "theoretical_loss": 3.3429748515101223, + "tokens_seen": 3074672640 + }, + { + "epoch": 1.31, + "learning_rate": 5.543029087261785e-05, + "loss": 1.1744, + "theoretical_loss": 3.3429694467780475, + "tokens_seen": 3074738176 + }, + { + "epoch": 1.31, + "learning_rate": 5.541424272818455e-05, + "loss": 1.2193, + "theoretical_loss": 3.342964042193424, + "tokens_seen": 3074803712 + }, + { + "epoch": 1.31, + "learning_rate": 5.539819458375126e-05, + "loss": 1.2073, + "theoretical_loss": 3.3429586377562455, + "tokens_seen": 3074869248 + }, + { + "epoch": 1.31, + "learning_rate": 5.538214643931796e-05, + "loss": 1.1225, + "theoretical_loss": 3.3429532334665044, + "tokens_seen": 3074934784 + }, + { + "epoch": 1.31, + "learning_rate": 5.536609829488466e-05, + "loss": 1.1451, + "theoretical_loss": 3.3429478293241934, + "tokens_seen": 3075000320 + }, + { + "epoch": 1.31, + "learning_rate": 5.535005015045136e-05, + "loss": 1.1168, + "theoretical_loss": 3.342942425329306, + "tokens_seen": 3075065856 + }, + { + "epoch": 1.31, + "learning_rate": 5.533400200601806e-05, + "loss": 1.2182, + "theoretical_loss": 3.342937021481834, + "tokens_seen": 3075131392 + }, + { + "epoch": 1.31, + "learning_rate": 5.531795386158476e-05, + "loss": 1.1772, + "theoretical_loss": 3.342931617781771, + "tokens_seen": 3075196928 + }, + { + "epoch": 1.31, + "learning_rate": 5.5301905717151456e-05, + "loss": 1.1397, + "theoretical_loss": 3.3429262142291094, + "tokens_seen": 3075262464 + }, + { + "epoch": 1.31, + "learning_rate": 5.528585757271816e-05, + "loss": 1.197, + "theoretical_loss": 3.3429208108238426, + "tokens_seen": 3075328000 + }, + { + "epoch": 1.31, + "learning_rate": 5.526980942828486e-05, + "loss": 1.1268, + "theoretical_loss": 3.342915407565963, + "tokens_seen": 3075393536 + }, + { + "epoch": 1.31, + "learning_rate": 5.525376128385156e-05, + "loss": 1.0859, + "theoretical_loss": 3.342910004455464, + "tokens_seen": 3075459072 + }, + { + "epoch": 1.31, + "learning_rate": 5.523771313941826e-05, + "loss": 1.2051, + "theoretical_loss": 3.342904601492337, + "tokens_seen": 3075524608 + }, + { + "epoch": 1.31, + "learning_rate": 5.522166499498496e-05, + "loss": 1.2034, + "theoretical_loss": 3.3428991986765766, + "tokens_seen": 3075590144 + }, + { + "epoch": 1.31, + "learning_rate": 5.5205616850551653e-05, + "loss": 1.1307, + "theoretical_loss": 3.3428937960081746, + "tokens_seen": 3075655680 + }, + { + "epoch": 1.31, + "learning_rate": 5.5189568706118354e-05, + "loss": 1.1827, + "theoretical_loss": 3.342888393487124, + "tokens_seen": 3075721216 + }, + { + "epoch": 1.31, + "learning_rate": 5.5173520561685055e-05, + "loss": 1.1263, + "theoretical_loss": 3.342882991113418, + "tokens_seen": 3075786752 + }, + { + "epoch": 1.31, + "learning_rate": 5.5157472417251755e-05, + "loss": 1.2035, + "theoretical_loss": 3.342877588887049, + "tokens_seen": 3075852288 + }, + { + "epoch": 1.31, + "learning_rate": 5.5141424272818456e-05, + "loss": 1.1322, + "theoretical_loss": 3.34287218680801, + "tokens_seen": 3075917824 + }, + { + "epoch": 1.31, + "learning_rate": 5.512537612838516e-05, + "loss": 1.2698, + "theoretical_loss": 3.342866784876294, + "tokens_seen": 3075983360 + }, + { + "epoch": 1.31, + "learning_rate": 5.510932798395185e-05, + "loss": 1.206, + "theoretical_loss": 3.342861383091894, + "tokens_seen": 3076048896 + }, + { + "epoch": 1.31, + "learning_rate": 5.5093279839518565e-05, + "loss": 1.114, + "theoretical_loss": 3.342855981454802, + "tokens_seen": 3076114432 + }, + { + "epoch": 1.31, + "learning_rate": 5.5077231695085265e-05, + "loss": 1.1132, + "theoretical_loss": 3.3428505799650114, + "tokens_seen": 3076179968 + }, + { + "epoch": 1.31, + "learning_rate": 5.506118355065196e-05, + "loss": 1.1745, + "theoretical_loss": 3.3428451786225155, + "tokens_seen": 3076245504 + }, + { + "epoch": 1.31, + "learning_rate": 5.504513540621866e-05, + "loss": 1.1739, + "theoretical_loss": 3.342839777427306, + "tokens_seen": 3076311040 + }, + { + "epoch": 1.31, + "learning_rate": 5.502908726178536e-05, + "loss": 1.2355, + "theoretical_loss": 3.3428343763793773, + "tokens_seen": 3076376576 + }, + { + "epoch": 1.31, + "learning_rate": 5.501303911735206e-05, + "loss": 1.138, + "theoretical_loss": 3.3428289754787213, + "tokens_seen": 3076442112 + }, + { + "epoch": 1.31, + "learning_rate": 5.499699097291876e-05, + "loss": 1.0409, + "theoretical_loss": 3.3428235747253305, + "tokens_seen": 3076507648 + }, + { + "epoch": 1.31, + "learning_rate": 5.498094282848546e-05, + "loss": 1.2659, + "theoretical_loss": 3.3428181741191985, + "tokens_seen": 3076573184 + }, + { + "epoch": 1.31, + "learning_rate": 5.4964894684052156e-05, + "loss": 1.0729, + "theoretical_loss": 3.3428127736603175, + "tokens_seen": 3076638720 + }, + { + "epoch": 1.31, + "learning_rate": 5.494884653961886e-05, + "loss": 1.1975, + "theoretical_loss": 3.342807373348681, + "tokens_seen": 3076704256 + }, + { + "epoch": 1.31, + "learning_rate": 5.493279839518556e-05, + "loss": 1.1931, + "theoretical_loss": 3.3428019731842813, + "tokens_seen": 3076769792 + }, + { + "epoch": 1.31, + "learning_rate": 5.491675025075226e-05, + "loss": 1.2492, + "theoretical_loss": 3.342796573167112, + "tokens_seen": 3076835328 + }, + { + "epoch": 1.31, + "learning_rate": 5.490070210631896e-05, + "loss": 1.1229, + "theoretical_loss": 3.3427911732971647, + "tokens_seen": 3076900864 + }, + { + "epoch": 1.31, + "learning_rate": 5.488465396188566e-05, + "loss": 1.2009, + "theoretical_loss": 3.342785773574433, + "tokens_seen": 3076966400 + }, + { + "epoch": 1.31, + "learning_rate": 5.486860581745235e-05, + "loss": 1.1383, + "theoretical_loss": 3.3427803739989104, + "tokens_seen": 3077031936 + }, + { + "epoch": 1.31, + "learning_rate": 5.4852557673019054e-05, + "loss": 1.1852, + "theoretical_loss": 3.3427749745705886, + "tokens_seen": 3077097472 + }, + { + "epoch": 1.31, + "learning_rate": 5.4836509528585755e-05, + "loss": 1.0829, + "theoretical_loss": 3.342769575289461, + "tokens_seen": 3077163008 + }, + { + "epoch": 1.31, + "learning_rate": 5.4820461384152455e-05, + "loss": 1.1492, + "theoretical_loss": 3.3427641761555202, + "tokens_seen": 3077228544 + }, + { + "epoch": 1.31, + "learning_rate": 5.480441323971916e-05, + "loss": 1.1371, + "theoretical_loss": 3.3427587771687595, + "tokens_seen": 3077294080 + }, + { + "epoch": 1.31, + "learning_rate": 5.478836509528586e-05, + "loss": 1.2252, + "theoretical_loss": 3.3427533783291716, + "tokens_seen": 3077359616 + }, + { + "epoch": 1.31, + "learning_rate": 5.4772316950852564e-05, + "loss": 1.2374, + "theoretical_loss": 3.342747979636749, + "tokens_seen": 3077425152 + }, + { + "epoch": 1.31, + "learning_rate": 5.4756268806419264e-05, + "loss": 1.1805, + "theoretical_loss": 3.342742581091485, + "tokens_seen": 3077490688 + }, + { + "epoch": 1.31, + "learning_rate": 5.4740220661985965e-05, + "loss": 1.1346, + "theoretical_loss": 3.342737182693372, + "tokens_seen": 3077556224 + }, + { + "epoch": 1.31, + "learning_rate": 5.4724172517552666e-05, + "loss": 1.2135, + "theoretical_loss": 3.3427317844424036, + "tokens_seen": 3077621760 + }, + { + "epoch": 1.31, + "learning_rate": 5.470812437311936e-05, + "loss": 1.0694, + "theoretical_loss": 3.3427263863385717, + "tokens_seen": 3077687296 + }, + { + "epoch": 1.31, + "learning_rate": 5.469207622868606e-05, + "loss": 1.2511, + "theoretical_loss": 3.3427209883818696, + "tokens_seen": 3077752832 + }, + { + "epoch": 1.31, + "learning_rate": 5.467602808425276e-05, + "loss": 1.3415, + "theoretical_loss": 3.3427155905722907, + "tokens_seen": 3077818368 + }, + { + "epoch": 1.31, + "learning_rate": 5.465997993981946e-05, + "loss": 1.079, + "theoretical_loss": 3.342710192909827, + "tokens_seen": 3077883904 + }, + { + "epoch": 1.31, + "learning_rate": 5.464393179538616e-05, + "loss": 1.1849, + "theoretical_loss": 3.3427047953944715, + "tokens_seen": 3077949440 + }, + { + "epoch": 1.31, + "learning_rate": 5.462788365095286e-05, + "loss": 1.2228, + "theoretical_loss": 3.342699398026218, + "tokens_seen": 3078014976 + }, + { + "epoch": 1.31, + "learning_rate": 5.461183550651956e-05, + "loss": 1.1232, + "theoretical_loss": 3.3426940008050576, + "tokens_seen": 3078080512 + }, + { + "epoch": 1.31, + "learning_rate": 5.459578736208626e-05, + "loss": 1.1581, + "theoretical_loss": 3.3426886037309846, + "tokens_seen": 3078146048 + }, + { + "epoch": 1.31, + "learning_rate": 5.457973921765296e-05, + "loss": 1.1299, + "theoretical_loss": 3.342683206803992, + "tokens_seen": 3078211584 + }, + { + "epoch": 1.31, + "learning_rate": 5.456369107321966e-05, + "loss": 1.2468, + "theoretical_loss": 3.342677810024071, + "tokens_seen": 3078277120 + }, + { + "epoch": 1.31, + "learning_rate": 5.454764292878636e-05, + "loss": 1.1502, + "theoretical_loss": 3.3426724133912162, + "tokens_seen": 3078342656 + }, + { + "epoch": 1.31, + "learning_rate": 5.453159478435306e-05, + "loss": 1.2123, + "theoretical_loss": 3.34266701690542, + "tokens_seen": 3078408192 + }, + { + "epoch": 1.31, + "learning_rate": 5.4515546639919754e-05, + "loss": 1.236, + "theoretical_loss": 3.342661620566675, + "tokens_seen": 3078473728 + }, + { + "epoch": 1.31, + "learning_rate": 5.449949849548647e-05, + "loss": 1.084, + "theoretical_loss": 3.342656224374974, + "tokens_seen": 3078539264 + }, + { + "epoch": 1.31, + "learning_rate": 5.448345035105317e-05, + "loss": 1.1139, + "theoretical_loss": 3.34265082833031, + "tokens_seen": 3078604800 + }, + { + "epoch": 1.31, + "learning_rate": 5.446740220661987e-05, + "loss": 1.2139, + "theoretical_loss": 3.3426454324326755, + "tokens_seen": 3078670336 + }, + { + "epoch": 1.31, + "learning_rate": 5.445135406218656e-05, + "loss": 1.0583, + "theoretical_loss": 3.3426400366820643, + "tokens_seen": 3078735872 + }, + { + "epoch": 1.31, + "learning_rate": 5.4435305917753264e-05, + "loss": 1.1785, + "theoretical_loss": 3.342634641078469, + "tokens_seen": 3078801408 + }, + { + "epoch": 1.31, + "learning_rate": 5.4419257773319964e-05, + "loss": 1.2047, + "theoretical_loss": 3.342629245621881, + "tokens_seen": 3078866944 + }, + { + "epoch": 1.31, + "learning_rate": 5.4403209628886665e-05, + "loss": 1.2294, + "theoretical_loss": 3.3426238503122954, + "tokens_seen": 3078932480 + }, + { + "epoch": 1.31, + "learning_rate": 5.4387161484453365e-05, + "loss": 1.2034, + "theoretical_loss": 3.3426184551497036, + "tokens_seen": 3078998016 + }, + { + "epoch": 1.31, + "learning_rate": 5.4371113340020066e-05, + "loss": 1.1708, + "theoretical_loss": 3.3426130601340986, + "tokens_seen": 3079063552 + }, + { + "epoch": 1.31, + "learning_rate": 5.435506519558676e-05, + "loss": 1.1294, + "theoretical_loss": 3.342607665265474, + "tokens_seen": 3079129088 + }, + { + "epoch": 1.31, + "learning_rate": 5.433901705115346e-05, + "loss": 1.1618, + "theoretical_loss": 3.342602270543822, + "tokens_seen": 3079194624 + }, + { + "epoch": 1.31, + "learning_rate": 5.432296890672016e-05, + "loss": 1.1723, + "theoretical_loss": 3.342596875969136, + "tokens_seen": 3079260160 + }, + { + "epoch": 1.31, + "learning_rate": 5.430692076228686e-05, + "loss": 1.098, + "theoretical_loss": 3.3425914815414077, + "tokens_seen": 3079325696 + }, + { + "epoch": 1.31, + "learning_rate": 5.429087261785356e-05, + "loss": 1.1815, + "theoretical_loss": 3.3425860872606314, + "tokens_seen": 3079391232 + }, + { + "epoch": 1.31, + "learning_rate": 5.427482447342026e-05, + "loss": 1.1867, + "theoretical_loss": 3.3425806931267994, + "tokens_seen": 3079456768 + }, + { + "epoch": 1.31, + "learning_rate": 5.425877632898696e-05, + "loss": 1.1177, + "theoretical_loss": 3.342575299139904, + "tokens_seen": 3079522304 + }, + { + "epoch": 1.31, + "learning_rate": 5.424272818455366e-05, + "loss": 1.147, + "theoretical_loss": 3.342569905299939, + "tokens_seen": 3079587840 + }, + { + "epoch": 1.31, + "learning_rate": 5.422668004012036e-05, + "loss": 1.2153, + "theoretical_loss": 3.342564511606897, + "tokens_seen": 3079653376 + }, + { + "epoch": 1.31, + "learning_rate": 5.421063189568706e-05, + "loss": 1.0482, + "theoretical_loss": 3.342559118060771, + "tokens_seen": 3079718912 + }, + { + "epoch": 1.31, + "learning_rate": 5.4194583751253766e-05, + "loss": 1.1543, + "theoretical_loss": 3.342553724661553, + "tokens_seen": 3079784448 + }, + { + "epoch": 1.31, + "learning_rate": 5.417853560682047e-05, + "loss": 1.1429, + "theoretical_loss": 3.342548331409237, + "tokens_seen": 3079849984 + }, + { + "epoch": 1.31, + "learning_rate": 5.416248746238717e-05, + "loss": 1.1746, + "theoretical_loss": 3.342542938303815, + "tokens_seen": 3079915520 + }, + { + "epoch": 1.31, + "learning_rate": 5.414643931795387e-05, + "loss": 1.2084, + "theoretical_loss": 3.3425375453452806, + "tokens_seen": 3079981056 + }, + { + "epoch": 1.31, + "learning_rate": 5.413039117352057e-05, + "loss": 1.1683, + "theoretical_loss": 3.342532152533626, + "tokens_seen": 3080046592 + }, + { + "epoch": 1.31, + "learning_rate": 5.411434302908727e-05, + "loss": 1.2201, + "theoretical_loss": 3.3425267598688446, + "tokens_seen": 3080112128 + }, + { + "epoch": 1.31, + "objective/train/docs_used": 1136241, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1470959186553955, + "objective/train/theoretical_loss": 3.342521367350929, + "objective/train/tokens_used": 3100637664, + "theoretical_loss": 3.342521367350929, + "tokens_seen": 3080177664 + }, + { + "epoch": 1.31, + "learning_rate": 5.409829488465396e-05, + "loss": 1.1471, + "theoretical_loss": 3.342521367350929, + "tokens_seen": 3080177664 + }, + { + "epoch": 1.31, + "learning_rate": 5.4082246740220664e-05, + "loss": 1.1457, + "theoretical_loss": 3.342515974979872, + "tokens_seen": 3080243200 + }, + { + "epoch": 1.31, + "learning_rate": 5.4066198595787365e-05, + "loss": 1.0898, + "theoretical_loss": 3.342510582755667, + "tokens_seen": 3080308736 + }, + { + "epoch": 1.31, + "learning_rate": 5.4050150451354065e-05, + "loss": 1.2584, + "theoretical_loss": 3.342505190678306, + "tokens_seen": 3080374272 + }, + { + "epoch": 1.31, + "learning_rate": 5.4034102306920766e-05, + "loss": 1.122, + "theoretical_loss": 3.342499798747783, + "tokens_seen": 3080439808 + }, + { + "epoch": 1.31, + "learning_rate": 5.4018054162487467e-05, + "loss": 1.1932, + "theoretical_loss": 3.34249440696409, + "tokens_seen": 3080505344 + }, + { + "epoch": 1.31, + "learning_rate": 5.400200601805416e-05, + "loss": 1.1241, + "theoretical_loss": 3.34248901532722, + "tokens_seen": 3080570880 + }, + { + "epoch": 1.31, + "learning_rate": 5.398595787362086e-05, + "loss": 1.2271, + "theoretical_loss": 3.342483623837166, + "tokens_seen": 3080636416 + }, + { + "epoch": 1.31, + "learning_rate": 5.396990972918756e-05, + "loss": 1.1386, + "theoretical_loss": 3.342478232493921, + "tokens_seen": 3080701952 + }, + { + "epoch": 1.31, + "learning_rate": 5.395386158475426e-05, + "loss": 1.2053, + "theoretical_loss": 3.342472841297478, + "tokens_seen": 3080767488 + }, + { + "epoch": 1.31, + "learning_rate": 5.393781344032096e-05, + "loss": 1.2103, + "theoretical_loss": 3.342467450247829, + "tokens_seen": 3080833024 + }, + { + "epoch": 1.31, + "learning_rate": 5.3921765295887664e-05, + "loss": 1.1407, + "theoretical_loss": 3.342462059344968, + "tokens_seen": 3080898560 + }, + { + "epoch": 1.31, + "learning_rate": 5.390571715145436e-05, + "loss": 1.2127, + "theoretical_loss": 3.3424566685888872, + "tokens_seen": 3080964096 + }, + { + "epoch": 1.31, + "learning_rate": 5.388966900702107e-05, + "loss": 1.0764, + "theoretical_loss": 3.34245127797958, + "tokens_seen": 3081029632 + }, + { + "epoch": 1.31, + "learning_rate": 5.387362086258777e-05, + "loss": 1.0794, + "theoretical_loss": 3.342445887517038, + "tokens_seen": 3081095168 + }, + { + "epoch": 1.31, + "learning_rate": 5.385757271815447e-05, + "loss": 1.1813, + "theoretical_loss": 3.342440497201256, + "tokens_seen": 3081160704 + }, + { + "epoch": 1.31, + "learning_rate": 5.384152457372117e-05, + "loss": 1.1595, + "theoretical_loss": 3.342435107032226, + "tokens_seen": 3081226240 + }, + { + "epoch": 1.31, + "learning_rate": 5.382547642928787e-05, + "loss": 1.1122, + "theoretical_loss": 3.3424297170099404, + "tokens_seen": 3081291776 + }, + { + "epoch": 1.31, + "learning_rate": 5.380942828485457e-05, + "loss": 1.1953, + "theoretical_loss": 3.3424243271343927, + "tokens_seen": 3081357312 + }, + { + "epoch": 1.31, + "learning_rate": 5.379338014042127e-05, + "loss": 1.0944, + "theoretical_loss": 3.3424189374055757, + "tokens_seen": 3081422848 + }, + { + "epoch": 1.31, + "learning_rate": 5.377733199598797e-05, + "loss": 1.199, + "theoretical_loss": 3.342413547823482, + "tokens_seen": 3081488384 + }, + { + "epoch": 1.31, + "learning_rate": 5.376128385155467e-05, + "loss": 1.1253, + "theoretical_loss": 3.3424081583881047, + "tokens_seen": 3081553920 + }, + { + "epoch": 1.31, + "learning_rate": 5.3745235707121364e-05, + "loss": 1.098, + "theoretical_loss": 3.3424027690994365, + "tokens_seen": 3081619456 + }, + { + "epoch": 1.31, + "learning_rate": 5.3729187562688064e-05, + "loss": 1.1557, + "theoretical_loss": 3.342397379957471, + "tokens_seen": 3081684992 + }, + { + "epoch": 1.31, + "learning_rate": 5.3713139418254765e-05, + "loss": 1.2403, + "theoretical_loss": 3.3423919909622, + "tokens_seen": 3081750528 + }, + { + "epoch": 1.31, + "learning_rate": 5.3697091273821466e-05, + "loss": 1.2032, + "theoretical_loss": 3.342386602113617, + "tokens_seen": 3081816064 + }, + { + "epoch": 1.31, + "learning_rate": 5.3681043129388166e-05, + "loss": 1.2784, + "theoretical_loss": 3.3423812134117146, + "tokens_seen": 3081881600 + }, + { + "epoch": 1.31, + "learning_rate": 5.366499498495487e-05, + "loss": 1.139, + "theoretical_loss": 3.342375824856486, + "tokens_seen": 3081947136 + }, + { + "epoch": 1.31, + "learning_rate": 5.364894684052156e-05, + "loss": 1.1174, + "theoretical_loss": 3.3423704364479243, + "tokens_seen": 3082012672 + }, + { + "epoch": 1.31, + "learning_rate": 5.363289869608826e-05, + "loss": 1.1981, + "theoretical_loss": 3.342365048186022, + "tokens_seen": 3082078208 + }, + { + "epoch": 1.31, + "learning_rate": 5.361685055165496e-05, + "loss": 1.1398, + "theoretical_loss": 3.3423596600707715, + "tokens_seen": 3082143744 + }, + { + "epoch": 1.31, + "learning_rate": 5.3600802407221676e-05, + "loss": 1.1374, + "theoretical_loss": 3.342354272102167, + "tokens_seen": 3082209280 + }, + { + "epoch": 1.31, + "learning_rate": 5.358475426278837e-05, + "loss": 1.0889, + "theoretical_loss": 3.3423488842802, + "tokens_seen": 3082274816 + }, + { + "epoch": 1.31, + "learning_rate": 5.356870611835507e-05, + "loss": 1.1917, + "theoretical_loss": 3.3423434966048644, + "tokens_seen": 3082340352 + }, + { + "epoch": 1.31, + "learning_rate": 5.355265797392177e-05, + "loss": 1.2546, + "theoretical_loss": 3.3423381090761524, + "tokens_seen": 3082405888 + }, + { + "epoch": 1.31, + "learning_rate": 5.353660982948847e-05, + "loss": 1.1684, + "theoretical_loss": 3.3423327216940573, + "tokens_seen": 3082471424 + }, + { + "epoch": 1.31, + "learning_rate": 5.352056168505517e-05, + "loss": 1.1133, + "theoretical_loss": 3.342327334458572, + "tokens_seen": 3082536960 + }, + { + "epoch": 1.31, + "learning_rate": 5.350451354062187e-05, + "loss": 1.2605, + "theoretical_loss": 3.3423219473696895, + "tokens_seen": 3082602496 + }, + { + "epoch": 1.31, + "learning_rate": 5.348846539618857e-05, + "loss": 1.1726, + "theoretical_loss": 3.342316560427402, + "tokens_seen": 3082668032 + }, + { + "epoch": 1.31, + "learning_rate": 5.347241725175527e-05, + "loss": 1.1278, + "theoretical_loss": 3.3423111736317033, + "tokens_seen": 3082733568 + }, + { + "epoch": 1.31, + "learning_rate": 5.345636910732197e-05, + "loss": 1.1731, + "theoretical_loss": 3.342305786982586, + "tokens_seen": 3082799104 + }, + { + "epoch": 1.31, + "learning_rate": 5.344032096288867e-05, + "loss": 1.2298, + "theoretical_loss": 3.3423004004800423, + "tokens_seen": 3082864640 + }, + { + "epoch": 1.31, + "learning_rate": 5.342427281845537e-05, + "loss": 1.1275, + "theoretical_loss": 3.3422950141240664, + "tokens_seen": 3082930176 + }, + { + "epoch": 1.31, + "learning_rate": 5.340822467402207e-05, + "loss": 1.1167, + "theoretical_loss": 3.3422896279146497, + "tokens_seen": 3082995712 + }, + { + "epoch": 1.31, + "learning_rate": 5.3392176529588764e-05, + "loss": 1.1536, + "theoretical_loss": 3.3422842418517864, + "tokens_seen": 3083061248 + }, + { + "epoch": 1.31, + "learning_rate": 5.3376128385155465e-05, + "loss": 1.0863, + "theoretical_loss": 3.3422788559354686, + "tokens_seen": 3083126784 + }, + { + "epoch": 1.31, + "learning_rate": 5.3360080240722165e-05, + "loss": 1.2545, + "theoretical_loss": 3.3422734701656895, + "tokens_seen": 3083192320 + }, + { + "epoch": 1.31, + "learning_rate": 5.3344032096288866e-05, + "loss": 1.1354, + "theoretical_loss": 3.342268084542442, + "tokens_seen": 3083257856 + }, + { + "epoch": 1.32, + "learning_rate": 5.332798395185557e-05, + "loss": 1.1853, + "theoretical_loss": 3.342262699065719, + "tokens_seen": 3083323392 + }, + { + "epoch": 1.32, + "learning_rate": 5.331193580742227e-05, + "loss": 1.0951, + "theoretical_loss": 3.342257313735513, + "tokens_seen": 3083388928 + }, + { + "epoch": 1.32, + "learning_rate": 5.3295887662988975e-05, + "loss": 1.2117, + "theoretical_loss": 3.3422519285518177, + "tokens_seen": 3083454464 + }, + { + "epoch": 1.32, + "learning_rate": 5.3279839518555675e-05, + "loss": 1.1341, + "theoretical_loss": 3.3422465435146256, + "tokens_seen": 3083520000 + }, + { + "epoch": 1.32, + "learning_rate": 5.3263791374122376e-05, + "loss": 1.0983, + "theoretical_loss": 3.342241158623929, + "tokens_seen": 3083585536 + }, + { + "epoch": 1.32, + "learning_rate": 5.324774322968907e-05, + "loss": 1.148, + "theoretical_loss": 3.3422357738797217, + "tokens_seen": 3083651072 + }, + { + "epoch": 1.32, + "learning_rate": 5.323169508525577e-05, + "loss": 1.2177, + "theoretical_loss": 3.3422303892819967, + "tokens_seen": 3083716608 + }, + { + "epoch": 1.32, + "learning_rate": 5.321564694082247e-05, + "loss": 1.1273, + "theoretical_loss": 3.3422250048307456, + "tokens_seen": 3083782144 + }, + { + "epoch": 1.32, + "learning_rate": 5.319959879638917e-05, + "loss": 1.1953, + "theoretical_loss": 3.342219620525963, + "tokens_seen": 3083847680 + }, + { + "epoch": 1.32, + "learning_rate": 5.318355065195587e-05, + "loss": 1.2745, + "theoretical_loss": 3.3422142363676404, + "tokens_seen": 3083913216 + }, + { + "epoch": 1.32, + "learning_rate": 5.316750250752257e-05, + "loss": 1.0938, + "theoretical_loss": 3.3422088523557716, + "tokens_seen": 3083978752 + }, + { + "epoch": 1.32, + "learning_rate": 5.315145436308927e-05, + "loss": 1.1988, + "theoretical_loss": 3.3422034684903488, + "tokens_seen": 3084044288 + }, + { + "epoch": 1.32, + "learning_rate": 5.313540621865597e-05, + "loss": 1.0431, + "theoretical_loss": 3.3421980847713657, + "tokens_seen": 3084109824 + }, + { + "epoch": 1.32, + "learning_rate": 5.311935807422267e-05, + "loss": 1.1074, + "theoretical_loss": 3.3421927011988144, + "tokens_seen": 3084175360 + }, + { + "epoch": 1.32, + "learning_rate": 5.310330992978937e-05, + "loss": 1.1583, + "theoretical_loss": 3.342187317772688, + "tokens_seen": 3084240896 + }, + { + "epoch": 1.32, + "learning_rate": 5.308726178535607e-05, + "loss": 1.05, + "theoretical_loss": 3.3421819344929804, + "tokens_seen": 3084306432 + }, + { + "epoch": 1.32, + "learning_rate": 5.307121364092277e-05, + "loss": 1.1638, + "theoretical_loss": 3.3421765513596835, + "tokens_seen": 3084371968 + }, + { + "epoch": 1.32, + "learning_rate": 5.3055165496489464e-05, + "loss": 1.1932, + "theoretical_loss": 3.34217116837279, + "tokens_seen": 3084437504 + }, + { + "epoch": 1.32, + "learning_rate": 5.3039117352056165e-05, + "loss": 1.1496, + "theoretical_loss": 3.3421657855322935, + "tokens_seen": 3084503040 + }, + { + "epoch": 1.32, + "learning_rate": 5.3023069207622865e-05, + "loss": 1.2086, + "theoretical_loss": 3.342160402838186, + "tokens_seen": 3084568576 + }, + { + "epoch": 1.32, + "learning_rate": 5.3007021063189566e-05, + "loss": 1.1892, + "theoretical_loss": 3.3421550202904617, + "tokens_seen": 3084634112 + }, + { + "epoch": 1.32, + "learning_rate": 5.299097291875627e-05, + "loss": 1.2774, + "theoretical_loss": 3.342149637889113, + "tokens_seen": 3084699648 + }, + { + "epoch": 1.32, + "learning_rate": 5.2974924774322974e-05, + "loss": 1.1798, + "theoretical_loss": 3.3421442556341323, + "tokens_seen": 3084765184 + }, + { + "epoch": 1.32, + "learning_rate": 5.2958876629889675e-05, + "loss": 1.0928, + "theoretical_loss": 3.342138873525513, + "tokens_seen": 3084830720 + }, + { + "epoch": 1.32, + "learning_rate": 5.2942828485456375e-05, + "loss": 1.3619, + "theoretical_loss": 3.3421334915632475, + "tokens_seen": 3084896256 + }, + { + "epoch": 1.32, + "learning_rate": 5.2926780341023076e-05, + "loss": 1.0868, + "theoretical_loss": 3.3421281097473297, + "tokens_seen": 3084961792 + }, + { + "epoch": 1.32, + "learning_rate": 5.2910732196589776e-05, + "loss": 1.1543, + "theoretical_loss": 3.3421227280777517, + "tokens_seen": 3085027328 + }, + { + "epoch": 1.32, + "learning_rate": 5.289468405215647e-05, + "loss": 1.2115, + "theoretical_loss": 3.342117346554506, + "tokens_seen": 3085092864 + }, + { + "epoch": 1.32, + "learning_rate": 5.287863590772317e-05, + "loss": 1.2512, + "theoretical_loss": 3.342111965177587, + "tokens_seen": 3085158400 + }, + { + "epoch": 1.32, + "learning_rate": 5.286258776328987e-05, + "loss": 1.1367, + "theoretical_loss": 3.342106583946986, + "tokens_seen": 3085223936 + }, + { + "epoch": 1.32, + "learning_rate": 5.284653961885657e-05, + "loss": 1.2015, + "theoretical_loss": 3.3421012028626973, + "tokens_seen": 3085289472 + }, + { + "epoch": 1.32, + "learning_rate": 5.283049147442327e-05, + "loss": 1.1289, + "theoretical_loss": 3.3420958219247128, + "tokens_seen": 3085355008 + }, + { + "epoch": 1.32, + "learning_rate": 5.2814443329989973e-05, + "loss": 1.1831, + "theoretical_loss": 3.342090441133026, + "tokens_seen": 3085420544 + }, + { + "epoch": 1.32, + "learning_rate": 5.279839518555667e-05, + "loss": 1.1592, + "theoretical_loss": 3.342085060487629, + "tokens_seen": 3085486080 + }, + { + "epoch": 1.32, + "learning_rate": 5.278234704112337e-05, + "loss": 1.1863, + "theoretical_loss": 3.342079679988516, + "tokens_seen": 3085551616 + }, + { + "epoch": 1.32, + "learning_rate": 5.276629889669007e-05, + "loss": 1.189, + "theoretical_loss": 3.342074299635679, + "tokens_seen": 3085617152 + }, + { + "epoch": 1.32, + "learning_rate": 5.275025075225677e-05, + "loss": 1.2755, + "theoretical_loss": 3.342068919429111, + "tokens_seen": 3085682688 + }, + { + "epoch": 1.32, + "learning_rate": 5.273420260782347e-05, + "loss": 1.1597, + "theoretical_loss": 3.342063539368805, + "tokens_seen": 3085748224 + }, + { + "epoch": 1.32, + "learning_rate": 5.271815446339017e-05, + "loss": 1.1635, + "theoretical_loss": 3.3420581594547545, + "tokens_seen": 3085813760 + }, + { + "epoch": 1.32, + "learning_rate": 5.270210631895688e-05, + "loss": 1.1478, + "theoretical_loss": 3.3420527796869512, + "tokens_seen": 3085879296 + }, + { + "epoch": 1.32, + "learning_rate": 5.268605817452358e-05, + "loss": 1.1601, + "theoretical_loss": 3.3420474000653893, + "tokens_seen": 3085944832 + }, + { + "epoch": 1.32, + "learning_rate": 5.267001003009028e-05, + "loss": 1.216, + "theoretical_loss": 3.342042020590061, + "tokens_seen": 3086010368 + }, + { + "epoch": 1.32, + "learning_rate": 5.265396188565698e-05, + "loss": 1.061, + "theoretical_loss": 3.3420366412609592, + "tokens_seen": 3086075904 + }, + { + "epoch": 1.32, + "learning_rate": 5.2637913741223674e-05, + "loss": 1.159, + "theoretical_loss": 3.342031262078077, + "tokens_seen": 3086141440 + }, + { + "epoch": 1.32, + "learning_rate": 5.2621865596790374e-05, + "loss": 1.2034, + "theoretical_loss": 3.3420258830414076, + "tokens_seen": 3086206976 + }, + { + "epoch": 1.32, + "learning_rate": 5.2605817452357075e-05, + "loss": 1.1714, + "theoretical_loss": 3.342020504150943, + "tokens_seen": 3086272512 + }, + { + "epoch": 1.32, + "learning_rate": 5.2589769307923776e-05, + "loss": 1.1639, + "theoretical_loss": 3.342015125406677, + "tokens_seen": 3086338048 + }, + { + "epoch": 1.32, + "learning_rate": 5.2573721163490476e-05, + "loss": 1.1587, + "theoretical_loss": 3.3420097468086025, + "tokens_seen": 3086403584 + }, + { + "epoch": 1.32, + "learning_rate": 5.255767301905718e-05, + "loss": 1.2219, + "theoretical_loss": 3.342004368356712, + "tokens_seen": 3086469120 + }, + { + "epoch": 1.32, + "learning_rate": 5.254162487462387e-05, + "loss": 1.2209, + "theoretical_loss": 3.3419989900509983, + "tokens_seen": 3086534656 + }, + { + "epoch": 1.32, + "learning_rate": 5.252557673019057e-05, + "loss": 1.2367, + "theoretical_loss": 3.3419936118914553, + "tokens_seen": 3086600192 + }, + { + "epoch": 1.32, + "learning_rate": 5.250952858575727e-05, + "loss": 1.2546, + "theoretical_loss": 3.341988233878075, + "tokens_seen": 3086665728 + }, + { + "epoch": 1.32, + "objective/train/docs_used": 1138607, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2358028888702393, + "objective/train/theoretical_loss": 3.34198285601085, + "objective/train/tokens_used": 3107191264, + "theoretical_loss": 3.34198285601085, + "tokens_seen": 3086731264 + }, + { + "epoch": 1.32, + "learning_rate": 5.249348044132397e-05, + "loss": 1.2358, + "theoretical_loss": 3.34198285601085, + "tokens_seen": 3086731264 + }, + { + "epoch": 1.32, + "learning_rate": 5.247743229689067e-05, + "loss": 1.1532, + "theoretical_loss": 3.3419774782897744, + "tokens_seen": 3086796800 + }, + { + "epoch": 1.32, + "learning_rate": 5.2461384152457374e-05, + "loss": 1.2067, + "theoretical_loss": 3.34197210071484, + "tokens_seen": 3086862336 + }, + { + "epoch": 1.32, + "learning_rate": 5.244533600802407e-05, + "loss": 1.1369, + "theoretical_loss": 3.341966723286041, + "tokens_seen": 3086927872 + }, + { + "epoch": 1.32, + "learning_rate": 5.242928786359077e-05, + "loss": 1.2141, + "theoretical_loss": 3.3419613460033695, + "tokens_seen": 3086993408 + }, + { + "epoch": 1.32, + "learning_rate": 5.241323971915747e-05, + "loss": 1.1171, + "theoretical_loss": 3.341955968866818, + "tokens_seen": 3087058944 + }, + { + "epoch": 1.32, + "learning_rate": 5.239719157472418e-05, + "loss": 1.1972, + "theoretical_loss": 3.34195059187638, + "tokens_seen": 3087124480 + }, + { + "epoch": 1.32, + "learning_rate": 5.238114343029088e-05, + "loss": 1.1223, + "theoretical_loss": 3.3419452150320486, + "tokens_seen": 3087190016 + }, + { + "epoch": 1.32, + "learning_rate": 5.236509528585758e-05, + "loss": 1.2801, + "theoretical_loss": 3.3419398383338166, + "tokens_seen": 3087255552 + }, + { + "epoch": 1.32, + "learning_rate": 5.234904714142428e-05, + "loss": 1.1387, + "theoretical_loss": 3.341934461781676, + "tokens_seen": 3087321088 + }, + { + "epoch": 1.32, + "learning_rate": 5.233299899699098e-05, + "loss": 1.2018, + "theoretical_loss": 3.3419290853756216, + "tokens_seen": 3087386624 + }, + { + "epoch": 1.32, + "learning_rate": 5.231695085255768e-05, + "loss": 1.0725, + "theoretical_loss": 3.341923709115645, + "tokens_seen": 3087452160 + }, + { + "epoch": 1.32, + "learning_rate": 5.230090270812438e-05, + "loss": 1.0767, + "theoretical_loss": 3.341918333001739, + "tokens_seen": 3087517696 + }, + { + "epoch": 1.32, + "learning_rate": 5.2284854563691074e-05, + "loss": 1.1107, + "theoretical_loss": 3.3419129570338972, + "tokens_seen": 3087583232 + }, + { + "epoch": 1.32, + "learning_rate": 5.2268806419257775e-05, + "loss": 1.1927, + "theoretical_loss": 3.341907581212112, + "tokens_seen": 3087648768 + }, + { + "epoch": 1.32, + "learning_rate": 5.2252758274824475e-05, + "loss": 1.1298, + "theoretical_loss": 3.341902205536377, + "tokens_seen": 3087714304 + }, + { + "epoch": 1.32, + "learning_rate": 5.2236710130391176e-05, + "loss": 1.2274, + "theoretical_loss": 3.3418968300066845, + "tokens_seen": 3087779840 + }, + { + "epoch": 1.32, + "learning_rate": 5.222066198595788e-05, + "loss": 1.1717, + "theoretical_loss": 3.341891454623028, + "tokens_seen": 3087845376 + }, + { + "epoch": 1.32, + "learning_rate": 5.220461384152458e-05, + "loss": 1.2025, + "theoretical_loss": 3.3418860793853997, + "tokens_seen": 3087910912 + }, + { + "epoch": 1.32, + "learning_rate": 5.218856569709127e-05, + "loss": 1.1317, + "theoretical_loss": 3.3418807042937932, + "tokens_seen": 3087976448 + }, + { + "epoch": 1.32, + "learning_rate": 5.217251755265797e-05, + "loss": 1.2732, + "theoretical_loss": 3.341875329348201, + "tokens_seen": 3088041984 + }, + { + "epoch": 1.32, + "learning_rate": 5.215646940822467e-05, + "loss": 1.1896, + "theoretical_loss": 3.341869954548616, + "tokens_seen": 3088107520 + }, + { + "epoch": 1.32, + "learning_rate": 5.214042126379137e-05, + "loss": 1.1742, + "theoretical_loss": 3.3418645798950317, + "tokens_seen": 3088173056 + }, + { + "epoch": 1.32, + "learning_rate": 5.2124373119358074e-05, + "loss": 1.168, + "theoretical_loss": 3.3418592053874407, + "tokens_seen": 3088238592 + }, + { + "epoch": 1.32, + "learning_rate": 5.2108324974924774e-05, + "loss": 1.1398, + "theoretical_loss": 3.3418538310258357, + "tokens_seen": 3088304128 + }, + { + "epoch": 1.32, + "learning_rate": 5.209227683049148e-05, + "loss": 1.061, + "theoretical_loss": 3.3418484568102103, + "tokens_seen": 3088369664 + }, + { + "epoch": 1.32, + "learning_rate": 5.207622868605818e-05, + "loss": 1.2118, + "theoretical_loss": 3.3418430827405565, + "tokens_seen": 3088435200 + }, + { + "epoch": 1.32, + "learning_rate": 5.206018054162488e-05, + "loss": 1.1861, + "theoretical_loss": 3.341837708816868, + "tokens_seen": 3088500736 + }, + { + "epoch": 1.32, + "learning_rate": 5.2044132397191584e-05, + "loss": 1.159, + "theoretical_loss": 3.341832335039137, + "tokens_seen": 3088566272 + }, + { + "epoch": 1.32, + "learning_rate": 5.202808425275828e-05, + "loss": 1.0829, + "theoretical_loss": 3.3418269614073575, + "tokens_seen": 3088631808 + }, + { + "epoch": 1.32, + "learning_rate": 5.201203610832498e-05, + "loss": 1.209, + "theoretical_loss": 3.341821587921521, + "tokens_seen": 3088697344 + }, + { + "epoch": 1.32, + "learning_rate": 5.199598796389168e-05, + "loss": 1.1283, + "theoretical_loss": 3.341816214581622, + "tokens_seen": 3088762880 + }, + { + "epoch": 1.32, + "learning_rate": 5.197993981945838e-05, + "loss": 1.0889, + "theoretical_loss": 3.3418108413876526, + "tokens_seen": 3088828416 + }, + { + "epoch": 1.32, + "learning_rate": 5.196389167502508e-05, + "loss": 1.197, + "theoretical_loss": 3.341805468339606, + "tokens_seen": 3088893952 + }, + { + "epoch": 1.32, + "learning_rate": 5.194784353059178e-05, + "loss": 1.1542, + "theoretical_loss": 3.341800095437475, + "tokens_seen": 3088959488 + }, + { + "epoch": 1.32, + "learning_rate": 5.1931795386158474e-05, + "loss": 1.1852, + "theoretical_loss": 3.341794722681252, + "tokens_seen": 3089025024 + }, + { + "epoch": 1.32, + "learning_rate": 5.1915747241725175e-05, + "loss": 1.1679, + "theoretical_loss": 3.341789350070931, + "tokens_seen": 3089090560 + }, + { + "epoch": 1.32, + "learning_rate": 5.1899699097291876e-05, + "loss": 1.1535, + "theoretical_loss": 3.341783977606504, + "tokens_seen": 3089156096 + }, + { + "epoch": 1.32, + "learning_rate": 5.1883650952858576e-05, + "loss": 1.1619, + "theoretical_loss": 3.3417786052879648, + "tokens_seen": 3089221632 + }, + { + "epoch": 1.32, + "learning_rate": 5.186760280842528e-05, + "loss": 1.2137, + "theoretical_loss": 3.3417732331153056, + "tokens_seen": 3089287168 + }, + { + "epoch": 1.32, + "learning_rate": 5.185155466399198e-05, + "loss": 1.0321, + "theoretical_loss": 3.34176786108852, + "tokens_seen": 3089352704 + }, + { + "epoch": 1.32, + "learning_rate": 5.183550651955867e-05, + "loss": 1.1793, + "theoretical_loss": 3.3417624892076003, + "tokens_seen": 3089418240 + }, + { + "epoch": 1.32, + "learning_rate": 5.181945837512537e-05, + "loss": 1.1945, + "theoretical_loss": 3.3417571174725396, + "tokens_seen": 3089483776 + }, + { + "epoch": 1.32, + "learning_rate": 5.180341023069207e-05, + "loss": 1.1954, + "theoretical_loss": 3.3417517458833315, + "tokens_seen": 3089549312 + }, + { + "epoch": 1.32, + "learning_rate": 5.178736208625879e-05, + "loss": 1.1767, + "theoretical_loss": 3.341746374439968, + "tokens_seen": 3089614848 + }, + { + "epoch": 1.32, + "learning_rate": 5.177131394182548e-05, + "loss": 1.1729, + "theoretical_loss": 3.3417410031424426, + "tokens_seen": 3089680384 + }, + { + "epoch": 1.32, + "learning_rate": 5.175526579739218e-05, + "loss": 1.1748, + "theoretical_loss": 3.3417356319907485, + "tokens_seen": 3089745920 + }, + { + "epoch": 1.32, + "learning_rate": 5.173921765295888e-05, + "loss": 1.1175, + "theoretical_loss": 3.3417302609848782, + "tokens_seen": 3089811456 + }, + { + "epoch": 1.32, + "learning_rate": 5.172316950852558e-05, + "loss": 1.1788, + "theoretical_loss": 3.341724890124824, + "tokens_seen": 3089876992 + }, + { + "epoch": 1.32, + "learning_rate": 5.170712136409228e-05, + "loss": 1.2125, + "theoretical_loss": 3.34171951941058, + "tokens_seen": 3089942528 + }, + { + "epoch": 1.32, + "learning_rate": 5.1691073219658984e-05, + "loss": 1.1774, + "theoretical_loss": 3.341714148842139, + "tokens_seen": 3090008064 + }, + { + "epoch": 1.32, + "learning_rate": 5.167502507522568e-05, + "loss": 1.1189, + "theoretical_loss": 3.341708778419494, + "tokens_seen": 3090073600 + }, + { + "epoch": 1.32, + "learning_rate": 5.165897693079238e-05, + "loss": 1.2165, + "theoretical_loss": 3.341703408142637, + "tokens_seen": 3090139136 + }, + { + "epoch": 1.32, + "learning_rate": 5.164292878635908e-05, + "loss": 1.2411, + "theoretical_loss": 3.3416980380115615, + "tokens_seen": 3090204672 + }, + { + "epoch": 1.32, + "learning_rate": 5.162688064192578e-05, + "loss": 1.1912, + "theoretical_loss": 3.3416926680262606, + "tokens_seen": 3090270208 + }, + { + "epoch": 1.32, + "learning_rate": 5.161083249749248e-05, + "loss": 1.1437, + "theoretical_loss": 3.3416872981867276, + "tokens_seen": 3090335744 + }, + { + "epoch": 1.32, + "learning_rate": 5.159478435305918e-05, + "loss": 1.1293, + "theoretical_loss": 3.341681928492955, + "tokens_seen": 3090401280 + }, + { + "epoch": 1.32, + "learning_rate": 5.1578736208625875e-05, + "loss": 1.207, + "theoretical_loss": 3.3416765589449353, + "tokens_seen": 3090466816 + }, + { + "epoch": 1.32, + "learning_rate": 5.1562688064192576e-05, + "loss": 1.2616, + "theoretical_loss": 3.3416711895426623, + "tokens_seen": 3090532352 + }, + { + "epoch": 1.32, + "learning_rate": 5.1546639919759276e-05, + "loss": 1.2653, + "theoretical_loss": 3.3416658202861282, + "tokens_seen": 3090597888 + }, + { + "epoch": 1.32, + "learning_rate": 5.153059177532598e-05, + "loss": 1.1647, + "theoretical_loss": 3.3416604511753265, + "tokens_seen": 3090663424 + }, + { + "epoch": 1.32, + "learning_rate": 5.151454363089268e-05, + "loss": 1.1879, + "theoretical_loss": 3.3416550822102504, + "tokens_seen": 3090728960 + }, + { + "epoch": 1.32, + "learning_rate": 5.1498495486459385e-05, + "loss": 1.0878, + "theoretical_loss": 3.341649713390892, + "tokens_seen": 3090794496 + }, + { + "epoch": 1.32, + "learning_rate": 5.1482447342026085e-05, + "loss": 1.0969, + "theoretical_loss": 3.341644344717245, + "tokens_seen": 3090860032 + }, + { + "epoch": 1.32, + "learning_rate": 5.1466399197592786e-05, + "loss": 1.2113, + "theoretical_loss": 3.341638976189302, + "tokens_seen": 3090925568 + }, + { + "epoch": 1.32, + "learning_rate": 5.145035105315949e-05, + "loss": 1.2875, + "theoretical_loss": 3.341633607807056, + "tokens_seen": 3090991104 + }, + { + "epoch": 1.32, + "learning_rate": 5.143430290872619e-05, + "loss": 1.1446, + "theoretical_loss": 3.3416282395704995, + "tokens_seen": 3091056640 + }, + { + "epoch": 1.32, + "learning_rate": 5.141825476429288e-05, + "loss": 1.1216, + "theoretical_loss": 3.3416228714796268, + "tokens_seen": 3091122176 + }, + { + "epoch": 1.32, + "learning_rate": 5.140220661985958e-05, + "loss": 1.2343, + "theoretical_loss": 3.3416175035344295, + "tokens_seen": 3091187712 + }, + { + "epoch": 1.32, + "learning_rate": 5.138615847542628e-05, + "loss": 1.193, + "theoretical_loss": 3.341612135734901, + "tokens_seen": 3091253248 + }, + { + "epoch": 1.32, + "learning_rate": 5.137011033099298e-05, + "loss": 1.2424, + "theoretical_loss": 3.3416067680810344, + "tokens_seen": 3091318784 + }, + { + "epoch": 1.32, + "learning_rate": 5.1354062186559684e-05, + "loss": 1.174, + "theoretical_loss": 3.3416014005728227, + "tokens_seen": 3091384320 + }, + { + "epoch": 1.32, + "learning_rate": 5.1338014042126384e-05, + "loss": 1.2019, + "theoretical_loss": 3.3415960332102586, + "tokens_seen": 3091449856 + }, + { + "epoch": 1.32, + "learning_rate": 5.132196589769308e-05, + "loss": 1.1494, + "theoretical_loss": 3.341590665993335, + "tokens_seen": 3091515392 + }, + { + "epoch": 1.32, + "learning_rate": 5.130591775325978e-05, + "loss": 1.2396, + "theoretical_loss": 3.3415852989220456, + "tokens_seen": 3091580928 + }, + { + "epoch": 1.32, + "learning_rate": 5.128986960882648e-05, + "loss": 1.2217, + "theoretical_loss": 3.3415799319963826, + "tokens_seen": 3091646464 + }, + { + "epoch": 1.32, + "learning_rate": 5.127382146439318e-05, + "loss": 1.2372, + "theoretical_loss": 3.341574565216339, + "tokens_seen": 3091712000 + }, + { + "epoch": 1.32, + "learning_rate": 5.125777331995988e-05, + "loss": 1.1435, + "theoretical_loss": 3.341569198581908, + "tokens_seen": 3091777536 + }, + { + "epoch": 1.32, + "learning_rate": 5.124172517552658e-05, + "loss": 1.2252, + "theoretical_loss": 3.3415638320930827, + "tokens_seen": 3091843072 + }, + { + "epoch": 1.32, + "learning_rate": 5.1225677031093275e-05, + "loss": 1.1388, + "theoretical_loss": 3.3415584657498556, + "tokens_seen": 3091908608 + }, + { + "epoch": 1.32, + "learning_rate": 5.1209628886659976e-05, + "loss": 1.1174, + "theoretical_loss": 3.34155309955222, + "tokens_seen": 3091974144 + }, + { + "epoch": 1.32, + "learning_rate": 5.119358074222669e-05, + "loss": 1.1457, + "theoretical_loss": 3.341547733500169, + "tokens_seen": 3092039680 + }, + { + "epoch": 1.32, + "learning_rate": 5.1177532597793384e-05, + "loss": 1.0758, + "theoretical_loss": 3.341542367593695, + "tokens_seen": 3092105216 + }, + { + "epoch": 1.32, + "learning_rate": 5.1161484453360085e-05, + "loss": 1.1772, + "theoretical_loss": 3.3415370018327915, + "tokens_seen": 3092170752 + }, + { + "epoch": 1.32, + "learning_rate": 5.1145436308926785e-05, + "loss": 1.0302, + "theoretical_loss": 3.3415316362174514, + "tokens_seen": 3092236288 + }, + { + "epoch": 1.32, + "learning_rate": 5.1129388164493486e-05, + "loss": 1.0927, + "theoretical_loss": 3.3415262707476674, + "tokens_seen": 3092301824 + }, + { + "epoch": 1.32, + "learning_rate": 5.1113340020060187e-05, + "loss": 1.221, + "theoretical_loss": 3.3415209054234327, + "tokens_seen": 3092367360 + }, + { + "epoch": 1.32, + "learning_rate": 5.109729187562689e-05, + "loss": 1.1412, + "theoretical_loss": 3.34151554024474, + "tokens_seen": 3092432896 + }, + { + "epoch": 1.32, + "learning_rate": 5.108124373119358e-05, + "loss": 1.2559, + "theoretical_loss": 3.3415101752115826, + "tokens_seen": 3092498432 + }, + { + "epoch": 1.32, + "learning_rate": 5.106519558676028e-05, + "loss": 1.2194, + "theoretical_loss": 3.3415048103239533, + "tokens_seen": 3092563968 + }, + { + "epoch": 1.32, + "learning_rate": 5.104914744232698e-05, + "loss": 1.1038, + "theoretical_loss": 3.3414994455818454, + "tokens_seen": 3092629504 + }, + { + "epoch": 1.32, + "learning_rate": 5.103309929789368e-05, + "loss": 1.0606, + "theoretical_loss": 3.3414940809852514, + "tokens_seen": 3092695040 + }, + { + "epoch": 1.32, + "learning_rate": 5.1017051153460384e-05, + "loss": 1.1386, + "theoretical_loss": 3.341488716534164, + "tokens_seen": 3092760576 + }, + { + "epoch": 1.32, + "learning_rate": 5.1001003009027084e-05, + "loss": 1.1271, + "theoretical_loss": 3.341483352228577, + "tokens_seen": 3092826112 + }, + { + "epoch": 1.32, + "learning_rate": 5.098495486459378e-05, + "loss": 1.143, + "theoretical_loss": 3.341477988068483, + "tokens_seen": 3092891648 + }, + { + "epoch": 1.32, + "learning_rate": 5.096890672016048e-05, + "loss": 1.1701, + "theoretical_loss": 3.341472624053875, + "tokens_seen": 3092957184 + }, + { + "epoch": 1.32, + "learning_rate": 5.095285857572718e-05, + "loss": 1.2529, + "theoretical_loss": 3.3414672601847455, + "tokens_seen": 3093022720 + }, + { + "epoch": 1.32, + "learning_rate": 5.093681043129388e-05, + "loss": 1.1755, + "theoretical_loss": 3.3414618964610883, + "tokens_seen": 3093088256 + }, + { + "epoch": 1.32, + "learning_rate": 5.092076228686058e-05, + "loss": 1.2281, + "theoretical_loss": 3.3414565328828956, + "tokens_seen": 3093153792 + }, + { + "epoch": 1.32, + "learning_rate": 5.090471414242728e-05, + "loss": 1.1279, + "theoretical_loss": 3.3414511694501607, + "tokens_seen": 3093219328 + }, + { + "epoch": 1.32, + "objective/train/docs_used": 1141358, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.0752753019332886, + "objective/train/theoretical_loss": 3.341445806162877, + "objective/train/tokens_used": 3113744864, + "theoretical_loss": 3.341445806162877, + "tokens_seen": 3093284864 + }, + { + "epoch": 1.32, + "learning_rate": 5.088866599799399e-05, + "loss": 1.0753, + "theoretical_loss": 3.341445806162877, + "tokens_seen": 3093284864 + }, + { + "epoch": 1.32, + "learning_rate": 5.087261785356069e-05, + "loss": 1.1041, + "theoretical_loss": 3.341440443021037, + "tokens_seen": 3093350400 + }, + { + "epoch": 1.32, + "learning_rate": 5.085656970912739e-05, + "loss": 1.1568, + "theoretical_loss": 3.3414350800246337, + "tokens_seen": 3093415936 + }, + { + "epoch": 1.32, + "learning_rate": 5.084052156469409e-05, + "loss": 1.1551, + "theoretical_loss": 3.3414297171736598, + "tokens_seen": 3093481472 + }, + { + "epoch": 1.32, + "learning_rate": 5.0824473420260784e-05, + "loss": 1.1434, + "theoretical_loss": 3.341424354468109, + "tokens_seen": 3093547008 + }, + { + "epoch": 1.32, + "learning_rate": 5.0808425275827485e-05, + "loss": 1.2294, + "theoretical_loss": 3.341418991907974, + "tokens_seen": 3093612544 + }, + { + "epoch": 1.32, + "learning_rate": 5.0792377131394186e-05, + "loss": 1.118, + "theoretical_loss": 3.3414136294932475, + "tokens_seen": 3093678080 + }, + { + "epoch": 1.32, + "learning_rate": 5.0776328986960886e-05, + "loss": 1.2257, + "theoretical_loss": 3.3414082672239225, + "tokens_seen": 3093743616 + }, + { + "epoch": 1.32, + "learning_rate": 5.076028084252759e-05, + "loss": 1.2289, + "theoretical_loss": 3.3414029050999923, + "tokens_seen": 3093809152 + }, + { + "epoch": 1.32, + "learning_rate": 5.074423269809429e-05, + "loss": 1.1638, + "theoretical_loss": 3.3413975431214493, + "tokens_seen": 3093874688 + }, + { + "epoch": 1.32, + "learning_rate": 5.072818455366098e-05, + "loss": 1.1114, + "theoretical_loss": 3.3413921812882874, + "tokens_seen": 3093940224 + }, + { + "epoch": 1.32, + "learning_rate": 5.071213640922768e-05, + "loss": 1.1111, + "theoretical_loss": 3.3413868196004985, + "tokens_seen": 3094005760 + }, + { + "epoch": 1.32, + "learning_rate": 5.069608826479438e-05, + "loss": 1.1967, + "theoretical_loss": 3.3413814580580765, + "tokens_seen": 3094071296 + }, + { + "epoch": 1.32, + "learning_rate": 5.068004012036108e-05, + "loss": 1.2114, + "theoretical_loss": 3.341376096661014, + "tokens_seen": 3094136832 + }, + { + "epoch": 1.32, + "learning_rate": 5.0663991975927784e-05, + "loss": 1.1759, + "theoretical_loss": 3.341370735409304, + "tokens_seen": 3094202368 + }, + { + "epoch": 1.32, + "learning_rate": 5.0647943831494485e-05, + "loss": 1.2946, + "theoretical_loss": 3.3413653743029395, + "tokens_seen": 3094267904 + }, + { + "epoch": 1.32, + "learning_rate": 5.063189568706118e-05, + "loss": 1.2052, + "theoretical_loss": 3.3413600133419132, + "tokens_seen": 3094333440 + }, + { + "epoch": 1.32, + "learning_rate": 5.061584754262788e-05, + "loss": 1.1704, + "theoretical_loss": 3.3413546525262188, + "tokens_seen": 3094398976 + }, + { + "epoch": 1.32, + "learning_rate": 5.059979939819459e-05, + "loss": 1.1002, + "theoretical_loss": 3.341349291855848, + "tokens_seen": 3094464512 + }, + { + "epoch": 1.32, + "learning_rate": 5.0583751253761294e-05, + "loss": 1.2045, + "theoretical_loss": 3.3413439313307953, + "tokens_seen": 3094530048 + }, + { + "epoch": 1.32, + "learning_rate": 5.056770310932799e-05, + "loss": 1.1079, + "theoretical_loss": 3.341338570951053, + "tokens_seen": 3094595584 + }, + { + "epoch": 1.32, + "learning_rate": 5.055165496489469e-05, + "loss": 1.1728, + "theoretical_loss": 3.3413332107166136, + "tokens_seen": 3094661120 + }, + { + "epoch": 1.32, + "learning_rate": 5.053560682046139e-05, + "loss": 1.0695, + "theoretical_loss": 3.341327850627471, + "tokens_seen": 3094726656 + }, + { + "epoch": 1.32, + "learning_rate": 5.051955867602809e-05, + "loss": 1.1385, + "theoretical_loss": 3.341322490683617, + "tokens_seen": 3094792192 + }, + { + "epoch": 1.32, + "learning_rate": 5.050351053159479e-05, + "loss": 1.0707, + "theoretical_loss": 3.341317130885046, + "tokens_seen": 3094857728 + }, + { + "epoch": 1.32, + "learning_rate": 5.048746238716149e-05, + "loss": 1.1788, + "theoretical_loss": 3.34131177123175, + "tokens_seen": 3094923264 + }, + { + "epoch": 1.32, + "learning_rate": 5.0471414242728185e-05, + "loss": 1.1108, + "theoretical_loss": 3.341306411723723, + "tokens_seen": 3094988800 + }, + { + "epoch": 1.32, + "learning_rate": 5.0455366098294885e-05, + "loss": 1.134, + "theoretical_loss": 3.3413010523609565, + "tokens_seen": 3095054336 + }, + { + "epoch": 1.32, + "learning_rate": 5.0439317953861586e-05, + "loss": 1.21, + "theoretical_loss": 3.3412956931434445, + "tokens_seen": 3095119872 + }, + { + "epoch": 1.32, + "learning_rate": 5.042326980942829e-05, + "loss": 1.2013, + "theoretical_loss": 3.3412903340711795, + "tokens_seen": 3095185408 + }, + { + "epoch": 1.32, + "learning_rate": 5.040722166499499e-05, + "loss": 1.1242, + "theoretical_loss": 3.341284975144155, + "tokens_seen": 3095250944 + }, + { + "epoch": 1.32, + "learning_rate": 5.039117352056169e-05, + "loss": 1.1022, + "theoretical_loss": 3.341279616362364, + "tokens_seen": 3095316480 + }, + { + "epoch": 1.32, + "learning_rate": 5.037512537612838e-05, + "loss": 1.2128, + "theoretical_loss": 3.341274257725799, + "tokens_seen": 3095382016 + }, + { + "epoch": 1.32, + "learning_rate": 5.035907723169508e-05, + "loss": 1.2246, + "theoretical_loss": 3.341268899234453, + "tokens_seen": 3095447552 + }, + { + "epoch": 1.32, + "learning_rate": 5.034302908726178e-05, + "loss": 1.1724, + "theoretical_loss": 3.341263540888319, + "tokens_seen": 3095513088 + }, + { + "epoch": 1.32, + "learning_rate": 5.0326980942828484e-05, + "loss": 1.1191, + "theoretical_loss": 3.341258182687391, + "tokens_seen": 3095578624 + }, + { + "epoch": 1.32, + "learning_rate": 5.0310932798395184e-05, + "loss": 1.1548, + "theoretical_loss": 3.3412528246316606, + "tokens_seen": 3095644160 + }, + { + "epoch": 1.32, + "learning_rate": 5.029488465396189e-05, + "loss": 1.1776, + "theoretical_loss": 3.3412474667211214, + "tokens_seen": 3095709696 + }, + { + "epoch": 1.32, + "learning_rate": 5.027883650952859e-05, + "loss": 1.1549, + "theoretical_loss": 3.3412421089557665, + "tokens_seen": 3095775232 + }, + { + "epoch": 1.32, + "learning_rate": 5.026278836509529e-05, + "loss": 1.1284, + "theoretical_loss": 3.3412367513355887, + "tokens_seen": 3095840768 + }, + { + "epoch": 1.32, + "learning_rate": 5.0246740220661994e-05, + "loss": 1.1896, + "theoretical_loss": 3.3412313938605807, + "tokens_seen": 3095906304 + }, + { + "epoch": 1.32, + "learning_rate": 5.0230692076228694e-05, + "loss": 1.1413, + "theoretical_loss": 3.3412260365307365, + "tokens_seen": 3095971840 + }, + { + "epoch": 1.32, + "learning_rate": 5.021464393179539e-05, + "loss": 1.0751, + "theoretical_loss": 3.3412206793460477, + "tokens_seen": 3096037376 + }, + { + "epoch": 1.32, + "learning_rate": 5.019859578736209e-05, + "loss": 1.1188, + "theoretical_loss": 3.3412153223065086, + "tokens_seen": 3096102912 + }, + { + "epoch": 1.32, + "learning_rate": 5.018254764292879e-05, + "loss": 1.1532, + "theoretical_loss": 3.3412099654121112, + "tokens_seen": 3096168448 + }, + { + "epoch": 1.32, + "learning_rate": 5.016649949849549e-05, + "loss": 1.1676, + "theoretical_loss": 3.3412046086628493, + "tokens_seen": 3096233984 + }, + { + "epoch": 1.32, + "learning_rate": 5.015045135406219e-05, + "loss": 1.1111, + "theoretical_loss": 3.3411992520587157, + "tokens_seen": 3096299520 + }, + { + "epoch": 1.32, + "learning_rate": 5.013440320962889e-05, + "loss": 1.1518, + "theoretical_loss": 3.341193895599703, + "tokens_seen": 3096365056 + }, + { + "epoch": 1.32, + "learning_rate": 5.0118355065195585e-05, + "loss": 1.1838, + "theoretical_loss": 3.3411885392858043, + "tokens_seen": 3096430592 + }, + { + "epoch": 1.32, + "learning_rate": 5.0102306920762286e-05, + "loss": 1.0718, + "theoretical_loss": 3.3411831831170127, + "tokens_seen": 3096496128 + }, + { + "epoch": 1.32, + "learning_rate": 5.0086258776328986e-05, + "loss": 1.2697, + "theoretical_loss": 3.341177827093321, + "tokens_seen": 3096561664 + }, + { + "epoch": 1.32, + "learning_rate": 5.007021063189569e-05, + "loss": 1.1866, + "theoretical_loss": 3.3411724712147226, + "tokens_seen": 3096627200 + }, + { + "epoch": 1.32, + "learning_rate": 5.005416248746239e-05, + "loss": 1.2046, + "theoretical_loss": 3.3411671154812104, + "tokens_seen": 3096692736 + }, + { + "epoch": 1.32, + "learning_rate": 5.003811434302909e-05, + "loss": 1.1254, + "theoretical_loss": 3.3411617598927776, + "tokens_seen": 3096758272 + }, + { + "epoch": 1.32, + "learning_rate": 5.002206619859578e-05, + "loss": 1.1136, + "theoretical_loss": 3.3411564044494164, + "tokens_seen": 3096823808 + }, + { + "epoch": 1.32, + "learning_rate": 5.000601805416248e-05, + "loss": 1.0761, + "theoretical_loss": 3.3411510491511205, + "tokens_seen": 3096889344 + }, + { + "epoch": 1.32, + "learning_rate": 4.998996990972919e-05, + "loss": 1.1268, + "theoretical_loss": 3.3411456939978827, + "tokens_seen": 3096954880 + }, + { + "epoch": 1.32, + "learning_rate": 4.997392176529589e-05, + "loss": 1.1436, + "theoretical_loss": 3.341140338989696, + "tokens_seen": 3097020416 + }, + { + "epoch": 1.32, + "learning_rate": 4.995787362086259e-05, + "loss": 1.2044, + "theoretical_loss": 3.341134984126553, + "tokens_seen": 3097085952 + }, + { + "epoch": 1.32, + "learning_rate": 4.9941825476429285e-05, + "loss": 1.2081, + "theoretical_loss": 3.3411296294084476, + "tokens_seen": 3097151488 + }, + { + "epoch": 1.32, + "learning_rate": 4.9925777331995986e-05, + "loss": 1.2034, + "theoretical_loss": 3.3411242748353724, + "tokens_seen": 3097217024 + }, + { + "epoch": 1.32, + "learning_rate": 4.9909729187562693e-05, + "loss": 1.1792, + "theoretical_loss": 3.34111892040732, + "tokens_seen": 3097282560 + }, + { + "epoch": 1.32, + "learning_rate": 4.9893681043129394e-05, + "loss": 1.1119, + "theoretical_loss": 3.341113566124284, + "tokens_seen": 3097348096 + }, + { + "epoch": 1.32, + "learning_rate": 4.9877632898696095e-05, + "loss": 1.2282, + "theoretical_loss": 3.3411082119862567, + "tokens_seen": 3097413632 + }, + { + "epoch": 1.32, + "learning_rate": 4.986158475426279e-05, + "loss": 1.0905, + "theoretical_loss": 3.341102857993232, + "tokens_seen": 3097479168 + }, + { + "epoch": 1.32, + "learning_rate": 4.984553660982949e-05, + "loss": 1.1536, + "theoretical_loss": 3.3410975041452025, + "tokens_seen": 3097544704 + }, + { + "epoch": 1.32, + "learning_rate": 4.982948846539619e-05, + "loss": 1.1698, + "theoretical_loss": 3.3410921504421607, + "tokens_seen": 3097610240 + }, + { + "epoch": 1.32, + "learning_rate": 4.981344032096289e-05, + "loss": 1.1854, + "theoretical_loss": 3.3410867968841, + "tokens_seen": 3097675776 + }, + { + "epoch": 1.32, + "learning_rate": 4.979739217652959e-05, + "loss": 1.1149, + "theoretical_loss": 3.3410814434710137, + "tokens_seen": 3097741312 + }, + { + "epoch": 1.32, + "learning_rate": 4.978134403209629e-05, + "loss": 1.1573, + "theoretical_loss": 3.3410760902028946, + "tokens_seen": 3097806848 + }, + { + "epoch": 1.32, + "learning_rate": 4.976529588766299e-05, + "loss": 1.0891, + "theoretical_loss": 3.3410707370797357, + "tokens_seen": 3097872384 + }, + { + "epoch": 1.32, + "learning_rate": 4.974924774322969e-05, + "loss": 1.1571, + "theoretical_loss": 3.3410653841015296, + "tokens_seen": 3097937920 + }, + { + "epoch": 1.32, + "learning_rate": 4.9733199598796394e-05, + "loss": 1.1503, + "theoretical_loss": 3.3410600312682703, + "tokens_seen": 3098003456 + }, + { + "epoch": 1.32, + "learning_rate": 4.9717151454363094e-05, + "loss": 1.2143, + "theoretical_loss": 3.3410546785799498, + "tokens_seen": 3098068992 + }, + { + "epoch": 1.32, + "learning_rate": 4.9701103309929795e-05, + "loss": 1.2104, + "theoretical_loss": 3.3410493260365612, + "tokens_seen": 3098134528 + }, + { + "epoch": 1.32, + "learning_rate": 4.968505516549649e-05, + "loss": 1.1781, + "theoretical_loss": 3.341043973638098, + "tokens_seen": 3098200064 + }, + { + "epoch": 1.32, + "learning_rate": 4.966900702106319e-05, + "loss": 1.1979, + "theoretical_loss": 3.3410386213845533, + "tokens_seen": 3098265600 + }, + { + "epoch": 1.32, + "learning_rate": 4.965295887662989e-05, + "loss": 1.1007, + "theoretical_loss": 3.34103326927592, + "tokens_seen": 3098331136 + }, + { + "epoch": 1.32, + "learning_rate": 4.963691073219659e-05, + "loss": 1.2086, + "theoretical_loss": 3.3410279173121906, + "tokens_seen": 3098396672 + }, + { + "epoch": 1.32, + "learning_rate": 4.96208625877633e-05, + "loss": 1.0884, + "theoretical_loss": 3.3410225654933585, + "tokens_seen": 3098462208 + }, + { + "epoch": 1.32, + "learning_rate": 4.960481444332999e-05, + "loss": 1.1734, + "theoretical_loss": 3.3410172138194167, + "tokens_seen": 3098527744 + }, + { + "epoch": 1.32, + "learning_rate": 4.958876629889669e-05, + "loss": 1.1885, + "theoretical_loss": 3.341011862290358, + "tokens_seen": 3098593280 + }, + { + "epoch": 1.32, + "learning_rate": 4.957271815446339e-05, + "loss": 1.1322, + "theoretical_loss": 3.3410065109061757, + "tokens_seen": 3098658816 + }, + { + "epoch": 1.32, + "learning_rate": 4.9556670010030094e-05, + "loss": 1.0747, + "theoretical_loss": 3.341001159666863, + "tokens_seen": 3098724352 + }, + { + "epoch": 1.32, + "learning_rate": 4.9540621865596795e-05, + "loss": 1.1745, + "theoretical_loss": 3.340995808572412, + "tokens_seen": 3098789888 + }, + { + "epoch": 1.32, + "learning_rate": 4.9524573721163495e-05, + "loss": 1.275, + "theoretical_loss": 3.340990457622817, + "tokens_seen": 3098855424 + }, + { + "epoch": 1.32, + "learning_rate": 4.950852557673019e-05, + "loss": 1.1499, + "theoretical_loss": 3.3409851068180703, + "tokens_seen": 3098920960 + }, + { + "epoch": 1.32, + "learning_rate": 4.949247743229689e-05, + "loss": 1.085, + "theoretical_loss": 3.3409797561581644, + "tokens_seen": 3098986496 + }, + { + "epoch": 1.32, + "learning_rate": 4.947642928786359e-05, + "loss": 1.0833, + "theoretical_loss": 3.3409744056430934, + "tokens_seen": 3099052032 + }, + { + "epoch": 1.32, + "learning_rate": 4.94603811434303e-05, + "loss": 1.1471, + "theoretical_loss": 3.3409690552728497, + "tokens_seen": 3099117568 + }, + { + "epoch": 1.32, + "learning_rate": 4.9444332998997e-05, + "loss": 1.1487, + "theoretical_loss": 3.3409637050474266, + "tokens_seen": 3099183104 + }, + { + "epoch": 1.32, + "learning_rate": 4.942828485456369e-05, + "loss": 1.211, + "theoretical_loss": 3.3409583549668165, + "tokens_seen": 3099248640 + }, + { + "epoch": 1.32, + "learning_rate": 4.941223671013039e-05, + "loss": 1.1379, + "theoretical_loss": 3.3409530050310132, + "tokens_seen": 3099314176 + }, + { + "epoch": 1.32, + "learning_rate": 4.9396188565697093e-05, + "loss": 1.1444, + "theoretical_loss": 3.3409476552400093, + "tokens_seen": 3099379712 + }, + { + "epoch": 1.32, + "learning_rate": 4.9380140421263794e-05, + "loss": 1.0342, + "theoretical_loss": 3.3409423055937975, + "tokens_seen": 3099445248 + }, + { + "epoch": 1.32, + "learning_rate": 4.9364092276830495e-05, + "loss": 1.2288, + "theoretical_loss": 3.3409369560923716, + "tokens_seen": 3099510784 + }, + { + "epoch": 1.32, + "learning_rate": 4.9348044132397195e-05, + "loss": 1.0199, + "theoretical_loss": 3.340931606735724, + "tokens_seen": 3099576320 + }, + { + "epoch": 1.32, + "learning_rate": 4.933199598796389e-05, + "loss": 1.1383, + "theoretical_loss": 3.3409262575238485, + "tokens_seen": 3099641856 + }, + { + "epoch": 1.32, + "learning_rate": 4.93159478435306e-05, + "loss": 1.1978, + "theoretical_loss": 3.340920908456737, + "tokens_seen": 3099707392 + }, + { + "epoch": 1.32, + "learning_rate": 4.92998996990973e-05, + "loss": 1.1277, + "theoretical_loss": 3.3409155595343836, + "tokens_seen": 3099772928 + }, + { + "epoch": 1.32, + "objective/train/docs_used": 1143720, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2005659341812134, + "objective/train/theoretical_loss": 3.3409102107567805, + "objective/train/tokens_used": 3120298464, + "theoretical_loss": 3.3409102107567805, + "tokens_seen": 3099838464 + }, + { + "epoch": 1.32, + "learning_rate": 4.9283851554664e-05, + "loss": 1.2006, + "theoretical_loss": 3.3409102107567805, + "tokens_seen": 3099838464 + }, + { + "epoch": 1.32, + "learning_rate": 4.926780341023069e-05, + "loss": 1.1498, + "theoretical_loss": 3.3409048621239212, + "tokens_seen": 3099904000 + }, + { + "epoch": 1.32, + "learning_rate": 4.925175526579739e-05, + "loss": 1.2071, + "theoretical_loss": 3.340899513635798, + "tokens_seen": 3099969536 + }, + { + "epoch": 1.32, + "learning_rate": 4.923570712136409e-05, + "loss": 1.2278, + "theoretical_loss": 3.340894165292405, + "tokens_seen": 3100035072 + }, + { + "epoch": 1.32, + "learning_rate": 4.9219658976930794e-05, + "loss": 1.1479, + "theoretical_loss": 3.340888817093735, + "tokens_seen": 3100100608 + }, + { + "epoch": 1.32, + "learning_rate": 4.9203610832497494e-05, + "loss": 1.126, + "theoretical_loss": 3.34088346903978, + "tokens_seen": 3100166144 + }, + { + "epoch": 1.32, + "learning_rate": 4.9187562688064195e-05, + "loss": 1.2181, + "theoretical_loss": 3.3408781211305345, + "tokens_seen": 3100231680 + }, + { + "epoch": 1.32, + "learning_rate": 4.9171514543630896e-05, + "loss": 1.1163, + "theoretical_loss": 3.3408727733659904, + "tokens_seen": 3100297216 + }, + { + "epoch": 1.32, + "learning_rate": 4.9155466399197596e-05, + "loss": 1.1692, + "theoretical_loss": 3.3408674257461413, + "tokens_seen": 3100362752 + }, + { + "epoch": 1.32, + "learning_rate": 4.91394182547643e-05, + "loss": 1.1113, + "theoretical_loss": 3.34086207827098, + "tokens_seen": 3100428288 + }, + { + "epoch": 1.32, + "learning_rate": 4.9123370110331e-05, + "loss": 1.2518, + "theoretical_loss": 3.3408567309404993, + "tokens_seen": 3100493824 + }, + { + "epoch": 1.32, + "learning_rate": 4.91073219658977e-05, + "loss": 1.2285, + "theoretical_loss": 3.3408513837546927, + "tokens_seen": 3100559360 + }, + { + "epoch": 1.32, + "learning_rate": 4.909127382146439e-05, + "loss": 1.1554, + "theoretical_loss": 3.340846036713553, + "tokens_seen": 3100624896 + }, + { + "epoch": 1.32, + "learning_rate": 4.907522567703109e-05, + "loss": 1.1958, + "theoretical_loss": 3.3408406898170733, + "tokens_seen": 3100690432 + }, + { + "epoch": 1.32, + "learning_rate": 4.905917753259779e-05, + "loss": 1.0885, + "theoretical_loss": 3.3408353430652467, + "tokens_seen": 3100755968 + }, + { + "epoch": 1.32, + "learning_rate": 4.9043129388164494e-05, + "loss": 1.0941, + "theoretical_loss": 3.3408299964580657, + "tokens_seen": 3100821504 + }, + { + "epoch": 1.32, + "learning_rate": 4.9027081243731194e-05, + "loss": 1.1593, + "theoretical_loss": 3.340824649995524, + "tokens_seen": 3100887040 + }, + { + "epoch": 1.32, + "learning_rate": 4.9011033099297895e-05, + "loss": 1.1379, + "theoretical_loss": 3.3408193036776144, + "tokens_seen": 3100952576 + }, + { + "epoch": 1.32, + "learning_rate": 4.8994984954864596e-05, + "loss": 1.2514, + "theoretical_loss": 3.34081395750433, + "tokens_seen": 3101018112 + }, + { + "epoch": 1.32, + "learning_rate": 4.8978936810431296e-05, + "loss": 1.1393, + "theoretical_loss": 3.340808611475664, + "tokens_seen": 3101083648 + }, + { + "epoch": 1.32, + "learning_rate": 4.8962888665998e-05, + "loss": 1.1543, + "theoretical_loss": 3.3408032655916085, + "tokens_seen": 3101149184 + }, + { + "epoch": 1.32, + "learning_rate": 4.89468405215647e-05, + "loss": 1.1343, + "theoretical_loss": 3.3407979198521573, + "tokens_seen": 3101214720 + }, + { + "epoch": 1.32, + "learning_rate": 4.89307923771314e-05, + "loss": 1.0796, + "theoretical_loss": 3.3407925742573035, + "tokens_seen": 3101280256 + }, + { + "epoch": 1.32, + "learning_rate": 4.891474423269809e-05, + "loss": 1.0394, + "theoretical_loss": 3.34078722880704, + "tokens_seen": 3101345792 + }, + { + "epoch": 1.32, + "learning_rate": 4.889869608826479e-05, + "loss": 1.211, + "theoretical_loss": 3.34078188350136, + "tokens_seen": 3101411328 + }, + { + "epoch": 1.32, + "learning_rate": 4.8882647943831493e-05, + "loss": 1.2744, + "theoretical_loss": 3.340776538340256, + "tokens_seen": 3101476864 + }, + { + "epoch": 1.32, + "learning_rate": 4.88665997993982e-05, + "loss": 1.2154, + "theoretical_loss": 3.3407711933237216, + "tokens_seen": 3101542400 + }, + { + "epoch": 1.32, + "learning_rate": 4.88505516549649e-05, + "loss": 1.1623, + "theoretical_loss": 3.340765848451749, + "tokens_seen": 3101607936 + }, + { + "epoch": 1.32, + "learning_rate": 4.8834503510531595e-05, + "loss": 1.2005, + "theoretical_loss": 3.3407605037243324, + "tokens_seen": 3101673472 + }, + { + "epoch": 1.32, + "learning_rate": 4.8818455366098296e-05, + "loss": 1.3005, + "theoretical_loss": 3.340755159141464, + "tokens_seen": 3101739008 + }, + { + "epoch": 1.32, + "learning_rate": 4.8802407221664997e-05, + "loss": 1.1838, + "theoretical_loss": 3.3407498147031376, + "tokens_seen": 3101804544 + }, + { + "epoch": 1.32, + "learning_rate": 4.87863590772317e-05, + "loss": 1.2467, + "theoretical_loss": 3.340744470409345, + "tokens_seen": 3101870080 + }, + { + "epoch": 1.32, + "learning_rate": 4.87703109327984e-05, + "loss": 1.2399, + "theoretical_loss": 3.3407391262600807, + "tokens_seen": 3101935616 + }, + { + "epoch": 1.32, + "learning_rate": 4.87542627883651e-05, + "loss": 1.277, + "theoretical_loss": 3.340733782255337, + "tokens_seen": 3102001152 + }, + { + "epoch": 1.32, + "learning_rate": 4.873821464393179e-05, + "loss": 1.1886, + "theoretical_loss": 3.340728438395107, + "tokens_seen": 3102066688 + }, + { + "epoch": 1.32, + "learning_rate": 4.87221664994985e-05, + "loss": 1.0987, + "theoretical_loss": 3.340723094679383, + "tokens_seen": 3102132224 + }, + { + "epoch": 1.32, + "learning_rate": 4.87061183550652e-05, + "loss": 1.1468, + "theoretical_loss": 3.3407177511081594, + "tokens_seen": 3102197760 + }, + { + "epoch": 1.32, + "learning_rate": 4.86900702106319e-05, + "loss": 1.209, + "theoretical_loss": 3.340712407681428, + "tokens_seen": 3102263296 + }, + { + "epoch": 1.32, + "learning_rate": 4.86740220661986e-05, + "loss": 1.146, + "theoretical_loss": 3.340707064399183, + "tokens_seen": 3102328832 + }, + { + "epoch": 1.32, + "learning_rate": 4.8657973921765296e-05, + "loss": 1.1334, + "theoretical_loss": 3.3407017212614165, + "tokens_seen": 3102394368 + }, + { + "epoch": 1.32, + "learning_rate": 4.8641925777331996e-05, + "loss": 1.2089, + "theoretical_loss": 3.340696378268122, + "tokens_seen": 3102459904 + }, + { + "epoch": 1.32, + "learning_rate": 4.86258776328987e-05, + "loss": 1.179, + "theoretical_loss": 3.3406910354192925, + "tokens_seen": 3102525440 + }, + { + "epoch": 1.32, + "learning_rate": 4.86098294884654e-05, + "loss": 1.202, + "theoretical_loss": 3.340685692714921, + "tokens_seen": 3102590976 + }, + { + "epoch": 1.32, + "learning_rate": 4.85937813440321e-05, + "loss": 1.1561, + "theoretical_loss": 3.3406803501550004, + "tokens_seen": 3102656512 + }, + { + "epoch": 1.32, + "learning_rate": 4.85777331995988e-05, + "loss": 1.2245, + "theoretical_loss": 3.340675007739524, + "tokens_seen": 3102722048 + }, + { + "epoch": 1.32, + "learning_rate": 4.85616850551655e-05, + "loss": 1.1841, + "theoretical_loss": 3.340669665468485, + "tokens_seen": 3102787584 + }, + { + "epoch": 1.32, + "learning_rate": 4.85456369107322e-05, + "loss": 1.195, + "theoretical_loss": 3.340664323341876, + "tokens_seen": 3102853120 + }, + { + "epoch": 1.32, + "learning_rate": 4.85295887662989e-05, + "loss": 1.22, + "theoretical_loss": 3.34065898135969, + "tokens_seen": 3102918656 + }, + { + "epoch": 1.32, + "learning_rate": 4.85135406218656e-05, + "loss": 1.1366, + "theoretical_loss": 3.34065363952192, + "tokens_seen": 3102984192 + }, + { + "epoch": 1.32, + "learning_rate": 4.84974924774323e-05, + "loss": 1.1784, + "theoretical_loss": 3.34064829782856, + "tokens_seen": 3103049728 + }, + { + "epoch": 1.32, + "learning_rate": 4.8481444332998996e-05, + "loss": 1.1773, + "theoretical_loss": 3.3406429562796016, + "tokens_seen": 3103115264 + }, + { + "epoch": 1.32, + "learning_rate": 4.8465396188565696e-05, + "loss": 1.1453, + "theoretical_loss": 3.340637614875039, + "tokens_seen": 3103180800 + }, + { + "epoch": 1.32, + "learning_rate": 4.84493480441324e-05, + "loss": 1.2506, + "theoretical_loss": 3.340632273614865, + "tokens_seen": 3103246336 + }, + { + "epoch": 1.32, + "learning_rate": 4.84332998996991e-05, + "loss": 1.2013, + "theoretical_loss": 3.340626932499072, + "tokens_seen": 3103311872 + }, + { + "epoch": 1.32, + "learning_rate": 4.8417251755265805e-05, + "loss": 1.223, + "theoretical_loss": 3.340621591527654, + "tokens_seen": 3103377408 + }, + { + "epoch": 1.32, + "learning_rate": 4.84012036108325e-05, + "loss": 1.1488, + "theoretical_loss": 3.340616250700603, + "tokens_seen": 3103442944 + }, + { + "epoch": 1.32, + "learning_rate": 4.83851554663992e-05, + "loss": 1.1535, + "theoretical_loss": 3.3406109100179133, + "tokens_seen": 3103508480 + }, + { + "epoch": 1.32, + "learning_rate": 4.83691073219659e-05, + "loss": 1.1216, + "theoretical_loss": 3.340605569479577, + "tokens_seen": 3103574016 + }, + { + "epoch": 1.32, + "learning_rate": 4.83530591775326e-05, + "loss": 1.2928, + "theoretical_loss": 3.340600229085587, + "tokens_seen": 3103639552 + }, + { + "epoch": 1.32, + "learning_rate": 4.83370110330993e-05, + "loss": 1.1798, + "theoretical_loss": 3.3405948888359376, + "tokens_seen": 3103705088 + }, + { + "epoch": 1.32, + "learning_rate": 4.8320962888666e-05, + "loss": 1.1533, + "theoretical_loss": 3.3405895487306205, + "tokens_seen": 3103770624 + }, + { + "epoch": 1.32, + "learning_rate": 4.8304914744232696e-05, + "loss": 1.1458, + "theoretical_loss": 3.34058420876963, + "tokens_seen": 3103836160 + }, + { + "epoch": 1.32, + "learning_rate": 4.8288866599799397e-05, + "loss": 1.2436, + "theoretical_loss": 3.3405788689529574, + "tokens_seen": 3103901696 + }, + { + "epoch": 1.32, + "learning_rate": 4.8272818455366104e-05, + "loss": 1.1504, + "theoretical_loss": 3.3405735292805976, + "tokens_seen": 3103967232 + }, + { + "epoch": 1.32, + "learning_rate": 4.8256770310932805e-05, + "loss": 1.143, + "theoretical_loss": 3.3405681897525423, + "tokens_seen": 3104032768 + }, + { + "epoch": 1.32, + "learning_rate": 4.8240722166499505e-05, + "loss": 1.1495, + "theoretical_loss": 3.3405628503687854, + "tokens_seen": 3104098304 + }, + { + "epoch": 1.32, + "learning_rate": 4.82246740220662e-05, + "loss": 1.1968, + "theoretical_loss": 3.3405575111293198, + "tokens_seen": 3104163840 + }, + { + "epoch": 1.32, + "learning_rate": 4.82086258776329e-05, + "loss": 1.1653, + "theoretical_loss": 3.340552172034138, + "tokens_seen": 3104229376 + }, + { + "epoch": 1.32, + "learning_rate": 4.81925777331996e-05, + "loss": 1.2043, + "theoretical_loss": 3.3405468330832337, + "tokens_seen": 3104294912 + }, + { + "epoch": 1.32, + "learning_rate": 4.81765295887663e-05, + "loss": 1.186, + "theoretical_loss": 3.3405414942765996, + "tokens_seen": 3104360448 + }, + { + "epoch": 1.32, + "learning_rate": 4.8160481444333e-05, + "loss": 1.1783, + "theoretical_loss": 3.3405361556142292, + "tokens_seen": 3104425984 + }, + { + "epoch": 1.32, + "learning_rate": 4.81444332998997e-05, + "loss": 1.2309, + "theoretical_loss": 3.3405308170961145, + "tokens_seen": 3104491520 + }, + { + "epoch": 1.32, + "learning_rate": 4.8128385155466396e-05, + "loss": 1.1869, + "theoretical_loss": 3.3405254787222503, + "tokens_seen": 3104557056 + }, + { + "epoch": 1.32, + "learning_rate": 4.8112337011033104e-05, + "loss": 1.2456, + "theoretical_loss": 3.340520140492628, + "tokens_seen": 3104622592 + }, + { + "epoch": 1.32, + "learning_rate": 4.8096288866599804e-05, + "loss": 1.2269, + "theoretical_loss": 3.3405148024072413, + "tokens_seen": 3104688128 + }, + { + "epoch": 1.32, + "learning_rate": 4.8080240722166505e-05, + "loss": 1.1094, + "theoretical_loss": 3.3405094644660833, + "tokens_seen": 3104753664 + }, + { + "epoch": 1.32, + "learning_rate": 4.8064192577733205e-05, + "loss": 1.2555, + "theoretical_loss": 3.3405041266691473, + "tokens_seen": 3104819200 + }, + { + "epoch": 1.32, + "learning_rate": 4.80481444332999e-05, + "loss": 1.2245, + "theoretical_loss": 3.3404987890164257, + "tokens_seen": 3104884736 + }, + { + "epoch": 1.32, + "learning_rate": 4.80320962888666e-05, + "loss": 1.1561, + "theoretical_loss": 3.3404934515079123, + "tokens_seen": 3104950272 + }, + { + "epoch": 1.32, + "learning_rate": 4.80160481444333e-05, + "loss": 1.2274, + "theoretical_loss": 3.3404881141435996, + "tokens_seen": 3105015808 + }, + { + "epoch": 1.32, + "learning_rate": 4.8e-05, + "loss": 1.2655, + "theoretical_loss": 3.340482776923481, + "tokens_seen": 3105081344 + }, + { + "epoch": 1.32, + "learning_rate": 4.79839518555667e-05, + "loss": 1.2082, + "theoretical_loss": 3.340477439847549, + "tokens_seen": 3105146880 + }, + { + "epoch": 1.32, + "learning_rate": 4.79679037111334e-05, + "loss": 1.1235, + "theoretical_loss": 3.3404721029157973, + "tokens_seen": 3105212416 + }, + { + "epoch": 1.32, + "learning_rate": 4.79518555667001e-05, + "loss": 1.2304, + "theoretical_loss": 3.340466766128219, + "tokens_seen": 3105277952 + }, + { + "epoch": 1.32, + "learning_rate": 4.7935807422266804e-05, + "loss": 1.1457, + "theoretical_loss": 3.3404614294848063, + "tokens_seen": 3105343488 + }, + { + "epoch": 1.32, + "learning_rate": 4.7919759277833504e-05, + "loss": 1.211, + "theoretical_loss": 3.3404560929855536, + "tokens_seen": 3105409024 + }, + { + "epoch": 1.32, + "learning_rate": 4.7903711133400205e-05, + "loss": 1.1761, + "theoretical_loss": 3.3404507566304527, + "tokens_seen": 3105474560 + }, + { + "epoch": 1.32, + "learning_rate": 4.7887662988966906e-05, + "loss": 1.3376, + "theoretical_loss": 3.3404454204194973, + "tokens_seen": 3105540096 + }, + { + "epoch": 1.32, + "learning_rate": 4.78716148445336e-05, + "loss": 1.1957, + "theoretical_loss": 3.3404400843526805, + "tokens_seen": 3105605632 + }, + { + "epoch": 1.32, + "learning_rate": 4.78555667001003e-05, + "loss": 1.1526, + "theoretical_loss": 3.3404347484299954, + "tokens_seen": 3105671168 + }, + { + "epoch": 1.32, + "learning_rate": 4.7839518555667e-05, + "loss": 1.1843, + "theoretical_loss": 3.3404294126514347, + "tokens_seen": 3105736704 + }, + { + "epoch": 1.32, + "learning_rate": 4.78234704112337e-05, + "loss": 1.0671, + "theoretical_loss": 3.3404240770169915, + "tokens_seen": 3105802240 + }, + { + "epoch": 1.32, + "learning_rate": 4.780742226680041e-05, + "loss": 1.1241, + "theoretical_loss": 3.3404187415266593, + "tokens_seen": 3105867776 + }, + { + "epoch": 1.32, + "learning_rate": 4.77913741223671e-05, + "loss": 1.2216, + "theoretical_loss": 3.340413406180431, + "tokens_seen": 3105933312 + }, + { + "epoch": 1.32, + "learning_rate": 4.77753259779338e-05, + "loss": 1.2066, + "theoretical_loss": 3.340408070978299, + "tokens_seen": 3105998848 + }, + { + "epoch": 1.32, + "learning_rate": 4.7759277833500504e-05, + "loss": 1.169, + "theoretical_loss": 3.3404027359202573, + "tokens_seen": 3106064384 + }, + { + "epoch": 1.32, + "learning_rate": 4.7743229689067205e-05, + "loss": 1.2066, + "theoretical_loss": 3.3403974010062987, + "tokens_seen": 3106129920 + }, + { + "epoch": 1.32, + "learning_rate": 4.7727181544633905e-05, + "loss": 1.1664, + "theoretical_loss": 3.340392066236416, + "tokens_seen": 3106195456 + }, + { + "epoch": 1.32, + "learning_rate": 4.7711133400200606e-05, + "loss": 1.174, + "theoretical_loss": 3.3403867316106024, + "tokens_seen": 3106260992 + }, + { + "epoch": 1.32, + "learning_rate": 4.76950852557673e-05, + "loss": 1.0096, + "theoretical_loss": 3.340381397128851, + "tokens_seen": 3106326528 + }, + { + "epoch": 1.32, + "objective/train/docs_used": 1146003, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.136634349822998, + "objective/train/theoretical_loss": 3.340376062791155, + "objective/train/tokens_used": 3126852064, + "theoretical_loss": 3.340376062791155, + "tokens_seen": 3106392064 + }, + { + "epoch": 1.32, + "learning_rate": 4.7679037111334e-05, + "loss": 1.1366, + "theoretical_loss": 3.340376062791155, + "tokens_seen": 3106392064 + }, + { + "epoch": 1.32, + "learning_rate": 4.766298896690071e-05, + "loss": 1.1354, + "theoretical_loss": 3.3403707285975073, + "tokens_seen": 3106457600 + }, + { + "epoch": 1.32, + "learning_rate": 4.764694082246741e-05, + "loss": 1.1631, + "theoretical_loss": 3.340365394547901, + "tokens_seen": 3106523136 + }, + { + "epoch": 1.32, + "learning_rate": 4.763089267803411e-05, + "loss": 1.1221, + "theoretical_loss": 3.340360060642329, + "tokens_seen": 3106588672 + }, + { + "epoch": 1.32, + "learning_rate": 4.76148445336008e-05, + "loss": 1.2073, + "theoretical_loss": 3.340354726880785, + "tokens_seen": 3106654208 + }, + { + "epoch": 1.32, + "learning_rate": 4.7598796389167504e-05, + "loss": 1.177, + "theoretical_loss": 3.3403493932632613, + "tokens_seen": 3106719744 + }, + { + "epoch": 1.32, + "learning_rate": 4.7582748244734204e-05, + "loss": 1.1357, + "theoretical_loss": 3.3403440597897514, + "tokens_seen": 3106785280 + }, + { + "epoch": 1.32, + "learning_rate": 4.7566700100300905e-05, + "loss": 1.2022, + "theoretical_loss": 3.3403387264602484, + "tokens_seen": 3106850816 + }, + { + "epoch": 1.32, + "learning_rate": 4.7550651955867605e-05, + "loss": 1.0784, + "theoretical_loss": 3.340333393274745, + "tokens_seen": 3106916352 + }, + { + "epoch": 1.32, + "learning_rate": 4.7534603811434306e-05, + "loss": 1.1258, + "theoretical_loss": 3.340328060233235, + "tokens_seen": 3106981888 + }, + { + "epoch": 1.32, + "learning_rate": 4.751855566700101e-05, + "loss": 1.1464, + "theoretical_loss": 3.3403227273357103, + "tokens_seen": 3107047424 + }, + { + "epoch": 1.32, + "learning_rate": 4.750250752256771e-05, + "loss": 1.2969, + "theoretical_loss": 3.340317394582165, + "tokens_seen": 3107112960 + }, + { + "epoch": 1.32, + "learning_rate": 4.748645937813441e-05, + "loss": 1.1149, + "theoretical_loss": 3.340312061972592, + "tokens_seen": 3107178496 + }, + { + "epoch": 1.32, + "learning_rate": 4.747041123370111e-05, + "loss": 1.1631, + "theoretical_loss": 3.340306729506984, + "tokens_seen": 3107244032 + }, + { + "epoch": 1.32, + "learning_rate": 4.745436308926781e-05, + "loss": 1.1728, + "theoretical_loss": 3.3403013971853346, + "tokens_seen": 3107309568 + }, + { + "epoch": 1.32, + "learning_rate": 4.74383149448345e-05, + "loss": 1.0922, + "theoretical_loss": 3.340296065007636, + "tokens_seen": 3107375104 + }, + { + "epoch": 1.32, + "learning_rate": 4.7422266800401204e-05, + "loss": 1.2056, + "theoretical_loss": 3.3402907329738825, + "tokens_seen": 3107440640 + }, + { + "epoch": 1.32, + "learning_rate": 4.7406218655967904e-05, + "loss": 1.1986, + "theoretical_loss": 3.340285401084066, + "tokens_seen": 3107506176 + }, + { + "epoch": 1.32, + "learning_rate": 4.7390170511534605e-05, + "loss": 1.2773, + "theoretical_loss": 3.340280069338181, + "tokens_seen": 3107571712 + }, + { + "epoch": 1.32, + "learning_rate": 4.7374122367101306e-05, + "loss": 1.1834, + "theoretical_loss": 3.3402747377362187, + "tokens_seen": 3107637248 + }, + { + "epoch": 1.32, + "learning_rate": 4.7358074222668006e-05, + "loss": 1.2086, + "theoretical_loss": 3.340269406278174, + "tokens_seen": 3107702784 + }, + { + "epoch": 1.32, + "learning_rate": 4.734202607823471e-05, + "loss": 1.2343, + "theoretical_loss": 3.340264074964039, + "tokens_seen": 3107768320 + }, + { + "epoch": 1.32, + "learning_rate": 4.732597793380141e-05, + "loss": 1.193, + "theoretical_loss": 3.3402587437938065, + "tokens_seen": 3107833856 + }, + { + "epoch": 1.32, + "learning_rate": 4.730992978936811e-05, + "loss": 1.1964, + "theoretical_loss": 3.34025341276747, + "tokens_seen": 3107899392 + }, + { + "epoch": 1.32, + "learning_rate": 4.729388164493481e-05, + "loss": 1.1318, + "theoretical_loss": 3.340248081885023, + "tokens_seen": 3107964928 + }, + { + "epoch": 1.32, + "learning_rate": 4.727783350050151e-05, + "loss": 1.2121, + "theoretical_loss": 3.340242751146458, + "tokens_seen": 3108030464 + }, + { + "epoch": 1.32, + "learning_rate": 4.72617853560682e-05, + "loss": 1.1603, + "theoretical_loss": 3.3402374205517686, + "tokens_seen": 3108096000 + }, + { + "epoch": 1.32, + "learning_rate": 4.7245737211634904e-05, + "loss": 1.1692, + "theoretical_loss": 3.340232090100947, + "tokens_seen": 3108161536 + }, + { + "epoch": 1.32, + "learning_rate": 4.7229689067201605e-05, + "loss": 1.199, + "theoretical_loss": 3.340226759793987, + "tokens_seen": 3108227072 + }, + { + "epoch": 1.32, + "learning_rate": 4.721364092276831e-05, + "loss": 1.1198, + "theoretical_loss": 3.3402214296308816, + "tokens_seen": 3108292608 + }, + { + "epoch": 1.32, + "learning_rate": 4.7197592778335006e-05, + "loss": 1.21, + "theoretical_loss": 3.340216099611624, + "tokens_seen": 3108358144 + }, + { + "epoch": 1.32, + "learning_rate": 4.7181544633901706e-05, + "loss": 1.1902, + "theoretical_loss": 3.340210769736207, + "tokens_seen": 3108423680 + }, + { + "epoch": 1.32, + "learning_rate": 4.716549648946841e-05, + "loss": 1.1059, + "theoretical_loss": 3.340205440004624, + "tokens_seen": 3108489216 + }, + { + "epoch": 1.32, + "learning_rate": 4.714944834503511e-05, + "loss": 1.1286, + "theoretical_loss": 3.3402001104168675, + "tokens_seen": 3108554752 + }, + { + "epoch": 1.32, + "learning_rate": 4.713340020060181e-05, + "loss": 1.1435, + "theoretical_loss": 3.340194780972931, + "tokens_seen": 3108620288 + }, + { + "epoch": 1.32, + "learning_rate": 4.711735205616851e-05, + "loss": 1.2384, + "theoretical_loss": 3.3401894516728077, + "tokens_seen": 3108685824 + }, + { + "epoch": 1.32, + "learning_rate": 4.71013039117352e-05, + "loss": 1.1779, + "theoretical_loss": 3.34018412251649, + "tokens_seen": 3108751360 + }, + { + "epoch": 1.32, + "learning_rate": 4.7085255767301904e-05, + "loss": 1.1766, + "theoretical_loss": 3.340178793503972, + "tokens_seen": 3108816896 + }, + { + "epoch": 1.32, + "learning_rate": 4.706920762286861e-05, + "loss": 1.1589, + "theoretical_loss": 3.3401734646352463, + "tokens_seen": 3108882432 + }, + { + "epoch": 1.32, + "learning_rate": 4.705315947843531e-05, + "loss": 1.1751, + "theoretical_loss": 3.340168135910306, + "tokens_seen": 3108947968 + }, + { + "epoch": 1.32, + "learning_rate": 4.703711133400201e-05, + "loss": 1.2334, + "theoretical_loss": 3.3401628073291443, + "tokens_seen": 3109013504 + }, + { + "epoch": 1.32, + "learning_rate": 4.7021063189568706e-05, + "loss": 1.1396, + "theoretical_loss": 3.340157478891754, + "tokens_seen": 3109079040 + }, + { + "epoch": 1.32, + "learning_rate": 4.700501504513541e-05, + "loss": 1.2459, + "theoretical_loss": 3.340152150598128, + "tokens_seen": 3109144576 + }, + { + "epoch": 1.32, + "learning_rate": 4.698896690070211e-05, + "loss": 1.2666, + "theoretical_loss": 3.34014682244826, + "tokens_seen": 3109210112 + }, + { + "epoch": 1.32, + "learning_rate": 4.697291875626881e-05, + "loss": 1.1347, + "theoretical_loss": 3.3401414944421433, + "tokens_seen": 3109275648 + }, + { + "epoch": 1.32, + "learning_rate": 4.695687061183551e-05, + "loss": 1.2473, + "theoretical_loss": 3.3401361665797697, + "tokens_seen": 3109341184 + }, + { + "epoch": 1.32, + "learning_rate": 4.694082246740221e-05, + "loss": 1.2194, + "theoretical_loss": 3.3401308388611337, + "tokens_seen": 3109406720 + }, + { + "epoch": 1.32, + "learning_rate": 4.69247743229689e-05, + "loss": 1.2312, + "theoretical_loss": 3.340125511286228, + "tokens_seen": 3109472256 + }, + { + "epoch": 1.32, + "learning_rate": 4.690872617853561e-05, + "loss": 1.1819, + "theoretical_loss": 3.340120183855045, + "tokens_seen": 3109537792 + }, + { + "epoch": 1.32, + "learning_rate": 4.689267803410231e-05, + "loss": 1.0969, + "theoretical_loss": 3.3401148565675784, + "tokens_seen": 3109603328 + }, + { + "epoch": 1.32, + "learning_rate": 4.687662988966901e-05, + "loss": 1.2059, + "theoretical_loss": 3.3401095294238217, + "tokens_seen": 3109668864 + }, + { + "epoch": 1.32, + "learning_rate": 4.686058174523571e-05, + "loss": 1.1142, + "theoretical_loss": 3.340104202423767, + "tokens_seen": 3109734400 + }, + { + "epoch": 1.32, + "learning_rate": 4.6844533600802406e-05, + "loss": 1.0943, + "theoretical_loss": 3.340098875567408, + "tokens_seen": 3109799936 + }, + { + "epoch": 1.32, + "learning_rate": 4.682848545636911e-05, + "loss": 1.2028, + "theoretical_loss": 3.3400935488547376, + "tokens_seen": 3109865472 + }, + { + "epoch": 1.32, + "learning_rate": 4.681243731193581e-05, + "loss": 1.1361, + "theoretical_loss": 3.340088222285749, + "tokens_seen": 3109931008 + }, + { + "epoch": 1.32, + "learning_rate": 4.679638916750251e-05, + "loss": 1.1038, + "theoretical_loss": 3.3400828958604354, + "tokens_seen": 3109996544 + }, + { + "epoch": 1.32, + "learning_rate": 4.678034102306921e-05, + "loss": 1.1441, + "theoretical_loss": 3.34007756957879, + "tokens_seen": 3110062080 + }, + { + "epoch": 1.32, + "learning_rate": 4.676429287863591e-05, + "loss": 1.1898, + "theoretical_loss": 3.3400722434408054, + "tokens_seen": 3110127616 + }, + { + "epoch": 1.32, + "learning_rate": 4.674824473420261e-05, + "loss": 1.1319, + "theoretical_loss": 3.340066917446475, + "tokens_seen": 3110193152 + }, + { + "epoch": 1.32, + "learning_rate": 4.673219658976931e-05, + "loss": 1.113, + "theoretical_loss": 3.3400615915957914, + "tokens_seen": 3110258688 + }, + { + "epoch": 1.32, + "learning_rate": 4.671614844533601e-05, + "loss": 1.204, + "theoretical_loss": 3.340056265888749, + "tokens_seen": 3110324224 + }, + { + "epoch": 1.32, + "learning_rate": 4.670010030090271e-05, + "loss": 1.1177, + "theoretical_loss": 3.3400509403253396, + "tokens_seen": 3110389760 + }, + { + "epoch": 1.32, + "learning_rate": 4.668405215646941e-05, + "loss": 1.1979, + "theoretical_loss": 3.340045614905557, + "tokens_seen": 3110455296 + }, + { + "epoch": 1.32, + "learning_rate": 4.6668004012036106e-05, + "loss": 1.1487, + "theoretical_loss": 3.340040289629394, + "tokens_seen": 3110520832 + }, + { + "epoch": 1.32, + "learning_rate": 4.665195586760281e-05, + "loss": 1.1981, + "theoretical_loss": 3.3400349644968435, + "tokens_seen": 3110586368 + }, + { + "epoch": 1.32, + "learning_rate": 4.663590772316951e-05, + "loss": 1.1165, + "theoretical_loss": 3.3400296395078994, + "tokens_seen": 3110651904 + }, + { + "epoch": 1.32, + "learning_rate": 4.6619859578736215e-05, + "loss": 1.2208, + "theoretical_loss": 3.340024314662554, + "tokens_seen": 3110717440 + }, + { + "epoch": 1.32, + "learning_rate": 4.6603811434302916e-05, + "loss": 1.115, + "theoretical_loss": 3.3400189899608006, + "tokens_seen": 3110782976 + }, + { + "epoch": 1.32, + "learning_rate": 4.658776328986961e-05, + "loss": 1.195, + "theoretical_loss": 3.3400136654026324, + "tokens_seen": 3110848512 + }, + { + "epoch": 1.32, + "learning_rate": 4.657171514543631e-05, + "loss": 1.1537, + "theoretical_loss": 3.3400083409880423, + "tokens_seen": 3110914048 + }, + { + "epoch": 1.32, + "learning_rate": 4.655566700100301e-05, + "loss": 1.0849, + "theoretical_loss": 3.3400030167170236, + "tokens_seen": 3110979584 + }, + { + "epoch": 1.32, + "learning_rate": 4.653961885656971e-05, + "loss": 1.1777, + "theoretical_loss": 3.33999769258957, + "tokens_seen": 3111045120 + }, + { + "epoch": 1.32, + "learning_rate": 4.652357071213641e-05, + "loss": 1.1919, + "theoretical_loss": 3.3399923686056736, + "tokens_seen": 3111110656 + }, + { + "epoch": 1.32, + "learning_rate": 4.650752256770311e-05, + "loss": 1.2327, + "theoretical_loss": 3.3399870447653277, + "tokens_seen": 3111176192 + }, + { + "epoch": 1.32, + "learning_rate": 4.649147442326981e-05, + "loss": 1.2004, + "theoretical_loss": 3.3399817210685256, + "tokens_seen": 3111241728 + }, + { + "epoch": 1.32, + "learning_rate": 4.647542627883651e-05, + "loss": 1.1186, + "theoretical_loss": 3.3399763975152608, + "tokens_seen": 3111307264 + }, + { + "epoch": 1.32, + "learning_rate": 4.6459378134403215e-05, + "loss": 1.1823, + "theoretical_loss": 3.339971074105526, + "tokens_seen": 3111372800 + }, + { + "epoch": 1.32, + "learning_rate": 4.6443329989969915e-05, + "loss": 1.0564, + "theoretical_loss": 3.339965750839314, + "tokens_seen": 3111438336 + }, + { + "epoch": 1.32, + "learning_rate": 4.6427281845536616e-05, + "loss": 1.0663, + "theoretical_loss": 3.3399604277166186, + "tokens_seen": 3111503872 + }, + { + "epoch": 1.32, + "learning_rate": 4.641123370110331e-05, + "loss": 1.2074, + "theoretical_loss": 3.3399551047374323, + "tokens_seen": 3111569408 + }, + { + "epoch": 1.32, + "learning_rate": 4.639518555667001e-05, + "loss": 1.1581, + "theoretical_loss": 3.3399497819017485, + "tokens_seen": 3111634944 + }, + { + "epoch": 1.32, + "learning_rate": 4.637913741223671e-05, + "loss": 1.1169, + "theoretical_loss": 3.33994445920956, + "tokens_seen": 3111700480 + }, + { + "epoch": 1.32, + "learning_rate": 4.636308926780341e-05, + "loss": 1.1955, + "theoretical_loss": 3.3399391366608606, + "tokens_seen": 3111766016 + }, + { + "epoch": 1.32, + "learning_rate": 4.634704112337011e-05, + "loss": 1.2154, + "theoretical_loss": 3.3399338142556427, + "tokens_seen": 3111831552 + }, + { + "epoch": 1.32, + "learning_rate": 4.633099297893681e-05, + "loss": 1.0964, + "theoretical_loss": 3.3399284919939, + "tokens_seen": 3111897088 + }, + { + "epoch": 1.32, + "learning_rate": 4.6314944834503514e-05, + "loss": 1.2859, + "theoretical_loss": 3.339923169875625, + "tokens_seen": 3111962624 + }, + { + "epoch": 1.32, + "learning_rate": 4.6298896690070214e-05, + "loss": 1.1419, + "theoretical_loss": 3.339917847900811, + "tokens_seen": 3112028160 + }, + { + "epoch": 1.32, + "learning_rate": 4.6282848545636915e-05, + "loss": 1.2496, + "theoretical_loss": 3.3399125260694515, + "tokens_seen": 3112093696 + }, + { + "epoch": 1.32, + "learning_rate": 4.6266800401203616e-05, + "loss": 1.1985, + "theoretical_loss": 3.3399072043815394, + "tokens_seen": 3112159232 + }, + { + "epoch": 1.32, + "learning_rate": 4.6250752256770316e-05, + "loss": 1.1753, + "theoretical_loss": 3.3399018828370677, + "tokens_seen": 3112224768 + }, + { + "epoch": 1.32, + "learning_rate": 4.623470411233701e-05, + "loss": 1.1707, + "theoretical_loss": 3.3398965614360296, + "tokens_seen": 3112290304 + }, + { + "epoch": 1.32, + "learning_rate": 4.621865596790371e-05, + "loss": 1.24, + "theoretical_loss": 3.3398912401784178, + "tokens_seen": 3112355840 + }, + { + "epoch": 1.32, + "learning_rate": 4.620260782347041e-05, + "loss": 1.1341, + "theoretical_loss": 3.3398859190642263, + "tokens_seen": 3112421376 + }, + { + "epoch": 1.32, + "learning_rate": 4.618655967903711e-05, + "loss": 1.2325, + "theoretical_loss": 3.339880598093447, + "tokens_seen": 3112486912 + }, + { + "epoch": 1.32, + "learning_rate": 4.617051153460382e-05, + "loss": 1.1379, + "theoretical_loss": 3.3398752772660742, + "tokens_seen": 3112552448 + }, + { + "epoch": 1.32, + "learning_rate": 4.615446339017051e-05, + "loss": 1.1433, + "theoretical_loss": 3.339869956582101, + "tokens_seen": 3112617984 + }, + { + "epoch": 1.32, + "learning_rate": 4.6138415245737214e-05, + "loss": 1.1595, + "theoretical_loss": 3.3398646360415194, + "tokens_seen": 3112683520 + }, + { + "epoch": 1.32, + "learning_rate": 4.6122367101303914e-05, + "loss": 1.0666, + "theoretical_loss": 3.3398593156443233, + "tokens_seen": 3112749056 + }, + { + "epoch": 1.32, + "learning_rate": 4.6106318956870615e-05, + "loss": 1.1295, + "theoretical_loss": 3.3398539953905058, + "tokens_seen": 3112814592 + }, + { + "epoch": 1.32, + "learning_rate": 4.6090270812437316e-05, + "loss": 1.186, + "theoretical_loss": 3.33984867528006, + "tokens_seen": 3112880128 + }, + { + "epoch": 1.32, + "objective/train/docs_used": 1148323, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2414441108703613, + "objective/train/theoretical_loss": 3.3398433553129783, + "objective/train/tokens_used": 3133405664, + "theoretical_loss": 3.3398433553129783, + "tokens_seen": 3112945664 + }, + { + "epoch": 1.32, + "learning_rate": 4.6074222668004016e-05, + "loss": 1.2414, + "theoretical_loss": 3.3398433553129783, + "tokens_seen": 3112945664 + }, + { + "epoch": 1.32, + "learning_rate": 4.605817452357071e-05, + "loss": 1.1383, + "theoretical_loss": 3.339838035489255, + "tokens_seen": 3113011200 + }, + { + "epoch": 1.32, + "learning_rate": 4.604212637913741e-05, + "loss": 1.0956, + "theoretical_loss": 3.3398327158088827, + "tokens_seen": 3113076736 + }, + { + "epoch": 1.32, + "learning_rate": 4.602607823470411e-05, + "loss": 1.188, + "theoretical_loss": 3.339827396271854, + "tokens_seen": 3113142272 + }, + { + "epoch": 1.32, + "learning_rate": 4.601003009027082e-05, + "loss": 1.1756, + "theoretical_loss": 3.3398220768781632, + "tokens_seen": 3113207808 + }, + { + "epoch": 1.32, + "learning_rate": 4.599398194583752e-05, + "loss": 1.2126, + "theoretical_loss": 3.339816757627802, + "tokens_seen": 3113273344 + }, + { + "epoch": 1.32, + "learning_rate": 4.5977933801404213e-05, + "loss": 1.2113, + "theoretical_loss": 3.3398114385207647, + "tokens_seen": 3113338880 + }, + { + "epoch": 1.32, + "learning_rate": 4.5961885656970914e-05, + "loss": 1.1776, + "theoretical_loss": 3.339806119557044, + "tokens_seen": 3113404416 + }, + { + "epoch": 1.32, + "learning_rate": 4.5945837512537615e-05, + "loss": 1.1555, + "theoretical_loss": 3.339800800736633, + "tokens_seen": 3113469952 + }, + { + "epoch": 1.32, + "learning_rate": 4.5929789368104315e-05, + "loss": 1.1602, + "theoretical_loss": 3.3397954820595244, + "tokens_seen": 3113535488 + }, + { + "epoch": 1.32, + "learning_rate": 4.5913741223671016e-05, + "loss": 1.0749, + "theoretical_loss": 3.3397901635257123, + "tokens_seen": 3113601024 + }, + { + "epoch": 1.32, + "learning_rate": 4.5897693079237717e-05, + "loss": 1.1342, + "theoretical_loss": 3.339784845135189, + "tokens_seen": 3113666560 + }, + { + "epoch": 1.32, + "learning_rate": 4.588164493480441e-05, + "loss": 1.1304, + "theoretical_loss": 3.3397795268879475, + "tokens_seen": 3113732096 + }, + { + "epoch": 1.32, + "learning_rate": 4.586559679037112e-05, + "loss": 1.2378, + "theoretical_loss": 3.339774208783982, + "tokens_seen": 3113797632 + }, + { + "epoch": 1.32, + "learning_rate": 4.584954864593782e-05, + "loss": 1.2623, + "theoretical_loss": 3.3397688908232843, + "tokens_seen": 3113863168 + }, + { + "epoch": 1.32, + "learning_rate": 4.583350050150452e-05, + "loss": 1.199, + "theoretical_loss": 3.3397635730058486, + "tokens_seen": 3113928704 + }, + { + "epoch": 1.32, + "learning_rate": 4.581745235707122e-05, + "loss": 1.1102, + "theoretical_loss": 3.339758255331667, + "tokens_seen": 3113994240 + }, + { + "epoch": 1.32, + "learning_rate": 4.5801404212637914e-05, + "loss": 1.2256, + "theoretical_loss": 3.339752937800734, + "tokens_seen": 3114059776 + }, + { + "epoch": 1.32, + "learning_rate": 4.5785356068204614e-05, + "loss": 1.1863, + "theoretical_loss": 3.3397476204130414, + "tokens_seen": 3114125312 + }, + { + "epoch": 1.32, + "learning_rate": 4.5769307923771315e-05, + "loss": 1.1272, + "theoretical_loss": 3.339742303168583, + "tokens_seen": 3114190848 + }, + { + "epoch": 1.32, + "learning_rate": 4.5753259779338016e-05, + "loss": 1.201, + "theoretical_loss": 3.3397369860673516, + "tokens_seen": 3114256384 + }, + { + "epoch": 1.32, + "learning_rate": 4.5737211634904716e-05, + "loss": 1.1282, + "theoretical_loss": 3.339731669109341, + "tokens_seen": 3114321920 + }, + { + "epoch": 1.32, + "learning_rate": 4.572116349047142e-05, + "loss": 1.2173, + "theoretical_loss": 3.339726352294543, + "tokens_seen": 3114387456 + }, + { + "epoch": 1.32, + "learning_rate": 4.570511534603812e-05, + "loss": 1.2393, + "theoretical_loss": 3.339721035622952, + "tokens_seen": 3114452992 + }, + { + "epoch": 1.32, + "learning_rate": 4.568906720160482e-05, + "loss": 1.1964, + "theoretical_loss": 3.3397157190945608, + "tokens_seen": 3114518528 + }, + { + "epoch": 1.32, + "learning_rate": 4.567301905717152e-05, + "loss": 1.1778, + "theoretical_loss": 3.3397104027093625, + "tokens_seen": 3114584064 + }, + { + "epoch": 1.32, + "learning_rate": 4.565697091273822e-05, + "loss": 1.2476, + "theoretical_loss": 3.33970508646735, + "tokens_seen": 3114649600 + }, + { + "epoch": 1.32, + "learning_rate": 4.564092276830492e-05, + "loss": 1.1452, + "theoretical_loss": 3.339699770368517, + "tokens_seen": 3114715136 + }, + { + "epoch": 1.32, + "learning_rate": 4.5624874623871614e-05, + "loss": 1.2294, + "theoretical_loss": 3.3396944544128555, + "tokens_seen": 3114780672 + }, + { + "epoch": 1.32, + "learning_rate": 4.5608826479438314e-05, + "loss": 1.1115, + "theoretical_loss": 3.33968913860036, + "tokens_seen": 3114846208 + }, + { + "epoch": 1.32, + "learning_rate": 4.5592778335005015e-05, + "loss": 1.1226, + "theoretical_loss": 3.339683822931023, + "tokens_seen": 3114911744 + }, + { + "epoch": 1.32, + "learning_rate": 4.5576730190571716e-05, + "loss": 1.2116, + "theoretical_loss": 3.339678507404837, + "tokens_seen": 3114977280 + }, + { + "epoch": 1.32, + "learning_rate": 4.556068204613842e-05, + "loss": 1.1232, + "theoretical_loss": 3.339673192021796, + "tokens_seen": 3115042816 + }, + { + "epoch": 1.32, + "learning_rate": 4.554463390170512e-05, + "loss": 1.1867, + "theoretical_loss": 3.339667876781893, + "tokens_seen": 3115108352 + }, + { + "epoch": 1.32, + "learning_rate": 4.552858575727182e-05, + "loss": 1.2343, + "theoretical_loss": 3.339662561685121, + "tokens_seen": 3115173888 + }, + { + "epoch": 1.32, + "learning_rate": 4.551253761283852e-05, + "loss": 1.2228, + "theoretical_loss": 3.3396572467314734, + "tokens_seen": 3115239424 + }, + { + "epoch": 1.32, + "learning_rate": 4.549648946840522e-05, + "loss": 1.1088, + "theoretical_loss": 3.3396519319209426, + "tokens_seen": 3115304960 + }, + { + "epoch": 1.32, + "learning_rate": 4.548044132397192e-05, + "loss": 1.1716, + "theoretical_loss": 3.339646617253522, + "tokens_seen": 3115370496 + }, + { + "epoch": 1.32, + "learning_rate": 4.546439317953862e-05, + "loss": 1.1643, + "theoretical_loss": 3.339641302729206, + "tokens_seen": 3115436032 + }, + { + "epoch": 1.32, + "learning_rate": 4.5448345035105314e-05, + "loss": 1.1328, + "theoretical_loss": 3.339635988347986, + "tokens_seen": 3115501568 + }, + { + "epoch": 1.32, + "learning_rate": 4.5432296890672015e-05, + "loss": 1.1638, + "theoretical_loss": 3.339630674109856, + "tokens_seen": 3115567104 + }, + { + "epoch": 1.32, + "learning_rate": 4.541624874623872e-05, + "loss": 1.1995, + "theoretical_loss": 3.339625360014809, + "tokens_seen": 3115632640 + }, + { + "epoch": 1.32, + "learning_rate": 4.540020060180542e-05, + "loss": 1.1656, + "theoretical_loss": 3.339620046062838, + "tokens_seen": 3115698176 + }, + { + "epoch": 1.32, + "learning_rate": 4.538415245737212e-05, + "loss": 1.1277, + "theoretical_loss": 3.3396147322539362, + "tokens_seen": 3115763712 + }, + { + "epoch": 1.32, + "learning_rate": 4.536810431293882e-05, + "loss": 1.1976, + "theoretical_loss": 3.339609418588097, + "tokens_seen": 3115829248 + }, + { + "epoch": 1.32, + "learning_rate": 4.535205616850552e-05, + "loss": 1.1675, + "theoretical_loss": 3.339604105065313, + "tokens_seen": 3115894784 + }, + { + "epoch": 1.32, + "learning_rate": 4.533600802407222e-05, + "loss": 1.1305, + "theoretical_loss": 3.339598791685578, + "tokens_seen": 3115960320 + }, + { + "epoch": 1.32, + "learning_rate": 4.531995987963892e-05, + "loss": 1.202, + "theoretical_loss": 3.3395934784488848, + "tokens_seen": 3116025856 + }, + { + "epoch": 1.32, + "learning_rate": 4.530391173520562e-05, + "loss": 1.2796, + "theoretical_loss": 3.3395881653552264, + "tokens_seen": 3116091392 + }, + { + "epoch": 1.32, + "learning_rate": 4.528786359077232e-05, + "loss": 1.1994, + "theoretical_loss": 3.339582852404596, + "tokens_seen": 3116156928 + }, + { + "epoch": 1.32, + "learning_rate": 4.5271815446339014e-05, + "loss": 1.1316, + "theoretical_loss": 3.339577539596987, + "tokens_seen": 3116222464 + }, + { + "epoch": 1.32, + "learning_rate": 4.525576730190572e-05, + "loss": 1.1499, + "theoretical_loss": 3.3395722269323924, + "tokens_seen": 3116288000 + }, + { + "epoch": 1.33, + "learning_rate": 4.523971915747242e-05, + "loss": 1.1637, + "theoretical_loss": 3.3395669144108053, + "tokens_seen": 3116353536 + }, + { + "epoch": 1.33, + "learning_rate": 4.522367101303912e-05, + "loss": 1.2011, + "theoretical_loss": 3.3395616020322185, + "tokens_seen": 3116419072 + }, + { + "epoch": 1.33, + "learning_rate": 4.520762286860582e-05, + "loss": 1.2835, + "theoretical_loss": 3.3395562897966258, + "tokens_seen": 3116484608 + }, + { + "epoch": 1.33, + "learning_rate": 4.519157472417252e-05, + "loss": 1.2427, + "theoretical_loss": 3.33955097770402, + "tokens_seen": 3116550144 + }, + { + "epoch": 1.33, + "learning_rate": 4.517552657973922e-05, + "loss": 1.1946, + "theoretical_loss": 3.3395456657543945, + "tokens_seen": 3116615680 + }, + { + "epoch": 1.33, + "learning_rate": 4.515947843530592e-05, + "loss": 1.3611, + "theoretical_loss": 3.339540353947742, + "tokens_seen": 3116681216 + }, + { + "epoch": 1.33, + "learning_rate": 4.514343029087262e-05, + "loss": 1.2289, + "theoretical_loss": 3.339535042284056, + "tokens_seen": 3116746752 + }, + { + "epoch": 1.33, + "learning_rate": 4.512738214643932e-05, + "loss": 1.156, + "theoretical_loss": 3.3395297307633296, + "tokens_seen": 3116812288 + }, + { + "epoch": 1.33, + "learning_rate": 4.511133400200602e-05, + "loss": 1.1643, + "theoretical_loss": 3.3395244193855556, + "tokens_seen": 3116877824 + }, + { + "epoch": 1.33, + "learning_rate": 4.509528585757272e-05, + "loss": 1.1986, + "theoretical_loss": 3.3395191081507276, + "tokens_seen": 3116943360 + }, + { + "epoch": 1.33, + "learning_rate": 4.507923771313942e-05, + "loss": 1.232, + "theoretical_loss": 3.3395137970588387, + "tokens_seen": 3117008896 + }, + { + "epoch": 1.33, + "learning_rate": 4.506318956870612e-05, + "loss": 1.1639, + "theoretical_loss": 3.3395084861098816, + "tokens_seen": 3117074432 + }, + { + "epoch": 1.33, + "learning_rate": 4.504714142427282e-05, + "loss": 1.236, + "theoretical_loss": 3.33950317530385, + "tokens_seen": 3117139968 + }, + { + "epoch": 1.33, + "learning_rate": 4.503109327983952e-05, + "loss": 1.1607, + "theoretical_loss": 3.339497864640737, + "tokens_seen": 3117205504 + }, + { + "epoch": 1.33, + "learning_rate": 4.501504513540622e-05, + "loss": 1.1626, + "theoretical_loss": 3.339492554120535, + "tokens_seen": 3117271040 + }, + { + "epoch": 1.33, + "learning_rate": 4.499899699097292e-05, + "loss": 1.0777, + "theoretical_loss": 3.339487243743238, + "tokens_seen": 3117336576 + }, + { + "epoch": 1.33, + "learning_rate": 4.498294884653962e-05, + "loss": 1.252, + "theoretical_loss": 3.3394819335088393, + "tokens_seen": 3117402112 + }, + { + "epoch": 1.33, + "learning_rate": 4.4966900702106326e-05, + "loss": 1.1372, + "theoretical_loss": 3.339476623417331, + "tokens_seen": 3117467648 + }, + { + "epoch": 1.33, + "learning_rate": 4.495085255767302e-05, + "loss": 1.2006, + "theoretical_loss": 3.339471313468707, + "tokens_seen": 3117533184 + }, + { + "epoch": 1.33, + "learning_rate": 4.493480441323972e-05, + "loss": 1.1313, + "theoretical_loss": 3.3394660036629604, + "tokens_seen": 3117598720 + }, + { + "epoch": 1.33, + "learning_rate": 4.491875626880642e-05, + "loss": 1.2175, + "theoretical_loss": 3.3394606940000844, + "tokens_seen": 3117664256 + }, + { + "epoch": 1.33, + "learning_rate": 4.490270812437312e-05, + "loss": 1.1423, + "theoretical_loss": 3.339455384480072, + "tokens_seen": 3117729792 + }, + { + "epoch": 1.33, + "learning_rate": 4.488665997993982e-05, + "loss": 1.1641, + "theoretical_loss": 3.339450075102916, + "tokens_seen": 3117795328 + }, + { + "epoch": 1.33, + "learning_rate": 4.487061183550652e-05, + "loss": 1.2514, + "theoretical_loss": 3.3394447658686106, + "tokens_seen": 3117860864 + }, + { + "epoch": 1.33, + "learning_rate": 4.485456369107322e-05, + "loss": 1.1944, + "theoretical_loss": 3.339439456777148, + "tokens_seen": 3117926400 + }, + { + "epoch": 1.33, + "learning_rate": 4.483851554663992e-05, + "loss": 1.2218, + "theoretical_loss": 3.339434147828521, + "tokens_seen": 3117991936 + }, + { + "epoch": 1.33, + "learning_rate": 4.482246740220662e-05, + "loss": 1.1808, + "theoretical_loss": 3.339428839022724, + "tokens_seen": 3118057472 + }, + { + "epoch": 1.33, + "learning_rate": 4.4806419257773326e-05, + "loss": 1.1948, + "theoretical_loss": 3.33942353035975, + "tokens_seen": 3118123008 + }, + { + "epoch": 1.33, + "learning_rate": 4.4790371113340026e-05, + "loss": 1.1328, + "theoretical_loss": 3.339418221839591, + "tokens_seen": 3118188544 + }, + { + "epoch": 1.33, + "learning_rate": 4.477432296890672e-05, + "loss": 1.2191, + "theoretical_loss": 3.3394129134622412, + "tokens_seen": 3118254080 + }, + { + "epoch": 1.33, + "learning_rate": 4.475827482447342e-05, + "loss": 1.1488, + "theoretical_loss": 3.3394076052276933, + "tokens_seen": 3118319616 + }, + { + "epoch": 1.33, + "learning_rate": 4.474222668004012e-05, + "loss": 1.1623, + "theoretical_loss": 3.3394022971359405, + "tokens_seen": 3118385152 + }, + { + "epoch": 1.33, + "learning_rate": 4.472617853560682e-05, + "loss": 1.2022, + "theoretical_loss": 3.3393969891869757, + "tokens_seen": 3118450688 + }, + { + "epoch": 1.33, + "learning_rate": 4.471013039117352e-05, + "loss": 1.1974, + "theoretical_loss": 3.339391681380793, + "tokens_seen": 3118516224 + }, + { + "epoch": 1.33, + "learning_rate": 4.4694082246740224e-05, + "loss": 1.1331, + "theoretical_loss": 3.339386373717385, + "tokens_seen": 3118581760 + }, + { + "epoch": 1.33, + "learning_rate": 4.467803410230692e-05, + "loss": 1.2149, + "theoretical_loss": 3.3393810661967445, + "tokens_seen": 3118647296 + }, + { + "epoch": 1.33, + "learning_rate": 4.4661985957873625e-05, + "loss": 1.1246, + "theoretical_loss": 3.3393757588188646, + "tokens_seen": 3118712832 + }, + { + "epoch": 1.33, + "learning_rate": 4.4645937813440325e-05, + "loss": 1.2424, + "theoretical_loss": 3.3393704515837395, + "tokens_seen": 3118778368 + }, + { + "epoch": 1.33, + "learning_rate": 4.4629889669007026e-05, + "loss": 1.2389, + "theoretical_loss": 3.339365144491361, + "tokens_seen": 3118843904 + }, + { + "epoch": 1.33, + "learning_rate": 4.461384152457373e-05, + "loss": 1.3159, + "theoretical_loss": 3.3393598375417235, + "tokens_seen": 3118909440 + }, + { + "epoch": 1.33, + "learning_rate": 4.459779338014042e-05, + "loss": 1.1812, + "theoretical_loss": 3.33935453073482, + "tokens_seen": 3118974976 + }, + { + "epoch": 1.33, + "learning_rate": 4.458174523570712e-05, + "loss": 1.2297, + "theoretical_loss": 3.3393492240706424, + "tokens_seen": 3119040512 + }, + { + "epoch": 1.33, + "learning_rate": 4.456569709127382e-05, + "loss": 1.211, + "theoretical_loss": 3.339343917549185, + "tokens_seen": 3119106048 + }, + { + "epoch": 1.33, + "learning_rate": 4.454964894684052e-05, + "loss": 1.1819, + "theoretical_loss": 3.339338611170441, + "tokens_seen": 3119171584 + }, + { + "epoch": 1.33, + "learning_rate": 4.453360080240722e-05, + "loss": 1.2349, + "theoretical_loss": 3.339333304934403, + "tokens_seen": 3119237120 + }, + { + "epoch": 1.33, + "learning_rate": 4.4517552657973924e-05, + "loss": 1.1883, + "theoretical_loss": 3.3393279988410645, + "tokens_seen": 3119302656 + }, + { + "epoch": 1.33, + "learning_rate": 4.4501504513540624e-05, + "loss": 1.2391, + "theoretical_loss": 3.3393226928904185, + "tokens_seen": 3119368192 + }, + { + "epoch": 1.33, + "learning_rate": 4.4485456369107325e-05, + "loss": 1.1152, + "theoretical_loss": 3.3393173870824584, + "tokens_seen": 3119433728 + }, + { + "epoch": 1.33, + "objective/train/docs_used": 1150757, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1945302486419678, + "objective/train/theoretical_loss": 3.339312081417177, + "objective/train/tokens_used": 3139959264, + "theoretical_loss": 3.339312081417177, + "tokens_seen": 3119499264 + }, + { + "epoch": 1.33, + "learning_rate": 4.4469408224674026e-05, + "loss": 1.1945, + "theoretical_loss": 3.339312081417177, + "tokens_seen": 3119499264 + }, + { + "epoch": 1.33, + "learning_rate": 4.4453360080240726e-05, + "loss": 1.1235, + "theoretical_loss": 3.339306775894568, + "tokens_seen": 3119564800 + }, + { + "epoch": 1.33, + "learning_rate": 4.443731193580743e-05, + "loss": 1.1593, + "theoretical_loss": 3.339301470514624, + "tokens_seen": 3119630336 + }, + { + "epoch": 1.33, + "learning_rate": 4.442126379137412e-05, + "loss": 1.1923, + "theoretical_loss": 3.3392961652773385, + "tokens_seen": 3119695872 + }, + { + "epoch": 1.33, + "learning_rate": 4.440521564694082e-05, + "loss": 1.2278, + "theoretical_loss": 3.3392908601827047, + "tokens_seen": 3119761408 + }, + { + "epoch": 1.33, + "learning_rate": 4.438916750250752e-05, + "loss": 1.1083, + "theoretical_loss": 3.339285555230716, + "tokens_seen": 3119826944 + }, + { + "epoch": 1.33, + "learning_rate": 4.437311935807422e-05, + "loss": 1.1688, + "theoretical_loss": 3.3392802504213646, + "tokens_seen": 3119892480 + }, + { + "epoch": 1.33, + "learning_rate": 4.435707121364093e-05, + "loss": 1.0407, + "theoretical_loss": 3.3392749457546445, + "tokens_seen": 3119958016 + }, + { + "epoch": 1.33, + "learning_rate": 4.4341023069207624e-05, + "loss": 1.2951, + "theoretical_loss": 3.3392696412305485, + "tokens_seen": 3120023552 + }, + { + "epoch": 1.33, + "learning_rate": 4.4324974924774325e-05, + "loss": 1.1083, + "theoretical_loss": 3.3392643368490704, + "tokens_seen": 3120089088 + }, + { + "epoch": 1.33, + "learning_rate": 4.4308926780341025e-05, + "loss": 1.1783, + "theoretical_loss": 3.3392590326102023, + "tokens_seen": 3120154624 + }, + { + "epoch": 1.33, + "learning_rate": 4.4292878635907726e-05, + "loss": 1.2501, + "theoretical_loss": 3.3392537285139383, + "tokens_seen": 3120220160 + }, + { + "epoch": 1.33, + "learning_rate": 4.4276830491474426e-05, + "loss": 1.0832, + "theoretical_loss": 3.3392484245602714, + "tokens_seen": 3120285696 + }, + { + "epoch": 1.33, + "learning_rate": 4.426078234704113e-05, + "loss": 1.238, + "theoretical_loss": 3.3392431207491944, + "tokens_seen": 3120351232 + }, + { + "epoch": 1.33, + "learning_rate": 4.424473420260782e-05, + "loss": 1.1878, + "theoretical_loss": 3.3392378170807007, + "tokens_seen": 3120416768 + }, + { + "epoch": 1.33, + "learning_rate": 4.422868605817452e-05, + "loss": 1.3357, + "theoretical_loss": 3.3392325135547836, + "tokens_seen": 3120482304 + }, + { + "epoch": 1.33, + "learning_rate": 4.421263791374123e-05, + "loss": 1.2456, + "theoretical_loss": 3.339227210171436, + "tokens_seen": 3120547840 + }, + { + "epoch": 1.33, + "learning_rate": 4.419658976930793e-05, + "loss": 1.2877, + "theoretical_loss": 3.3392219069306512, + "tokens_seen": 3120613376 + }, + { + "epoch": 1.33, + "learning_rate": 4.418054162487463e-05, + "loss": 1.18, + "theoretical_loss": 3.3392166038324227, + "tokens_seen": 3120678912 + }, + { + "epoch": 1.33, + "learning_rate": 4.4164493480441324e-05, + "loss": 1.077, + "theoretical_loss": 3.339211300876743, + "tokens_seen": 3120744448 + }, + { + "epoch": 1.33, + "learning_rate": 4.4148445336008025e-05, + "loss": 1.1497, + "theoretical_loss": 3.3392059980636053, + "tokens_seen": 3120809984 + }, + { + "epoch": 1.33, + "learning_rate": 4.4132397191574725e-05, + "loss": 1.0091, + "theoretical_loss": 3.3392006953930036, + "tokens_seen": 3120875520 + }, + { + "epoch": 1.33, + "learning_rate": 4.4116349047141426e-05, + "loss": 1.1044, + "theoretical_loss": 3.3391953928649305, + "tokens_seen": 3120941056 + }, + { + "epoch": 1.33, + "learning_rate": 4.410030090270813e-05, + "loss": 1.0779, + "theoretical_loss": 3.3391900904793794, + "tokens_seen": 3121006592 + }, + { + "epoch": 1.33, + "learning_rate": 4.408425275827483e-05, + "loss": 1.2061, + "theoretical_loss": 3.3391847882363432, + "tokens_seen": 3121072128 + }, + { + "epoch": 1.33, + "learning_rate": 4.406820461384153e-05, + "loss": 1.1715, + "theoretical_loss": 3.339179486135815, + "tokens_seen": 3121137664 + }, + { + "epoch": 1.33, + "learning_rate": 4.405215646940823e-05, + "loss": 1.2164, + "theoretical_loss": 3.3391741841777884, + "tokens_seen": 3121203200 + }, + { + "epoch": 1.33, + "learning_rate": 4.403610832497493e-05, + "loss": 1.2403, + "theoretical_loss": 3.3391688823622565, + "tokens_seen": 3121268736 + }, + { + "epoch": 1.33, + "learning_rate": 4.402006018054163e-05, + "loss": 1.1572, + "theoretical_loss": 3.3391635806892124, + "tokens_seen": 3121334272 + }, + { + "epoch": 1.33, + "learning_rate": 4.400401203610833e-05, + "loss": 1.1712, + "theoretical_loss": 3.3391582791586485, + "tokens_seen": 3121399808 + }, + { + "epoch": 1.33, + "learning_rate": 4.3987963891675024e-05, + "loss": 1.0682, + "theoretical_loss": 3.3391529777705595, + "tokens_seen": 3121465344 + }, + { + "epoch": 1.33, + "learning_rate": 4.3971915747241725e-05, + "loss": 1.1172, + "theoretical_loss": 3.3391476765249375, + "tokens_seen": 3121530880 + }, + { + "epoch": 1.33, + "learning_rate": 4.3955867602808426e-05, + "loss": 1.1232, + "theoretical_loss": 3.339142375421776, + "tokens_seen": 3121596416 + }, + { + "epoch": 1.33, + "learning_rate": 4.3939819458375126e-05, + "loss": 1.1417, + "theoretical_loss": 3.339137074461068, + "tokens_seen": 3121661952 + }, + { + "epoch": 1.33, + "learning_rate": 4.392377131394183e-05, + "loss": 1.1114, + "theoretical_loss": 3.3391317736428068, + "tokens_seen": 3121727488 + }, + { + "epoch": 1.33, + "learning_rate": 4.390772316950853e-05, + "loss": 1.2306, + "theoretical_loss": 3.339126472966986, + "tokens_seen": 3121793024 + }, + { + "epoch": 1.33, + "learning_rate": 4.389167502507523e-05, + "loss": 1.2525, + "theoretical_loss": 3.339121172433598, + "tokens_seen": 3121858560 + }, + { + "epoch": 1.33, + "learning_rate": 4.387562688064193e-05, + "loss": 1.1365, + "theoretical_loss": 3.339115872042637, + "tokens_seen": 3121924096 + }, + { + "epoch": 1.33, + "learning_rate": 4.385957873620863e-05, + "loss": 1.1756, + "theoretical_loss": 3.3391105717940945, + "tokens_seen": 3121989632 + }, + { + "epoch": 1.33, + "learning_rate": 4.384353059177533e-05, + "loss": 1.1776, + "theoretical_loss": 3.3391052716879654, + "tokens_seen": 3122055168 + }, + { + "epoch": 1.33, + "learning_rate": 4.382748244734203e-05, + "loss": 1.2311, + "theoretical_loss": 3.3390999717242424, + "tokens_seen": 3122120704 + }, + { + "epoch": 1.33, + "learning_rate": 4.3811434302908725e-05, + "loss": 1.2388, + "theoretical_loss": 3.339094671902918, + "tokens_seen": 3122186240 + }, + { + "epoch": 1.33, + "learning_rate": 4.3795386158475425e-05, + "loss": 1.2057, + "theoretical_loss": 3.3390893722239863, + "tokens_seen": 3122251776 + }, + { + "epoch": 1.33, + "learning_rate": 4.3779338014042126e-05, + "loss": 1.204, + "theoretical_loss": 3.33908407268744, + "tokens_seen": 3122317312 + }, + { + "epoch": 1.33, + "learning_rate": 4.376328986960883e-05, + "loss": 1.2557, + "theoretical_loss": 3.339078773293272, + "tokens_seen": 3122382848 + }, + { + "epoch": 1.33, + "learning_rate": 4.3747241725175534e-05, + "loss": 1.12, + "theoretical_loss": 3.3390734740414763, + "tokens_seen": 3122448384 + }, + { + "epoch": 1.33, + "learning_rate": 4.373119358074223e-05, + "loss": 1.2392, + "theoretical_loss": 3.3390681749320454, + "tokens_seen": 3122513920 + }, + { + "epoch": 1.33, + "learning_rate": 4.371514543630893e-05, + "loss": 1.1693, + "theoretical_loss": 3.3390628759649728, + "tokens_seen": 3122579456 + }, + { + "epoch": 1.33, + "learning_rate": 4.369909729187563e-05, + "loss": 1.1481, + "theoretical_loss": 3.3390575771402515, + "tokens_seen": 3122644992 + }, + { + "epoch": 1.33, + "learning_rate": 4.368304914744233e-05, + "loss": 1.175, + "theoretical_loss": 3.339052278457875, + "tokens_seen": 3122710528 + }, + { + "epoch": 1.33, + "learning_rate": 4.366700100300903e-05, + "loss": 1.1565, + "theoretical_loss": 3.3390469799178364, + "tokens_seen": 3122776064 + }, + { + "epoch": 1.33, + "learning_rate": 4.365095285857573e-05, + "loss": 1.137, + "theoretical_loss": 3.3390416815201287, + "tokens_seen": 3122841600 + }, + { + "epoch": 1.33, + "learning_rate": 4.3634904714142425e-05, + "loss": 1.1434, + "theoretical_loss": 3.339036383264745, + "tokens_seen": 3122907136 + }, + { + "epoch": 1.33, + "learning_rate": 4.3618856569709125e-05, + "loss": 1.2469, + "theoretical_loss": 3.339031085151679, + "tokens_seen": 3122972672 + }, + { + "epoch": 1.33, + "learning_rate": 4.360280842527583e-05, + "loss": 1.1609, + "theoretical_loss": 3.339025787180923, + "tokens_seen": 3123038208 + }, + { + "epoch": 1.33, + "learning_rate": 4.3586760280842533e-05, + "loss": 1.2061, + "theoretical_loss": 3.339020489352471, + "tokens_seen": 3123103744 + }, + { + "epoch": 1.33, + "learning_rate": 4.3570712136409234e-05, + "loss": 1.0778, + "theoretical_loss": 3.339015191666316, + "tokens_seen": 3123169280 + }, + { + "epoch": 1.33, + "learning_rate": 4.355466399197593e-05, + "loss": 1.1807, + "theoretical_loss": 3.339009894122451, + "tokens_seen": 3123234816 + }, + { + "epoch": 1.33, + "learning_rate": 4.353861584754263e-05, + "loss": 1.149, + "theoretical_loss": 3.3390045967208692, + "tokens_seen": 3123300352 + }, + { + "epoch": 1.33, + "learning_rate": 4.352256770310933e-05, + "loss": 1.1876, + "theoretical_loss": 3.3389992994615643, + "tokens_seen": 3123365888 + }, + { + "epoch": 1.33, + "learning_rate": 4.350651955867603e-05, + "loss": 1.2098, + "theoretical_loss": 3.338994002344529, + "tokens_seen": 3123431424 + }, + { + "epoch": 1.33, + "learning_rate": 4.349047141424273e-05, + "loss": 1.2211, + "theoretical_loss": 3.3389887053697564, + "tokens_seen": 3123496960 + }, + { + "epoch": 1.33, + "learning_rate": 4.347442326980943e-05, + "loss": 1.2552, + "theoretical_loss": 3.3389834085372403, + "tokens_seen": 3123562496 + }, + { + "epoch": 1.33, + "learning_rate": 4.345837512537613e-05, + "loss": 1.1767, + "theoretical_loss": 3.338978111846973, + "tokens_seen": 3123628032 + }, + { + "epoch": 1.33, + "learning_rate": 4.344232698094283e-05, + "loss": 1.173, + "theoretical_loss": 3.3389728152989484, + "tokens_seen": 3123693568 + }, + { + "epoch": 1.33, + "learning_rate": 4.342627883650953e-05, + "loss": 1.0791, + "theoretical_loss": 3.33896751889316, + "tokens_seen": 3123759104 + }, + { + "epoch": 1.33, + "learning_rate": 4.3410230692076234e-05, + "loss": 1.2425, + "theoretical_loss": 3.3389622226295996, + "tokens_seen": 3123824640 + }, + { + "epoch": 1.33, + "learning_rate": 4.3394182547642934e-05, + "loss": 1.2149, + "theoretical_loss": 3.3389569265082617, + "tokens_seen": 3123890176 + }, + { + "epoch": 1.33, + "learning_rate": 4.337813440320963e-05, + "loss": 1.2022, + "theoretical_loss": 3.338951630529139, + "tokens_seen": 3123955712 + }, + { + "epoch": 1.33, + "learning_rate": 4.336208625877633e-05, + "loss": 1.1929, + "theoretical_loss": 3.338946334692225, + "tokens_seen": 3124021248 + }, + { + "epoch": 1.33, + "learning_rate": 4.334603811434303e-05, + "loss": 1.1555, + "theoretical_loss": 3.338941038997512, + "tokens_seen": 3124086784 + }, + { + "epoch": 1.33, + "learning_rate": 4.332998996990973e-05, + "loss": 1.1837, + "theoretical_loss": 3.3389357434449947, + "tokens_seen": 3124152320 + }, + { + "epoch": 1.33, + "learning_rate": 4.331394182547643e-05, + "loss": 1.1779, + "theoretical_loss": 3.338930448034665, + "tokens_seen": 3124217856 + }, + { + "epoch": 1.33, + "learning_rate": 4.329789368104313e-05, + "loss": 1.2077, + "theoretical_loss": 3.3389251527665165, + "tokens_seen": 3124283392 + }, + { + "epoch": 1.33, + "learning_rate": 4.328184553660983e-05, + "loss": 1.1474, + "theoretical_loss": 3.3389198576405428, + "tokens_seen": 3124348928 + }, + { + "epoch": 1.33, + "learning_rate": 4.326579739217653e-05, + "loss": 1.1035, + "theoretical_loss": 3.3389145626567367, + "tokens_seen": 3124414464 + }, + { + "epoch": 1.33, + "learning_rate": 4.324974924774323e-05, + "loss": 1.145, + "theoretical_loss": 3.338909267815091, + "tokens_seen": 3124480000 + }, + { + "epoch": 1.33, + "learning_rate": 4.3233701103309934e-05, + "loss": 1.1316, + "theoretical_loss": 3.3389039731156, + "tokens_seen": 3124545536 + }, + { + "epoch": 1.33, + "learning_rate": 4.321765295887663e-05, + "loss": 1.2802, + "theoretical_loss": 3.338898678558256, + "tokens_seen": 3124611072 + }, + { + "epoch": 1.33, + "learning_rate": 4.320160481444333e-05, + "loss": 1.0489, + "theoretical_loss": 3.3388933841430526, + "tokens_seen": 3124676608 + }, + { + "epoch": 1.33, + "learning_rate": 4.318555667001003e-05, + "loss": 1.0956, + "theoretical_loss": 3.3388880898699824, + "tokens_seen": 3124742144 + }, + { + "epoch": 1.33, + "learning_rate": 4.316950852557673e-05, + "loss": 1.2845, + "theoretical_loss": 3.33888279573904, + "tokens_seen": 3124807680 + }, + { + "epoch": 1.33, + "learning_rate": 4.315346038114344e-05, + "loss": 1.208, + "theoretical_loss": 3.3388775017502166, + "tokens_seen": 3124873216 + }, + { + "epoch": 1.33, + "learning_rate": 4.313741223671013e-05, + "loss": 1.0659, + "theoretical_loss": 3.338872207903507, + "tokens_seen": 3124938752 + }, + { + "epoch": 1.33, + "learning_rate": 4.312136409227683e-05, + "loss": 1.0942, + "theoretical_loss": 3.338866914198904, + "tokens_seen": 3125004288 + }, + { + "epoch": 1.33, + "learning_rate": 4.310531594784353e-05, + "loss": 1.1323, + "theoretical_loss": 3.3388616206364006, + "tokens_seen": 3125069824 + }, + { + "epoch": 1.33, + "learning_rate": 4.308926780341023e-05, + "loss": 1.1797, + "theoretical_loss": 3.33885632721599, + "tokens_seen": 3125135360 + }, + { + "epoch": 1.33, + "learning_rate": 4.3073219658976933e-05, + "loss": 1.1497, + "theoretical_loss": 3.338851033937666, + "tokens_seen": 3125200896 + }, + { + "epoch": 1.33, + "learning_rate": 4.3057171514543634e-05, + "loss": 1.1267, + "theoretical_loss": 3.3388457408014207, + "tokens_seen": 3125266432 + }, + { + "epoch": 1.33, + "learning_rate": 4.304112337011033e-05, + "loss": 1.1525, + "theoretical_loss": 3.3388404478072484, + "tokens_seen": 3125331968 + }, + { + "epoch": 1.33, + "learning_rate": 4.302507522567703e-05, + "loss": 1.1795, + "theoretical_loss": 3.3388351549551416, + "tokens_seen": 3125397504 + }, + { + "epoch": 1.33, + "learning_rate": 4.3009027081243736e-05, + "loss": 1.0844, + "theoretical_loss": 3.3388298622450936, + "tokens_seen": 3125463040 + }, + { + "epoch": 1.33, + "learning_rate": 4.2992978936810437e-05, + "loss": 1.291, + "theoretical_loss": 3.3388245696770977, + "tokens_seen": 3125528576 + }, + { + "epoch": 1.33, + "learning_rate": 4.297693079237714e-05, + "loss": 1.1589, + "theoretical_loss": 3.3388192772511474, + "tokens_seen": 3125594112 + }, + { + "epoch": 1.33, + "learning_rate": 4.296088264794383e-05, + "loss": 1.1483, + "theoretical_loss": 3.338813984967236, + "tokens_seen": 3125659648 + }, + { + "epoch": 1.33, + "learning_rate": 4.294483450351053e-05, + "loss": 1.1131, + "theoretical_loss": 3.3388086928253555, + "tokens_seen": 3125725184 + }, + { + "epoch": 1.33, + "learning_rate": 4.292878635907723e-05, + "loss": 1.1895, + "theoretical_loss": 3.3388034008255008, + "tokens_seen": 3125790720 + }, + { + "epoch": 1.33, + "learning_rate": 4.291273821464393e-05, + "loss": 1.2296, + "theoretical_loss": 3.338798108967664, + "tokens_seen": 3125856256 + }, + { + "epoch": 1.33, + "learning_rate": 4.2896690070210634e-05, + "loss": 1.1068, + "theoretical_loss": 3.3387928172518384, + "tokens_seen": 3125921792 + }, + { + "epoch": 1.33, + "learning_rate": 4.2880641925777334e-05, + "loss": 1.2505, + "theoretical_loss": 3.3387875256780175, + "tokens_seen": 3125987328 + }, + { + "epoch": 1.33, + "objective/train/docs_used": 1153490, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1381617784500122, + "objective/train/theoretical_loss": 3.3387822342461946, + "objective/train/tokens_used": 3146512864, + "theoretical_loss": 3.3387822342461946, + "tokens_seen": 3126052864 + }, + { + "epoch": 1.33, + "learning_rate": 4.2864593781344035e-05, + "loss": 1.1382, + "theoretical_loss": 3.3387822342461946, + "tokens_seen": 3126052864 + }, + { + "epoch": 1.33, + "learning_rate": 4.2848545636910736e-05, + "loss": 1.2155, + "theoretical_loss": 3.338776942956363, + "tokens_seen": 3126118400 + }, + { + "epoch": 1.33, + "learning_rate": 4.2832497492477436e-05, + "loss": 1.0624, + "theoretical_loss": 3.338771651808515, + "tokens_seen": 3126183936 + }, + { + "epoch": 1.33, + "learning_rate": 4.281644934804414e-05, + "loss": 1.145, + "theoretical_loss": 3.3387663608026448, + "tokens_seen": 3126249472 + }, + { + "epoch": 1.33, + "learning_rate": 4.280040120361084e-05, + "loss": 1.1956, + "theoretical_loss": 3.3387610699387453, + "tokens_seen": 3126315008 + }, + { + "epoch": 1.33, + "learning_rate": 4.278435305917753e-05, + "loss": 1.2439, + "theoretical_loss": 3.3387557792168097, + "tokens_seen": 3126380544 + }, + { + "epoch": 1.33, + "learning_rate": 4.276830491474423e-05, + "loss": 1.2137, + "theoretical_loss": 3.3387504886368315, + "tokens_seen": 3126446080 + }, + { + "epoch": 1.33, + "learning_rate": 4.275225677031093e-05, + "loss": 1.176, + "theoretical_loss": 3.338745198198803, + "tokens_seen": 3126511616 + }, + { + "epoch": 1.33, + "learning_rate": 4.273620862587763e-05, + "loss": 1.1848, + "theoretical_loss": 3.3387399079027187, + "tokens_seen": 3126577152 + }, + { + "epoch": 1.33, + "learning_rate": 4.2720160481444334e-05, + "loss": 1.1243, + "theoretical_loss": 3.3387346177485706, + "tokens_seen": 3126642688 + }, + { + "epoch": 1.33, + "learning_rate": 4.2704112337011034e-05, + "loss": 1.1737, + "theoretical_loss": 3.3387293277363526, + "tokens_seen": 3126708224 + }, + { + "epoch": 1.33, + "learning_rate": 4.2688064192577735e-05, + "loss": 1.209, + "theoretical_loss": 3.3387240378660583, + "tokens_seen": 3126773760 + }, + { + "epoch": 1.33, + "learning_rate": 4.2672016048144436e-05, + "loss": 1.202, + "theoretical_loss": 3.33871874813768, + "tokens_seen": 3126839296 + }, + { + "epoch": 1.33, + "learning_rate": 4.2655967903711136e-05, + "loss": 1.099, + "theoretical_loss": 3.3387134585512115, + "tokens_seen": 3126904832 + }, + { + "epoch": 1.33, + "learning_rate": 4.263991975927784e-05, + "loss": 1.2073, + "theoretical_loss": 3.3387081691066456, + "tokens_seen": 3126970368 + }, + { + "epoch": 1.33, + "learning_rate": 4.262387161484454e-05, + "loss": 1.2376, + "theoretical_loss": 3.338702879803976, + "tokens_seen": 3127035904 + }, + { + "epoch": 1.33, + "learning_rate": 4.260782347041123e-05, + "loss": 1.1414, + "theoretical_loss": 3.3386975906431955, + "tokens_seen": 3127101440 + }, + { + "epoch": 1.33, + "learning_rate": 4.259177532597793e-05, + "loss": 1.0096, + "theoretical_loss": 3.3386923016242975, + "tokens_seen": 3127166976 + }, + { + "epoch": 1.33, + "learning_rate": 4.257572718154463e-05, + "loss": 1.1173, + "theoretical_loss": 3.3386870127472754, + "tokens_seen": 3127232512 + }, + { + "epoch": 1.33, + "learning_rate": 4.255967903711134e-05, + "loss": 1.1564, + "theoretical_loss": 3.3386817240121225, + "tokens_seen": 3127298048 + }, + { + "epoch": 1.33, + "learning_rate": 4.254363089267804e-05, + "loss": 1.1346, + "theoretical_loss": 3.338676435418831, + "tokens_seen": 3127363584 + }, + { + "epoch": 1.33, + "learning_rate": 4.2527582748244735e-05, + "loss": 1.2421, + "theoretical_loss": 3.3386711469673958, + "tokens_seen": 3127429120 + }, + { + "epoch": 1.33, + "learning_rate": 4.2511534603811435e-05, + "loss": 1.19, + "theoretical_loss": 3.3386658586578086, + "tokens_seen": 3127494656 + }, + { + "epoch": 1.33, + "learning_rate": 4.2495486459378136e-05, + "loss": 1.2619, + "theoretical_loss": 3.3386605704900636, + "tokens_seen": 3127560192 + }, + { + "epoch": 1.33, + "learning_rate": 4.2479438314944837e-05, + "loss": 1.1202, + "theoretical_loss": 3.3386552824641536, + "tokens_seen": 3127625728 + }, + { + "epoch": 1.33, + "learning_rate": 4.246339017051154e-05, + "loss": 1.185, + "theoretical_loss": 3.338649994580072, + "tokens_seen": 3127691264 + }, + { + "epoch": 1.33, + "learning_rate": 4.244734202607824e-05, + "loss": 1.1349, + "theoretical_loss": 3.338644706837812, + "tokens_seen": 3127756800 + }, + { + "epoch": 1.33, + "learning_rate": 4.243129388164493e-05, + "loss": 1.2605, + "theoretical_loss": 3.3386394192373663, + "tokens_seen": 3127822336 + }, + { + "epoch": 1.33, + "learning_rate": 4.241524573721164e-05, + "loss": 1.3047, + "theoretical_loss": 3.3386341317787287, + "tokens_seen": 3127887872 + }, + { + "epoch": 1.33, + "learning_rate": 4.239919759277834e-05, + "loss": 1.1595, + "theoretical_loss": 3.3386288444618923, + "tokens_seen": 3127953408 + }, + { + "epoch": 1.33, + "learning_rate": 4.238314944834504e-05, + "loss": 1.1766, + "theoretical_loss": 3.3386235572868506, + "tokens_seen": 3128018944 + }, + { + "epoch": 1.33, + "learning_rate": 4.236710130391174e-05, + "loss": 1.1226, + "theoretical_loss": 3.338618270253596, + "tokens_seen": 3128084480 + }, + { + "epoch": 1.33, + "learning_rate": 4.2351053159478435e-05, + "loss": 1.3681, + "theoretical_loss": 3.3386129833621228, + "tokens_seen": 3128150016 + }, + { + "epoch": 1.33, + "learning_rate": 4.2335005015045135e-05, + "loss": 1.1548, + "theoretical_loss": 3.3386076966124234, + "tokens_seen": 3128215552 + }, + { + "epoch": 1.33, + "learning_rate": 4.2318956870611836e-05, + "loss": 1.1545, + "theoretical_loss": 3.3386024100044915, + "tokens_seen": 3128281088 + }, + { + "epoch": 1.33, + "learning_rate": 4.230290872617854e-05, + "loss": 1.1944, + "theoretical_loss": 3.33859712353832, + "tokens_seen": 3128346624 + }, + { + "epoch": 1.33, + "learning_rate": 4.228686058174524e-05, + "loss": 1.1589, + "theoretical_loss": 3.3385918372139023, + "tokens_seen": 3128412160 + }, + { + "epoch": 1.33, + "learning_rate": 4.227081243731194e-05, + "loss": 1.0632, + "theoretical_loss": 3.3385865510312316, + "tokens_seen": 3128477696 + }, + { + "epoch": 1.33, + "learning_rate": 4.225476429287864e-05, + "loss": 1.2406, + "theoretical_loss": 3.3385812649903013, + "tokens_seen": 3128543232 + }, + { + "epoch": 1.33, + "learning_rate": 4.223871614844534e-05, + "loss": 1.1998, + "theoretical_loss": 3.3385759790911043, + "tokens_seen": 3128608768 + }, + { + "epoch": 1.33, + "learning_rate": 4.222266800401204e-05, + "loss": 1.1176, + "theoretical_loss": 3.338570693333634, + "tokens_seen": 3128674304 + }, + { + "epoch": 1.33, + "learning_rate": 4.220661985957874e-05, + "loss": 1.199, + "theoretical_loss": 3.338565407717884, + "tokens_seen": 3128739840 + }, + { + "epoch": 1.33, + "learning_rate": 4.219057171514544e-05, + "loss": 1.1673, + "theoretical_loss": 3.338560122243847, + "tokens_seen": 3128805376 + }, + { + "epoch": 1.33, + "learning_rate": 4.2174523570712135e-05, + "loss": 1.1565, + "theoretical_loss": 3.3385548369115163, + "tokens_seen": 3128870912 + }, + { + "epoch": 1.33, + "learning_rate": 4.2158475426278836e-05, + "loss": 1.0944, + "theoretical_loss": 3.338549551720885, + "tokens_seen": 3128936448 + }, + { + "epoch": 1.33, + "learning_rate": 4.2142427281845536e-05, + "loss": 1.184, + "theoretical_loss": 3.338544266671947, + "tokens_seen": 3129001984 + }, + { + "epoch": 1.33, + "learning_rate": 4.212637913741224e-05, + "loss": 1.2725, + "theoretical_loss": 3.338538981764695, + "tokens_seen": 3129067520 + }, + { + "epoch": 1.33, + "learning_rate": 4.2110330992978944e-05, + "loss": 1.1566, + "theoretical_loss": 3.338533696999122, + "tokens_seen": 3129133056 + }, + { + "epoch": 1.33, + "learning_rate": 4.209428284854564e-05, + "loss": 1.2254, + "theoretical_loss": 3.338528412375222, + "tokens_seen": 3129198592 + }, + { + "epoch": 1.33, + "learning_rate": 4.207823470411234e-05, + "loss": 1.1983, + "theoretical_loss": 3.3385231278929877, + "tokens_seen": 3129264128 + }, + { + "epoch": 1.33, + "learning_rate": 4.206218655967904e-05, + "loss": 1.0799, + "theoretical_loss": 3.3385178435524123, + "tokens_seen": 3129329664 + }, + { + "epoch": 1.33, + "learning_rate": 4.204613841524574e-05, + "loss": 1.2779, + "theoretical_loss": 3.3385125593534895, + "tokens_seen": 3129395200 + }, + { + "epoch": 1.33, + "learning_rate": 4.203009027081244e-05, + "loss": 1.105, + "theoretical_loss": 3.3385072752962115, + "tokens_seen": 3129460736 + }, + { + "epoch": 1.33, + "learning_rate": 4.201404212637914e-05, + "loss": 1.1408, + "theoretical_loss": 3.338501991380573, + "tokens_seen": 3129526272 + }, + { + "epoch": 1.33, + "learning_rate": 4.1997993981945835e-05, + "loss": 1.1601, + "theoretical_loss": 3.338496707606566, + "tokens_seen": 3129591808 + }, + { + "epoch": 1.33, + "learning_rate": 4.1981945837512536e-05, + "loss": 1.1901, + "theoretical_loss": 3.3384914239741845, + "tokens_seen": 3129657344 + }, + { + "epoch": 1.33, + "learning_rate": 4.196589769307924e-05, + "loss": 1.2051, + "theoretical_loss": 3.3384861404834214, + "tokens_seen": 3129722880 + }, + { + "epoch": 1.33, + "learning_rate": 4.1949849548645944e-05, + "loss": 1.1412, + "theoretical_loss": 3.33848085713427, + "tokens_seen": 3129788416 + }, + { + "epoch": 1.33, + "learning_rate": 4.1933801404212645e-05, + "loss": 1.0725, + "theoretical_loss": 3.338475573926724, + "tokens_seen": 3129853952 + }, + { + "epoch": 1.33, + "learning_rate": 4.191775325977934e-05, + "loss": 1.1778, + "theoretical_loss": 3.3384702908607755, + "tokens_seen": 3129919488 + }, + { + "epoch": 1.33, + "learning_rate": 4.190170511534604e-05, + "loss": 1.1293, + "theoretical_loss": 3.3384650079364184, + "tokens_seen": 3129985024 + }, + { + "epoch": 1.33, + "learning_rate": 4.188565697091274e-05, + "loss": 1.1645, + "theoretical_loss": 3.3384597251536463, + "tokens_seen": 3130050560 + }, + { + "epoch": 1.33, + "learning_rate": 4.186960882647944e-05, + "loss": 1.2439, + "theoretical_loss": 3.338454442512452, + "tokens_seen": 3130116096 + }, + { + "epoch": 1.33, + "learning_rate": 4.185356068204614e-05, + "loss": 1.2181, + "theoretical_loss": 3.338449160012829, + "tokens_seen": 3130181632 + }, + { + "epoch": 1.33, + "learning_rate": 4.183751253761284e-05, + "loss": 1.0952, + "theoretical_loss": 3.33844387765477, + "tokens_seen": 3130247168 + }, + { + "epoch": 1.33, + "learning_rate": 4.1821464393179535e-05, + "loss": 1.1698, + "theoretical_loss": 3.338438595438269, + "tokens_seen": 3130312704 + }, + { + "epoch": 1.33, + "learning_rate": 4.180541624874624e-05, + "loss": 1.147, + "theoretical_loss": 3.3384333133633186, + "tokens_seen": 3130378240 + }, + { + "epoch": 1.33, + "learning_rate": 4.1789368104312944e-05, + "loss": 1.205, + "theoretical_loss": 3.338428031429913, + "tokens_seen": 3130443776 + }, + { + "epoch": 1.33, + "learning_rate": 4.1773319959879644e-05, + "loss": 1.1984, + "theoretical_loss": 3.3384227496380436, + "tokens_seen": 3130509312 + }, + { + "epoch": 1.33, + "learning_rate": 4.1757271815446345e-05, + "loss": 1.1311, + "theoretical_loss": 3.3384174679877057, + "tokens_seen": 3130574848 + }, + { + "epoch": 1.33, + "learning_rate": 4.174122367101304e-05, + "loss": 1.2255, + "theoretical_loss": 3.3384121864788914, + "tokens_seen": 3130640384 + }, + { + "epoch": 1.33, + "learning_rate": 4.172517552657974e-05, + "loss": 1.1184, + "theoretical_loss": 3.338406905111594, + "tokens_seen": 3130705920 + }, + { + "epoch": 1.33, + "learning_rate": 4.170912738214644e-05, + "loss": 1.1423, + "theoretical_loss": 3.338401623885807, + "tokens_seen": 3130771456 + }, + { + "epoch": 1.33, + "learning_rate": 4.169307923771314e-05, + "loss": 1.2255, + "theoretical_loss": 3.338396342801524, + "tokens_seen": 3130836992 + }, + { + "epoch": 1.33, + "learning_rate": 4.167703109327984e-05, + "loss": 1.2307, + "theoretical_loss": 3.338391061858738, + "tokens_seen": 3130902528 + }, + { + "epoch": 1.33, + "learning_rate": 4.166098294884654e-05, + "loss": 1.2883, + "theoretical_loss": 3.338385781057441, + "tokens_seen": 3130968064 + }, + { + "epoch": 1.33, + "learning_rate": 4.164493480441324e-05, + "loss": 1.2043, + "theoretical_loss": 3.338380500397628, + "tokens_seen": 3131033600 + }, + { + "epoch": 1.33, + "learning_rate": 4.162888665997994e-05, + "loss": 1.1748, + "theoretical_loss": 3.3383752198792918, + "tokens_seen": 3131099136 + }, + { + "epoch": 1.33, + "learning_rate": 4.1612838515546644e-05, + "loss": 1.3125, + "theoretical_loss": 3.3383699395024253, + "tokens_seen": 3131164672 + }, + { + "epoch": 1.33, + "learning_rate": 4.1596790371113344e-05, + "loss": 1.2353, + "theoretical_loss": 3.3383646592670218, + "tokens_seen": 3131230208 + }, + { + "epoch": 1.33, + "learning_rate": 4.1580742226680045e-05, + "loss": 1.2016, + "theoretical_loss": 3.3383593791730743, + "tokens_seen": 3131295744 + }, + { + "epoch": 1.33, + "learning_rate": 4.156469408224674e-05, + "loss": 1.2045, + "theoretical_loss": 3.3383540992205765, + "tokens_seen": 3131361280 + }, + { + "epoch": 1.33, + "learning_rate": 4.154864593781344e-05, + "loss": 1.2203, + "theoretical_loss": 3.338348819409522, + "tokens_seen": 3131426816 + }, + { + "epoch": 1.33, + "learning_rate": 4.153259779338014e-05, + "loss": 1.1836, + "theoretical_loss": 3.338343539739903, + "tokens_seen": 3131492352 + }, + { + "epoch": 1.33, + "learning_rate": 4.151654964894684e-05, + "loss": 1.1133, + "theoretical_loss": 3.3383382602117138, + "tokens_seen": 3131557888 + }, + { + "epoch": 1.33, + "learning_rate": 4.150050150451355e-05, + "loss": 1.2, + "theoretical_loss": 3.338332980824947, + "tokens_seen": 3131623424 + }, + { + "epoch": 1.33, + "learning_rate": 4.148445336008024e-05, + "loss": 1.1817, + "theoretical_loss": 3.338327701579596, + "tokens_seen": 3131688960 + }, + { + "epoch": 1.33, + "learning_rate": 4.146840521564694e-05, + "loss": 1.2762, + "theoretical_loss": 3.338322422475654, + "tokens_seen": 3131754496 + }, + { + "epoch": 1.33, + "learning_rate": 4.145235707121364e-05, + "loss": 1.1864, + "theoretical_loss": 3.3383171435131143, + "tokens_seen": 3131820032 + }, + { + "epoch": 1.33, + "learning_rate": 4.1436308926780344e-05, + "loss": 1.22, + "theoretical_loss": 3.3383118646919705, + "tokens_seen": 3131885568 + }, + { + "epoch": 1.33, + "learning_rate": 4.1420260782347045e-05, + "loss": 1.3897, + "theoretical_loss": 3.338306586012215, + "tokens_seen": 3131951104 + }, + { + "epoch": 1.33, + "learning_rate": 4.1404212637913745e-05, + "loss": 1.182, + "theoretical_loss": 3.3383013074738423, + "tokens_seen": 3132016640 + }, + { + "epoch": 1.33, + "learning_rate": 4.138816449348044e-05, + "loss": 1.1754, + "theoretical_loss": 3.3382960290768446, + "tokens_seen": 3132082176 + }, + { + "epoch": 1.33, + "learning_rate": 4.137211634904714e-05, + "loss": 1.1222, + "theoretical_loss": 3.338290750821215, + "tokens_seen": 3132147712 + }, + { + "epoch": 1.33, + "learning_rate": 4.135606820461385e-05, + "loss": 1.2127, + "theoretical_loss": 3.338285472706948, + "tokens_seen": 3132213248 + }, + { + "epoch": 1.33, + "learning_rate": 4.134002006018055e-05, + "loss": 1.1177, + "theoretical_loss": 3.338280194734036, + "tokens_seen": 3132278784 + }, + { + "epoch": 1.33, + "learning_rate": 4.132397191574724e-05, + "loss": 1.1571, + "theoretical_loss": 3.3382749169024724, + "tokens_seen": 3132344320 + }, + { + "epoch": 1.33, + "learning_rate": 4.130792377131394e-05, + "loss": 1.2574, + "theoretical_loss": 3.33826963921225, + "tokens_seen": 3132409856 + }, + { + "epoch": 1.33, + "learning_rate": 4.129187562688064e-05, + "loss": 1.1485, + "theoretical_loss": 3.338264361663363, + "tokens_seen": 3132475392 + }, + { + "epoch": 1.33, + "learning_rate": 4.1275827482447343e-05, + "loss": 1.1677, + "theoretical_loss": 3.338259084255804, + "tokens_seen": 3132540928 + }, + { + "epoch": 1.33, + "objective/train/docs_used": 1155919, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1551398038864136, + "objective/train/theoretical_loss": 3.338253806989566, + "objective/train/tokens_used": 3153066464, + "theoretical_loss": 3.338253806989566, + "tokens_seen": 3132606464 + }, + { + "epoch": 1.33, + "learning_rate": 4.1259779338014044e-05, + "loss": 1.1551, + "theoretical_loss": 3.338253806989566, + "tokens_seen": 3132606464 + }, + { + "epoch": 1.33, + "learning_rate": 4.1243731193580745e-05, + "loss": 1.1859, + "theoretical_loss": 3.3382485298646434, + "tokens_seen": 3132672000 + }, + { + "epoch": 1.33, + "learning_rate": 4.122768304914744e-05, + "loss": 1.1904, + "theoretical_loss": 3.3382432528810284, + "tokens_seen": 3132737536 + }, + { + "epoch": 1.33, + "learning_rate": 4.1211634904714146e-05, + "loss": 1.1916, + "theoretical_loss": 3.338237976038714, + "tokens_seen": 3132803072 + }, + { + "epoch": 1.33, + "learning_rate": 4.119558676028085e-05, + "loss": 1.1908, + "theoretical_loss": 3.338232699337695, + "tokens_seen": 3132868608 + }, + { + "epoch": 1.33, + "learning_rate": 4.117953861584755e-05, + "loss": 1.2637, + "theoretical_loss": 3.3382274227779636, + "tokens_seen": 3132934144 + }, + { + "epoch": 1.33, + "learning_rate": 4.116349047141425e-05, + "loss": 1.2335, + "theoretical_loss": 3.3382221463595125, + "tokens_seen": 3132999680 + }, + { + "epoch": 1.33, + "learning_rate": 4.114744232698094e-05, + "loss": 1.1434, + "theoretical_loss": 3.338216870082336, + "tokens_seen": 3133065216 + }, + { + "epoch": 1.33, + "learning_rate": 4.113139418254764e-05, + "loss": 1.1995, + "theoretical_loss": 3.3382115939464274, + "tokens_seen": 3133130752 + }, + { + "epoch": 1.33, + "learning_rate": 4.111534603811434e-05, + "loss": 1.2673, + "theoretical_loss": 3.338206317951779, + "tokens_seen": 3133196288 + }, + { + "epoch": 1.33, + "learning_rate": 4.1099297893681044e-05, + "loss": 1.1901, + "theoretical_loss": 3.338201042098385, + "tokens_seen": 3133261824 + }, + { + "epoch": 1.33, + "learning_rate": 4.1083249749247744e-05, + "loss": 1.1573, + "theoretical_loss": 3.338195766386238, + "tokens_seen": 3133327360 + }, + { + "epoch": 1.33, + "learning_rate": 4.1067201604814445e-05, + "loss": 1.1656, + "theoretical_loss": 3.3381904908153315, + "tokens_seen": 3133392896 + }, + { + "epoch": 1.33, + "learning_rate": 4.1051153460381146e-05, + "loss": 1.154, + "theoretical_loss": 3.338185215385659, + "tokens_seen": 3133458432 + }, + { + "epoch": 1.33, + "learning_rate": 4.1035105315947846e-05, + "loss": 1.1647, + "theoretical_loss": 3.3381799400972136, + "tokens_seen": 3133523968 + }, + { + "epoch": 1.33, + "learning_rate": 4.101905717151455e-05, + "loss": 1.1492, + "theoretical_loss": 3.3381746649499884, + "tokens_seen": 3133589504 + }, + { + "epoch": 1.33, + "learning_rate": 4.100300902708125e-05, + "loss": 1.183, + "theoretical_loss": 3.3381693899439773, + "tokens_seen": 3133655040 + }, + { + "epoch": 1.33, + "learning_rate": 4.098696088264795e-05, + "loss": 1.1241, + "theoretical_loss": 3.3381641150791723, + "tokens_seen": 3133720576 + }, + { + "epoch": 1.33, + "learning_rate": 4.097091273821464e-05, + "loss": 1.2517, + "theoretical_loss": 3.338158840355568, + "tokens_seen": 3133786112 + }, + { + "epoch": 1.33, + "learning_rate": 4.095486459378134e-05, + "loss": 1.1556, + "theoretical_loss": 3.338153565773157, + "tokens_seen": 3133851648 + }, + { + "epoch": 1.33, + "learning_rate": 4.093881644934804e-05, + "loss": 1.2606, + "theoretical_loss": 3.3381482913319327, + "tokens_seen": 3133917184 + }, + { + "epoch": 1.33, + "learning_rate": 4.0922768304914744e-05, + "loss": 1.1991, + "theoretical_loss": 3.338143017031888, + "tokens_seen": 3133982720 + }, + { + "epoch": 1.33, + "learning_rate": 4.090672016048145e-05, + "loss": 1.1957, + "theoretical_loss": 3.338137742873017, + "tokens_seen": 3134048256 + }, + { + "epoch": 1.33, + "learning_rate": 4.0890672016048145e-05, + "loss": 1.1825, + "theoretical_loss": 3.3381324688553122, + "tokens_seen": 3134113792 + }, + { + "epoch": 1.33, + "learning_rate": 4.0874623871614846e-05, + "loss": 1.1371, + "theoretical_loss": 3.3381271949787674, + "tokens_seen": 3134179328 + }, + { + "epoch": 1.33, + "learning_rate": 4.0858575727181546e-05, + "loss": 1.2306, + "theoretical_loss": 3.3381219212433755, + "tokens_seen": 3134244864 + }, + { + "epoch": 1.33, + "learning_rate": 4.084252758274825e-05, + "loss": 1.1863, + "theoretical_loss": 3.3381166476491297, + "tokens_seen": 3134310400 + }, + { + "epoch": 1.33, + "learning_rate": 4.082647943831495e-05, + "loss": 1.2301, + "theoretical_loss": 3.338111374196024, + "tokens_seen": 3134375936 + }, + { + "epoch": 1.33, + "learning_rate": 4.081043129388165e-05, + "loss": 1.1163, + "theoretical_loss": 3.3381061008840507, + "tokens_seen": 3134441472 + }, + { + "epoch": 1.33, + "learning_rate": 4.079438314944834e-05, + "loss": 1.2676, + "theoretical_loss": 3.3381008277132036, + "tokens_seen": 3134507008 + }, + { + "epoch": 1.33, + "learning_rate": 4.077833500501504e-05, + "loss": 1.1643, + "theoretical_loss": 3.338095554683476, + "tokens_seen": 3134572544 + }, + { + "epoch": 1.33, + "learning_rate": 4.076228686058175e-05, + "loss": 1.1474, + "theoretical_loss": 3.3380902817948606, + "tokens_seen": 3134638080 + }, + { + "epoch": 1.33, + "learning_rate": 4.074623871614845e-05, + "loss": 1.2323, + "theoretical_loss": 3.3380850090473517, + "tokens_seen": 3134703616 + }, + { + "epoch": 1.33, + "learning_rate": 4.073019057171515e-05, + "loss": 1.173, + "theoretical_loss": 3.3380797364409416, + "tokens_seen": 3134769152 + }, + { + "epoch": 1.33, + "learning_rate": 4.0714142427281845e-05, + "loss": 1.2619, + "theoretical_loss": 3.3380744639756244, + "tokens_seen": 3134834688 + }, + { + "epoch": 1.33, + "learning_rate": 4.0698094282848546e-05, + "loss": 1.1483, + "theoretical_loss": 3.338069191651393, + "tokens_seen": 3134900224 + }, + { + "epoch": 1.33, + "learning_rate": 4.068204613841525e-05, + "loss": 1.2105, + "theoretical_loss": 3.33806391946824, + "tokens_seen": 3134965760 + }, + { + "epoch": 1.33, + "learning_rate": 4.066599799398195e-05, + "loss": 1.1703, + "theoretical_loss": 3.33805864742616, + "tokens_seen": 3135031296 + }, + { + "epoch": 1.33, + "learning_rate": 4.064994984954865e-05, + "loss": 1.1388, + "theoretical_loss": 3.338053375525145, + "tokens_seen": 3135096832 + }, + { + "epoch": 1.33, + "learning_rate": 4.063390170511535e-05, + "loss": 1.2627, + "theoretical_loss": 3.3380481037651895, + "tokens_seen": 3135162368 + }, + { + "epoch": 1.33, + "learning_rate": 4.061785356068204e-05, + "loss": 1.1392, + "theoretical_loss": 3.338042832146286, + "tokens_seen": 3135227904 + }, + { + "epoch": 1.33, + "learning_rate": 4.060180541624875e-05, + "loss": 1.1844, + "theoretical_loss": 3.338037560668427, + "tokens_seen": 3135293440 + }, + { + "epoch": 1.33, + "learning_rate": 4.058575727181545e-05, + "loss": 1.0876, + "theoretical_loss": 3.3380322893316077, + "tokens_seen": 3135358976 + }, + { + "epoch": 1.33, + "learning_rate": 4.056970912738215e-05, + "loss": 1.1616, + "theoretical_loss": 3.3380270181358203, + "tokens_seen": 3135424512 + }, + { + "epoch": 1.33, + "learning_rate": 4.055366098294885e-05, + "loss": 1.0921, + "theoretical_loss": 3.338021747081058, + "tokens_seen": 3135490048 + }, + { + "epoch": 1.33, + "learning_rate": 4.0537612838515546e-05, + "loss": 1.1665, + "theoretical_loss": 3.338016476167314, + "tokens_seen": 3135555584 + }, + { + "epoch": 1.33, + "learning_rate": 4.0521564694082246e-05, + "loss": 1.1963, + "theoretical_loss": 3.338011205394582, + "tokens_seen": 3135621120 + }, + { + "epoch": 1.33, + "learning_rate": 4.050551654964895e-05, + "loss": 1.181, + "theoretical_loss": 3.338005934762855, + "tokens_seen": 3135686656 + }, + { + "epoch": 1.33, + "learning_rate": 4.048946840521565e-05, + "loss": 1.1539, + "theoretical_loss": 3.3380006642721263, + "tokens_seen": 3135752192 + }, + { + "epoch": 1.33, + "learning_rate": 4.047342026078235e-05, + "loss": 1.2009, + "theoretical_loss": 3.3379953939223896, + "tokens_seen": 3135817728 + }, + { + "epoch": 1.33, + "learning_rate": 4.045737211634905e-05, + "loss": 1.1689, + "theoretical_loss": 3.3379901237136376, + "tokens_seen": 3135883264 + }, + { + "epoch": 1.33, + "learning_rate": 4.044132397191575e-05, + "loss": 1.1412, + "theoretical_loss": 3.337984853645864, + "tokens_seen": 3135948800 + }, + { + "epoch": 1.33, + "learning_rate": 4.042527582748245e-05, + "loss": 1.1855, + "theoretical_loss": 3.337979583719062, + "tokens_seen": 3136014336 + }, + { + "epoch": 1.33, + "learning_rate": 4.040922768304915e-05, + "loss": 1.1148, + "theoretical_loss": 3.3379743139332243, + "tokens_seen": 3136079872 + }, + { + "epoch": 1.33, + "learning_rate": 4.039317953861585e-05, + "loss": 1.113, + "theoretical_loss": 3.3379690442883447, + "tokens_seen": 3136145408 + }, + { + "epoch": 1.33, + "learning_rate": 4.037713139418255e-05, + "loss": 1.1611, + "theoretical_loss": 3.337963774784417, + "tokens_seen": 3136210944 + }, + { + "epoch": 1.33, + "learning_rate": 4.0361083249749246e-05, + "loss": 1.1938, + "theoretical_loss": 3.3379585054214336, + "tokens_seen": 3136276480 + }, + { + "epoch": 1.33, + "learning_rate": 4.0345035105315946e-05, + "loss": 1.0824, + "theoretical_loss": 3.3379532361993878, + "tokens_seen": 3136342016 + }, + { + "epoch": 1.33, + "learning_rate": 4.032898696088265e-05, + "loss": 1.196, + "theoretical_loss": 3.3379479671182737, + "tokens_seen": 3136407552 + }, + { + "epoch": 1.33, + "learning_rate": 4.0312938816449354e-05, + "loss": 1.0629, + "theoretical_loss": 3.337942698178084, + "tokens_seen": 3136473088 + }, + { + "epoch": 1.33, + "learning_rate": 4.0296890672016055e-05, + "loss": 1.1011, + "theoretical_loss": 3.3379374293788118, + "tokens_seen": 3136538624 + }, + { + "epoch": 1.33, + "learning_rate": 4.028084252758275e-05, + "loss": 1.1021, + "theoretical_loss": 3.337932160720451, + "tokens_seen": 3136604160 + }, + { + "epoch": 1.33, + "learning_rate": 4.026479438314945e-05, + "loss": 1.0737, + "theoretical_loss": 3.3379268922029945, + "tokens_seen": 3136669696 + }, + { + "epoch": 1.33, + "learning_rate": 4.024874623871615e-05, + "loss": 1.1691, + "theoretical_loss": 3.3379216238264355, + "tokens_seen": 3136735232 + }, + { + "epoch": 1.33, + "learning_rate": 4.023269809428285e-05, + "loss": 1.1465, + "theoretical_loss": 3.337916355590768, + "tokens_seen": 3136800768 + }, + { + "epoch": 1.33, + "learning_rate": 4.021664994984955e-05, + "loss": 1.1463, + "theoretical_loss": 3.337911087495984, + "tokens_seen": 3136866304 + }, + { + "epoch": 1.33, + "learning_rate": 4.020060180541625e-05, + "loss": 1.2361, + "theoretical_loss": 3.337905819542078, + "tokens_seen": 3136931840 + }, + { + "epoch": 1.33, + "learning_rate": 4.0184553660982946e-05, + "loss": 1.1523, + "theoretical_loss": 3.3379005517290423, + "tokens_seen": 3136997376 + }, + { + "epoch": 1.33, + "learning_rate": 4.0168505516549647e-05, + "loss": 1.1503, + "theoretical_loss": 3.337895284056871, + "tokens_seen": 3137062912 + }, + { + "epoch": 1.33, + "learning_rate": 4.0152457372116354e-05, + "loss": 1.1458, + "theoretical_loss": 3.3378900165255567, + "tokens_seen": 3137128448 + }, + { + "epoch": 1.33, + "learning_rate": 4.0136409227683055e-05, + "loss": 1.2089, + "theoretical_loss": 3.3378847491350934, + "tokens_seen": 3137193984 + }, + { + "epoch": 1.33, + "learning_rate": 4.0120361083249755e-05, + "loss": 1.2119, + "theoretical_loss": 3.3378794818854742, + "tokens_seen": 3137259520 + }, + { + "epoch": 1.33, + "learning_rate": 4.010431293881645e-05, + "loss": 1.1618, + "theoretical_loss": 3.337874214776692, + "tokens_seen": 3137325056 + }, + { + "epoch": 1.33, + "learning_rate": 4.008826479438315e-05, + "loss": 1.1886, + "theoretical_loss": 3.3378689478087407, + "tokens_seen": 3137390592 + }, + { + "epoch": 1.33, + "learning_rate": 4.007221664994985e-05, + "loss": 1.2587, + "theoretical_loss": 3.337863680981613, + "tokens_seen": 3137456128 + }, + { + "epoch": 1.33, + "learning_rate": 4.005616850551655e-05, + "loss": 1.2101, + "theoretical_loss": 3.3378584142953023, + "tokens_seen": 3137521664 + }, + { + "epoch": 1.33, + "learning_rate": 4.004012036108325e-05, + "loss": 1.1865, + "theoretical_loss": 3.337853147749802, + "tokens_seen": 3137587200 + }, + { + "epoch": 1.33, + "learning_rate": 4.002407221664995e-05, + "loss": 1.2365, + "theoretical_loss": 3.337847881345106, + "tokens_seen": 3137652736 + }, + { + "epoch": 1.33, + "learning_rate": 4.000802407221665e-05, + "loss": 1.0941, + "theoretical_loss": 3.337842615081206, + "tokens_seen": 3137718272 + }, + { + "epoch": 1.33, + "learning_rate": 3.9991975927783354e-05, + "loss": 1.217, + "theoretical_loss": 3.337837348958097, + "tokens_seen": 3137783808 + }, + { + "epoch": 1.33, + "learning_rate": 3.9975927783350054e-05, + "loss": 1.1901, + "theoretical_loss": 3.3378320829757717, + "tokens_seen": 3137849344 + }, + { + "epoch": 1.33, + "learning_rate": 3.9959879638916755e-05, + "loss": 1.0957, + "theoretical_loss": 3.337826817134223, + "tokens_seen": 3137914880 + }, + { + "epoch": 1.33, + "learning_rate": 3.9943831494483455e-05, + "loss": 1.1744, + "theoretical_loss": 3.3378215514334446, + "tokens_seen": 3137980416 + }, + { + "epoch": 1.33, + "learning_rate": 3.992778335005015e-05, + "loss": 1.0899, + "theoretical_loss": 3.33781628587343, + "tokens_seen": 3138045952 + }, + { + "epoch": 1.33, + "learning_rate": 3.991173520561685e-05, + "loss": 1.241, + "theoretical_loss": 3.3378110204541716, + "tokens_seen": 3138111488 + }, + { + "epoch": 1.33, + "learning_rate": 3.989568706118355e-05, + "loss": 1.2028, + "theoretical_loss": 3.3378057551756637, + "tokens_seen": 3138177024 + }, + { + "epoch": 1.33, + "learning_rate": 3.987963891675025e-05, + "loss": 1.1382, + "theoretical_loss": 3.337800490037899, + "tokens_seen": 3138242560 + }, + { + "epoch": 1.33, + "learning_rate": 3.986359077231696e-05, + "loss": 1.1933, + "theoretical_loss": 3.337795225040871, + "tokens_seen": 3138308096 + }, + { + "epoch": 1.33, + "learning_rate": 3.984754262788365e-05, + "loss": 1.1995, + "theoretical_loss": 3.337789960184573, + "tokens_seen": 3138373632 + }, + { + "epoch": 1.33, + "learning_rate": 3.983149448345035e-05, + "loss": 1.1365, + "theoretical_loss": 3.3377846954689985, + "tokens_seen": 3138439168 + }, + { + "epoch": 1.33, + "learning_rate": 3.9815446339017054e-05, + "loss": 1.1663, + "theoretical_loss": 3.33777943089414, + "tokens_seen": 3138504704 + }, + { + "epoch": 1.33, + "learning_rate": 3.9799398194583754e-05, + "loss": 1.1179, + "theoretical_loss": 3.337774166459992, + "tokens_seen": 3138570240 + }, + { + "epoch": 1.33, + "learning_rate": 3.9783350050150455e-05, + "loss": 1.2464, + "theoretical_loss": 3.3377689021665473, + "tokens_seen": 3138635776 + }, + { + "epoch": 1.33, + "learning_rate": 3.9767301905717156e-05, + "loss": 1.0805, + "theoretical_loss": 3.3377636380137985, + "tokens_seen": 3138701312 + }, + { + "epoch": 1.33, + "learning_rate": 3.975125376128385e-05, + "loss": 1.2292, + "theoretical_loss": 3.33775837400174, + "tokens_seen": 3138766848 + }, + { + "epoch": 1.33, + "learning_rate": 3.973520561685055e-05, + "loss": 1.2002, + "theoretical_loss": 3.337753110130364, + "tokens_seen": 3138832384 + }, + { + "epoch": 1.33, + "learning_rate": 3.971915747241725e-05, + "loss": 1.1949, + "theoretical_loss": 3.3377478463996653, + "tokens_seen": 3138897920 + }, + { + "epoch": 1.33, + "learning_rate": 3.970310932798396e-05, + "loss": 1.1699, + "theoretical_loss": 3.3377425828096356, + "tokens_seen": 3138963456 + }, + { + "epoch": 1.33, + "learning_rate": 3.968706118355066e-05, + "loss": 1.2387, + "theoretical_loss": 3.337737319360269, + "tokens_seen": 3139028992 + }, + { + "epoch": 1.33, + "learning_rate": 3.967101303911735e-05, + "loss": 1.1638, + "theoretical_loss": 3.337732056051559, + "tokens_seen": 3139094528 + }, + { + "epoch": 1.33, + "objective/train/docs_used": 1158089, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.266603708267212, + "objective/train/theoretical_loss": 3.3377267928834984, + "objective/train/tokens_used": 3159620064, + "theoretical_loss": 3.3377267928834984, + "tokens_seen": 3139160064 + }, + { + "epoch": 1.33, + "learning_rate": 3.965496489468405e-05, + "loss": 1.2666, + "theoretical_loss": 3.3377267928834984, + "tokens_seen": 3139160064 + }, + { + "epoch": 1.33, + "learning_rate": 3.9638916750250754e-05, + "loss": 1.1443, + "theoretical_loss": 3.3377215298560805, + "tokens_seen": 3139225600 + }, + { + "epoch": 1.33, + "learning_rate": 3.9622868605817455e-05, + "loss": 1.2338, + "theoretical_loss": 3.337716266969299, + "tokens_seen": 3139291136 + }, + { + "epoch": 1.33, + "learning_rate": 3.9606820461384155e-05, + "loss": 1.1877, + "theoretical_loss": 3.3377110042231473, + "tokens_seen": 3139356672 + }, + { + "epoch": 1.33, + "learning_rate": 3.9590772316950856e-05, + "loss": 1.1732, + "theoretical_loss": 3.337705741617618, + "tokens_seen": 3139422208 + }, + { + "epoch": 1.33, + "learning_rate": 3.957472417251755e-05, + "loss": 1.2213, + "theoretical_loss": 3.337700479152705, + "tokens_seen": 3139487744 + }, + { + "epoch": 1.33, + "learning_rate": 3.955867602808426e-05, + "loss": 1.1735, + "theoretical_loss": 3.3376952168284015, + "tokens_seen": 3139553280 + }, + { + "epoch": 1.33, + "learning_rate": 3.954262788365096e-05, + "loss": 1.208, + "theoretical_loss": 3.337689954644701, + "tokens_seen": 3139618816 + }, + { + "epoch": 1.33, + "learning_rate": 3.952657973921766e-05, + "loss": 1.1965, + "theoretical_loss": 3.3376846926015964, + "tokens_seen": 3139684352 + }, + { + "epoch": 1.33, + "learning_rate": 3.951053159478436e-05, + "loss": 1.1269, + "theoretical_loss": 3.337679430699081, + "tokens_seen": 3139749888 + }, + { + "epoch": 1.33, + "learning_rate": 3.949448345035105e-05, + "loss": 1.2411, + "theoretical_loss": 3.3376741689371485, + "tokens_seen": 3139815424 + }, + { + "epoch": 1.33, + "learning_rate": 3.9478435305917754e-05, + "loss": 1.1625, + "theoretical_loss": 3.337668907315792, + "tokens_seen": 3139880960 + }, + { + "epoch": 1.33, + "learning_rate": 3.9462387161484454e-05, + "loss": 1.111, + "theoretical_loss": 3.3376636458350046, + "tokens_seen": 3139946496 + }, + { + "epoch": 1.33, + "learning_rate": 3.9446339017051155e-05, + "loss": 1.2077, + "theoretical_loss": 3.33765838449478, + "tokens_seen": 3140012032 + }, + { + "epoch": 1.33, + "learning_rate": 3.9430290872617855e-05, + "loss": 1.1019, + "theoretical_loss": 3.3376531232951114, + "tokens_seen": 3140077568 + }, + { + "epoch": 1.33, + "learning_rate": 3.9414242728184556e-05, + "loss": 1.1504, + "theoretical_loss": 3.3376478622359915, + "tokens_seen": 3140143104 + }, + { + "epoch": 1.33, + "learning_rate": 3.939819458375126e-05, + "loss": 1.1435, + "theoretical_loss": 3.3376426013174147, + "tokens_seen": 3140208640 + }, + { + "epoch": 1.33, + "learning_rate": 3.938214643931796e-05, + "loss": 1.2299, + "theoretical_loss": 3.3376373405393736, + "tokens_seen": 3140274176 + }, + { + "epoch": 1.33, + "learning_rate": 3.936609829488466e-05, + "loss": 1.1135, + "theoretical_loss": 3.3376320799018617, + "tokens_seen": 3140339712 + }, + { + "epoch": 1.33, + "learning_rate": 3.935005015045136e-05, + "loss": 1.2065, + "theoretical_loss": 3.3376268194048726, + "tokens_seen": 3140405248 + }, + { + "epoch": 1.33, + "learning_rate": 3.933400200601805e-05, + "loss": 1.1706, + "theoretical_loss": 3.337621559048399, + "tokens_seen": 3140470784 + }, + { + "epoch": 1.33, + "learning_rate": 3.931795386158475e-05, + "loss": 1.2039, + "theoretical_loss": 3.3376162988324345, + "tokens_seen": 3140536320 + }, + { + "epoch": 1.33, + "learning_rate": 3.9301905717151454e-05, + "loss": 1.223, + "theoretical_loss": 3.3376110387569726, + "tokens_seen": 3140601856 + }, + { + "epoch": 1.33, + "learning_rate": 3.9285857572718154e-05, + "loss": 1.1342, + "theoretical_loss": 3.337605778822006, + "tokens_seen": 3140667392 + }, + { + "epoch": 1.33, + "learning_rate": 3.9269809428284855e-05, + "loss": 1.2289, + "theoretical_loss": 3.337600519027529, + "tokens_seen": 3140732928 + }, + { + "epoch": 1.33, + "learning_rate": 3.9253761283851556e-05, + "loss": 1.2171, + "theoretical_loss": 3.337595259373534, + "tokens_seen": 3140798464 + }, + { + "epoch": 1.33, + "learning_rate": 3.9237713139418256e-05, + "loss": 1.1518, + "theoretical_loss": 3.337589999860015, + "tokens_seen": 3140864000 + }, + { + "epoch": 1.33, + "learning_rate": 3.922166499498496e-05, + "loss": 1.1926, + "theoretical_loss": 3.3375847404869647, + "tokens_seen": 3140929536 + }, + { + "epoch": 1.33, + "learning_rate": 3.920561685055166e-05, + "loss": 1.1114, + "theoretical_loss": 3.337579481254377, + "tokens_seen": 3140995072 + }, + { + "epoch": 1.33, + "learning_rate": 3.918956870611836e-05, + "loss": 1.2504, + "theoretical_loss": 3.337574222162245, + "tokens_seen": 3141060608 + }, + { + "epoch": 1.33, + "learning_rate": 3.917352056168506e-05, + "loss": 1.1219, + "theoretical_loss": 3.337568963210562, + "tokens_seen": 3141126144 + }, + { + "epoch": 1.33, + "learning_rate": 3.915747241725175e-05, + "loss": 1.1005, + "theoretical_loss": 3.337563704399321, + "tokens_seen": 3141191680 + }, + { + "epoch": 1.33, + "learning_rate": 3.914142427281845e-05, + "loss": 1.2498, + "theoretical_loss": 3.337558445728516, + "tokens_seen": 3141257216 + }, + { + "epoch": 1.33, + "learning_rate": 3.9125376128385154e-05, + "loss": 1.1805, + "theoretical_loss": 3.3375531871981394, + "tokens_seen": 3141322752 + }, + { + "epoch": 1.33, + "learning_rate": 3.910932798395186e-05, + "loss": 1.1911, + "theoretical_loss": 3.3375479288081853, + "tokens_seen": 3141388288 + }, + { + "epoch": 1.33, + "learning_rate": 3.909327983951856e-05, + "loss": 1.0711, + "theoretical_loss": 3.337542670558647, + "tokens_seen": 3141453824 + }, + { + "epoch": 1.33, + "learning_rate": 3.9077231695085256e-05, + "loss": 1.2503, + "theoretical_loss": 3.3375374124495174, + "tokens_seen": 3141519360 + }, + { + "epoch": 1.33, + "learning_rate": 3.9061183550651957e-05, + "loss": 1.1582, + "theoretical_loss": 3.33753215448079, + "tokens_seen": 3141584896 + }, + { + "epoch": 1.33, + "learning_rate": 3.904513540621866e-05, + "loss": 1.3287, + "theoretical_loss": 3.337526896652458, + "tokens_seen": 3141650432 + }, + { + "epoch": 1.33, + "learning_rate": 3.902908726178536e-05, + "loss": 1.1867, + "theoretical_loss": 3.337521638964515, + "tokens_seen": 3141715968 + }, + { + "epoch": 1.33, + "learning_rate": 3.901303911735206e-05, + "loss": 1.2078, + "theoretical_loss": 3.3375163814169544, + "tokens_seen": 3141781504 + }, + { + "epoch": 1.33, + "learning_rate": 3.899699097291876e-05, + "loss": 1.1523, + "theoretical_loss": 3.337511124009769, + "tokens_seen": 3141847040 + }, + { + "epoch": 1.33, + "learning_rate": 3.898094282848545e-05, + "loss": 1.0809, + "theoretical_loss": 3.3375058667429522, + "tokens_seen": 3141912576 + }, + { + "epoch": 1.33, + "learning_rate": 3.8964894684052154e-05, + "loss": 1.1423, + "theoretical_loss": 3.337500609616498, + "tokens_seen": 3141978112 + }, + { + "epoch": 1.33, + "learning_rate": 3.894884653961886e-05, + "loss": 1.073, + "theoretical_loss": 3.337495352630399, + "tokens_seen": 3142043648 + }, + { + "epoch": 1.33, + "learning_rate": 3.893279839518556e-05, + "loss": 1.18, + "theoretical_loss": 3.3374900957846494, + "tokens_seen": 3142109184 + }, + { + "epoch": 1.33, + "learning_rate": 3.891675025075226e-05, + "loss": 1.164, + "theoretical_loss": 3.3374848390792415, + "tokens_seen": 3142174720 + }, + { + "epoch": 1.33, + "learning_rate": 3.8900702106318956e-05, + "loss": 1.2098, + "theoretical_loss": 3.337479582514169, + "tokens_seen": 3142240256 + }, + { + "epoch": 1.33, + "learning_rate": 3.888465396188566e-05, + "loss": 1.1348, + "theoretical_loss": 3.3374743260894255, + "tokens_seen": 3142305792 + }, + { + "epoch": 1.33, + "learning_rate": 3.886860581745236e-05, + "loss": 1.0567, + "theoretical_loss": 3.3374690698050036, + "tokens_seen": 3142371328 + }, + { + "epoch": 1.33, + "learning_rate": 3.885255767301906e-05, + "loss": 1.1584, + "theoretical_loss": 3.3374638136608974, + "tokens_seen": 3142436864 + }, + { + "epoch": 1.33, + "learning_rate": 3.883650952858576e-05, + "loss": 1.1285, + "theoretical_loss": 3.3374585576571003, + "tokens_seen": 3142502400 + }, + { + "epoch": 1.33, + "learning_rate": 3.882046138415246e-05, + "loss": 1.1404, + "theoretical_loss": 3.337453301793605, + "tokens_seen": 3142567936 + }, + { + "epoch": 1.33, + "learning_rate": 3.880441323971916e-05, + "loss": 1.1921, + "theoretical_loss": 3.337448046070405, + "tokens_seen": 3142633472 + }, + { + "epoch": 1.33, + "learning_rate": 3.878836509528586e-05, + "loss": 1.0931, + "theoretical_loss": 3.3374427904874944, + "tokens_seen": 3142699008 + }, + { + "epoch": 1.33, + "learning_rate": 3.877231695085256e-05, + "loss": 1.2736, + "theoretical_loss": 3.3374375350448653, + "tokens_seen": 3142764544 + }, + { + "epoch": 1.33, + "learning_rate": 3.875626880641926e-05, + "loss": 1.1423, + "theoretical_loss": 3.3374322797425116, + "tokens_seen": 3142830080 + }, + { + "epoch": 1.33, + "learning_rate": 3.874022066198596e-05, + "loss": 1.1297, + "theoretical_loss": 3.3374270245804265, + "tokens_seen": 3142895616 + }, + { + "epoch": 1.33, + "learning_rate": 3.8724172517552656e-05, + "loss": 1.1165, + "theoretical_loss": 3.337421769558604, + "tokens_seen": 3142961152 + }, + { + "epoch": 1.33, + "learning_rate": 3.870812437311936e-05, + "loss": 1.1768, + "theoretical_loss": 3.3374165146770367, + "tokens_seen": 3143026688 + }, + { + "epoch": 1.33, + "learning_rate": 3.869207622868606e-05, + "loss": 1.2266, + "theoretical_loss": 3.3374112599357177, + "tokens_seen": 3143092224 + }, + { + "epoch": 1.33, + "learning_rate": 3.867602808425276e-05, + "loss": 1.2896, + "theoretical_loss": 3.3374060053346417, + "tokens_seen": 3143157760 + }, + { + "epoch": 1.33, + "learning_rate": 3.8659979939819466e-05, + "loss": 1.2367, + "theoretical_loss": 3.3374007508738, + "tokens_seen": 3143223296 + }, + { + "epoch": 1.33, + "learning_rate": 3.864393179538616e-05, + "loss": 1.1715, + "theoretical_loss": 3.337395496553188, + "tokens_seen": 3143288832 + }, + { + "epoch": 1.33, + "learning_rate": 3.862788365095286e-05, + "loss": 1.1793, + "theoretical_loss": 3.3373902423727975, + "tokens_seen": 3143354368 + }, + { + "epoch": 1.33, + "learning_rate": 3.861183550651956e-05, + "loss": 1.2323, + "theoretical_loss": 3.337384988332623, + "tokens_seen": 3143419904 + }, + { + "epoch": 1.33, + "learning_rate": 3.859578736208626e-05, + "loss": 1.167, + "theoretical_loss": 3.3373797344326563, + "tokens_seen": 3143485440 + }, + { + "epoch": 1.33, + "learning_rate": 3.857973921765296e-05, + "loss": 1.1643, + "theoretical_loss": 3.337374480672892, + "tokens_seen": 3143550976 + }, + { + "epoch": 1.33, + "learning_rate": 3.856369107321966e-05, + "loss": 1.1177, + "theoretical_loss": 3.337369227053324, + "tokens_seen": 3143616512 + }, + { + "epoch": 1.33, + "learning_rate": 3.8547642928786356e-05, + "loss": 1.1381, + "theoretical_loss": 3.337363973573944, + "tokens_seen": 3143682048 + }, + { + "epoch": 1.33, + "learning_rate": 3.853159478435306e-05, + "loss": 1.2347, + "theoretical_loss": 3.337358720234746, + "tokens_seen": 3143747584 + }, + { + "epoch": 1.33, + "learning_rate": 3.851554663991976e-05, + "loss": 1.1754, + "theoretical_loss": 3.3373534670357237, + "tokens_seen": 3143813120 + }, + { + "epoch": 1.33, + "learning_rate": 3.8499498495486465e-05, + "loss": 1.1175, + "theoretical_loss": 3.33734821397687, + "tokens_seen": 3143878656 + }, + { + "epoch": 1.33, + "learning_rate": 3.8483450351053166e-05, + "loss": 1.141, + "theoretical_loss": 3.3373429610581784, + "tokens_seen": 3143944192 + }, + { + "epoch": 1.33, + "learning_rate": 3.846740220661986e-05, + "loss": 1.1995, + "theoretical_loss": 3.3373377082796423, + "tokens_seen": 3144009728 + }, + { + "epoch": 1.33, + "learning_rate": 3.845135406218656e-05, + "loss": 1.139, + "theoretical_loss": 3.337332455641255, + "tokens_seen": 3144075264 + }, + { + "epoch": 1.33, + "learning_rate": 3.843530591775326e-05, + "loss": 1.1213, + "theoretical_loss": 3.3373272031430097, + "tokens_seen": 3144140800 + }, + { + "epoch": 1.33, + "learning_rate": 3.841925777331996e-05, + "loss": 1.0629, + "theoretical_loss": 3.3373219507849, + "tokens_seen": 3144206336 + }, + { + "epoch": 1.33, + "learning_rate": 3.840320962888666e-05, + "loss": 1.185, + "theoretical_loss": 3.337316698566919, + "tokens_seen": 3144271872 + }, + { + "epoch": 1.33, + "learning_rate": 3.838716148445336e-05, + "loss": 1.2452, + "theoretical_loss": 3.3373114464890605, + "tokens_seen": 3144337408 + }, + { + "epoch": 1.33, + "learning_rate": 3.837111334002006e-05, + "loss": 1.0698, + "theoretical_loss": 3.337306194551317, + "tokens_seen": 3144402944 + }, + { + "epoch": 1.33, + "learning_rate": 3.8355065195586764e-05, + "loss": 1.1306, + "theoretical_loss": 3.337300942753682, + "tokens_seen": 3144468480 + }, + { + "epoch": 1.33, + "learning_rate": 3.8339017051153465e-05, + "loss": 1.2181, + "theoretical_loss": 3.33729569109615, + "tokens_seen": 3144534016 + }, + { + "epoch": 1.33, + "learning_rate": 3.8322968906720165e-05, + "loss": 1.2095, + "theoretical_loss": 3.3372904395787133, + "tokens_seen": 3144599552 + }, + { + "epoch": 1.33, + "learning_rate": 3.8306920762286866e-05, + "loss": 1.2799, + "theoretical_loss": 3.337285188201365, + "tokens_seen": 3144665088 + }, + { + "epoch": 1.33, + "learning_rate": 3.829087261785356e-05, + "loss": 1.075, + "theoretical_loss": 3.3372799369640993, + "tokens_seen": 3144730624 + }, + { + "epoch": 1.33, + "learning_rate": 3.827482447342026e-05, + "loss": 1.3199, + "theoretical_loss": 3.337274685866909, + "tokens_seen": 3144796160 + }, + { + "epoch": 1.33, + "learning_rate": 3.825877632898696e-05, + "loss": 1.0534, + "theoretical_loss": 3.337269434909788, + "tokens_seen": 3144861696 + }, + { + "epoch": 1.33, + "learning_rate": 3.824272818455366e-05, + "loss": 1.2756, + "theoretical_loss": 3.337264184092729, + "tokens_seen": 3144927232 + }, + { + "epoch": 1.33, + "learning_rate": 3.822668004012036e-05, + "loss": 1.2317, + "theoretical_loss": 3.337258933415725, + "tokens_seen": 3144992768 + }, + { + "epoch": 1.33, + "learning_rate": 3.821063189568706e-05, + "loss": 1.29, + "theoretical_loss": 3.3372536828787704, + "tokens_seen": 3145058304 + }, + { + "epoch": 1.33, + "learning_rate": 3.8194583751253764e-05, + "loss": 1.2075, + "theoretical_loss": 3.337248432481858, + "tokens_seen": 3145123840 + }, + { + "epoch": 1.33, + "learning_rate": 3.8178535606820464e-05, + "loss": 1.1577, + "theoretical_loss": 3.3372431822249813, + "tokens_seen": 3145189376 + }, + { + "epoch": 1.33, + "learning_rate": 3.8162487462387165e-05, + "loss": 1.0892, + "theoretical_loss": 3.3372379321081334, + "tokens_seen": 3145254912 + }, + { + "epoch": 1.33, + "learning_rate": 3.8146439317953866e-05, + "loss": 1.1739, + "theoretical_loss": 3.337232682131308, + "tokens_seen": 3145320448 + }, + { + "epoch": 1.33, + "learning_rate": 3.8130391173520566e-05, + "loss": 1.1266, + "theoretical_loss": 3.337227432294498, + "tokens_seen": 3145385984 + }, + { + "epoch": 1.33, + "learning_rate": 3.811434302908726e-05, + "loss": 1.1362, + "theoretical_loss": 3.337222182597697, + "tokens_seen": 3145451520 + }, + { + "epoch": 1.33, + "learning_rate": 3.809829488465396e-05, + "loss": 1.1768, + "theoretical_loss": 3.3372169330408985, + "tokens_seen": 3145517056 + }, + { + "epoch": 1.33, + "learning_rate": 3.808224674022066e-05, + "loss": 1.1142, + "theoretical_loss": 3.337211683624096, + "tokens_seen": 3145582592 + }, + { + "epoch": 1.33, + "learning_rate": 3.806619859578736e-05, + "loss": 1.1784, + "theoretical_loss": 3.3372064343472823, + "tokens_seen": 3145648128 + }, + { + "epoch": 1.33, + "objective/train/docs_used": 1160435, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1147791147232056, + "objective/train/theoretical_loss": 3.337201185210451, + "objective/train/tokens_used": 3166173664, + "theoretical_loss": 3.337201185210451, + "tokens_seen": 3145713664 + }, + { + "epoch": 1.33, + "learning_rate": 3.805015045135407e-05, + "loss": 1.1148, + "theoretical_loss": 3.337201185210451, + "tokens_seen": 3145713664 + }, + { + "epoch": 1.33, + "learning_rate": 3.803410230692076e-05, + "loss": 1.1596, + "theoretical_loss": 3.337195936213595, + "tokens_seen": 3145779200 + }, + { + "epoch": 1.33, + "learning_rate": 3.8018054162487464e-05, + "loss": 1.1378, + "theoretical_loss": 3.3371906873567085, + "tokens_seen": 3145844736 + }, + { + "epoch": 1.33, + "learning_rate": 3.8002006018054165e-05, + "loss": 1.0826, + "theoretical_loss": 3.337185438639785, + "tokens_seen": 3145910272 + }, + { + "epoch": 1.33, + "learning_rate": 3.7985957873620865e-05, + "loss": 1.0652, + "theoretical_loss": 3.3371801900628166, + "tokens_seen": 3145975808 + }, + { + "epoch": 1.33, + "learning_rate": 3.7969909729187566e-05, + "loss": 1.1059, + "theoretical_loss": 3.3371749416257974, + "tokens_seen": 3146041344 + }, + { + "epoch": 1.33, + "learning_rate": 3.7953861584754266e-05, + "loss": 1.2109, + "theoretical_loss": 3.337169693328721, + "tokens_seen": 3146106880 + }, + { + "epoch": 1.33, + "learning_rate": 3.793781344032096e-05, + "loss": 1.1491, + "theoretical_loss": 3.33716444517158, + "tokens_seen": 3146172416 + }, + { + "epoch": 1.33, + "learning_rate": 3.792176529588766e-05, + "loss": 1.1818, + "theoretical_loss": 3.3371591971543686, + "tokens_seen": 3146237952 + }, + { + "epoch": 1.33, + "learning_rate": 3.790571715145437e-05, + "loss": 1.1824, + "theoretical_loss": 3.33715394927708, + "tokens_seen": 3146303488 + }, + { + "epoch": 1.33, + "learning_rate": 3.788966900702107e-05, + "loss": 1.1081, + "theoretical_loss": 3.337148701539707, + "tokens_seen": 3146369024 + }, + { + "epoch": 1.33, + "learning_rate": 3.787362086258777e-05, + "loss": 1.108, + "theoretical_loss": 3.337143453942243, + "tokens_seen": 3146434560 + }, + { + "epoch": 1.33, + "learning_rate": 3.7857572718154463e-05, + "loss": 1.1207, + "theoretical_loss": 3.337138206484682, + "tokens_seen": 3146500096 + }, + { + "epoch": 1.33, + "learning_rate": 3.7841524573721164e-05, + "loss": 1.1652, + "theoretical_loss": 3.3371329591670174, + "tokens_seen": 3146565632 + }, + { + "epoch": 1.33, + "learning_rate": 3.7825476429287865e-05, + "loss": 1.2504, + "theoretical_loss": 3.3371277119892415, + "tokens_seen": 3146631168 + }, + { + "epoch": 1.33, + "learning_rate": 3.7809428284854565e-05, + "loss": 1.1908, + "theoretical_loss": 3.3371224649513485, + "tokens_seen": 3146696704 + }, + { + "epoch": 1.33, + "learning_rate": 3.7793380140421266e-05, + "loss": 1.1987, + "theoretical_loss": 3.337117218053332, + "tokens_seen": 3146762240 + }, + { + "epoch": 1.33, + "learning_rate": 3.777733199598797e-05, + "loss": 1.1282, + "theoretical_loss": 3.3371119712951844, + "tokens_seen": 3146827776 + }, + { + "epoch": 1.33, + "learning_rate": 3.776128385155467e-05, + "loss": 1.2392, + "theoretical_loss": 3.3371067246768997, + "tokens_seen": 3146893312 + }, + { + "epoch": 1.33, + "learning_rate": 3.774523570712137e-05, + "loss": 1.1085, + "theoretical_loss": 3.337101478198471, + "tokens_seen": 3146958848 + }, + { + "epoch": 1.33, + "learning_rate": 3.772918756268807e-05, + "loss": 1.0898, + "theoretical_loss": 3.337096231859892, + "tokens_seen": 3147024384 + }, + { + "epoch": 1.33, + "learning_rate": 3.771313941825477e-05, + "loss": 1.1005, + "theoretical_loss": 3.337090985661156, + "tokens_seen": 3147089920 + }, + { + "epoch": 1.33, + "learning_rate": 3.769709127382147e-05, + "loss": 1.1357, + "theoretical_loss": 3.337085739602256, + "tokens_seen": 3147155456 + }, + { + "epoch": 1.33, + "learning_rate": 3.7681043129388164e-05, + "loss": 1.1654, + "theoretical_loss": 3.3370804936831857, + "tokens_seen": 3147220992 + }, + { + "epoch": 1.33, + "learning_rate": 3.7664994984954864e-05, + "loss": 1.2367, + "theoretical_loss": 3.3370752479039383, + "tokens_seen": 3147286528 + }, + { + "epoch": 1.33, + "learning_rate": 3.7648946840521565e-05, + "loss": 1.171, + "theoretical_loss": 3.3370700022645075, + "tokens_seen": 3147352064 + }, + { + "epoch": 1.33, + "learning_rate": 3.7632898696088266e-05, + "loss": 1.2378, + "theoretical_loss": 3.337064756764886, + "tokens_seen": 3147417600 + }, + { + "epoch": 1.33, + "learning_rate": 3.7616850551654966e-05, + "loss": 1.1599, + "theoretical_loss": 3.3370595114050676, + "tokens_seen": 3147483136 + }, + { + "epoch": 1.33, + "learning_rate": 3.760080240722167e-05, + "loss": 1.1877, + "theoretical_loss": 3.3370542661850457, + "tokens_seen": 3147548672 + }, + { + "epoch": 1.33, + "learning_rate": 3.758475426278837e-05, + "loss": 1.2412, + "theoretical_loss": 3.3370490211048134, + "tokens_seen": 3147614208 + }, + { + "epoch": 1.33, + "learning_rate": 3.756870611835507e-05, + "loss": 1.284, + "theoretical_loss": 3.3370437761643643, + "tokens_seen": 3147679744 + }, + { + "epoch": 1.33, + "learning_rate": 3.755265797392177e-05, + "loss": 1.1412, + "theoretical_loss": 3.337038531363692, + "tokens_seen": 3147745280 + }, + { + "epoch": 1.33, + "learning_rate": 3.753660982948847e-05, + "loss": 1.2478, + "theoretical_loss": 3.3370332867027894, + "tokens_seen": 3147810816 + }, + { + "epoch": 1.33, + "learning_rate": 3.752056168505517e-05, + "loss": 1.2139, + "theoretical_loss": 3.33702804218165, + "tokens_seen": 3147876352 + }, + { + "epoch": 1.33, + "learning_rate": 3.7504513540621864e-05, + "loss": 1.2028, + "theoretical_loss": 3.337022797800267, + "tokens_seen": 3147941888 + }, + { + "epoch": 1.33, + "learning_rate": 3.7488465396188564e-05, + "loss": 1.1805, + "theoretical_loss": 3.337017553558634, + "tokens_seen": 3148007424 + }, + { + "epoch": 1.33, + "learning_rate": 3.7472417251755265e-05, + "loss": 1.1749, + "theoretical_loss": 3.3370123094567443, + "tokens_seen": 3148072960 + }, + { + "epoch": 1.33, + "learning_rate": 3.745636910732197e-05, + "loss": 1.1653, + "theoretical_loss": 3.3370070654945914, + "tokens_seen": 3148138496 + }, + { + "epoch": 1.33, + "learning_rate": 3.7440320962888666e-05, + "loss": 1.1844, + "theoretical_loss": 3.3370018216721684, + "tokens_seen": 3148204032 + }, + { + "epoch": 1.33, + "learning_rate": 3.742427281845537e-05, + "loss": 1.1719, + "theoretical_loss": 3.336996577989469, + "tokens_seen": 3148269568 + }, + { + "epoch": 1.33, + "learning_rate": 3.740822467402207e-05, + "loss": 1.1319, + "theoretical_loss": 3.3369913344464863, + "tokens_seen": 3148335104 + }, + { + "epoch": 1.33, + "learning_rate": 3.739217652958877e-05, + "loss": 1.1648, + "theoretical_loss": 3.336986091043214, + "tokens_seen": 3148400640 + }, + { + "epoch": 1.33, + "learning_rate": 3.737612838515547e-05, + "loss": 1.1874, + "theoretical_loss": 3.336980847779645, + "tokens_seen": 3148466176 + }, + { + "epoch": 1.33, + "learning_rate": 3.736008024072217e-05, + "loss": 1.1678, + "theoretical_loss": 3.336975604655773, + "tokens_seen": 3148531712 + }, + { + "epoch": 1.33, + "learning_rate": 3.7344032096288863e-05, + "loss": 1.1435, + "theoretical_loss": 3.3369703616715913, + "tokens_seen": 3148597248 + }, + { + "epoch": 1.33, + "learning_rate": 3.7327983951855564e-05, + "loss": 1.1424, + "theoretical_loss": 3.336965118827093, + "tokens_seen": 3148662784 + }, + { + "epoch": 1.33, + "learning_rate": 3.731193580742227e-05, + "loss": 1.2193, + "theoretical_loss": 3.336959876122272, + "tokens_seen": 3148728320 + }, + { + "epoch": 1.33, + "learning_rate": 3.729588766298897e-05, + "loss": 1.1966, + "theoretical_loss": 3.336954633557121, + "tokens_seen": 3148793856 + }, + { + "epoch": 1.33, + "learning_rate": 3.727983951855567e-05, + "loss": 1.1781, + "theoretical_loss": 3.336949391131634, + "tokens_seen": 3148859392 + }, + { + "epoch": 1.33, + "learning_rate": 3.7263791374122367e-05, + "loss": 1.1883, + "theoretical_loss": 3.336944148845804, + "tokens_seen": 3148924928 + }, + { + "epoch": 1.33, + "learning_rate": 3.724774322968907e-05, + "loss": 1.2011, + "theoretical_loss": 3.3369389066996247, + "tokens_seen": 3148990464 + }, + { + "epoch": 1.33, + "learning_rate": 3.723169508525577e-05, + "loss": 1.1949, + "theoretical_loss": 3.336933664693089, + "tokens_seen": 3149056000 + }, + { + "epoch": 1.33, + "learning_rate": 3.721564694082247e-05, + "loss": 1.134, + "theoretical_loss": 3.3369284228261913, + "tokens_seen": 3149121536 + }, + { + "epoch": 1.33, + "learning_rate": 3.719959879638917e-05, + "loss": 1.2086, + "theoretical_loss": 3.3369231810989235, + "tokens_seen": 3149187072 + }, + { + "epoch": 1.33, + "learning_rate": 3.718355065195587e-05, + "loss": 1.1859, + "theoretical_loss": 3.33691793951128, + "tokens_seen": 3149252608 + }, + { + "epoch": 1.34, + "learning_rate": 3.7167502507522564e-05, + "loss": 1.1615, + "theoretical_loss": 3.3369126980632537, + "tokens_seen": 3149318144 + }, + { + "epoch": 1.34, + "learning_rate": 3.715145436308927e-05, + "loss": 1.1323, + "theoretical_loss": 3.336907456754838, + "tokens_seen": 3149383680 + }, + { + "epoch": 1.34, + "learning_rate": 3.713540621865597e-05, + "loss": 1.1763, + "theoretical_loss": 3.3369022155860266, + "tokens_seen": 3149449216 + }, + { + "epoch": 1.34, + "learning_rate": 3.711935807422267e-05, + "loss": 1.1031, + "theoretical_loss": 3.336896974556813, + "tokens_seen": 3149514752 + }, + { + "epoch": 1.34, + "learning_rate": 3.710330992978937e-05, + "loss": 1.1769, + "theoretical_loss": 3.3368917336671897, + "tokens_seen": 3149580288 + }, + { + "epoch": 1.34, + "learning_rate": 3.708726178535607e-05, + "loss": 1.2737, + "theoretical_loss": 3.3368864929171513, + "tokens_seen": 3149645824 + }, + { + "epoch": 1.34, + "learning_rate": 3.707121364092277e-05, + "loss": 1.1656, + "theoretical_loss": 3.33688125230669, + "tokens_seen": 3149711360 + }, + { + "epoch": 1.34, + "learning_rate": 3.705516549648947e-05, + "loss": 1.1548, + "theoretical_loss": 3.3368760118358, + "tokens_seen": 3149776896 + }, + { + "epoch": 1.34, + "learning_rate": 3.703911735205617e-05, + "loss": 1.0623, + "theoretical_loss": 3.3368707715044748, + "tokens_seen": 3149842432 + }, + { + "epoch": 1.34, + "learning_rate": 3.702306920762287e-05, + "loss": 1.0773, + "theoretical_loss": 3.3368655313127067, + "tokens_seen": 3149907968 + }, + { + "epoch": 1.34, + "learning_rate": 3.700702106318957e-05, + "loss": 1.3073, + "theoretical_loss": 3.33686029126049, + "tokens_seen": 3149973504 + }, + { + "epoch": 1.34, + "learning_rate": 3.699097291875627e-05, + "loss": 1.1894, + "theoretical_loss": 3.3368550513478175, + "tokens_seen": 3150039040 + }, + { + "epoch": 1.34, + "learning_rate": 3.697492477432297e-05, + "loss": 1.1549, + "theoretical_loss": 3.3368498115746834, + "tokens_seen": 3150104576 + }, + { + "epoch": 1.34, + "learning_rate": 3.695887662988967e-05, + "loss": 1.1264, + "theoretical_loss": 3.3368445719410804, + "tokens_seen": 3150170112 + }, + { + "epoch": 1.34, + "learning_rate": 3.694282848545637e-05, + "loss": 1.2348, + "theoretical_loss": 3.3368393324470023, + "tokens_seen": 3150235648 + }, + { + "epoch": 1.34, + "learning_rate": 3.692678034102307e-05, + "loss": 1.1643, + "theoretical_loss": 3.336834093092442, + "tokens_seen": 3150301184 + }, + { + "epoch": 1.34, + "learning_rate": 3.691073219658977e-05, + "loss": 1.2326, + "theoretical_loss": 3.336828853877393, + "tokens_seen": 3150366720 + }, + { + "epoch": 1.34, + "learning_rate": 3.689468405215647e-05, + "loss": 1.2778, + "theoretical_loss": 3.3368236148018493, + "tokens_seen": 3150432256 + }, + { + "epoch": 1.34, + "learning_rate": 3.687863590772317e-05, + "loss": 1.1511, + "theoretical_loss": 3.3368183758658034, + "tokens_seen": 3150497792 + }, + { + "epoch": 1.34, + "learning_rate": 3.686258776328987e-05, + "loss": 1.3216, + "theoretical_loss": 3.336813137069249, + "tokens_seen": 3150563328 + }, + { + "epoch": 1.34, + "learning_rate": 3.6846539618856576e-05, + "loss": 1.1411, + "theoretical_loss": 3.33680789841218, + "tokens_seen": 3150628864 + }, + { + "epoch": 1.34, + "learning_rate": 3.683049147442327e-05, + "loss": 1.2384, + "theoretical_loss": 3.3368026598945892, + "tokens_seen": 3150694400 + }, + { + "epoch": 1.34, + "learning_rate": 3.681444332998997e-05, + "loss": 1.1579, + "theoretical_loss": 3.33679742151647, + "tokens_seen": 3150759936 + }, + { + "epoch": 1.34, + "learning_rate": 3.679839518555667e-05, + "loss": 1.1317, + "theoretical_loss": 3.336792183277816, + "tokens_seen": 3150825472 + }, + { + "epoch": 1.34, + "learning_rate": 3.678234704112337e-05, + "loss": 1.1967, + "theoretical_loss": 3.3367869451786207, + "tokens_seen": 3150891008 + }, + { + "epoch": 1.34, + "learning_rate": 3.676629889669007e-05, + "loss": 1.1466, + "theoretical_loss": 3.336781707218877, + "tokens_seen": 3150956544 + }, + { + "epoch": 1.34, + "learning_rate": 3.675025075225677e-05, + "loss": 1.1808, + "theoretical_loss": 3.336776469398579, + "tokens_seen": 3151022080 + }, + { + "epoch": 1.34, + "learning_rate": 3.673420260782347e-05, + "loss": 1.177, + "theoretical_loss": 3.3367712317177194, + "tokens_seen": 3151087616 + }, + { + "epoch": 1.34, + "learning_rate": 3.671815446339017e-05, + "loss": 1.1788, + "theoretical_loss": 3.336765994176292, + "tokens_seen": 3151153152 + }, + { + "epoch": 1.34, + "learning_rate": 3.6702106318956875e-05, + "loss": 1.101, + "theoretical_loss": 3.33676075677429, + "tokens_seen": 3151218688 + }, + { + "epoch": 1.34, + "learning_rate": 3.6686058174523576e-05, + "loss": 1.1517, + "theoretical_loss": 3.3367555195117067, + "tokens_seen": 3151284224 + }, + { + "epoch": 1.34, + "learning_rate": 3.6670010030090277e-05, + "loss": 1.2949, + "theoretical_loss": 3.336750282388536, + "tokens_seen": 3151349760 + }, + { + "epoch": 1.34, + "learning_rate": 3.665396188565697e-05, + "loss": 1.1295, + "theoretical_loss": 3.3367450454047702, + "tokens_seen": 3151415296 + }, + { + "epoch": 1.34, + "learning_rate": 3.663791374122367e-05, + "loss": 1.1895, + "theoretical_loss": 3.336739808560404, + "tokens_seen": 3151480832 + }, + { + "epoch": 1.34, + "learning_rate": 3.662186559679037e-05, + "loss": 1.1836, + "theoretical_loss": 3.3367345718554304, + "tokens_seen": 3151546368 + }, + { + "epoch": 1.34, + "learning_rate": 3.660581745235707e-05, + "loss": 1.1929, + "theoretical_loss": 3.336729335289842, + "tokens_seen": 3151611904 + }, + { + "epoch": 1.34, + "learning_rate": 3.658976930792377e-05, + "loss": 1.1386, + "theoretical_loss": 3.336724098863633, + "tokens_seen": 3151677440 + }, + { + "epoch": 1.34, + "learning_rate": 3.6573721163490474e-05, + "loss": 1.1974, + "theoretical_loss": 3.336718862576797, + "tokens_seen": 3151742976 + }, + { + "epoch": 1.34, + "learning_rate": 3.6557673019057174e-05, + "loss": 1.1529, + "theoretical_loss": 3.3367136264293267, + "tokens_seen": 3151808512 + }, + { + "epoch": 1.34, + "learning_rate": 3.6541624874623875e-05, + "loss": 1.17, + "theoretical_loss": 3.3367083904212156, + "tokens_seen": 3151874048 + }, + { + "epoch": 1.34, + "learning_rate": 3.6525576730190575e-05, + "loss": 1.1607, + "theoretical_loss": 3.3367031545524575, + "tokens_seen": 3151939584 + }, + { + "epoch": 1.34, + "learning_rate": 3.6509528585757276e-05, + "loss": 1.2122, + "theoretical_loss": 3.3366979188230452, + "tokens_seen": 3152005120 + }, + { + "epoch": 1.34, + "learning_rate": 3.649348044132398e-05, + "loss": 1.1608, + "theoretical_loss": 3.3366926832329726, + "tokens_seen": 3152070656 + }, + { + "epoch": 1.34, + "learning_rate": 3.647743229689067e-05, + "loss": 1.1387, + "theoretical_loss": 3.3366874477822335, + "tokens_seen": 3152136192 + }, + { + "epoch": 1.34, + "learning_rate": 3.646138415245737e-05, + "loss": 1.1189, + "theoretical_loss": 3.33668221247082, + "tokens_seen": 3152201728 + }, + { + "epoch": 1.34, + "objective/train/docs_used": 1162761, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2654554843902588, + "objective/train/theoretical_loss": 3.3366769772987266, + "objective/train/tokens_used": 3172727264, + "theoretical_loss": 3.3366769772987266, + "tokens_seen": 3152267264 + }, + { + "epoch": 1.34, + "learning_rate": 3.644533600802407e-05, + "loss": 1.2655, + "theoretical_loss": 3.3366769772987266, + "tokens_seen": 3152267264 + }, + { + "epoch": 1.34, + "learning_rate": 3.642928786359077e-05, + "loss": 1.2164, + "theoretical_loss": 3.3366717422659464, + "tokens_seen": 3152332800 + }, + { + "epoch": 1.34, + "learning_rate": 3.641323971915747e-05, + "loss": 1.1709, + "theoretical_loss": 3.336666507372472, + "tokens_seen": 3152398336 + }, + { + "epoch": 1.34, + "learning_rate": 3.6397191574724174e-05, + "loss": 1.1823, + "theoretical_loss": 3.3366612726182985, + "tokens_seen": 3152463872 + }, + { + "epoch": 1.34, + "learning_rate": 3.6381143430290874e-05, + "loss": 1.195, + "theoretical_loss": 3.336656038003418, + "tokens_seen": 3152529408 + }, + { + "epoch": 1.34, + "learning_rate": 3.6365095285857575e-05, + "loss": 1.1876, + "theoretical_loss": 3.336650803527824, + "tokens_seen": 3152594944 + }, + { + "epoch": 1.34, + "learning_rate": 3.6349047141424276e-05, + "loss": 1.0606, + "theoretical_loss": 3.3366455691915102, + "tokens_seen": 3152660480 + }, + { + "epoch": 1.34, + "learning_rate": 3.6332998996990976e-05, + "loss": 1.176, + "theoretical_loss": 3.33664033499447, + "tokens_seen": 3152726016 + }, + { + "epoch": 1.34, + "learning_rate": 3.631695085255768e-05, + "loss": 1.1517, + "theoretical_loss": 3.3366351009366966, + "tokens_seen": 3152791552 + }, + { + "epoch": 1.34, + "learning_rate": 3.630090270812437e-05, + "loss": 1.1766, + "theoretical_loss": 3.3366298670181838, + "tokens_seen": 3152857088 + }, + { + "epoch": 1.34, + "learning_rate": 3.628485456369107e-05, + "loss": 1.1802, + "theoretical_loss": 3.3366246332389244, + "tokens_seen": 3152922624 + }, + { + "epoch": 1.34, + "learning_rate": 3.626880641925777e-05, + "loss": 1.2617, + "theoretical_loss": 3.3366193995989124, + "tokens_seen": 3152988160 + }, + { + "epoch": 1.34, + "learning_rate": 3.625275827482448e-05, + "loss": 1.3123, + "theoretical_loss": 3.3366141660981405, + "tokens_seen": 3153053696 + }, + { + "epoch": 1.34, + "learning_rate": 3.623671013039118e-05, + "loss": 1.2453, + "theoretical_loss": 3.336608932736603, + "tokens_seen": 3153119232 + }, + { + "epoch": 1.34, + "learning_rate": 3.6220661985957874e-05, + "loss": 1.1948, + "theoretical_loss": 3.3366036995142925, + "tokens_seen": 3153184768 + }, + { + "epoch": 1.34, + "learning_rate": 3.6204613841524575e-05, + "loss": 1.2868, + "theoretical_loss": 3.336598466431203, + "tokens_seen": 3153250304 + }, + { + "epoch": 1.34, + "learning_rate": 3.6188565697091275e-05, + "loss": 1.22, + "theoretical_loss": 3.3365932334873274, + "tokens_seen": 3153315840 + }, + { + "epoch": 1.34, + "learning_rate": 3.6172517552657976e-05, + "loss": 1.1954, + "theoretical_loss": 3.336588000682659, + "tokens_seen": 3153381376 + }, + { + "epoch": 1.34, + "learning_rate": 3.6156469408224677e-05, + "loss": 1.1976, + "theoretical_loss": 3.336582768017192, + "tokens_seen": 3153446912 + }, + { + "epoch": 1.34, + "learning_rate": 3.614042126379138e-05, + "loss": 1.1679, + "theoretical_loss": 3.3365775354909193, + "tokens_seen": 3153512448 + }, + { + "epoch": 1.34, + "learning_rate": 3.612437311935807e-05, + "loss": 1.2894, + "theoretical_loss": 3.3365723031038343, + "tokens_seen": 3153577984 + }, + { + "epoch": 1.34, + "learning_rate": 3.610832497492478e-05, + "loss": 1.1681, + "theoretical_loss": 3.3365670708559305, + "tokens_seen": 3153643520 + }, + { + "epoch": 1.34, + "learning_rate": 3.609227683049148e-05, + "loss": 1.0981, + "theoretical_loss": 3.336561838747201, + "tokens_seen": 3153709056 + }, + { + "epoch": 1.34, + "learning_rate": 3.607622868605818e-05, + "loss": 1.234, + "theoretical_loss": 3.33655660677764, + "tokens_seen": 3153774592 + }, + { + "epoch": 1.34, + "learning_rate": 3.606018054162488e-05, + "loss": 1.1038, + "theoretical_loss": 3.33655137494724, + "tokens_seen": 3153840128 + }, + { + "epoch": 1.34, + "learning_rate": 3.6044132397191574e-05, + "loss": 1.1266, + "theoretical_loss": 3.336546143255995, + "tokens_seen": 3153905664 + }, + { + "epoch": 1.34, + "learning_rate": 3.6028084252758275e-05, + "loss": 1.1712, + "theoretical_loss": 3.3365409117038975, + "tokens_seen": 3153971200 + }, + { + "epoch": 1.34, + "learning_rate": 3.6012036108324975e-05, + "loss": 1.201, + "theoretical_loss": 3.3365356802909423, + "tokens_seen": 3154036736 + }, + { + "epoch": 1.34, + "learning_rate": 3.5995987963891676e-05, + "loss": 1.1404, + "theoretical_loss": 3.336530449017122, + "tokens_seen": 3154102272 + }, + { + "epoch": 1.34, + "learning_rate": 3.597993981945838e-05, + "loss": 1.0624, + "theoretical_loss": 3.3365252178824294, + "tokens_seen": 3154167808 + }, + { + "epoch": 1.34, + "learning_rate": 3.596389167502508e-05, + "loss": 1.1592, + "theoretical_loss": 3.3365199868868594, + "tokens_seen": 3154233344 + }, + { + "epoch": 1.34, + "learning_rate": 3.594784353059178e-05, + "loss": 1.2808, + "theoretical_loss": 3.3365147560304043, + "tokens_seen": 3154298880 + }, + { + "epoch": 1.34, + "learning_rate": 3.593179538615848e-05, + "loss": 1.2312, + "theoretical_loss": 3.3365095253130583, + "tokens_seen": 3154364416 + }, + { + "epoch": 1.34, + "learning_rate": 3.591574724172518e-05, + "loss": 1.1715, + "theoretical_loss": 3.336504294734814, + "tokens_seen": 3154429952 + }, + { + "epoch": 1.34, + "learning_rate": 3.589969909729188e-05, + "loss": 1.1901, + "theoretical_loss": 3.3364990642956647, + "tokens_seen": 3154495488 + }, + { + "epoch": 1.34, + "learning_rate": 3.588365095285858e-05, + "loss": 1.1227, + "theoretical_loss": 3.336493833995605, + "tokens_seen": 3154561024 + }, + { + "epoch": 1.34, + "learning_rate": 3.5867602808425274e-05, + "loss": 1.1521, + "theoretical_loss": 3.3364886038346273, + "tokens_seen": 3154626560 + }, + { + "epoch": 1.34, + "learning_rate": 3.5851554663991975e-05, + "loss": 1.1782, + "theoretical_loss": 3.336483373812725, + "tokens_seen": 3154692096 + }, + { + "epoch": 1.34, + "learning_rate": 3.5835506519558676e-05, + "loss": 1.1357, + "theoretical_loss": 3.336478143929892, + "tokens_seen": 3154757632 + }, + { + "epoch": 1.34, + "learning_rate": 3.5819458375125376e-05, + "loss": 1.1724, + "theoretical_loss": 3.336472914186122, + "tokens_seen": 3154823168 + }, + { + "epoch": 1.34, + "learning_rate": 3.5803410230692084e-05, + "loss": 1.162, + "theoretical_loss": 3.3364676845814074, + "tokens_seen": 3154888704 + }, + { + "epoch": 1.34, + "learning_rate": 3.578736208625878e-05, + "loss": 1.2051, + "theoretical_loss": 3.336462455115742, + "tokens_seen": 3154954240 + }, + { + "epoch": 1.34, + "learning_rate": 3.577131394182548e-05, + "loss": 1.0862, + "theoretical_loss": 3.3364572257891196, + "tokens_seen": 3155019776 + }, + { + "epoch": 1.34, + "learning_rate": 3.575526579739218e-05, + "loss": 1.2423, + "theoretical_loss": 3.3364519966015336, + "tokens_seen": 3155085312 + }, + { + "epoch": 1.34, + "learning_rate": 3.573921765295888e-05, + "loss": 1.0559, + "theoretical_loss": 3.336446767552977, + "tokens_seen": 3155150848 + }, + { + "epoch": 1.34, + "learning_rate": 3.572316950852558e-05, + "loss": 1.1058, + "theoretical_loss": 3.3364415386434434, + "tokens_seen": 3155216384 + }, + { + "epoch": 1.34, + "learning_rate": 3.570712136409228e-05, + "loss": 1.1814, + "theoretical_loss": 3.3364363098729264, + "tokens_seen": 3155281920 + }, + { + "epoch": 1.34, + "learning_rate": 3.5691073219658975e-05, + "loss": 1.2058, + "theoretical_loss": 3.3364310812414186, + "tokens_seen": 3155347456 + }, + { + "epoch": 1.34, + "learning_rate": 3.5675025075225675e-05, + "loss": 1.1783, + "theoretical_loss": 3.336425852748915, + "tokens_seen": 3155412992 + }, + { + "epoch": 1.34, + "learning_rate": 3.565897693079238e-05, + "loss": 1.0329, + "theoretical_loss": 3.3364206243954073, + "tokens_seen": 3155478528 + }, + { + "epoch": 1.34, + "learning_rate": 3.564292878635908e-05, + "loss": 1.1204, + "theoretical_loss": 3.33641539618089, + "tokens_seen": 3155544064 + }, + { + "epoch": 1.34, + "learning_rate": 3.5626880641925784e-05, + "loss": 1.1565, + "theoretical_loss": 3.336410168105356, + "tokens_seen": 3155609600 + }, + { + "epoch": 1.34, + "learning_rate": 3.561083249749248e-05, + "loss": 1.1076, + "theoretical_loss": 3.336404940168799, + "tokens_seen": 3155675136 + }, + { + "epoch": 1.34, + "learning_rate": 3.559478435305918e-05, + "loss": 1.1436, + "theoretical_loss": 3.336399712371213, + "tokens_seen": 3155740672 + }, + { + "epoch": 1.34, + "learning_rate": 3.557873620862588e-05, + "loss": 1.1899, + "theoretical_loss": 3.33639448471259, + "tokens_seen": 3155806208 + }, + { + "epoch": 1.34, + "learning_rate": 3.556268806419258e-05, + "loss": 1.2478, + "theoretical_loss": 3.3363892571929243, + "tokens_seen": 3155871744 + }, + { + "epoch": 1.34, + "learning_rate": 3.554663991975928e-05, + "loss": 1.1507, + "theoretical_loss": 3.3363840298122094, + "tokens_seen": 3155937280 + }, + { + "epoch": 1.34, + "learning_rate": 3.553059177532598e-05, + "loss": 1.0901, + "theoretical_loss": 3.3363788025704384, + "tokens_seen": 3156002816 + }, + { + "epoch": 1.34, + "learning_rate": 3.5514543630892675e-05, + "loss": 1.2347, + "theoretical_loss": 3.336373575467605, + "tokens_seen": 3156068352 + }, + { + "epoch": 1.34, + "learning_rate": 3.549849548645938e-05, + "loss": 1.0577, + "theoretical_loss": 3.3363683485037026, + "tokens_seen": 3156133888 + }, + { + "epoch": 1.34, + "learning_rate": 3.548244734202608e-05, + "loss": 1.1935, + "theoretical_loss": 3.3363631216787244, + "tokens_seen": 3156199424 + }, + { + "epoch": 1.34, + "learning_rate": 3.5466399197592783e-05, + "loss": 1.0768, + "theoretical_loss": 3.3363578949926636, + "tokens_seen": 3156264960 + }, + { + "epoch": 1.34, + "learning_rate": 3.545035105315948e-05, + "loss": 1.3132, + "theoretical_loss": 3.336352668445514, + "tokens_seen": 3156330496 + }, + { + "epoch": 1.34, + "learning_rate": 3.543430290872618e-05, + "loss": 1.1292, + "theoretical_loss": 3.336347442037269, + "tokens_seen": 3156396032 + }, + { + "epoch": 1.34, + "learning_rate": 3.541825476429288e-05, + "loss": 1.1542, + "theoretical_loss": 3.3363422157679223, + "tokens_seen": 3156461568 + }, + { + "epoch": 1.34, + "learning_rate": 3.540220661985958e-05, + "loss": 1.1207, + "theoretical_loss": 3.336336989637467, + "tokens_seen": 3156527104 + }, + { + "epoch": 1.34, + "learning_rate": 3.538615847542628e-05, + "loss": 1.1705, + "theoretical_loss": 3.3363317636458962, + "tokens_seen": 3156592640 + }, + { + "epoch": 1.34, + "learning_rate": 3.537011033099298e-05, + "loss": 1.1372, + "theoretical_loss": 3.336326537793204, + "tokens_seen": 3156658176 + }, + { + "epoch": 1.34, + "learning_rate": 3.535406218655968e-05, + "loss": 1.1396, + "theoretical_loss": 3.3363213120793835, + "tokens_seen": 3156723712 + }, + { + "epoch": 1.34, + "learning_rate": 3.533801404212638e-05, + "loss": 1.1838, + "theoretical_loss": 3.3363160865044277, + "tokens_seen": 3156789248 + }, + { + "epoch": 1.34, + "learning_rate": 3.532196589769308e-05, + "loss": 1.1871, + "theoretical_loss": 3.3363108610683305, + "tokens_seen": 3156854784 + }, + { + "epoch": 1.34, + "learning_rate": 3.530591775325978e-05, + "loss": 1.2099, + "theoretical_loss": 3.336305635771086, + "tokens_seen": 3156920320 + }, + { + "epoch": 1.34, + "learning_rate": 3.5289869608826484e-05, + "loss": 1.2009, + "theoretical_loss": 3.336300410612686, + "tokens_seen": 3156985856 + }, + { + "epoch": 1.34, + "learning_rate": 3.527382146439318e-05, + "loss": 1.2105, + "theoretical_loss": 3.3362951855931255, + "tokens_seen": 3157051392 + }, + { + "epoch": 1.34, + "learning_rate": 3.525777331995988e-05, + "loss": 1.2808, + "theoretical_loss": 3.336289960712397, + "tokens_seen": 3157116928 + }, + { + "epoch": 1.34, + "learning_rate": 3.524172517552658e-05, + "loss": 1.2154, + "theoretical_loss": 3.3362847359704944, + "tokens_seen": 3157182464 + }, + { + "epoch": 1.34, + "learning_rate": 3.522567703109328e-05, + "loss": 1.1375, + "theoretical_loss": 3.3362795113674104, + "tokens_seen": 3157248000 + }, + { + "epoch": 1.34, + "learning_rate": 3.520962888665999e-05, + "loss": 1.2137, + "theoretical_loss": 3.336274286903139, + "tokens_seen": 3157313536 + }, + { + "epoch": 1.34, + "learning_rate": 3.519358074222668e-05, + "loss": 1.1195, + "theoretical_loss": 3.3362690625776743, + "tokens_seen": 3157379072 + }, + { + "epoch": 1.34, + "learning_rate": 3.517753259779338e-05, + "loss": 1.1908, + "theoretical_loss": 3.3362638383910084, + "tokens_seen": 3157444608 + }, + { + "epoch": 1.34, + "learning_rate": 3.516148445336008e-05, + "loss": 1.1458, + "theoretical_loss": 3.3362586143431354, + "tokens_seen": 3157510144 + }, + { + "epoch": 1.34, + "learning_rate": 3.514543630892678e-05, + "loss": 1.2339, + "theoretical_loss": 3.336253390434049, + "tokens_seen": 3157575680 + }, + { + "epoch": 1.34, + "learning_rate": 3.512938816449348e-05, + "loss": 1.1322, + "theoretical_loss": 3.336248166663742, + "tokens_seen": 3157641216 + }, + { + "epoch": 1.34, + "learning_rate": 3.5113340020060184e-05, + "loss": 1.1794, + "theoretical_loss": 3.3362429430322083, + "tokens_seen": 3157706752 + }, + { + "epoch": 1.34, + "learning_rate": 3.509729187562688e-05, + "loss": 1.1475, + "theoretical_loss": 3.336237719539441, + "tokens_seen": 3157772288 + }, + { + "epoch": 1.34, + "learning_rate": 3.508124373119358e-05, + "loss": 1.2587, + "theoretical_loss": 3.336232496185434, + "tokens_seen": 3157837824 + }, + { + "epoch": 1.34, + "learning_rate": 3.506519558676028e-05, + "loss": 1.2422, + "theoretical_loss": 3.33622727297018, + "tokens_seen": 3157903360 + }, + { + "epoch": 1.34, + "learning_rate": 3.5049147442326986e-05, + "loss": 1.1714, + "theoretical_loss": 3.336222049893673, + "tokens_seen": 3157968896 + }, + { + "epoch": 1.34, + "learning_rate": 3.503309929789369e-05, + "loss": 1.1323, + "theoretical_loss": 3.3362168269559067, + "tokens_seen": 3158034432 + }, + { + "epoch": 1.34, + "learning_rate": 3.501705115346038e-05, + "loss": 1.2129, + "theoretical_loss": 3.3362116041568735, + "tokens_seen": 3158099968 + }, + { + "epoch": 1.34, + "learning_rate": 3.500100300902708e-05, + "loss": 1.083, + "theoretical_loss": 3.3362063814965683, + "tokens_seen": 3158165504 + }, + { + "epoch": 1.34, + "learning_rate": 3.498495486459378e-05, + "loss": 1.1684, + "theoretical_loss": 3.336201158974983, + "tokens_seen": 3158231040 + }, + { + "epoch": 1.34, + "learning_rate": 3.496890672016048e-05, + "loss": 1.2342, + "theoretical_loss": 3.3361959365921123, + "tokens_seen": 3158296576 + }, + { + "epoch": 1.34, + "learning_rate": 3.4952858575727183e-05, + "loss": 1.12, + "theoretical_loss": 3.336190714347949, + "tokens_seen": 3158362112 + }, + { + "epoch": 1.34, + "learning_rate": 3.4936810431293884e-05, + "loss": 1.201, + "theoretical_loss": 3.3361854922424863, + "tokens_seen": 3158427648 + }, + { + "epoch": 1.34, + "learning_rate": 3.492076228686058e-05, + "loss": 1.1753, + "theoretical_loss": 3.336180270275718, + "tokens_seen": 3158493184 + }, + { + "epoch": 1.34, + "learning_rate": 3.4904714142427285e-05, + "loss": 1.125, + "theoretical_loss": 3.336175048447638, + "tokens_seen": 3158558720 + }, + { + "epoch": 1.34, + "learning_rate": 3.4888665997993986e-05, + "loss": 1.1559, + "theoretical_loss": 3.3361698267582387, + "tokens_seen": 3158624256 + }, + { + "epoch": 1.34, + "learning_rate": 3.4872617853560687e-05, + "loss": 1.181, + "theoretical_loss": 3.3361646052075145, + "tokens_seen": 3158689792 + }, + { + "epoch": 1.34, + "learning_rate": 3.485656970912739e-05, + "loss": 1.1604, + "theoretical_loss": 3.336159383795458, + "tokens_seen": 3158755328 + }, + { + "epoch": 1.34, + "objective/train/docs_used": 1165287, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.0911651849746704, + "objective/train/theoretical_loss": 3.336154162522063, + "objective/train/tokens_used": 3179280864, + "theoretical_loss": 3.336154162522063, + "tokens_seen": 3158820864 + }, + { + "epoch": 1.34, + "learning_rate": 3.484052156469408e-05, + "loss": 1.0912, + "theoretical_loss": 3.336154162522063, + "tokens_seen": 3158820864 + }, + { + "epoch": 1.34, + "learning_rate": 3.482447342026078e-05, + "loss": 1.1788, + "theoretical_loss": 3.3361489413873238, + "tokens_seen": 3158886400 + }, + { + "epoch": 1.34, + "learning_rate": 3.480842527582748e-05, + "loss": 1.0736, + "theoretical_loss": 3.3361437203912323, + "tokens_seen": 3158951936 + }, + { + "epoch": 1.34, + "learning_rate": 3.479237713139418e-05, + "loss": 1.1706, + "theoretical_loss": 3.3361384995337833, + "tokens_seen": 3159017472 + }, + { + "epoch": 1.34, + "learning_rate": 3.4776328986960884e-05, + "loss": 1.1356, + "theoretical_loss": 3.336133278814969, + "tokens_seen": 3159083008 + }, + { + "epoch": 1.34, + "learning_rate": 3.4760280842527584e-05, + "loss": 1.1175, + "theoretical_loss": 3.3361280582347836, + "tokens_seen": 3159148544 + }, + { + "epoch": 1.34, + "learning_rate": 3.4744232698094285e-05, + "loss": 1.1503, + "theoretical_loss": 3.336122837793221, + "tokens_seen": 3159214080 + }, + { + "epoch": 1.34, + "learning_rate": 3.4728184553660986e-05, + "loss": 1.1293, + "theoretical_loss": 3.336117617490274, + "tokens_seen": 3159279616 + }, + { + "epoch": 1.34, + "learning_rate": 3.4712136409227686e-05, + "loss": 1.1073, + "theoretical_loss": 3.3361123973259352, + "tokens_seen": 3159345152 + }, + { + "epoch": 1.34, + "learning_rate": 3.469608826479439e-05, + "loss": 1.1906, + "theoretical_loss": 3.3361071773001996, + "tokens_seen": 3159410688 + }, + { + "epoch": 1.34, + "learning_rate": 3.468004012036109e-05, + "loss": 1.1653, + "theoretical_loss": 3.33610195741306, + "tokens_seen": 3159476224 + }, + { + "epoch": 1.34, + "learning_rate": 3.466399197592778e-05, + "loss": 1.196, + "theoretical_loss": 3.33609673766451, + "tokens_seen": 3159541760 + }, + { + "epoch": 1.34, + "learning_rate": 3.464794383149448e-05, + "loss": 1.1945, + "theoretical_loss": 3.3360915180545425, + "tokens_seen": 3159607296 + }, + { + "epoch": 1.34, + "learning_rate": 3.463189568706118e-05, + "loss": 1.3013, + "theoretical_loss": 3.3360862985831514, + "tokens_seen": 3159672832 + }, + { + "epoch": 1.34, + "learning_rate": 3.461584754262788e-05, + "loss": 1.2314, + "theoretical_loss": 3.3360810792503304, + "tokens_seen": 3159738368 + }, + { + "epoch": 1.34, + "learning_rate": 3.459979939819459e-05, + "loss": 1.3074, + "theoretical_loss": 3.3360758600560723, + "tokens_seen": 3159803904 + }, + { + "epoch": 1.34, + "learning_rate": 3.4583751253761284e-05, + "loss": 1.1412, + "theoretical_loss": 3.336070641000371, + "tokens_seen": 3159869440 + }, + { + "epoch": 1.34, + "learning_rate": 3.4567703109327985e-05, + "loss": 1.1288, + "theoretical_loss": 3.33606542208322, + "tokens_seen": 3159934976 + }, + { + "epoch": 1.34, + "learning_rate": 3.4551654964894686e-05, + "loss": 1.1996, + "theoretical_loss": 3.336060203304612, + "tokens_seen": 3160000512 + }, + { + "epoch": 1.34, + "learning_rate": 3.4535606820461386e-05, + "loss": 1.0976, + "theoretical_loss": 3.3360549846645418, + "tokens_seen": 3160066048 + }, + { + "epoch": 1.34, + "learning_rate": 3.451955867602809e-05, + "loss": 1.2527, + "theoretical_loss": 3.3360497661630015, + "tokens_seen": 3160131584 + }, + { + "epoch": 1.34, + "learning_rate": 3.450351053159479e-05, + "loss": 1.0849, + "theoretical_loss": 3.3360445477999856, + "tokens_seen": 3160197120 + }, + { + "epoch": 1.34, + "learning_rate": 3.448746238716148e-05, + "loss": 1.2134, + "theoretical_loss": 3.3360393295754864, + "tokens_seen": 3160262656 + }, + { + "epoch": 1.34, + "learning_rate": 3.447141424272818e-05, + "loss": 1.1626, + "theoretical_loss": 3.3360341114894987, + "tokens_seen": 3160328192 + }, + { + "epoch": 1.34, + "learning_rate": 3.445536609829489e-05, + "loss": 1.0835, + "theoretical_loss": 3.3360288935420153, + "tokens_seen": 3160393728 + }, + { + "epoch": 1.34, + "learning_rate": 3.443931795386159e-05, + "loss": 1.1587, + "theoretical_loss": 3.336023675733029, + "tokens_seen": 3160459264 + }, + { + "epoch": 1.34, + "learning_rate": 3.442326980942829e-05, + "loss": 1.1289, + "theoretical_loss": 3.3360184580625343, + "tokens_seen": 3160524800 + }, + { + "epoch": 1.34, + "learning_rate": 3.4407221664994985e-05, + "loss": 1.3024, + "theoretical_loss": 3.3360132405305243, + "tokens_seen": 3160590336 + }, + { + "epoch": 1.34, + "learning_rate": 3.4391173520561685e-05, + "loss": 1.1293, + "theoretical_loss": 3.336008023136992, + "tokens_seen": 3160655872 + }, + { + "epoch": 1.34, + "learning_rate": 3.4375125376128386e-05, + "loss": 1.0988, + "theoretical_loss": 3.3360028058819315, + "tokens_seen": 3160721408 + }, + { + "epoch": 1.34, + "learning_rate": 3.4359077231695087e-05, + "loss": 1.0846, + "theoretical_loss": 3.335997588765336, + "tokens_seen": 3160786944 + }, + { + "epoch": 1.34, + "learning_rate": 3.434302908726179e-05, + "loss": 1.17, + "theoretical_loss": 3.335992371787199, + "tokens_seen": 3160852480 + }, + { + "epoch": 1.34, + "learning_rate": 3.432698094282849e-05, + "loss": 1.2212, + "theoretical_loss": 3.3359871549475137, + "tokens_seen": 3160918016 + }, + { + "epoch": 1.34, + "learning_rate": 3.431093279839518e-05, + "loss": 1.1973, + "theoretical_loss": 3.335981938246274, + "tokens_seen": 3160983552 + }, + { + "epoch": 1.34, + "learning_rate": 3.429488465396189e-05, + "loss": 1.1839, + "theoretical_loss": 3.335976721683473, + "tokens_seen": 3161049088 + }, + { + "epoch": 1.34, + "learning_rate": 3.427883650952859e-05, + "loss": 1.1737, + "theoretical_loss": 3.335971505259104, + "tokens_seen": 3161114624 + }, + { + "epoch": 1.34, + "learning_rate": 3.426278836509529e-05, + "loss": 1.1367, + "theoretical_loss": 3.335966288973161, + "tokens_seen": 3161180160 + }, + { + "epoch": 1.34, + "learning_rate": 3.424674022066199e-05, + "loss": 1.1415, + "theoretical_loss": 3.3359610728256373, + "tokens_seen": 3161245696 + }, + { + "epoch": 1.34, + "learning_rate": 3.4230692076228685e-05, + "loss": 1.1754, + "theoretical_loss": 3.335955856816526, + "tokens_seen": 3161311232 + }, + { + "epoch": 1.34, + "learning_rate": 3.4214643931795386e-05, + "loss": 1.0266, + "theoretical_loss": 3.335950640945821, + "tokens_seen": 3161376768 + }, + { + "epoch": 1.34, + "learning_rate": 3.4198595787362086e-05, + "loss": 1.2, + "theoretical_loss": 3.3359454252135157, + "tokens_seen": 3161442304 + }, + { + "epoch": 1.34, + "learning_rate": 3.418254764292879e-05, + "loss": 1.2208, + "theoretical_loss": 3.335940209619603, + "tokens_seen": 3161507840 + }, + { + "epoch": 1.34, + "learning_rate": 3.416649949849549e-05, + "loss": 1.1121, + "theoretical_loss": 3.335934994164077, + "tokens_seen": 3161573376 + }, + { + "epoch": 1.34, + "learning_rate": 3.415045135406219e-05, + "loss": 1.2522, + "theoretical_loss": 3.3359297788469306, + "tokens_seen": 3161638912 + }, + { + "epoch": 1.34, + "learning_rate": 3.413440320962889e-05, + "loss": 1.1846, + "theoretical_loss": 3.335924563668158, + "tokens_seen": 3161704448 + }, + { + "epoch": 1.34, + "learning_rate": 3.411835506519559e-05, + "loss": 1.2616, + "theoretical_loss": 3.335919348627752, + "tokens_seen": 3161769984 + }, + { + "epoch": 1.34, + "learning_rate": 3.410230692076229e-05, + "loss": 1.1892, + "theoretical_loss": 3.3359141337257068, + "tokens_seen": 3161835520 + }, + { + "epoch": 1.34, + "learning_rate": 3.408625877632899e-05, + "loss": 1.2165, + "theoretical_loss": 3.335908918962015, + "tokens_seen": 3161901056 + }, + { + "epoch": 1.34, + "learning_rate": 3.407021063189569e-05, + "loss": 1.1066, + "theoretical_loss": 3.3359037043366704, + "tokens_seen": 3161966592 + }, + { + "epoch": 1.34, + "learning_rate": 3.4054162487462385e-05, + "loss": 1.2728, + "theoretical_loss": 3.3358984898496664, + "tokens_seen": 3162032128 + }, + { + "epoch": 1.34, + "learning_rate": 3.4038114343029086e-05, + "loss": 1.1859, + "theoretical_loss": 3.3358932755009967, + "tokens_seen": 3162097664 + }, + { + "epoch": 1.34, + "learning_rate": 3.4022066198595786e-05, + "loss": 1.1748, + "theoretical_loss": 3.335888061290655, + "tokens_seen": 3162163200 + }, + { + "epoch": 1.34, + "learning_rate": 3.4006018054162494e-05, + "loss": 1.2033, + "theoretical_loss": 3.3358828472186337, + "tokens_seen": 3162228736 + }, + { + "epoch": 1.34, + "learning_rate": 3.3989969909729194e-05, + "loss": 1.1653, + "theoretical_loss": 3.3358776332849276, + "tokens_seen": 3162294272 + }, + { + "epoch": 1.34, + "learning_rate": 3.397392176529589e-05, + "loss": 1.1592, + "theoretical_loss": 3.335872419489529, + "tokens_seen": 3162359808 + }, + { + "epoch": 1.34, + "learning_rate": 3.395787362086259e-05, + "loss": 1.1794, + "theoretical_loss": 3.3358672058324323, + "tokens_seen": 3162425344 + }, + { + "epoch": 1.34, + "learning_rate": 3.394182547642929e-05, + "loss": 1.1718, + "theoretical_loss": 3.3358619923136303, + "tokens_seen": 3162490880 + }, + { + "epoch": 1.34, + "learning_rate": 3.392577733199599e-05, + "loss": 1.1427, + "theoretical_loss": 3.335856778933117, + "tokens_seen": 3162556416 + }, + { + "epoch": 1.34, + "learning_rate": 3.390972918756269e-05, + "loss": 1.0713, + "theoretical_loss": 3.335851565690885, + "tokens_seen": 3162621952 + }, + { + "epoch": 1.34, + "learning_rate": 3.389368104312939e-05, + "loss": 1.1414, + "theoretical_loss": 3.3358463525869286, + "tokens_seen": 3162687488 + }, + { + "epoch": 1.34, + "learning_rate": 3.3877632898696085e-05, + "loss": 1.1484, + "theoretical_loss": 3.335841139621241, + "tokens_seen": 3162753024 + }, + { + "epoch": 1.34, + "learning_rate": 3.3861584754262786e-05, + "loss": 1.1413, + "theoretical_loss": 3.3358359267938162, + "tokens_seen": 3162818560 + }, + { + "epoch": 1.34, + "learning_rate": 3.384553660982949e-05, + "loss": 1.1892, + "theoretical_loss": 3.3358307141046466, + "tokens_seen": 3162884096 + }, + { + "epoch": 1.34, + "learning_rate": 3.3829488465396194e-05, + "loss": 1.2214, + "theoretical_loss": 3.3358255015537264, + "tokens_seen": 3162949632 + }, + { + "epoch": 1.34, + "learning_rate": 3.3813440320962895e-05, + "loss": 1.1328, + "theoretical_loss": 3.335820289141049, + "tokens_seen": 3163015168 + }, + { + "epoch": 1.34, + "learning_rate": 3.379739217652959e-05, + "loss": 1.1206, + "theoretical_loss": 3.3358150768666075, + "tokens_seen": 3163080704 + }, + { + "epoch": 1.34, + "learning_rate": 3.378134403209629e-05, + "loss": 1.2831, + "theoretical_loss": 3.335809864730396, + "tokens_seen": 3163146240 + }, + { + "epoch": 1.34, + "learning_rate": 3.376529588766299e-05, + "loss": 1.1644, + "theoretical_loss": 3.3358046527324072, + "tokens_seen": 3163211776 + }, + { + "epoch": 1.34, + "learning_rate": 3.374924774322969e-05, + "loss": 1.1506, + "theoretical_loss": 3.335799440872635, + "tokens_seen": 3163277312 + }, + { + "epoch": 1.34, + "learning_rate": 3.373319959879639e-05, + "loss": 1.1574, + "theoretical_loss": 3.335794229151073, + "tokens_seen": 3163342848 + }, + { + "epoch": 1.34, + "learning_rate": 3.371715145436309e-05, + "loss": 1.1203, + "theoretical_loss": 3.3357890175677145, + "tokens_seen": 3163408384 + }, + { + "epoch": 1.34, + "learning_rate": 3.370110330992979e-05, + "loss": 1.1506, + "theoretical_loss": 3.335783806122553, + "tokens_seen": 3163473920 + }, + { + "epoch": 1.34, + "learning_rate": 3.368505516549649e-05, + "loss": 1.0955, + "theoretical_loss": 3.335778594815582, + "tokens_seen": 3163539456 + }, + { + "epoch": 1.34, + "learning_rate": 3.3669007021063194e-05, + "loss": 1.1411, + "theoretical_loss": 3.335773383646795, + "tokens_seen": 3163604992 + }, + { + "epoch": 1.34, + "learning_rate": 3.3652958876629894e-05, + "loss": 1.2584, + "theoretical_loss": 3.335768172616185, + "tokens_seen": 3163670528 + }, + { + "epoch": 1.34, + "learning_rate": 3.3636910732196595e-05, + "loss": 1.1126, + "theoretical_loss": 3.3357629617237463, + "tokens_seen": 3163736064 + }, + { + "epoch": 1.34, + "learning_rate": 3.362086258776329e-05, + "loss": 1.1659, + "theoretical_loss": 3.3357577509694716, + "tokens_seen": 3163801600 + }, + { + "epoch": 1.34, + "learning_rate": 3.360481444332999e-05, + "loss": 1.1616, + "theoretical_loss": 3.3357525403533552, + "tokens_seen": 3163867136 + }, + { + "epoch": 1.34, + "learning_rate": 3.358876629889669e-05, + "loss": 1.1847, + "theoretical_loss": 3.3357473298753897, + "tokens_seen": 3163932672 + }, + { + "epoch": 1.34, + "learning_rate": 3.357271815446339e-05, + "loss": 1.0675, + "theoretical_loss": 3.335742119535569, + "tokens_seen": 3163998208 + }, + { + "epoch": 1.34, + "learning_rate": 3.355667001003009e-05, + "loss": 1.1901, + "theoretical_loss": 3.335736909333887, + "tokens_seen": 3164063744 + }, + { + "epoch": 1.34, + "learning_rate": 3.354062186559679e-05, + "loss": 1.2019, + "theoretical_loss": 3.3357316992703363, + "tokens_seen": 3164129280 + }, + { + "epoch": 1.34, + "learning_rate": 3.352457372116349e-05, + "loss": 1.2567, + "theoretical_loss": 3.3357264893449106, + "tokens_seen": 3164194816 + }, + { + "epoch": 1.34, + "learning_rate": 3.350852557673019e-05, + "loss": 1.1878, + "theoretical_loss": 3.335721279557604, + "tokens_seen": 3164260352 + }, + { + "epoch": 1.34, + "learning_rate": 3.3492477432296894e-05, + "loss": 1.1995, + "theoretical_loss": 3.3357160699084094, + "tokens_seen": 3164325888 + }, + { + "epoch": 1.34, + "learning_rate": 3.3476429287863594e-05, + "loss": 1.1444, + "theoretical_loss": 3.3357108603973207, + "tokens_seen": 3164391424 + }, + { + "epoch": 1.34, + "learning_rate": 3.346038114343029e-05, + "loss": 1.09, + "theoretical_loss": 3.335705651024331, + "tokens_seen": 3164456960 + }, + { + "epoch": 1.34, + "learning_rate": 3.344433299899699e-05, + "loss": 1.2172, + "theoretical_loss": 3.3357004417894336, + "tokens_seen": 3164522496 + }, + { + "epoch": 1.34, + "learning_rate": 3.342828485456369e-05, + "loss": 1.1842, + "theoretical_loss": 3.335695232692623, + "tokens_seen": 3164588032 + }, + { + "epoch": 1.34, + "learning_rate": 3.341223671013039e-05, + "loss": 1.2199, + "theoretical_loss": 3.335690023733891, + "tokens_seen": 3164653568 + }, + { + "epoch": 1.34, + "learning_rate": 3.33961885656971e-05, + "loss": 1.2081, + "theoretical_loss": 3.3356848149132325, + "tokens_seen": 3164719104 + }, + { + "epoch": 1.34, + "learning_rate": 3.338014042126379e-05, + "loss": 1.1515, + "theoretical_loss": 3.3356796062306406, + "tokens_seen": 3164784640 + }, + { + "epoch": 1.34, + "learning_rate": 3.336409227683049e-05, + "loss": 1.1771, + "theoretical_loss": 3.3356743976861085, + "tokens_seen": 3164850176 + }, + { + "epoch": 1.34, + "learning_rate": 3.334804413239719e-05, + "loss": 1.2411, + "theoretical_loss": 3.33566918927963, + "tokens_seen": 3164915712 + }, + { + "epoch": 1.34, + "learning_rate": 3.333199598796389e-05, + "loss": 1.1392, + "theoretical_loss": 3.3356639810111983, + "tokens_seen": 3164981248 + }, + { + "epoch": 1.34, + "learning_rate": 3.3315947843530594e-05, + "loss": 1.1504, + "theoretical_loss": 3.3356587728808074, + "tokens_seen": 3165046784 + }, + { + "epoch": 1.34, + "learning_rate": 3.3299899699097295e-05, + "loss": 1.1769, + "theoretical_loss": 3.33565356488845, + "tokens_seen": 3165112320 + }, + { + "epoch": 1.34, + "learning_rate": 3.328385155466399e-05, + "loss": 1.1621, + "theoretical_loss": 3.3356483570341204, + "tokens_seen": 3165177856 + }, + { + "epoch": 1.34, + "learning_rate": 3.326780341023069e-05, + "loss": 1.2307, + "theoretical_loss": 3.3356431493178116, + "tokens_seen": 3165243392 + }, + { + "epoch": 1.34, + "learning_rate": 3.3251755265797396e-05, + "loss": 1.1391, + "theoretical_loss": 3.3356379417395168, + "tokens_seen": 3165308928 + }, + { + "epoch": 1.34, + "objective/train/docs_used": 1167588, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2408559322357178, + "objective/train/theoretical_loss": 3.3356327342992307, + "objective/train/tokens_used": 3185834464, + "theoretical_loss": 3.3356327342992307, + "tokens_seen": 3165374464 + }, + { + "epoch": 1.34, + "learning_rate": 3.32357071213641e-05, + "loss": 1.2409, + "theoretical_loss": 3.3356327342992307, + "tokens_seen": 3165374464 + }, + { + "epoch": 1.34, + "learning_rate": 3.32196589769308e-05, + "loss": 1.2076, + "theoretical_loss": 3.3356275269969453, + "tokens_seen": 3165440000 + }, + { + "epoch": 1.34, + "learning_rate": 3.320361083249749e-05, + "loss": 1.2365, + "theoretical_loss": 3.335622319832655, + "tokens_seen": 3165505536 + }, + { + "epoch": 1.34, + "learning_rate": 3.318756268806419e-05, + "loss": 1.2666, + "theoretical_loss": 3.3356171128063528, + "tokens_seen": 3165571072 + }, + { + "epoch": 1.34, + "learning_rate": 3.317151454363089e-05, + "loss": 1.0758, + "theoretical_loss": 3.3356119059180327, + "tokens_seen": 3165636608 + }, + { + "epoch": 1.34, + "learning_rate": 3.3155466399197594e-05, + "loss": 1.2015, + "theoretical_loss": 3.3356066991676876, + "tokens_seen": 3165702144 + }, + { + "epoch": 1.34, + "learning_rate": 3.3139418254764294e-05, + "loss": 1.2302, + "theoretical_loss": 3.3356014925553112, + "tokens_seen": 3165767680 + }, + { + "epoch": 1.34, + "learning_rate": 3.3123370110330995e-05, + "loss": 1.2027, + "theoretical_loss": 3.3355962860808974, + "tokens_seen": 3165833216 + }, + { + "epoch": 1.34, + "learning_rate": 3.3107321965897695e-05, + "loss": 1.1698, + "theoretical_loss": 3.3355910797444395, + "tokens_seen": 3165898752 + }, + { + "epoch": 1.34, + "learning_rate": 3.3091273821464396e-05, + "loss": 1.3161, + "theoretical_loss": 3.3355858735459307, + "tokens_seen": 3165964288 + }, + { + "epoch": 1.34, + "learning_rate": 3.30752256770311e-05, + "loss": 1.1907, + "theoretical_loss": 3.3355806674853645, + "tokens_seen": 3166029824 + }, + { + "epoch": 1.34, + "learning_rate": 3.30591775325978e-05, + "loss": 1.0648, + "theoretical_loss": 3.3355754615627347, + "tokens_seen": 3166095360 + }, + { + "epoch": 1.34, + "learning_rate": 3.30431293881645e-05, + "loss": 1.1315, + "theoretical_loss": 3.3355702557780345, + "tokens_seen": 3166160896 + }, + { + "epoch": 1.34, + "learning_rate": 3.302708124373119e-05, + "loss": 1.1885, + "theoretical_loss": 3.3355650501312573, + "tokens_seen": 3166226432 + }, + { + "epoch": 1.34, + "learning_rate": 3.301103309929789e-05, + "loss": 1.1301, + "theoretical_loss": 3.3355598446223973, + "tokens_seen": 3166291968 + }, + { + "epoch": 1.34, + "learning_rate": 3.299498495486459e-05, + "loss": 1.1803, + "theoretical_loss": 3.335554639251447, + "tokens_seen": 3166357504 + }, + { + "epoch": 1.34, + "learning_rate": 3.2978936810431294e-05, + "loss": 1.308, + "theoretical_loss": 3.3355494340184006, + "tokens_seen": 3166423040 + }, + { + "epoch": 1.34, + "learning_rate": 3.2962888665997994e-05, + "loss": 1.2666, + "theoretical_loss": 3.3355442289232515, + "tokens_seen": 3166488576 + }, + { + "epoch": 1.34, + "learning_rate": 3.2946840521564695e-05, + "loss": 1.2268, + "theoretical_loss": 3.335539023965993, + "tokens_seen": 3166554112 + }, + { + "epoch": 1.34, + "learning_rate": 3.2930792377131396e-05, + "loss": 1.1572, + "theoretical_loss": 3.335533819146619, + "tokens_seen": 3166619648 + }, + { + "epoch": 1.34, + "learning_rate": 3.2914744232698096e-05, + "loss": 1.1306, + "theoretical_loss": 3.335528614465122, + "tokens_seen": 3166685184 + }, + { + "epoch": 1.34, + "learning_rate": 3.28986960882648e-05, + "loss": 1.1551, + "theoretical_loss": 3.3355234099214965, + "tokens_seen": 3166750720 + }, + { + "epoch": 1.34, + "learning_rate": 3.28826479438315e-05, + "loss": 1.1971, + "theoretical_loss": 3.3355182055157355, + "tokens_seen": 3166816256 + }, + { + "epoch": 1.34, + "learning_rate": 3.28665997993982e-05, + "loss": 1.209, + "theoretical_loss": 3.3355130012478327, + "tokens_seen": 3166881792 + }, + { + "epoch": 1.34, + "learning_rate": 3.285055165496489e-05, + "loss": 1.0832, + "theoretical_loss": 3.335507797117782, + "tokens_seen": 3166947328 + }, + { + "epoch": 1.34, + "learning_rate": 3.283450351053159e-05, + "loss": 1.0828, + "theoretical_loss": 3.3355025931255757, + "tokens_seen": 3167012864 + }, + { + "epoch": 1.34, + "learning_rate": 3.281845536609829e-05, + "loss": 1.3082, + "theoretical_loss": 3.3354973892712083, + "tokens_seen": 3167078400 + }, + { + "epoch": 1.34, + "learning_rate": 3.2802407221665e-05, + "loss": 1.1081, + "theoretical_loss": 3.3354921855546733, + "tokens_seen": 3167143936 + }, + { + "epoch": 1.34, + "learning_rate": 3.27863590772317e-05, + "loss": 1.2318, + "theoretical_loss": 3.3354869819759636, + "tokens_seen": 3167209472 + }, + { + "epoch": 1.34, + "learning_rate": 3.2770310932798395e-05, + "loss": 1.0667, + "theoretical_loss": 3.3354817785350734, + "tokens_seen": 3167275008 + }, + { + "epoch": 1.34, + "learning_rate": 3.2754262788365096e-05, + "loss": 1.2345, + "theoretical_loss": 3.3354765752319953, + "tokens_seen": 3167340544 + }, + { + "epoch": 1.34, + "learning_rate": 3.2738214643931796e-05, + "loss": 1.1266, + "theoretical_loss": 3.3354713720667237, + "tokens_seen": 3167406080 + }, + { + "epoch": 1.34, + "learning_rate": 3.27221664994985e-05, + "loss": 1.1783, + "theoretical_loss": 3.3354661690392517, + "tokens_seen": 3167471616 + }, + { + "epoch": 1.34, + "learning_rate": 3.27061183550652e-05, + "loss": 1.2173, + "theoretical_loss": 3.3354609661495727, + "tokens_seen": 3167537152 + }, + { + "epoch": 1.34, + "learning_rate": 3.26900702106319e-05, + "loss": 1.1465, + "theoretical_loss": 3.3354557633976807, + "tokens_seen": 3167602688 + }, + { + "epoch": 1.34, + "learning_rate": 3.267402206619859e-05, + "loss": 1.1488, + "theoretical_loss": 3.3354505607835683, + "tokens_seen": 3167668224 + }, + { + "epoch": 1.34, + "learning_rate": 3.265797392176529e-05, + "loss": 1.1208, + "theoretical_loss": 3.3354453583072297, + "tokens_seen": 3167733760 + }, + { + "epoch": 1.34, + "learning_rate": 3.2641925777332e-05, + "loss": 1.1591, + "theoretical_loss": 3.335440155968658, + "tokens_seen": 3167799296 + }, + { + "epoch": 1.34, + "learning_rate": 3.26258776328987e-05, + "loss": 1.0798, + "theoretical_loss": 3.335434953767847, + "tokens_seen": 3167864832 + }, + { + "epoch": 1.34, + "learning_rate": 3.26098294884654e-05, + "loss": 1.2162, + "theoretical_loss": 3.3354297517047904, + "tokens_seen": 3167930368 + }, + { + "epoch": 1.34, + "learning_rate": 3.2593781344032095e-05, + "loss": 1.2682, + "theoretical_loss": 3.3354245497794817, + "tokens_seen": 3167995904 + }, + { + "epoch": 1.34, + "learning_rate": 3.2577733199598796e-05, + "loss": 1.2239, + "theoretical_loss": 3.3354193479919134, + "tokens_seen": 3168061440 + }, + { + "epoch": 1.34, + "learning_rate": 3.25616850551655e-05, + "loss": 1.1118, + "theoretical_loss": 3.3354141463420803, + "tokens_seen": 3168126976 + }, + { + "epoch": 1.34, + "learning_rate": 3.25456369107322e-05, + "loss": 1.1595, + "theoretical_loss": 3.3354089448299753, + "tokens_seen": 3168192512 + }, + { + "epoch": 1.34, + "learning_rate": 3.25295887662989e-05, + "loss": 1.1056, + "theoretical_loss": 3.3354037434555917, + "tokens_seen": 3168258048 + }, + { + "epoch": 1.34, + "learning_rate": 3.25135406218656e-05, + "loss": 1.1608, + "theoretical_loss": 3.335398542218923, + "tokens_seen": 3168323584 + }, + { + "epoch": 1.34, + "learning_rate": 3.24974924774323e-05, + "loss": 1.1466, + "theoretical_loss": 3.3353933411199637, + "tokens_seen": 3168389120 + }, + { + "epoch": 1.34, + "learning_rate": 3.2481444332999e-05, + "loss": 1.1954, + "theoretical_loss": 3.335388140158706, + "tokens_seen": 3168454656 + }, + { + "epoch": 1.34, + "learning_rate": 3.24653961885657e-05, + "loss": 1.1774, + "theoretical_loss": 3.335382939335144, + "tokens_seen": 3168520192 + }, + { + "epoch": 1.34, + "learning_rate": 3.24493480441324e-05, + "loss": 1.1903, + "theoretical_loss": 3.3353777386492713, + "tokens_seen": 3168585728 + }, + { + "epoch": 1.34, + "learning_rate": 3.24332998996991e-05, + "loss": 1.1056, + "theoretical_loss": 3.3353725381010815, + "tokens_seen": 3168651264 + }, + { + "epoch": 1.34, + "learning_rate": 3.2417251755265796e-05, + "loss": 1.1734, + "theoretical_loss": 3.3353673376905677, + "tokens_seen": 3168716800 + }, + { + "epoch": 1.34, + "learning_rate": 3.2401203610832496e-05, + "loss": 1.3335, + "theoretical_loss": 3.3353621374177234, + "tokens_seen": 3168782336 + }, + { + "epoch": 1.34, + "learning_rate": 3.23851554663992e-05, + "loss": 1.1654, + "theoretical_loss": 3.3353569372825422, + "tokens_seen": 3168847872 + }, + { + "epoch": 1.34, + "learning_rate": 3.23691073219659e-05, + "loss": 1.1756, + "theoretical_loss": 3.335351737285018, + "tokens_seen": 3168913408 + }, + { + "epoch": 1.34, + "learning_rate": 3.2353059177532605e-05, + "loss": 1.1804, + "theoretical_loss": 3.335346537425144, + "tokens_seen": 3168978944 + }, + { + "epoch": 1.34, + "learning_rate": 3.23370110330993e-05, + "loss": 1.154, + "theoretical_loss": 3.3353413377029137, + "tokens_seen": 3169044480 + }, + { + "epoch": 1.34, + "learning_rate": 3.2320962888666e-05, + "loss": 1.119, + "theoretical_loss": 3.3353361381183206, + "tokens_seen": 3169110016 + }, + { + "epoch": 1.34, + "learning_rate": 3.23049147442327e-05, + "loss": 1.2494, + "theoretical_loss": 3.3353309386713583, + "tokens_seen": 3169175552 + }, + { + "epoch": 1.34, + "learning_rate": 3.22888665997994e-05, + "loss": 1.064, + "theoretical_loss": 3.3353257393620206, + "tokens_seen": 3169241088 + }, + { + "epoch": 1.34, + "learning_rate": 3.22728184553661e-05, + "loss": 1.2932, + "theoretical_loss": 3.3353205401903003, + "tokens_seen": 3169306624 + }, + { + "epoch": 1.34, + "learning_rate": 3.22567703109328e-05, + "loss": 1.1703, + "theoretical_loss": 3.3353153411561913, + "tokens_seen": 3169372160 + }, + { + "epoch": 1.34, + "learning_rate": 3.2240722166499496e-05, + "loss": 1.1298, + "theoretical_loss": 3.335310142259687, + "tokens_seen": 3169437696 + }, + { + "epoch": 1.34, + "learning_rate": 3.2224674022066196e-05, + "loss": 1.1305, + "theoretical_loss": 3.3353049435007813, + "tokens_seen": 3169503232 + }, + { + "epoch": 1.34, + "learning_rate": 3.22086258776329e-05, + "loss": 1.1713, + "theoretical_loss": 3.3352997448794675, + "tokens_seen": 3169568768 + }, + { + "epoch": 1.34, + "learning_rate": 3.2192577733199604e-05, + "loss": 1.2744, + "theoretical_loss": 3.3352945463957386, + "tokens_seen": 3169634304 + }, + { + "epoch": 1.34, + "learning_rate": 3.2176529588766305e-05, + "loss": 1.1638, + "theoretical_loss": 3.3352893480495887, + "tokens_seen": 3169699840 + }, + { + "epoch": 1.34, + "learning_rate": 3.2160481444333e-05, + "loss": 1.2046, + "theoretical_loss": 3.3352841498410113, + "tokens_seen": 3169765376 + }, + { + "epoch": 1.34, + "learning_rate": 3.21444332998997e-05, + "loss": 1.2535, + "theoretical_loss": 3.3352789517699994, + "tokens_seen": 3169830912 + }, + { + "epoch": 1.34, + "learning_rate": 3.21283851554664e-05, + "loss": 1.2226, + "theoretical_loss": 3.3352737538365473, + "tokens_seen": 3169896448 + }, + { + "epoch": 1.34, + "learning_rate": 3.21123370110331e-05, + "loss": 1.1744, + "theoretical_loss": 3.3352685560406483, + "tokens_seen": 3169961984 + }, + { + "epoch": 1.34, + "learning_rate": 3.20962888665998e-05, + "loss": 1.3025, + "theoretical_loss": 3.3352633583822953, + "tokens_seen": 3170027520 + }, + { + "epoch": 1.34, + "learning_rate": 3.20802407221665e-05, + "loss": 1.1585, + "theoretical_loss": 3.3352581608614824, + "tokens_seen": 3170093056 + }, + { + "epoch": 1.34, + "learning_rate": 3.2064192577733196e-05, + "loss": 1.2007, + "theoretical_loss": 3.335252963478203, + "tokens_seen": 3170158592 + }, + { + "epoch": 1.34, + "learning_rate": 3.2048144433299903e-05, + "loss": 1.1037, + "theoretical_loss": 3.3352477662324507, + "tokens_seen": 3170224128 + }, + { + "epoch": 1.34, + "learning_rate": 3.2032096288866604e-05, + "loss": 1.1667, + "theoretical_loss": 3.3352425691242185, + "tokens_seen": 3170289664 + }, + { + "epoch": 1.34, + "learning_rate": 3.2016048144433305e-05, + "loss": 1.1399, + "theoretical_loss": 3.3352373721535007, + "tokens_seen": 3170355200 + }, + { + "epoch": 1.34, + "learning_rate": 3.2000000000000005e-05, + "loss": 1.2113, + "theoretical_loss": 3.3352321753202903, + "tokens_seen": 3170420736 + }, + { + "epoch": 1.34, + "learning_rate": 3.19839518555667e-05, + "loss": 1.2453, + "theoretical_loss": 3.335226978624581, + "tokens_seen": 3170486272 + }, + { + "epoch": 1.34, + "learning_rate": 3.19679037111334e-05, + "loss": 1.2192, + "theoretical_loss": 3.335221782066366, + "tokens_seen": 3170551808 + }, + { + "epoch": 1.34, + "learning_rate": 3.19518555667001e-05, + "loss": 1.1426, + "theoretical_loss": 3.3352165856456395, + "tokens_seen": 3170617344 + }, + { + "epoch": 1.34, + "learning_rate": 3.19358074222668e-05, + "loss": 1.2069, + "theoretical_loss": 3.3352113893623945, + "tokens_seen": 3170682880 + }, + { + "epoch": 1.34, + "learning_rate": 3.19197592778335e-05, + "loss": 1.1756, + "theoretical_loss": 3.3352061932166244, + "tokens_seen": 3170748416 + }, + { + "epoch": 1.34, + "learning_rate": 3.19037111334002e-05, + "loss": 1.1814, + "theoretical_loss": 3.3352009972083234, + "tokens_seen": 3170813952 + }, + { + "epoch": 1.34, + "learning_rate": 3.18876629889669e-05, + "loss": 1.2544, + "theoretical_loss": 3.335195801337484, + "tokens_seen": 3170879488 + }, + { + "epoch": 1.34, + "learning_rate": 3.1871614844533604e-05, + "loss": 1.151, + "theoretical_loss": 3.335190605604101, + "tokens_seen": 3170945024 + }, + { + "epoch": 1.34, + "learning_rate": 3.1855566700100304e-05, + "loss": 1.0326, + "theoretical_loss": 3.3351854100081666, + "tokens_seen": 3171010560 + }, + { + "epoch": 1.34, + "learning_rate": 3.1839518555667005e-05, + "loss": 1.2546, + "theoretical_loss": 3.335180214549675, + "tokens_seen": 3171076096 + }, + { + "epoch": 1.34, + "learning_rate": 3.1823470411233706e-05, + "loss": 1.1364, + "theoretical_loss": 3.33517501922862, + "tokens_seen": 3171141632 + }, + { + "epoch": 1.34, + "learning_rate": 3.18074222668004e-05, + "loss": 1.1795, + "theoretical_loss": 3.335169824044995, + "tokens_seen": 3171207168 + }, + { + "epoch": 1.34, + "learning_rate": 3.17913741223671e-05, + "loss": 1.1827, + "theoretical_loss": 3.335164628998793, + "tokens_seen": 3171272704 + }, + { + "epoch": 1.34, + "learning_rate": 3.17753259779338e-05, + "loss": 1.2853, + "theoretical_loss": 3.3351594340900075, + "tokens_seen": 3171338240 + }, + { + "epoch": 1.34, + "learning_rate": 3.17592778335005e-05, + "loss": 1.2774, + "theoretical_loss": 3.335154239318633, + "tokens_seen": 3171403776 + }, + { + "epoch": 1.34, + "learning_rate": 3.174322968906721e-05, + "loss": 1.1228, + "theoretical_loss": 3.335149044684662, + "tokens_seen": 3171469312 + }, + { + "epoch": 1.34, + "learning_rate": 3.17271815446339e-05, + "loss": 1.106, + "theoretical_loss": 3.3351438501880883, + "tokens_seen": 3171534848 + }, + { + "epoch": 1.34, + "learning_rate": 3.17111334002006e-05, + "loss": 1.0962, + "theoretical_loss": 3.335138655828906, + "tokens_seen": 3171600384 + }, + { + "epoch": 1.34, + "learning_rate": 3.1695085255767304e-05, + "loss": 1.09, + "theoretical_loss": 3.335133461607108, + "tokens_seen": 3171665920 + }, + { + "epoch": 1.34, + "learning_rate": 3.1679037111334004e-05, + "loss": 1.0563, + "theoretical_loss": 3.335128267522688, + "tokens_seen": 3171731456 + }, + { + "epoch": 1.34, + "learning_rate": 3.1662988966900705e-05, + "loss": 1.1819, + "theoretical_loss": 3.3351230735756396, + "tokens_seen": 3171796992 + }, + { + "epoch": 1.34, + "learning_rate": 3.1646940822467406e-05, + "loss": 1.1379, + "theoretical_loss": 3.335117879765956, + "tokens_seen": 3171862528 + }, + { + "epoch": 1.34, + "objective/train/docs_used": 1169840, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.178410530090332, + "objective/train/theoretical_loss": 3.3351126860936313, + "objective/train/tokens_used": 3192388064, + "theoretical_loss": 3.3351126860936313, + "tokens_seen": 3171928064 + }, + { + "epoch": 1.34, + "learning_rate": 3.16308926780341e-05, + "loss": 1.1784, + "theoretical_loss": 3.3351126860936313, + "tokens_seen": 3171928064 + }, + { + "epoch": 1.34, + "learning_rate": 3.16148445336008e-05, + "loss": 1.2027, + "theoretical_loss": 3.3351074925586586, + "tokens_seen": 3171993600 + }, + { + "epoch": 1.34, + "learning_rate": 3.159879638916751e-05, + "loss": 1.1575, + "theoretical_loss": 3.3351022991610315, + "tokens_seen": 3172059136 + }, + { + "epoch": 1.34, + "learning_rate": 3.158274824473421e-05, + "loss": 1.1776, + "theoretical_loss": 3.3350971059007435, + "tokens_seen": 3172124672 + }, + { + "epoch": 1.34, + "learning_rate": 3.156670010030091e-05, + "loss": 1.1864, + "theoretical_loss": 3.3350919127777887, + "tokens_seen": 3172190208 + }, + { + "epoch": 1.34, + "learning_rate": 3.15506519558676e-05, + "loss": 1.1086, + "theoretical_loss": 3.3350867197921596, + "tokens_seen": 3172255744 + }, + { + "epoch": 1.34, + "learning_rate": 3.1534603811434303e-05, + "loss": 1.1495, + "theoretical_loss": 3.3350815269438505, + "tokens_seen": 3172321280 + }, + { + "epoch": 1.34, + "learning_rate": 3.1518555667001004e-05, + "loss": 1.2137, + "theoretical_loss": 3.3350763342328547, + "tokens_seen": 3172386816 + }, + { + "epoch": 1.34, + "learning_rate": 3.1502507522567705e-05, + "loss": 1.1491, + "theoretical_loss": 3.335071141659166, + "tokens_seen": 3172452352 + }, + { + "epoch": 1.34, + "learning_rate": 3.1486459378134405e-05, + "loss": 1.1762, + "theoretical_loss": 3.335065949222777, + "tokens_seen": 3172517888 + }, + { + "epoch": 1.34, + "learning_rate": 3.1470411233701106e-05, + "loss": 1.0253, + "theoretical_loss": 3.3350607569236823, + "tokens_seen": 3172583424 + }, + { + "epoch": 1.34, + "learning_rate": 3.1454363089267807e-05, + "loss": 1.1235, + "theoretical_loss": 3.335055564761875, + "tokens_seen": 3172648960 + }, + { + "epoch": 1.34, + "learning_rate": 3.143831494483451e-05, + "loss": 1.2637, + "theoretical_loss": 3.335050372737349, + "tokens_seen": 3172714496 + }, + { + "epoch": 1.34, + "learning_rate": 3.142226680040121e-05, + "loss": 1.0365, + "theoretical_loss": 3.335045180850097, + "tokens_seen": 3172780032 + }, + { + "epoch": 1.34, + "learning_rate": 3.140621865596791e-05, + "loss": 1.1211, + "theoretical_loss": 3.335039989100113, + "tokens_seen": 3172845568 + }, + { + "epoch": 1.34, + "learning_rate": 3.13901705115346e-05, + "loss": 1.1403, + "theoretical_loss": 3.335034797487391, + "tokens_seen": 3172911104 + }, + { + "epoch": 1.34, + "learning_rate": 3.13741223671013e-05, + "loss": 1.1816, + "theoretical_loss": 3.335029606011924, + "tokens_seen": 3172976640 + }, + { + "epoch": 1.34, + "learning_rate": 3.1358074222668004e-05, + "loss": 1.1358, + "theoretical_loss": 3.335024414673706, + "tokens_seen": 3173042176 + }, + { + "epoch": 1.34, + "learning_rate": 3.1342026078234704e-05, + "loss": 1.1864, + "theoretical_loss": 3.3350192234727296, + "tokens_seen": 3173107712 + }, + { + "epoch": 1.34, + "learning_rate": 3.1325977933801405e-05, + "loss": 1.1402, + "theoretical_loss": 3.335014032408989, + "tokens_seen": 3173173248 + }, + { + "epoch": 1.34, + "learning_rate": 3.1309929789368106e-05, + "loss": 1.2001, + "theoretical_loss": 3.3350088414824777, + "tokens_seen": 3173238784 + }, + { + "epoch": 1.34, + "learning_rate": 3.1293881644934806e-05, + "loss": 1.131, + "theoretical_loss": 3.3350036506931895, + "tokens_seen": 3173304320 + }, + { + "epoch": 1.34, + "learning_rate": 3.127783350050151e-05, + "loss": 1.1867, + "theoretical_loss": 3.3349984600411173, + "tokens_seen": 3173369856 + }, + { + "epoch": 1.34, + "learning_rate": 3.126178535606821e-05, + "loss": 1.2013, + "theoretical_loss": 3.3349932695262554, + "tokens_seen": 3173435392 + }, + { + "epoch": 1.34, + "learning_rate": 3.124573721163491e-05, + "loss": 1.1539, + "theoretical_loss": 3.334988079148596, + "tokens_seen": 3173500928 + }, + { + "epoch": 1.34, + "learning_rate": 3.122968906720161e-05, + "loss": 1.196, + "theoretical_loss": 3.3349828889081348, + "tokens_seen": 3173566464 + }, + { + "epoch": 1.34, + "learning_rate": 3.12136409227683e-05, + "loss": 1.1789, + "theoretical_loss": 3.334977698804863, + "tokens_seen": 3173632000 + }, + { + "epoch": 1.34, + "learning_rate": 3.1197592778335e-05, + "loss": 1.1926, + "theoretical_loss": 3.334972508838776, + "tokens_seen": 3173697536 + }, + { + "epoch": 1.34, + "learning_rate": 3.1181544633901704e-05, + "loss": 1.2449, + "theoretical_loss": 3.3349673190098663, + "tokens_seen": 3173763072 + }, + { + "epoch": 1.34, + "learning_rate": 3.1165496489468404e-05, + "loss": 1.2224, + "theoretical_loss": 3.3349621293181277, + "tokens_seen": 3173828608 + }, + { + "epoch": 1.34, + "learning_rate": 3.114944834503511e-05, + "loss": 1.2, + "theoretical_loss": 3.334956939763554, + "tokens_seen": 3173894144 + }, + { + "epoch": 1.34, + "learning_rate": 3.1133400200601806e-05, + "loss": 1.2617, + "theoretical_loss": 3.3349517503461383, + "tokens_seen": 3173959680 + }, + { + "epoch": 1.34, + "learning_rate": 3.1117352056168506e-05, + "loss": 1.1828, + "theoretical_loss": 3.334946561065874, + "tokens_seen": 3174025216 + }, + { + "epoch": 1.34, + "learning_rate": 3.110130391173521e-05, + "loss": 1.1241, + "theoretical_loss": 3.334941371922756, + "tokens_seen": 3174090752 + }, + { + "epoch": 1.34, + "learning_rate": 3.108525576730191e-05, + "loss": 1.1897, + "theoretical_loss": 3.334936182916776, + "tokens_seen": 3174156288 + }, + { + "epoch": 1.34, + "learning_rate": 3.106920762286861e-05, + "loss": 1.1228, + "theoretical_loss": 3.3349309940479284, + "tokens_seen": 3174221824 + }, + { + "epoch": 1.34, + "learning_rate": 3.105315947843531e-05, + "loss": 1.1667, + "theoretical_loss": 3.334925805316207, + "tokens_seen": 3174287360 + }, + { + "epoch": 1.34, + "learning_rate": 3.1037111334002e-05, + "loss": 1.1996, + "theoretical_loss": 3.3349206167216052, + "tokens_seen": 3174352896 + }, + { + "epoch": 1.34, + "learning_rate": 3.10210631895687e-05, + "loss": 1.19, + "theoretical_loss": 3.3349154282641162, + "tokens_seen": 3174418432 + }, + { + "epoch": 1.34, + "learning_rate": 3.100501504513541e-05, + "loss": 1.3054, + "theoretical_loss": 3.334910239943734, + "tokens_seen": 3174483968 + }, + { + "epoch": 1.34, + "learning_rate": 3.098896690070211e-05, + "loss": 1.1893, + "theoretical_loss": 3.3349050517604515, + "tokens_seen": 3174549504 + }, + { + "epoch": 1.34, + "learning_rate": 3.097291875626881e-05, + "loss": 1.1528, + "theoretical_loss": 3.334899863714263, + "tokens_seen": 3174615040 + }, + { + "epoch": 1.34, + "learning_rate": 3.0956870611835506e-05, + "loss": 1.1237, + "theoretical_loss": 3.3348946758051614, + "tokens_seen": 3174680576 + }, + { + "epoch": 1.34, + "learning_rate": 3.0940822467402207e-05, + "loss": 1.117, + "theoretical_loss": 3.3348894880331406, + "tokens_seen": 3174746112 + }, + { + "epoch": 1.34, + "learning_rate": 3.092477432296891e-05, + "loss": 1.0973, + "theoretical_loss": 3.3348843003981945, + "tokens_seen": 3174811648 + }, + { + "epoch": 1.34, + "learning_rate": 3.090872617853561e-05, + "loss": 1.2287, + "theoretical_loss": 3.3348791129003157, + "tokens_seen": 3174877184 + }, + { + "epoch": 1.34, + "learning_rate": 3.089267803410231e-05, + "loss": 1.1783, + "theoretical_loss": 3.334873925539499, + "tokens_seen": 3174942720 + }, + { + "epoch": 1.34, + "learning_rate": 3.087662988966901e-05, + "loss": 1.2074, + "theoretical_loss": 3.3348687383157367, + "tokens_seen": 3175008256 + }, + { + "epoch": 1.34, + "learning_rate": 3.08605817452357e-05, + "loss": 1.2333, + "theoretical_loss": 3.334863551229023, + "tokens_seen": 3175073792 + }, + { + "epoch": 1.34, + "learning_rate": 3.084453360080241e-05, + "loss": 1.1471, + "theoretical_loss": 3.3348583642793512, + "tokens_seen": 3175139328 + }, + { + "epoch": 1.34, + "learning_rate": 3.082848545636911e-05, + "loss": 1.1002, + "theoretical_loss": 3.334853177466715, + "tokens_seen": 3175204864 + }, + { + "epoch": 1.34, + "learning_rate": 3.081243731193581e-05, + "loss": 1.0984, + "theoretical_loss": 3.3348479907911086, + "tokens_seen": 3175270400 + }, + { + "epoch": 1.34, + "learning_rate": 3.079638916750251e-05, + "loss": 1.089, + "theoretical_loss": 3.334842804252524, + "tokens_seen": 3175335936 + }, + { + "epoch": 1.34, + "learning_rate": 3.0780341023069206e-05, + "loss": 1.2237, + "theoretical_loss": 3.3348376178509564, + "tokens_seen": 3175401472 + }, + { + "epoch": 1.34, + "learning_rate": 3.076429287863591e-05, + "loss": 1.1552, + "theoretical_loss": 3.334832431586398, + "tokens_seen": 3175467008 + }, + { + "epoch": 1.34, + "learning_rate": 3.074824473420261e-05, + "loss": 1.2074, + "theoretical_loss": 3.3348272454588432, + "tokens_seen": 3175532544 + }, + { + "epoch": 1.34, + "learning_rate": 3.073219658976931e-05, + "loss": 1.2827, + "theoretical_loss": 3.3348220594682854, + "tokens_seen": 3175598080 + }, + { + "epoch": 1.34, + "learning_rate": 3.071614844533601e-05, + "loss": 1.1745, + "theoretical_loss": 3.3348168736147183, + "tokens_seen": 3175663616 + }, + { + "epoch": 1.34, + "learning_rate": 3.070010030090271e-05, + "loss": 1.2303, + "theoretical_loss": 3.334811687898135, + "tokens_seen": 3175729152 + }, + { + "epoch": 1.34, + "learning_rate": 3.068405215646941e-05, + "loss": 1.1489, + "theoretical_loss": 3.3348065023185294, + "tokens_seen": 3175794688 + }, + { + "epoch": 1.34, + "learning_rate": 3.066800401203611e-05, + "loss": 1.194, + "theoretical_loss": 3.3348013168758945, + "tokens_seen": 3175860224 + }, + { + "epoch": 1.34, + "learning_rate": 3.065195586760281e-05, + "loss": 1.1932, + "theoretical_loss": 3.334796131570225, + "tokens_seen": 3175925760 + }, + { + "epoch": 1.34, + "learning_rate": 3.063590772316951e-05, + "loss": 1.1076, + "theoretical_loss": 3.334790946401513, + "tokens_seen": 3175991296 + }, + { + "epoch": 1.34, + "learning_rate": 3.061985957873621e-05, + "loss": 1.2343, + "theoretical_loss": 3.334785761369753, + "tokens_seen": 3176056832 + }, + { + "epoch": 1.34, + "learning_rate": 3.0603811434302906e-05, + "loss": 1.1855, + "theoretical_loss": 3.3347805764749388, + "tokens_seen": 3176122368 + }, + { + "epoch": 1.34, + "learning_rate": 3.058776328986961e-05, + "loss": 1.1672, + "theoretical_loss": 3.334775391717063, + "tokens_seen": 3176187904 + }, + { + "epoch": 1.34, + "learning_rate": 3.057171514543631e-05, + "loss": 1.245, + "theoretical_loss": 3.33477020709612, + "tokens_seen": 3176253440 + }, + { + "epoch": 1.34, + "learning_rate": 3.055566700100301e-05, + "loss": 1.1086, + "theoretical_loss": 3.334765022612103, + "tokens_seen": 3176318976 + }, + { + "epoch": 1.34, + "learning_rate": 3.0539618856569716e-05, + "loss": 1.2527, + "theoretical_loss": 3.334759838265005, + "tokens_seen": 3176384512 + }, + { + "epoch": 1.34, + "learning_rate": 3.052357071213641e-05, + "loss": 1.2395, + "theoretical_loss": 3.3347546540548207, + "tokens_seen": 3176450048 + }, + { + "epoch": 1.34, + "learning_rate": 3.050752256770311e-05, + "loss": 1.2037, + "theoretical_loss": 3.3347494699815434, + "tokens_seen": 3176515584 + }, + { + "epoch": 1.34, + "learning_rate": 3.049147442326981e-05, + "loss": 1.1724, + "theoretical_loss": 3.334744286045166, + "tokens_seen": 3176581120 + }, + { + "epoch": 1.34, + "learning_rate": 3.047542627883651e-05, + "loss": 1.2571, + "theoretical_loss": 3.3347391022456825, + "tokens_seen": 3176646656 + }, + { + "epoch": 1.34, + "learning_rate": 3.045937813440321e-05, + "loss": 1.1711, + "theoretical_loss": 3.334733918583086, + "tokens_seen": 3176712192 + }, + { + "epoch": 1.34, + "learning_rate": 3.044332998996991e-05, + "loss": 1.1897, + "theoretical_loss": 3.334728735057371, + "tokens_seen": 3176777728 + }, + { + "epoch": 1.34, + "learning_rate": 3.042728184553661e-05, + "loss": 1.1507, + "theoretical_loss": 3.3347235516685303, + "tokens_seen": 3176843264 + }, + { + "epoch": 1.34, + "learning_rate": 3.0411233701103307e-05, + "loss": 1.1169, + "theoretical_loss": 3.3347183684165578, + "tokens_seen": 3176908800 + }, + { + "epoch": 1.34, + "learning_rate": 3.0395185556670015e-05, + "loss": 1.1549, + "theoretical_loss": 3.334713185301447, + "tokens_seen": 3176974336 + }, + { + "epoch": 1.34, + "learning_rate": 3.0379137412236712e-05, + "loss": 0.9841, + "theoretical_loss": 3.334708002323191, + "tokens_seen": 3177039872 + }, + { + "epoch": 1.34, + "learning_rate": 3.0363089267803412e-05, + "loss": 1.1627, + "theoretical_loss": 3.334702819481784, + "tokens_seen": 3177105408 + }, + { + "epoch": 1.34, + "learning_rate": 3.0347041123370113e-05, + "loss": 1.1315, + "theoretical_loss": 3.3346976367772196, + "tokens_seen": 3177170944 + }, + { + "epoch": 1.34, + "learning_rate": 3.033099297893681e-05, + "loss": 1.2254, + "theoretical_loss": 3.334692454209491, + "tokens_seen": 3177236480 + }, + { + "epoch": 1.34, + "learning_rate": 3.031494483450351e-05, + "loss": 1.1257, + "theoretical_loss": 3.3346872717785914, + "tokens_seen": 3177302016 + }, + { + "epoch": 1.34, + "learning_rate": 3.029889669007021e-05, + "loss": 1.1585, + "theoretical_loss": 3.3346820894845153, + "tokens_seen": 3177367552 + }, + { + "epoch": 1.34, + "learning_rate": 3.028284854563691e-05, + "loss": 1.2486, + "theoretical_loss": 3.3346769073272555, + "tokens_seen": 3177433088 + }, + { + "epoch": 1.34, + "learning_rate": 3.026680040120361e-05, + "loss": 1.1948, + "theoretical_loss": 3.334671725306806, + "tokens_seen": 3177498624 + }, + { + "epoch": 1.34, + "learning_rate": 3.0250752256770314e-05, + "loss": 1.108, + "theoretical_loss": 3.33466654342316, + "tokens_seen": 3177564160 + }, + { + "epoch": 1.34, + "learning_rate": 3.0234704112337014e-05, + "loss": 1.1466, + "theoretical_loss": 3.3346613616763117, + "tokens_seen": 3177629696 + }, + { + "epoch": 1.34, + "learning_rate": 3.0218655967903715e-05, + "loss": 1.2483, + "theoretical_loss": 3.334656180066254, + "tokens_seen": 3177695232 + }, + { + "epoch": 1.34, + "learning_rate": 3.0202607823470412e-05, + "loss": 1.1783, + "theoretical_loss": 3.3346509985929806, + "tokens_seen": 3177760768 + }, + { + "epoch": 1.34, + "learning_rate": 3.0186559679037113e-05, + "loss": 1.1761, + "theoretical_loss": 3.3346458172564857, + "tokens_seen": 3177826304 + }, + { + "epoch": 1.34, + "learning_rate": 3.0170511534603813e-05, + "loss": 1.1744, + "theoretical_loss": 3.3346406360567618, + "tokens_seen": 3177891840 + }, + { + "epoch": 1.34, + "learning_rate": 3.015446339017051e-05, + "loss": 1.1721, + "theoretical_loss": 3.334635454993803, + "tokens_seen": 3177957376 + }, + { + "epoch": 1.34, + "learning_rate": 3.013841524573721e-05, + "loss": 1.1607, + "theoretical_loss": 3.3346302740676035, + "tokens_seen": 3178022912 + }, + { + "epoch": 1.34, + "learning_rate": 3.0122367101303912e-05, + "loss": 1.0259, + "theoretical_loss": 3.334625093278156, + "tokens_seen": 3178088448 + }, + { + "epoch": 1.34, + "learning_rate": 3.010631895687061e-05, + "loss": 1.0844, + "theoretical_loss": 3.3346199126254543, + "tokens_seen": 3178153984 + }, + { + "epoch": 1.34, + "learning_rate": 3.0090270812437316e-05, + "loss": 1.141, + "theoretical_loss": 3.334614732109492, + "tokens_seen": 3178219520 + }, + { + "epoch": 1.34, + "learning_rate": 3.0074222668004014e-05, + "loss": 1.1319, + "theoretical_loss": 3.3346095517302627, + "tokens_seen": 3178285056 + }, + { + "epoch": 1.34, + "learning_rate": 3.0058174523570714e-05, + "loss": 1.1666, + "theoretical_loss": 3.33460437148776, + "tokens_seen": 3178350592 + }, + { + "epoch": 1.34, + "learning_rate": 3.0042126379137415e-05, + "loss": 1.2147, + "theoretical_loss": 3.3345991913819772, + "tokens_seen": 3178416128 + }, + { + "epoch": 1.34, + "objective/train/docs_used": 1172218, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.142197847366333, + "objective/train/theoretical_loss": 3.334594011412908, + "objective/train/tokens_used": 3198941664, + "theoretical_loss": 3.334594011412908, + "tokens_seen": 3178481664 + }, + { + "epoch": 1.34, + "learning_rate": 3.0026078234704112e-05, + "loss": 1.1422, + "theoretical_loss": 3.334594011412908, + "tokens_seen": 3178481664 + }, + { + "epoch": 1.34, + "learning_rate": 3.0010030090270813e-05, + "loss": 1.2311, + "theoretical_loss": 3.3345888315805468, + "tokens_seen": 3178547200 + }, + { + "epoch": 1.34, + "learning_rate": 2.9993981945837513e-05, + "loss": 1.1747, + "theoretical_loss": 3.334583651884886, + "tokens_seen": 3178612736 + }, + { + "epoch": 1.34, + "learning_rate": 2.997793380140421e-05, + "loss": 1.2476, + "theoretical_loss": 3.3345784723259193, + "tokens_seen": 3178678272 + }, + { + "epoch": 1.34, + "learning_rate": 2.996188565697091e-05, + "loss": 1.1073, + "theoretical_loss": 3.334573292903641, + "tokens_seen": 3178743808 + }, + { + "epoch": 1.34, + "learning_rate": 2.9945837512537615e-05, + "loss": 1.1916, + "theoretical_loss": 3.3345681136180443, + "tokens_seen": 3178809344 + }, + { + "epoch": 1.34, + "learning_rate": 2.9929789368104316e-05, + "loss": 1.1529, + "theoretical_loss": 3.3345629344691226, + "tokens_seen": 3178874880 + }, + { + "epoch": 1.34, + "learning_rate": 2.9913741223671017e-05, + "loss": 1.0848, + "theoretical_loss": 3.3345577554568697, + "tokens_seen": 3178940416 + }, + { + "epoch": 1.34, + "learning_rate": 2.9897693079237714e-05, + "loss": 1.1308, + "theoretical_loss": 3.334552576581279, + "tokens_seen": 3179005952 + }, + { + "epoch": 1.34, + "learning_rate": 2.9881644934804415e-05, + "loss": 1.1237, + "theoretical_loss": 3.334547397842344, + "tokens_seen": 3179071488 + }, + { + "epoch": 1.34, + "learning_rate": 2.9865596790371115e-05, + "loss": 1.183, + "theoretical_loss": 3.334542219240059, + "tokens_seen": 3179137024 + }, + { + "epoch": 1.34, + "learning_rate": 2.9849548645937812e-05, + "loss": 1.1176, + "theoretical_loss": 3.334537040774417, + "tokens_seen": 3179202560 + }, + { + "epoch": 1.34, + "learning_rate": 2.9833500501504513e-05, + "loss": 1.1994, + "theoretical_loss": 3.3345318624454108, + "tokens_seen": 3179268096 + }, + { + "epoch": 1.34, + "learning_rate": 2.9817452357071214e-05, + "loss": 1.2067, + "theoretical_loss": 3.3345266842530354, + "tokens_seen": 3179333632 + }, + { + "epoch": 1.34, + "learning_rate": 2.9801404212637918e-05, + "loss": 1.1427, + "theoretical_loss": 3.3345215061972837, + "tokens_seen": 3179399168 + }, + { + "epoch": 1.34, + "learning_rate": 2.978535606820462e-05, + "loss": 1.1463, + "theoretical_loss": 3.3345163282781494, + "tokens_seen": 3179464704 + }, + { + "epoch": 1.34, + "learning_rate": 2.9769307923771316e-05, + "loss": 1.133, + "theoretical_loss": 3.3345111504956257, + "tokens_seen": 3179530240 + }, + { + "epoch": 1.34, + "learning_rate": 2.9753259779338016e-05, + "loss": 1.2082, + "theoretical_loss": 3.3345059728497066, + "tokens_seen": 3179595776 + }, + { + "epoch": 1.34, + "learning_rate": 2.9737211634904717e-05, + "loss": 1.1397, + "theoretical_loss": 3.334500795340386, + "tokens_seen": 3179661312 + }, + { + "epoch": 1.34, + "learning_rate": 2.9721163490471414e-05, + "loss": 1.1207, + "theoretical_loss": 3.3344956179676566, + "tokens_seen": 3179726848 + }, + { + "epoch": 1.34, + "learning_rate": 2.9705115346038115e-05, + "loss": 1.1139, + "theoretical_loss": 3.334490440731513, + "tokens_seen": 3179792384 + }, + { + "epoch": 1.34, + "learning_rate": 2.9689067201604815e-05, + "loss": 1.2047, + "theoretical_loss": 3.3344852636319473, + "tokens_seen": 3179857920 + }, + { + "epoch": 1.34, + "learning_rate": 2.9673019057171513e-05, + "loss": 1.198, + "theoretical_loss": 3.334480086668955, + "tokens_seen": 3179923456 + }, + { + "epoch": 1.34, + "learning_rate": 2.9656970912738213e-05, + "loss": 1.1942, + "theoretical_loss": 3.334474909842528, + "tokens_seen": 3179988992 + }, + { + "epoch": 1.34, + "learning_rate": 2.9640922768304917e-05, + "loss": 1.1583, + "theoretical_loss": 3.3344697331526607, + "tokens_seen": 3180054528 + }, + { + "epoch": 1.34, + "learning_rate": 2.9624874623871618e-05, + "loss": 1.2529, + "theoretical_loss": 3.3344645565993467, + "tokens_seen": 3180120064 + }, + { + "epoch": 1.34, + "learning_rate": 2.960882647943832e-05, + "loss": 1.2353, + "theoretical_loss": 3.3344593801825795, + "tokens_seen": 3180185600 + }, + { + "epoch": 1.34, + "learning_rate": 2.9592778335005016e-05, + "loss": 1.1844, + "theoretical_loss": 3.334454203902353, + "tokens_seen": 3180251136 + }, + { + "epoch": 1.34, + "learning_rate": 2.9576730190571716e-05, + "loss": 1.1852, + "theoretical_loss": 3.3344490277586596, + "tokens_seen": 3180316672 + }, + { + "epoch": 1.34, + "learning_rate": 2.9560682046138417e-05, + "loss": 1.1723, + "theoretical_loss": 3.334443851751494, + "tokens_seen": 3180382208 + }, + { + "epoch": 1.34, + "learning_rate": 2.9544633901705114e-05, + "loss": 1.0594, + "theoretical_loss": 3.3344386758808495, + "tokens_seen": 3180447744 + }, + { + "epoch": 1.34, + "learning_rate": 2.9528585757271815e-05, + "loss": 1.1117, + "theoretical_loss": 3.33443350014672, + "tokens_seen": 3180513280 + }, + { + "epoch": 1.34, + "learning_rate": 2.9512537612838516e-05, + "loss": 1.1859, + "theoretical_loss": 3.3344283245490987, + "tokens_seen": 3180578816 + }, + { + "epoch": 1.34, + "learning_rate": 2.949648946840522e-05, + "loss": 1.2293, + "theoretical_loss": 3.3344231490879785, + "tokens_seen": 3180644352 + }, + { + "epoch": 1.34, + "learning_rate": 2.948044132397192e-05, + "loss": 1.1481, + "theoretical_loss": 3.3344179737633546, + "tokens_seen": 3180709888 + }, + { + "epoch": 1.34, + "learning_rate": 2.9464393179538617e-05, + "loss": 1.2389, + "theoretical_loss": 3.334412798575219, + "tokens_seen": 3180775424 + }, + { + "epoch": 1.34, + "learning_rate": 2.9448345035105318e-05, + "loss": 1.268, + "theoretical_loss": 3.3344076235235667, + "tokens_seen": 3180840960 + }, + { + "epoch": 1.34, + "learning_rate": 2.943229689067202e-05, + "loss": 1.2049, + "theoretical_loss": 3.3344024486083903, + "tokens_seen": 3180906496 + }, + { + "epoch": 1.34, + "learning_rate": 2.9416248746238716e-05, + "loss": 1.1931, + "theoretical_loss": 3.3343972738296834, + "tokens_seen": 3180972032 + }, + { + "epoch": 1.34, + "learning_rate": 2.9400200601805417e-05, + "loss": 1.1498, + "theoretical_loss": 3.3343920991874403, + "tokens_seen": 3181037568 + }, + { + "epoch": 1.34, + "learning_rate": 2.9384152457372117e-05, + "loss": 1.1911, + "theoretical_loss": 3.334386924681654, + "tokens_seen": 3181103104 + }, + { + "epoch": 1.34, + "learning_rate": 2.9368104312938815e-05, + "loss": 1.2163, + "theoretical_loss": 3.3343817503123185, + "tokens_seen": 3181168640 + }, + { + "epoch": 1.34, + "learning_rate": 2.9352056168505522e-05, + "loss": 1.1665, + "theoretical_loss": 3.3343765760794266, + "tokens_seen": 3181234176 + }, + { + "epoch": 1.34, + "learning_rate": 2.933600802407222e-05, + "loss": 1.163, + "theoretical_loss": 3.334371401982973, + "tokens_seen": 3181299712 + }, + { + "epoch": 1.34, + "learning_rate": 2.931995987963892e-05, + "loss": 1.1164, + "theoretical_loss": 3.3343662280229505, + "tokens_seen": 3181365248 + }, + { + "epoch": 1.34, + "learning_rate": 2.930391173520562e-05, + "loss": 1.1651, + "theoretical_loss": 3.334361054199353, + "tokens_seen": 3181430784 + }, + { + "epoch": 1.34, + "learning_rate": 2.9287863590772318e-05, + "loss": 1.1978, + "theoretical_loss": 3.334355880512174, + "tokens_seen": 3181496320 + }, + { + "epoch": 1.34, + "learning_rate": 2.927181544633902e-05, + "loss": 1.1444, + "theoretical_loss": 3.3343507069614065, + "tokens_seen": 3181561856 + }, + { + "epoch": 1.34, + "learning_rate": 2.925576730190572e-05, + "loss": 1.2554, + "theoretical_loss": 3.3343455335470455, + "tokens_seen": 3181627392 + }, + { + "epoch": 1.34, + "learning_rate": 2.9239719157472416e-05, + "loss": 1.0708, + "theoretical_loss": 3.3343403602690835, + "tokens_seen": 3181692928 + }, + { + "epoch": 1.34, + "learning_rate": 2.9223671013039117e-05, + "loss": 1.1768, + "theoretical_loss": 3.334335187127514, + "tokens_seen": 3181758464 + }, + { + "epoch": 1.34, + "learning_rate": 2.9207622868605817e-05, + "loss": 1.2019, + "theoretical_loss": 3.3343300141223318, + "tokens_seen": 3181824000 + }, + { + "epoch": 1.34, + "learning_rate": 2.919157472417252e-05, + "loss": 1.2175, + "theoretical_loss": 3.334324841253529, + "tokens_seen": 3181889536 + }, + { + "epoch": 1.34, + "learning_rate": 2.9175526579739222e-05, + "loss": 1.1772, + "theoretical_loss": 3.3343196685211, + "tokens_seen": 3181955072 + }, + { + "epoch": 1.34, + "learning_rate": 2.915947843530592e-05, + "loss": 1.114, + "theoretical_loss": 3.334314495925039, + "tokens_seen": 3182020608 + }, + { + "epoch": 1.34, + "learning_rate": 2.914343029087262e-05, + "loss": 1.202, + "theoretical_loss": 3.334309323465338, + "tokens_seen": 3182086144 + }, + { + "epoch": 1.34, + "learning_rate": 2.912738214643932e-05, + "loss": 1.1128, + "theoretical_loss": 3.3343041511419917, + "tokens_seen": 3182151680 + }, + { + "epoch": 1.34, + "learning_rate": 2.9111334002006018e-05, + "loss": 1.1765, + "theoretical_loss": 3.3342989789549935, + "tokens_seen": 3182217216 + }, + { + "epoch": 1.34, + "learning_rate": 2.909528585757272e-05, + "loss": 1.1755, + "theoretical_loss": 3.334293806904337, + "tokens_seen": 3182282752 + }, + { + "epoch": 1.35, + "learning_rate": 2.907923771313942e-05, + "loss": 1.1582, + "theoretical_loss": 3.3342886349900156, + "tokens_seen": 3182348288 + }, + { + "epoch": 1.35, + "learning_rate": 2.9063189568706116e-05, + "loss": 1.1733, + "theoretical_loss": 3.334283463212023, + "tokens_seen": 3182413824 + }, + { + "epoch": 1.35, + "learning_rate": 2.9047141424272824e-05, + "loss": 1.155, + "theoretical_loss": 3.334278291570353, + "tokens_seen": 3182479360 + }, + { + "epoch": 1.35, + "learning_rate": 2.903109327983952e-05, + "loss": 1.1646, + "theoretical_loss": 3.3342731200649993, + "tokens_seen": 3182544896 + }, + { + "epoch": 1.35, + "learning_rate": 2.9015045135406222e-05, + "loss": 1.2498, + "theoretical_loss": 3.3342679486959548, + "tokens_seen": 3182610432 + }, + { + "epoch": 1.35, + "learning_rate": 2.8998996990972922e-05, + "loss": 1.2444, + "theoretical_loss": 3.3342627774632136, + "tokens_seen": 3182675968 + }, + { + "epoch": 1.35, + "learning_rate": 2.898294884653962e-05, + "loss": 1.1507, + "theoretical_loss": 3.3342576063667697, + "tokens_seen": 3182741504 + }, + { + "epoch": 1.35, + "learning_rate": 2.896690070210632e-05, + "loss": 1.148, + "theoretical_loss": 3.334252435406616, + "tokens_seen": 3182807040 + }, + { + "epoch": 1.35, + "learning_rate": 2.895085255767302e-05, + "loss": 1.0731, + "theoretical_loss": 3.3342472645827463, + "tokens_seen": 3182872576 + }, + { + "epoch": 1.35, + "learning_rate": 2.8934804413239718e-05, + "loss": 1.1626, + "theoretical_loss": 3.3342420938951545, + "tokens_seen": 3182938112 + }, + { + "epoch": 1.35, + "learning_rate": 2.891875626880642e-05, + "loss": 1.2, + "theoretical_loss": 3.3342369233438336, + "tokens_seen": 3183003648 + }, + { + "epoch": 1.35, + "learning_rate": 2.8902708124373123e-05, + "loss": 1.155, + "theoretical_loss": 3.334231752928778, + "tokens_seen": 3183069184 + }, + { + "epoch": 1.35, + "learning_rate": 2.8886659979939823e-05, + "loss": 1.1792, + "theoretical_loss": 3.3342265826499804, + "tokens_seen": 3183134720 + }, + { + "epoch": 1.35, + "learning_rate": 2.8870611835506524e-05, + "loss": 1.2428, + "theoretical_loss": 3.334221412507435, + "tokens_seen": 3183200256 + }, + { + "epoch": 1.35, + "learning_rate": 2.885456369107322e-05, + "loss": 1.1476, + "theoretical_loss": 3.3342162425011352, + "tokens_seen": 3183265792 + }, + { + "epoch": 1.35, + "learning_rate": 2.8838515546639922e-05, + "loss": 1.1645, + "theoretical_loss": 3.334211072631075, + "tokens_seen": 3183331328 + }, + { + "epoch": 1.35, + "learning_rate": 2.8822467402206623e-05, + "loss": 1.2017, + "theoretical_loss": 3.3342059028972475, + "tokens_seen": 3183396864 + }, + { + "epoch": 1.35, + "learning_rate": 2.880641925777332e-05, + "loss": 1.2262, + "theoretical_loss": 3.334200733299647, + "tokens_seen": 3183462400 + }, + { + "epoch": 1.35, + "learning_rate": 2.879037111334002e-05, + "loss": 1.1873, + "theoretical_loss": 3.3341955638382657, + "tokens_seen": 3183527936 + }, + { + "epoch": 1.35, + "learning_rate": 2.877432296890672e-05, + "loss": 1.2049, + "theoretical_loss": 3.3341903945130986, + "tokens_seen": 3183593472 + }, + { + "epoch": 1.35, + "learning_rate": 2.875827482447342e-05, + "loss": 1.1434, + "theoretical_loss": 3.3341852253241386, + "tokens_seen": 3183659008 + }, + { + "epoch": 1.35, + "learning_rate": 2.8742226680040126e-05, + "loss": 1.1489, + "theoretical_loss": 3.3341800562713795, + "tokens_seen": 3183724544 + }, + { + "epoch": 1.35, + "learning_rate": 2.8726178535606823e-05, + "loss": 1.1649, + "theoretical_loss": 3.3341748873548154, + "tokens_seen": 3183790080 + }, + { + "epoch": 1.35, + "learning_rate": 2.8710130391173524e-05, + "loss": 1.2537, + "theoretical_loss": 3.3341697185744392, + "tokens_seen": 3183855616 + }, + { + "epoch": 1.35, + "learning_rate": 2.8694082246740224e-05, + "loss": 1.1908, + "theoretical_loss": 3.334164549930245, + "tokens_seen": 3183921152 + }, + { + "epoch": 1.35, + "learning_rate": 2.867803410230692e-05, + "loss": 1.0904, + "theoretical_loss": 3.3341593814222255, + "tokens_seen": 3183986688 + }, + { + "epoch": 1.35, + "learning_rate": 2.8661985957873622e-05, + "loss": 1.2066, + "theoretical_loss": 3.3341542130503754, + "tokens_seen": 3184052224 + }, + { + "epoch": 1.35, + "learning_rate": 2.8645937813440323e-05, + "loss": 1.2161, + "theoretical_loss": 3.3341490448146875, + "tokens_seen": 3184117760 + }, + { + "epoch": 1.35, + "learning_rate": 2.862988966900702e-05, + "loss": 1.1816, + "theoretical_loss": 3.334143876715156, + "tokens_seen": 3184183296 + }, + { + "epoch": 1.35, + "learning_rate": 2.861384152457372e-05, + "loss": 1.1297, + "theoretical_loss": 3.3341387087517744, + "tokens_seen": 3184248832 + }, + { + "epoch": 1.35, + "learning_rate": 2.8597793380140425e-05, + "loss": 1.1626, + "theoretical_loss": 3.3341335409245363, + "tokens_seen": 3184314368 + }, + { + "epoch": 1.35, + "learning_rate": 2.8581745235707125e-05, + "loss": 1.1684, + "theoretical_loss": 3.334128373233435, + "tokens_seen": 3184379904 + }, + { + "epoch": 1.35, + "learning_rate": 2.8565697091273823e-05, + "loss": 1.1619, + "theoretical_loss": 3.3341232056784644, + "tokens_seen": 3184445440 + }, + { + "epoch": 1.35, + "learning_rate": 2.8549648946840523e-05, + "loss": 1.1774, + "theoretical_loss": 3.334118038259618, + "tokens_seen": 3184510976 + }, + { + "epoch": 1.35, + "learning_rate": 2.8533600802407224e-05, + "loss": 1.1114, + "theoretical_loss": 3.3341128709768895, + "tokens_seen": 3184576512 + }, + { + "epoch": 1.35, + "learning_rate": 2.851755265797392e-05, + "loss": 1.2351, + "theoretical_loss": 3.334107703830272, + "tokens_seen": 3184642048 + }, + { + "epoch": 1.35, + "learning_rate": 2.850150451354062e-05, + "loss": 1.1988, + "theoretical_loss": 3.3341025368197603, + "tokens_seen": 3184707584 + }, + { + "epoch": 1.35, + "learning_rate": 2.8485456369107322e-05, + "loss": 1.1916, + "theoretical_loss": 3.334097369945347, + "tokens_seen": 3184773120 + }, + { + "epoch": 1.35, + "learning_rate": 2.846940822467402e-05, + "loss": 1.206, + "theoretical_loss": 3.3340922032070264, + "tokens_seen": 3184838656 + }, + { + "epoch": 1.35, + "learning_rate": 2.8453360080240727e-05, + "loss": 1.232, + "theoretical_loss": 3.3340870366047914, + "tokens_seen": 3184904192 + }, + { + "epoch": 1.35, + "learning_rate": 2.8437311935807424e-05, + "loss": 1.1833, + "theoretical_loss": 3.3340818701386357, + "tokens_seen": 3184969728 + }, + { + "epoch": 1.35, + "objective/train/docs_used": 1174500, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.140540361404419, + "objective/train/theoretical_loss": 3.3340767038085533, + "objective/train/tokens_used": 3205495264, + "theoretical_loss": 3.3340767038085533, + "tokens_seen": 3185035264 + }, + { + "epoch": 1.35, + "learning_rate": 2.8421263791374125e-05, + "loss": 1.1405, + "theoretical_loss": 3.3340767038085533, + "tokens_seen": 3185035264 + }, + { + "epoch": 1.35, + "learning_rate": 2.8405215646940825e-05, + "loss": 1.1939, + "theoretical_loss": 3.3340715376145376, + "tokens_seen": 3185100800 + }, + { + "epoch": 1.35, + "learning_rate": 2.8389167502507523e-05, + "loss": 1.1937, + "theoretical_loss": 3.334066371556583, + "tokens_seen": 3185166336 + }, + { + "epoch": 1.35, + "learning_rate": 2.8373119358074223e-05, + "loss": 1.2489, + "theoretical_loss": 3.3340612056346814, + "tokens_seen": 3185231872 + }, + { + "epoch": 1.35, + "learning_rate": 2.8357071213640924e-05, + "loss": 1.1688, + "theoretical_loss": 3.3340560398488277, + "tokens_seen": 3185297408 + }, + { + "epoch": 1.35, + "learning_rate": 2.834102306920762e-05, + "loss": 1.2874, + "theoretical_loss": 3.334050874199016, + "tokens_seen": 3185362944 + }, + { + "epoch": 1.35, + "learning_rate": 2.8324974924774322e-05, + "loss": 1.2308, + "theoretical_loss": 3.334045708685238, + "tokens_seen": 3185428480 + }, + { + "epoch": 1.35, + "learning_rate": 2.8308926780341023e-05, + "loss": 1.3049, + "theoretical_loss": 3.334040543307489, + "tokens_seen": 3185494016 + }, + { + "epoch": 1.35, + "learning_rate": 2.8292878635907727e-05, + "loss": 1.2386, + "theoretical_loss": 3.3340353780657623, + "tokens_seen": 3185559552 + }, + { + "epoch": 1.35, + "learning_rate": 2.8276830491474427e-05, + "loss": 1.2258, + "theoretical_loss": 3.334030212960051, + "tokens_seen": 3185625088 + }, + { + "epoch": 1.35, + "learning_rate": 2.8260782347041124e-05, + "loss": 1.2842, + "theoretical_loss": 3.3340250479903495, + "tokens_seen": 3185690624 + }, + { + "epoch": 1.35, + "learning_rate": 2.8244734202607825e-05, + "loss": 1.221, + "theoretical_loss": 3.3340198831566505, + "tokens_seen": 3185756160 + }, + { + "epoch": 1.35, + "learning_rate": 2.8228686058174526e-05, + "loss": 1.2435, + "theoretical_loss": 3.334014718458948, + "tokens_seen": 3185821696 + }, + { + "epoch": 1.35, + "learning_rate": 2.8212637913741223e-05, + "loss": 1.1951, + "theoretical_loss": 3.3340095538972365, + "tokens_seen": 3185887232 + }, + { + "epoch": 1.35, + "learning_rate": 2.8196589769307924e-05, + "loss": 1.2539, + "theoretical_loss": 3.334004389471508, + "tokens_seen": 3185952768 + }, + { + "epoch": 1.35, + "learning_rate": 2.8180541624874624e-05, + "loss": 1.1397, + "theoretical_loss": 3.333999225181757, + "tokens_seen": 3186018304 + }, + { + "epoch": 1.35, + "learning_rate": 2.816449348044132e-05, + "loss": 1.2421, + "theoretical_loss": 3.3339940610279775, + "tokens_seen": 3186083840 + }, + { + "epoch": 1.35, + "learning_rate": 2.814844533600803e-05, + "loss": 1.1841, + "theoretical_loss": 3.3339888970101628, + "tokens_seen": 3186149376 + }, + { + "epoch": 1.35, + "learning_rate": 2.8132397191574726e-05, + "loss": 1.2635, + "theoretical_loss": 3.333983733128306, + "tokens_seen": 3186214912 + }, + { + "epoch": 1.35, + "learning_rate": 2.8116349047141427e-05, + "loss": 1.1303, + "theoretical_loss": 3.3339785693824013, + "tokens_seen": 3186280448 + }, + { + "epoch": 1.35, + "learning_rate": 2.8100300902708127e-05, + "loss": 1.1553, + "theoretical_loss": 3.333973405772442, + "tokens_seen": 3186345984 + }, + { + "epoch": 1.35, + "learning_rate": 2.8084252758274825e-05, + "loss": 1.1286, + "theoretical_loss": 3.333968242298422, + "tokens_seen": 3186411520 + }, + { + "epoch": 1.35, + "learning_rate": 2.8068204613841525e-05, + "loss": 1.2337, + "theoretical_loss": 3.333963078960335, + "tokens_seen": 3186477056 + }, + { + "epoch": 1.35, + "learning_rate": 2.8052156469408226e-05, + "loss": 1.2218, + "theoretical_loss": 3.333957915758174, + "tokens_seen": 3186542592 + }, + { + "epoch": 1.35, + "learning_rate": 2.8036108324974923e-05, + "loss": 1.2237, + "theoretical_loss": 3.3339527526919333, + "tokens_seen": 3186608128 + }, + { + "epoch": 1.35, + "learning_rate": 2.8020060180541624e-05, + "loss": 1.1916, + "theoretical_loss": 3.3339475897616064, + "tokens_seen": 3186673664 + }, + { + "epoch": 1.35, + "learning_rate": 2.8004012036108324e-05, + "loss": 1.1124, + "theoretical_loss": 3.333942426967187, + "tokens_seen": 3186739200 + }, + { + "epoch": 1.35, + "learning_rate": 2.798796389167503e-05, + "loss": 1.1864, + "theoretical_loss": 3.333937264308668, + "tokens_seen": 3186804736 + }, + { + "epoch": 1.35, + "learning_rate": 2.797191574724173e-05, + "loss": 1.2821, + "theoretical_loss": 3.333932101786044, + "tokens_seen": 3186870272 + }, + { + "epoch": 1.35, + "learning_rate": 2.7955867602808426e-05, + "loss": 1.1755, + "theoretical_loss": 3.333926939399308, + "tokens_seen": 3186935808 + }, + { + "epoch": 1.35, + "learning_rate": 2.7939819458375127e-05, + "loss": 1.1162, + "theoretical_loss": 3.333921777148454, + "tokens_seen": 3187001344 + }, + { + "epoch": 1.35, + "learning_rate": 2.7923771313941828e-05, + "loss": 1.1041, + "theoretical_loss": 3.333916615033475, + "tokens_seen": 3187066880 + }, + { + "epoch": 1.35, + "learning_rate": 2.7907723169508525e-05, + "loss": 1.182, + "theoretical_loss": 3.333911453054366, + "tokens_seen": 3187132416 + }, + { + "epoch": 1.35, + "learning_rate": 2.7891675025075225e-05, + "loss": 1.1585, + "theoretical_loss": 3.333906291211119, + "tokens_seen": 3187197952 + }, + { + "epoch": 1.35, + "learning_rate": 2.7875626880641926e-05, + "loss": 1.2023, + "theoretical_loss": 3.3339011295037286, + "tokens_seen": 3187263488 + }, + { + "epoch": 1.35, + "learning_rate": 2.7859578736208623e-05, + "loss": 1.3159, + "theoretical_loss": 3.3338959679321882, + "tokens_seen": 3187329024 + }, + { + "epoch": 1.35, + "learning_rate": 2.784353059177533e-05, + "loss": 1.1751, + "theoretical_loss": 3.333890806496491, + "tokens_seen": 3187394560 + }, + { + "epoch": 1.35, + "learning_rate": 2.7827482447342028e-05, + "loss": 1.1749, + "theoretical_loss": 3.3338856451966317, + "tokens_seen": 3187460096 + }, + { + "epoch": 1.35, + "learning_rate": 2.781143430290873e-05, + "loss": 1.187, + "theoretical_loss": 3.333880484032603, + "tokens_seen": 3187525632 + }, + { + "epoch": 1.35, + "learning_rate": 2.779538615847543e-05, + "loss": 1.1748, + "theoretical_loss": 3.333875323004399, + "tokens_seen": 3187591168 + }, + { + "epoch": 1.35, + "learning_rate": 2.7779338014042127e-05, + "loss": 1.1911, + "theoretical_loss": 3.333870162112013, + "tokens_seen": 3187656704 + }, + { + "epoch": 1.35, + "learning_rate": 2.7763289869608827e-05, + "loss": 1.1011, + "theoretical_loss": 3.3338650013554387, + "tokens_seen": 3187722240 + }, + { + "epoch": 1.35, + "learning_rate": 2.7747241725175528e-05, + "loss": 1.2096, + "theoretical_loss": 3.33385984073467, + "tokens_seen": 3187787776 + }, + { + "epoch": 1.35, + "learning_rate": 2.7731193580742225e-05, + "loss": 1.1976, + "theoretical_loss": 3.3338546802496998, + "tokens_seen": 3187853312 + }, + { + "epoch": 1.35, + "learning_rate": 2.7715145436308926e-05, + "loss": 1.1868, + "theoretical_loss": 3.333849519900523, + "tokens_seen": 3187918848 + }, + { + "epoch": 1.35, + "learning_rate": 2.769909729187563e-05, + "loss": 1.1778, + "theoretical_loss": 3.333844359687132, + "tokens_seen": 3187984384 + }, + { + "epoch": 1.35, + "learning_rate": 2.768304914744233e-05, + "loss": 1.1459, + "theoretical_loss": 3.333839199609521, + "tokens_seen": 3188049920 + }, + { + "epoch": 1.35, + "learning_rate": 2.766700100300903e-05, + "loss": 1.1384, + "theoretical_loss": 3.3338340396676838, + "tokens_seen": 3188115456 + }, + { + "epoch": 1.35, + "learning_rate": 2.7650952858575728e-05, + "loss": 1.2302, + "theoretical_loss": 3.3338288798616142, + "tokens_seen": 3188180992 + }, + { + "epoch": 1.35, + "learning_rate": 2.763490471414243e-05, + "loss": 1.2422, + "theoretical_loss": 3.333823720191305, + "tokens_seen": 3188246528 + }, + { + "epoch": 1.35, + "learning_rate": 2.761885656970913e-05, + "loss": 1.1784, + "theoretical_loss": 3.3338185606567503, + "tokens_seen": 3188312064 + }, + { + "epoch": 1.35, + "learning_rate": 2.7602808425275827e-05, + "loss": 1.1702, + "theoretical_loss": 3.333813401257944, + "tokens_seen": 3188377600 + }, + { + "epoch": 1.35, + "learning_rate": 2.7586760280842527e-05, + "loss": 1.1525, + "theoretical_loss": 3.333808241994879, + "tokens_seen": 3188443136 + }, + { + "epoch": 1.35, + "learning_rate": 2.7570712136409228e-05, + "loss": 1.3038, + "theoretical_loss": 3.33380308286755, + "tokens_seen": 3188508672 + }, + { + "epoch": 1.35, + "learning_rate": 2.7554663991975925e-05, + "loss": 1.2047, + "theoretical_loss": 3.3337979238759496, + "tokens_seen": 3188574208 + }, + { + "epoch": 1.35, + "learning_rate": 2.7538615847542633e-05, + "loss": 1.0656, + "theoretical_loss": 3.333792765020072, + "tokens_seen": 3188639744 + }, + { + "epoch": 1.35, + "learning_rate": 2.752256770310933e-05, + "loss": 1.1386, + "theoretical_loss": 3.3337876062999108, + "tokens_seen": 3188705280 + }, + { + "epoch": 1.35, + "learning_rate": 2.750651955867603e-05, + "loss": 1.162, + "theoretical_loss": 3.3337824477154596, + "tokens_seen": 3188770816 + }, + { + "epoch": 1.35, + "learning_rate": 2.749047141424273e-05, + "loss": 1.1302, + "theoretical_loss": 3.333777289266712, + "tokens_seen": 3188836352 + }, + { + "epoch": 1.35, + "learning_rate": 2.747442326980943e-05, + "loss": 1.1101, + "theoretical_loss": 3.3337721309536614, + "tokens_seen": 3188901888 + }, + { + "epoch": 1.35, + "learning_rate": 2.745837512537613e-05, + "loss": 1.1791, + "theoretical_loss": 3.333766972776302, + "tokens_seen": 3188967424 + }, + { + "epoch": 1.35, + "learning_rate": 2.744232698094283e-05, + "loss": 1.1713, + "theoretical_loss": 3.3337618147346273, + "tokens_seen": 3189032960 + }, + { + "epoch": 1.35, + "learning_rate": 2.7426278836509527e-05, + "loss": 1.1208, + "theoretical_loss": 3.3337566568286303, + "tokens_seen": 3189098496 + }, + { + "epoch": 1.35, + "learning_rate": 2.7410230692076228e-05, + "loss": 1.1476, + "theoretical_loss": 3.3337514990583053, + "tokens_seen": 3189164032 + }, + { + "epoch": 1.35, + "learning_rate": 2.739418254764293e-05, + "loss": 1.2081, + "theoretical_loss": 3.333746341423646, + "tokens_seen": 3189229568 + }, + { + "epoch": 1.35, + "learning_rate": 2.7378134403209632e-05, + "loss": 1.0551, + "theoretical_loss": 3.3337411839246456, + "tokens_seen": 3189295104 + }, + { + "epoch": 1.35, + "learning_rate": 2.7362086258776333e-05, + "loss": 1.1898, + "theoretical_loss": 3.333736026561298, + "tokens_seen": 3189360640 + }, + { + "epoch": 1.35, + "learning_rate": 2.734603811434303e-05, + "loss": 1.1792, + "theoretical_loss": 3.333730869333597, + "tokens_seen": 3189426176 + }, + { + "epoch": 1.35, + "learning_rate": 2.732998996990973e-05, + "loss": 1.1267, + "theoretical_loss": 3.3337257122415362, + "tokens_seen": 3189491712 + }, + { + "epoch": 1.35, + "learning_rate": 2.731394182547643e-05, + "loss": 1.1784, + "theoretical_loss": 3.3337205552851086, + "tokens_seen": 3189557248 + }, + { + "epoch": 1.35, + "learning_rate": 2.729789368104313e-05, + "loss": 1.191, + "theoretical_loss": 3.3337153984643084, + "tokens_seen": 3189622784 + }, + { + "epoch": 1.35, + "learning_rate": 2.728184553660983e-05, + "loss": 1.1052, + "theoretical_loss": 3.333710241779129, + "tokens_seen": 3189688320 + }, + { + "epoch": 1.35, + "learning_rate": 2.726579739217653e-05, + "loss": 1.2089, + "theoretical_loss": 3.333705085229565, + "tokens_seen": 3189753856 + }, + { + "epoch": 1.35, + "learning_rate": 2.7249749247743234e-05, + "loss": 1.1485, + "theoretical_loss": 3.3336999288156086, + "tokens_seen": 3189819392 + }, + { + "epoch": 1.35, + "learning_rate": 2.7233701103309935e-05, + "loss": 1.1938, + "theoretical_loss": 3.3336947725372545, + "tokens_seen": 3189884928 + }, + { + "epoch": 1.35, + "learning_rate": 2.7217652958876632e-05, + "loss": 1.1622, + "theoretical_loss": 3.333689616394496, + "tokens_seen": 3189950464 + }, + { + "epoch": 1.35, + "learning_rate": 2.7201604814443332e-05, + "loss": 1.217, + "theoretical_loss": 3.3336844603873264, + "tokens_seen": 3190016000 + }, + { + "epoch": 1.35, + "learning_rate": 2.7185556670010033e-05, + "loss": 1.1787, + "theoretical_loss": 3.33367930451574, + "tokens_seen": 3190081536 + }, + { + "epoch": 1.35, + "learning_rate": 2.716950852557673e-05, + "loss": 1.29, + "theoretical_loss": 3.33367414877973, + "tokens_seen": 3190147072 + }, + { + "epoch": 1.35, + "learning_rate": 2.715346038114343e-05, + "loss": 1.1974, + "theoretical_loss": 3.33366899317929, + "tokens_seen": 3190212608 + }, + { + "epoch": 1.35, + "learning_rate": 2.713741223671013e-05, + "loss": 1.1677, + "theoretical_loss": 3.333663837714414, + "tokens_seen": 3190278144 + }, + { + "epoch": 1.35, + "learning_rate": 2.712136409227683e-05, + "loss": 1.1628, + "theoretical_loss": 3.3336586823850958, + "tokens_seen": 3190343680 + }, + { + "epoch": 1.35, + "learning_rate": 2.710531594784353e-05, + "loss": 1.2065, + "theoretical_loss": 3.333653527191328, + "tokens_seen": 3190409216 + }, + { + "epoch": 1.35, + "learning_rate": 2.7089267803410233e-05, + "loss": 1.1703, + "theoretical_loss": 3.3336483721331054, + "tokens_seen": 3190474752 + }, + { + "epoch": 1.35, + "learning_rate": 2.7073219658976934e-05, + "loss": 1.1195, + "theoretical_loss": 3.3336432172104216, + "tokens_seen": 3190540288 + }, + { + "epoch": 1.35, + "learning_rate": 2.7057171514543635e-05, + "loss": 1.1839, + "theoretical_loss": 3.333638062423269, + "tokens_seen": 3190605824 + }, + { + "epoch": 1.35, + "learning_rate": 2.7041123370110332e-05, + "loss": 1.1848, + "theoretical_loss": 3.333632907771643, + "tokens_seen": 3190671360 + }, + { + "epoch": 1.35, + "learning_rate": 2.7025075225677033e-05, + "loss": 1.1965, + "theoretical_loss": 3.3336277532555356, + "tokens_seen": 3190736896 + }, + { + "epoch": 1.35, + "learning_rate": 2.7009027081243733e-05, + "loss": 1.2415, + "theoretical_loss": 3.3336225988749417, + "tokens_seen": 3190802432 + }, + { + "epoch": 1.35, + "learning_rate": 2.699297893681043e-05, + "loss": 1.1666, + "theoretical_loss": 3.3336174446298545, + "tokens_seen": 3190867968 + }, + { + "epoch": 1.35, + "learning_rate": 2.697693079237713e-05, + "loss": 1.2059, + "theoretical_loss": 3.333612290520268, + "tokens_seen": 3190933504 + }, + { + "epoch": 1.35, + "learning_rate": 2.6960882647943832e-05, + "loss": 1.2271, + "theoretical_loss": 3.333607136546175, + "tokens_seen": 3190999040 + }, + { + "epoch": 1.35, + "learning_rate": 2.6944834503510536e-05, + "loss": 1.1712, + "theoretical_loss": 3.3336019827075694, + "tokens_seen": 3191064576 + }, + { + "epoch": 1.35, + "learning_rate": 2.6928786359077236e-05, + "loss": 1.2669, + "theoretical_loss": 3.3335968290044455, + "tokens_seen": 3191130112 + }, + { + "epoch": 1.35, + "learning_rate": 2.6912738214643934e-05, + "loss": 1.156, + "theoretical_loss": 3.3335916754367965, + "tokens_seen": 3191195648 + }, + { + "epoch": 1.35, + "learning_rate": 2.6896690070210634e-05, + "loss": 1.2287, + "theoretical_loss": 3.3335865220046164, + "tokens_seen": 3191261184 + }, + { + "epoch": 1.35, + "learning_rate": 2.6880641925777335e-05, + "loss": 1.0932, + "theoretical_loss": 3.3335813687078986, + "tokens_seen": 3191326720 + }, + { + "epoch": 1.35, + "learning_rate": 2.6864593781344032e-05, + "loss": 1.2138, + "theoretical_loss": 3.3335762155466364, + "tokens_seen": 3191392256 + }, + { + "epoch": 1.35, + "learning_rate": 2.6848545636910733e-05, + "loss": 1.2165, + "theoretical_loss": 3.3335710625208237, + "tokens_seen": 3191457792 + }, + { + "epoch": 1.35, + "learning_rate": 2.6832497492477433e-05, + "loss": 1.1817, + "theoretical_loss": 3.3335659096304546, + "tokens_seen": 3191523328 + }, + { + "epoch": 1.35, + "objective/train/docs_used": 1177192, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1170263290405273, + "objective/train/theoretical_loss": 3.3335607568755226, + "objective/train/tokens_used": 3212048864, + "theoretical_loss": 3.3335607568755226, + "tokens_seen": 3191588864 + }, + { + "epoch": 1.35, + "learning_rate": 2.681644934804413e-05, + "loss": 1.117, + "theoretical_loss": 3.3335607568755226, + "tokens_seen": 3191588864 + }, + { + "epoch": 1.35, + "learning_rate": 2.6800401203610838e-05, + "loss": 1.1931, + "theoretical_loss": 3.333555604256021, + "tokens_seen": 3191654400 + }, + { + "epoch": 1.35, + "learning_rate": 2.6784353059177535e-05, + "loss": 1.2027, + "theoretical_loss": 3.333550451771943, + "tokens_seen": 3191719936 + }, + { + "epoch": 1.35, + "learning_rate": 2.6768304914744236e-05, + "loss": 1.1606, + "theoretical_loss": 3.3335452994232835, + "tokens_seen": 3191785472 + }, + { + "epoch": 1.35, + "learning_rate": 2.6752256770310937e-05, + "loss": 1.1208, + "theoretical_loss": 3.3335401472100354, + "tokens_seen": 3191851008 + }, + { + "epoch": 1.35, + "learning_rate": 2.6736208625877634e-05, + "loss": 1.1018, + "theoretical_loss": 3.333534995132193, + "tokens_seen": 3191916544 + }, + { + "epoch": 1.35, + "learning_rate": 2.6720160481444335e-05, + "loss": 1.1866, + "theoretical_loss": 3.3335298431897487, + "tokens_seen": 3191982080 + }, + { + "epoch": 1.35, + "learning_rate": 2.6704112337011035e-05, + "loss": 1.102, + "theoretical_loss": 3.3335246913826975, + "tokens_seen": 3192047616 + }, + { + "epoch": 1.35, + "learning_rate": 2.6688064192577732e-05, + "loss": 1.2369, + "theoretical_loss": 3.333519539711032, + "tokens_seen": 3192113152 + }, + { + "epoch": 1.35, + "learning_rate": 2.6672016048144433e-05, + "loss": 1.1288, + "theoretical_loss": 3.333514388174747, + "tokens_seen": 3192178688 + }, + { + "epoch": 1.35, + "learning_rate": 2.6655967903711134e-05, + "loss": 1.1969, + "theoretical_loss": 3.3335092367738355, + "tokens_seen": 3192244224 + }, + { + "epoch": 1.35, + "learning_rate": 2.6639919759277838e-05, + "loss": 1.1364, + "theoretical_loss": 3.3335040855082907, + "tokens_seen": 3192309760 + }, + { + "epoch": 1.35, + "learning_rate": 2.6623871614844535e-05, + "loss": 1.1426, + "theoretical_loss": 3.333498934378107, + "tokens_seen": 3192375296 + }, + { + "epoch": 1.35, + "learning_rate": 2.6607823470411236e-05, + "loss": 1.2038, + "theoretical_loss": 3.333493783383278, + "tokens_seen": 3192440832 + }, + { + "epoch": 1.35, + "learning_rate": 2.6591775325977936e-05, + "loss": 1.1236, + "theoretical_loss": 3.333488632523797, + "tokens_seen": 3192506368 + }, + { + "epoch": 1.35, + "learning_rate": 2.6575727181544633e-05, + "loss": 1.2372, + "theoretical_loss": 3.3334834817996577, + "tokens_seen": 3192571904 + }, + { + "epoch": 1.35, + "learning_rate": 2.6559679037111334e-05, + "loss": 1.098, + "theoretical_loss": 3.3334783312108542, + "tokens_seen": 3192637440 + }, + { + "epoch": 1.35, + "learning_rate": 2.6543630892678035e-05, + "loss": 1.1909, + "theoretical_loss": 3.33347318075738, + "tokens_seen": 3192702976 + }, + { + "epoch": 1.35, + "learning_rate": 2.6527582748244732e-05, + "loss": 1.1991, + "theoretical_loss": 3.3334680304392283, + "tokens_seen": 3192768512 + }, + { + "epoch": 1.35, + "learning_rate": 2.6511534603811433e-05, + "loss": 1.1959, + "theoretical_loss": 3.333462880256393, + "tokens_seen": 3192834048 + }, + { + "epoch": 1.35, + "learning_rate": 2.6495486459378137e-05, + "loss": 1.176, + "theoretical_loss": 3.333457730208868, + "tokens_seen": 3192899584 + }, + { + "epoch": 1.35, + "learning_rate": 2.6479438314944837e-05, + "loss": 1.341, + "theoretical_loss": 3.3334525802966475, + "tokens_seen": 3192965120 + }, + { + "epoch": 1.35, + "learning_rate": 2.6463390170511538e-05, + "loss": 1.1349, + "theoretical_loss": 3.333447430519724, + "tokens_seen": 3193030656 + }, + { + "epoch": 1.35, + "learning_rate": 2.6447342026078235e-05, + "loss": 1.1719, + "theoretical_loss": 3.3334422808780912, + "tokens_seen": 3193096192 + }, + { + "epoch": 1.35, + "learning_rate": 2.6431293881644936e-05, + "loss": 1.1313, + "theoretical_loss": 3.333437131371744, + "tokens_seen": 3193161728 + }, + { + "epoch": 1.35, + "learning_rate": 2.6415245737211636e-05, + "loss": 1.2111, + "theoretical_loss": 3.333431982000675, + "tokens_seen": 3193227264 + }, + { + "epoch": 1.35, + "learning_rate": 2.6399197592778334e-05, + "loss": 1.22, + "theoretical_loss": 3.3334268327648786, + "tokens_seen": 3193292800 + }, + { + "epoch": 1.35, + "learning_rate": 2.6383149448345034e-05, + "loss": 1.274, + "theoretical_loss": 3.3334216836643478, + "tokens_seen": 3193358336 + }, + { + "epoch": 1.35, + "learning_rate": 2.6367101303911735e-05, + "loss": 1.1335, + "theoretical_loss": 3.3334165346990767, + "tokens_seen": 3193423872 + }, + { + "epoch": 1.35, + "learning_rate": 2.635105315947844e-05, + "loss": 1.1315, + "theoretical_loss": 3.3334113858690584, + "tokens_seen": 3193489408 + }, + { + "epoch": 1.35, + "learning_rate": 2.633500501504514e-05, + "loss": 1.1079, + "theoretical_loss": 3.3334062371742874, + "tokens_seen": 3193554944 + }, + { + "epoch": 1.35, + "learning_rate": 2.6318956870611837e-05, + "loss": 1.1602, + "theoretical_loss": 3.3334010886147567, + "tokens_seen": 3193620480 + }, + { + "epoch": 1.35, + "learning_rate": 2.6302908726178537e-05, + "loss": 1.165, + "theoretical_loss": 3.33339594019046, + "tokens_seen": 3193686016 + }, + { + "epoch": 1.35, + "learning_rate": 2.6286860581745238e-05, + "loss": 1.2327, + "theoretical_loss": 3.3333907919013916, + "tokens_seen": 3193751552 + }, + { + "epoch": 1.35, + "learning_rate": 2.6270812437311935e-05, + "loss": 1.2351, + "theoretical_loss": 3.333385643747545, + "tokens_seen": 3193817088 + }, + { + "epoch": 1.35, + "learning_rate": 2.6254764292878636e-05, + "loss": 1.124, + "theoretical_loss": 3.333380495728913, + "tokens_seen": 3193882624 + }, + { + "epoch": 1.35, + "learning_rate": 2.6238716148445337e-05, + "loss": 1.1859, + "theoretical_loss": 3.3333753478454904, + "tokens_seen": 3193948160 + }, + { + "epoch": 1.35, + "learning_rate": 2.6222668004012034e-05, + "loss": 1.1589, + "theoretical_loss": 3.3333702000972703, + "tokens_seen": 3194013696 + }, + { + "epoch": 1.35, + "learning_rate": 2.6206619859578735e-05, + "loss": 1.2319, + "theoretical_loss": 3.3333650524842464, + "tokens_seen": 3194079232 + }, + { + "epoch": 1.35, + "learning_rate": 2.619057171514544e-05, + "loss": 1.1626, + "theoretical_loss": 3.3333599050064127, + "tokens_seen": 3194144768 + }, + { + "epoch": 1.35, + "learning_rate": 2.617452357071214e-05, + "loss": 1.1489, + "theoretical_loss": 3.3333547576637623, + "tokens_seen": 3194210304 + }, + { + "epoch": 1.35, + "learning_rate": 2.615847542627884e-05, + "loss": 1.1932, + "theoretical_loss": 3.333349610456289, + "tokens_seen": 3194275840 + }, + { + "epoch": 1.35, + "learning_rate": 2.6142427281845537e-05, + "loss": 1.1067, + "theoretical_loss": 3.333344463383987, + "tokens_seen": 3194341376 + }, + { + "epoch": 1.35, + "learning_rate": 2.6126379137412238e-05, + "loss": 1.2632, + "theoretical_loss": 3.3333393164468497, + "tokens_seen": 3194406912 + }, + { + "epoch": 1.35, + "learning_rate": 2.611033099297894e-05, + "loss": 1.1635, + "theoretical_loss": 3.333334169644871, + "tokens_seen": 3194472448 + }, + { + "epoch": 1.35, + "learning_rate": 2.6094282848545636e-05, + "loss": 1.1114, + "theoretical_loss": 3.333329022978044, + "tokens_seen": 3194537984 + }, + { + "epoch": 1.35, + "learning_rate": 2.6078234704112336e-05, + "loss": 1.0966, + "theoretical_loss": 3.3333238764463626, + "tokens_seen": 3194603520 + }, + { + "epoch": 1.35, + "learning_rate": 2.6062186559679037e-05, + "loss": 1.1519, + "theoretical_loss": 3.3333187300498204, + "tokens_seen": 3194669056 + }, + { + "epoch": 1.35, + "learning_rate": 2.604613841524574e-05, + "loss": 1.1245, + "theoretical_loss": 3.3333135837884114, + "tokens_seen": 3194734592 + }, + { + "epoch": 1.35, + "learning_rate": 2.603009027081244e-05, + "loss": 1.125, + "theoretical_loss": 3.3333084376621294, + "tokens_seen": 3194800128 + }, + { + "epoch": 1.35, + "learning_rate": 2.601404212637914e-05, + "loss": 1.2199, + "theoretical_loss": 3.3333032916709677, + "tokens_seen": 3194865664 + }, + { + "epoch": 1.35, + "learning_rate": 2.599799398194584e-05, + "loss": 1.2316, + "theoretical_loss": 3.33329814581492, + "tokens_seen": 3194931200 + }, + { + "epoch": 1.35, + "learning_rate": 2.598194583751254e-05, + "loss": 1.1898, + "theoretical_loss": 3.3332930000939798, + "tokens_seen": 3194996736 + }, + { + "epoch": 1.35, + "learning_rate": 2.5965897693079237e-05, + "loss": 1.2386, + "theoretical_loss": 3.3332878545081415, + "tokens_seen": 3195062272 + }, + { + "epoch": 1.35, + "learning_rate": 2.5949849548645938e-05, + "loss": 1.2893, + "theoretical_loss": 3.333282709057398, + "tokens_seen": 3195127808 + }, + { + "epoch": 1.35, + "learning_rate": 2.593380140421264e-05, + "loss": 1.1367, + "theoretical_loss": 3.3332775637417438, + "tokens_seen": 3195193344 + }, + { + "epoch": 1.35, + "learning_rate": 2.5917753259779336e-05, + "loss": 1.2754, + "theoretical_loss": 3.3332724185611715, + "tokens_seen": 3195258880 + }, + { + "epoch": 1.35, + "learning_rate": 2.5901705115346036e-05, + "loss": 1.1783, + "theoretical_loss": 3.3332672735156756, + "tokens_seen": 3195324416 + }, + { + "epoch": 1.35, + "learning_rate": 2.588565697091274e-05, + "loss": 1.188, + "theoretical_loss": 3.3332621286052495, + "tokens_seen": 3195389952 + }, + { + "epoch": 1.35, + "learning_rate": 2.586960882647944e-05, + "loss": 1.1576, + "theoretical_loss": 3.3332569838298873, + "tokens_seen": 3195455488 + }, + { + "epoch": 1.35, + "learning_rate": 2.585356068204614e-05, + "loss": 1.1693, + "theoretical_loss": 3.333251839189582, + "tokens_seen": 3195521024 + }, + { + "epoch": 1.35, + "learning_rate": 2.583751253761284e-05, + "loss": 1.2983, + "theoretical_loss": 3.3332466946843278, + "tokens_seen": 3195586560 + }, + { + "epoch": 1.35, + "learning_rate": 2.582146439317954e-05, + "loss": 1.2104, + "theoretical_loss": 3.333241550314118, + "tokens_seen": 3195652096 + }, + { + "epoch": 1.35, + "learning_rate": 2.580541624874624e-05, + "loss": 1.2786, + "theoretical_loss": 3.333236406078947, + "tokens_seen": 3195717632 + }, + { + "epoch": 1.35, + "learning_rate": 2.5789368104312937e-05, + "loss": 1.2186, + "theoretical_loss": 3.333231261978807, + "tokens_seen": 3195783168 + }, + { + "epoch": 1.35, + "learning_rate": 2.5773319959879638e-05, + "loss": 1.2312, + "theoretical_loss": 3.3332261180136937, + "tokens_seen": 3195848704 + }, + { + "epoch": 1.35, + "learning_rate": 2.575727181544634e-05, + "loss": 1.2026, + "theoretical_loss": 3.333220974183599, + "tokens_seen": 3195914240 + }, + { + "epoch": 1.35, + "learning_rate": 2.5741223671013043e-05, + "loss": 1.0349, + "theoretical_loss": 3.3332158304885176, + "tokens_seen": 3195979776 + }, + { + "epoch": 1.35, + "learning_rate": 2.5725175526579743e-05, + "loss": 1.16, + "theoretical_loss": 3.3332106869284432, + "tokens_seen": 3196045312 + }, + { + "epoch": 1.35, + "learning_rate": 2.570912738214644e-05, + "loss": 1.2023, + "theoretical_loss": 3.333205543503369, + "tokens_seen": 3196110848 + }, + { + "epoch": 1.35, + "learning_rate": 2.569307923771314e-05, + "loss": 1.2397, + "theoretical_loss": 3.333200400213289, + "tokens_seen": 3196176384 + }, + { + "epoch": 1.35, + "learning_rate": 2.5677031093279842e-05, + "loss": 1.2099, + "theoretical_loss": 3.3331952570581964, + "tokens_seen": 3196241920 + }, + { + "epoch": 1.35, + "learning_rate": 2.566098294884654e-05, + "loss": 1.129, + "theoretical_loss": 3.3331901140380853, + "tokens_seen": 3196307456 + }, + { + "epoch": 1.35, + "learning_rate": 2.564493480441324e-05, + "loss": 1.1244, + "theoretical_loss": 3.3331849711529498, + "tokens_seen": 3196372992 + }, + { + "epoch": 1.35, + "learning_rate": 2.562888665997994e-05, + "loss": 1.1554, + "theoretical_loss": 3.3331798284027827, + "tokens_seen": 3196438528 + }, + { + "epoch": 1.35, + "learning_rate": 2.5612838515546638e-05, + "loss": 1.1006, + "theoretical_loss": 3.3331746857875784, + "tokens_seen": 3196504064 + }, + { + "epoch": 1.35, + "learning_rate": 2.5596790371113345e-05, + "loss": 1.0714, + "theoretical_loss": 3.3331695433073305, + "tokens_seen": 3196569600 + }, + { + "epoch": 1.35, + "learning_rate": 2.5580742226680042e-05, + "loss": 1.1864, + "theoretical_loss": 3.333164400962032, + "tokens_seen": 3196635136 + }, + { + "epoch": 1.35, + "learning_rate": 2.5564694082246743e-05, + "loss": 1.2438, + "theoretical_loss": 3.3331592587516776, + "tokens_seen": 3196700672 + }, + { + "epoch": 1.35, + "learning_rate": 2.5548645937813444e-05, + "loss": 1.2742, + "theoretical_loss": 3.3331541166762606, + "tokens_seen": 3196766208 + }, + { + "epoch": 1.35, + "learning_rate": 2.553259779338014e-05, + "loss": 1.0778, + "theoretical_loss": 3.3331489747357743, + "tokens_seen": 3196831744 + }, + { + "epoch": 1.35, + "learning_rate": 2.551654964894684e-05, + "loss": 1.1102, + "theoretical_loss": 3.3331438329302125, + "tokens_seen": 3196897280 + }, + { + "epoch": 1.35, + "learning_rate": 2.5500501504513542e-05, + "loss": 1.2343, + "theoretical_loss": 3.3331386912595695, + "tokens_seen": 3196962816 + }, + { + "epoch": 1.35, + "learning_rate": 2.548445336008024e-05, + "loss": 1.1115, + "theoretical_loss": 3.3331335497238386, + "tokens_seen": 3197028352 + }, + { + "epoch": 1.35, + "learning_rate": 2.546840521564694e-05, + "loss": 1.1353, + "theoretical_loss": 3.3331284083230135, + "tokens_seen": 3197093888 + }, + { + "epoch": 1.35, + "learning_rate": 2.545235707121364e-05, + "loss": 1.2479, + "theoretical_loss": 3.3331232670570876, + "tokens_seen": 3197159424 + }, + { + "epoch": 1.35, + "learning_rate": 2.5436308926780345e-05, + "loss": 1.1591, + "theoretical_loss": 3.333118125926055, + "tokens_seen": 3197224960 + }, + { + "epoch": 1.35, + "learning_rate": 2.5420260782347045e-05, + "loss": 1.0968, + "theoretical_loss": 3.3331129849299095, + "tokens_seen": 3197290496 + }, + { + "epoch": 1.35, + "learning_rate": 2.5404212637913743e-05, + "loss": 1.1294, + "theoretical_loss": 3.3331078440686444, + "tokens_seen": 3197356032 + }, + { + "epoch": 1.35, + "learning_rate": 2.5388164493480443e-05, + "loss": 1.1911, + "theoretical_loss": 3.3331027033422536, + "tokens_seen": 3197421568 + }, + { + "epoch": 1.35, + "learning_rate": 2.5372116349047144e-05, + "loss": 1.1506, + "theoretical_loss": 3.3330975627507304, + "tokens_seen": 3197487104 + }, + { + "epoch": 1.35, + "learning_rate": 2.535606820461384e-05, + "loss": 1.202, + "theoretical_loss": 3.333092422294069, + "tokens_seen": 3197552640 + }, + { + "epoch": 1.35, + "learning_rate": 2.534002006018054e-05, + "loss": 1.1722, + "theoretical_loss": 3.3330872819722637, + "tokens_seen": 3197618176 + }, + { + "epoch": 1.35, + "learning_rate": 2.5323971915747242e-05, + "loss": 1.1451, + "theoretical_loss": 3.3330821417853067, + "tokens_seen": 3197683712 + }, + { + "epoch": 1.35, + "learning_rate": 2.530792377131394e-05, + "loss": 1.2472, + "theoretical_loss": 3.3330770017331925, + "tokens_seen": 3197749248 + }, + { + "epoch": 1.35, + "learning_rate": 2.5291875626880647e-05, + "loss": 1.2187, + "theoretical_loss": 3.333071861815915, + "tokens_seen": 3197814784 + }, + { + "epoch": 1.35, + "learning_rate": 2.5275827482447344e-05, + "loss": 1.2004, + "theoretical_loss": 3.3330667220334673, + "tokens_seen": 3197880320 + }, + { + "epoch": 1.35, + "learning_rate": 2.5259779338014045e-05, + "loss": 1.21, + "theoretical_loss": 3.333061582385844, + "tokens_seen": 3197945856 + }, + { + "epoch": 1.35, + "learning_rate": 2.5243731193580745e-05, + "loss": 1.1623, + "theoretical_loss": 3.333056442873038, + "tokens_seen": 3198011392 + }, + { + "epoch": 1.35, + "learning_rate": 2.5227683049147443e-05, + "loss": 1.047, + "theoretical_loss": 3.333051303495043, + "tokens_seen": 3198076928 + }, + { + "epoch": 1.35, + "objective/train/docs_used": 1179560, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1728415489196777, + "objective/train/theoretical_loss": 3.3330461642518534, + "objective/train/tokens_used": 3218602464, + "theoretical_loss": 3.3330461642518534, + "tokens_seen": 3198142464 + }, + { + "epoch": 1.35, + "learning_rate": 2.5211634904714143e-05, + "loss": 1.1728, + "theoretical_loss": 3.3330461642518534, + "tokens_seen": 3198142464 + }, + { + "epoch": 1.35, + "learning_rate": 2.5195586760280844e-05, + "loss": 1.1486, + "theoretical_loss": 3.333041025143462, + "tokens_seen": 3198208000 + }, + { + "epoch": 1.35, + "learning_rate": 2.517953861584754e-05, + "loss": 1.1839, + "theoretical_loss": 3.333035886169863, + "tokens_seen": 3198273536 + }, + { + "epoch": 1.35, + "learning_rate": 2.5163490471414242e-05, + "loss": 1.2974, + "theoretical_loss": 3.3330307473310503, + "tokens_seen": 3198339072 + }, + { + "epoch": 1.35, + "learning_rate": 2.5147442326980946e-05, + "loss": 1.196, + "theoretical_loss": 3.3330256086270174, + "tokens_seen": 3198404608 + }, + { + "epoch": 1.35, + "learning_rate": 2.5131394182547647e-05, + "loss": 1.1944, + "theoretical_loss": 3.333020470057758, + "tokens_seen": 3198470144 + }, + { + "epoch": 1.35, + "learning_rate": 2.5115346038114347e-05, + "loss": 1.1133, + "theoretical_loss": 3.3330153316232654, + "tokens_seen": 3198535680 + }, + { + "epoch": 1.35, + "learning_rate": 2.5099297893681044e-05, + "loss": 1.1414, + "theoretical_loss": 3.333010193323534, + "tokens_seen": 3198601216 + }, + { + "epoch": 1.35, + "learning_rate": 2.5083249749247745e-05, + "loss": 1.265, + "theoretical_loss": 3.3330050551585573, + "tokens_seen": 3198666752 + }, + { + "epoch": 1.35, + "learning_rate": 2.5067201604814446e-05, + "loss": 1.2, + "theoretical_loss": 3.3329999171283284, + "tokens_seen": 3198732288 + }, + { + "epoch": 1.35, + "learning_rate": 2.5051153460381143e-05, + "loss": 1.1693, + "theoretical_loss": 3.332994779232842, + "tokens_seen": 3198797824 + }, + { + "epoch": 1.35, + "learning_rate": 2.5035105315947844e-05, + "loss": 1.1026, + "theoretical_loss": 3.332989641472091, + "tokens_seen": 3198863360 + }, + { + "epoch": 1.35, + "learning_rate": 2.5019057171514544e-05, + "loss": 1.0373, + "theoretical_loss": 3.3329845038460695, + "tokens_seen": 3198928896 + }, + { + "epoch": 1.35, + "learning_rate": 2.500300902708124e-05, + "loss": 1.1583, + "theoretical_loss": 3.332979366354771, + "tokens_seen": 3198994432 + }, + { + "epoch": 1.35, + "learning_rate": 2.4986960882647945e-05, + "loss": 1.189, + "theoretical_loss": 3.3329742289981894, + "tokens_seen": 3199059968 + }, + { + "epoch": 1.35, + "learning_rate": 2.4970912738214643e-05, + "loss": 1.0693, + "theoretical_loss": 3.3329690917763184, + "tokens_seen": 3199125504 + }, + { + "epoch": 1.35, + "learning_rate": 2.4954864593781347e-05, + "loss": 1.1411, + "theoretical_loss": 3.3329639546891516, + "tokens_seen": 3199191040 + }, + { + "epoch": 1.35, + "learning_rate": 2.4938816449348047e-05, + "loss": 1.1762, + "theoretical_loss": 3.3329588177366825, + "tokens_seen": 3199256576 + }, + { + "epoch": 1.35, + "learning_rate": 2.4922768304914745e-05, + "loss": 1.1937, + "theoretical_loss": 3.3329536809189055, + "tokens_seen": 3199322112 + }, + { + "epoch": 1.35, + "learning_rate": 2.4906720160481445e-05, + "loss": 1.1706, + "theoretical_loss": 3.3329485442358138, + "tokens_seen": 3199387648 + }, + { + "epoch": 1.35, + "learning_rate": 2.4890672016048146e-05, + "loss": 1.1835, + "theoretical_loss": 3.3329434076874005, + "tokens_seen": 3199453184 + }, + { + "epoch": 1.35, + "learning_rate": 2.4874623871614847e-05, + "loss": 1.1553, + "theoretical_loss": 3.332938271273661, + "tokens_seen": 3199518720 + }, + { + "epoch": 1.35, + "learning_rate": 2.4858575727181547e-05, + "loss": 1.1987, + "theoretical_loss": 3.332933134994587, + "tokens_seen": 3199584256 + }, + { + "epoch": 1.35, + "learning_rate": 2.4842527582748244e-05, + "loss": 1.0929, + "theoretical_loss": 3.332927998850174, + "tokens_seen": 3199649792 + }, + { + "epoch": 1.35, + "learning_rate": 2.4826479438314945e-05, + "loss": 1.1966, + "theoretical_loss": 3.3329228628404146, + "tokens_seen": 3199715328 + }, + { + "epoch": 1.35, + "learning_rate": 2.481043129388165e-05, + "loss": 1.2217, + "theoretical_loss": 3.3329177269653028, + "tokens_seen": 3199780864 + }, + { + "epoch": 1.35, + "learning_rate": 2.4794383149448346e-05, + "loss": 1.2207, + "theoretical_loss": 3.3329125912248325, + "tokens_seen": 3199846400 + }, + { + "epoch": 1.35, + "learning_rate": 2.4778335005015047e-05, + "loss": 1.1664, + "theoretical_loss": 3.332907455618997, + "tokens_seen": 3199911936 + }, + { + "epoch": 1.35, + "learning_rate": 2.4762286860581748e-05, + "loss": 1.2357, + "theoretical_loss": 3.3329023201477903, + "tokens_seen": 3199977472 + }, + { + "epoch": 1.35, + "learning_rate": 2.4746238716148445e-05, + "loss": 1.146, + "theoretical_loss": 3.3328971848112063, + "tokens_seen": 3200043008 + }, + { + "epoch": 1.35, + "learning_rate": 2.473019057171515e-05, + "loss": 1.1499, + "theoretical_loss": 3.3328920496092382, + "tokens_seen": 3200108544 + }, + { + "epoch": 1.35, + "learning_rate": 2.4714142427281846e-05, + "loss": 1.0943, + "theoretical_loss": 3.3328869145418802, + "tokens_seen": 3200174080 + }, + { + "epoch": 1.35, + "learning_rate": 2.4698094282848547e-05, + "loss": 1.1827, + "theoretical_loss": 3.332881779609126, + "tokens_seen": 3200239616 + }, + { + "epoch": 1.35, + "learning_rate": 2.4682046138415247e-05, + "loss": 1.1625, + "theoretical_loss": 3.332876644810969, + "tokens_seen": 3200305152 + }, + { + "epoch": 1.35, + "learning_rate": 2.4665997993981945e-05, + "loss": 1.2415, + "theoretical_loss": 3.332871510147403, + "tokens_seen": 3200370688 + }, + { + "epoch": 1.35, + "learning_rate": 2.464994984954865e-05, + "loss": 1.149, + "theoretical_loss": 3.332866375618422, + "tokens_seen": 3200436224 + }, + { + "epoch": 1.35, + "learning_rate": 2.4633901705115346e-05, + "loss": 1.2287, + "theoretical_loss": 3.332861241224019, + "tokens_seen": 3200501760 + }, + { + "epoch": 1.35, + "learning_rate": 2.4617853560682047e-05, + "loss": 1.1856, + "theoretical_loss": 3.332856106964188, + "tokens_seen": 3200567296 + }, + { + "epoch": 1.35, + "learning_rate": 2.4601805416248747e-05, + "loss": 1.2106, + "theoretical_loss": 3.3328509728389237, + "tokens_seen": 3200632832 + }, + { + "epoch": 1.35, + "learning_rate": 2.4585757271815448e-05, + "loss": 1.1279, + "theoretical_loss": 3.3328458388482187, + "tokens_seen": 3200698368 + }, + { + "epoch": 1.35, + "learning_rate": 2.456970912738215e-05, + "loss": 1.2205, + "theoretical_loss": 3.332840704992067, + "tokens_seen": 3200763904 + }, + { + "epoch": 1.35, + "learning_rate": 2.455366098294885e-05, + "loss": 1.1511, + "theoretical_loss": 3.3328355712704623, + "tokens_seen": 3200829440 + }, + { + "epoch": 1.35, + "learning_rate": 2.4537612838515546e-05, + "loss": 1.0237, + "theoretical_loss": 3.3328304376833984, + "tokens_seen": 3200894976 + }, + { + "epoch": 1.35, + "learning_rate": 2.4521564694082247e-05, + "loss": 1.1338, + "theoretical_loss": 3.3328253042308695, + "tokens_seen": 3200960512 + }, + { + "epoch": 1.35, + "learning_rate": 2.4505516549648948e-05, + "loss": 1.2098, + "theoretical_loss": 3.3328201709128686, + "tokens_seen": 3201026048 + }, + { + "epoch": 1.35, + "learning_rate": 2.4489468405215648e-05, + "loss": 0.9904, + "theoretical_loss": 3.3328150377293895, + "tokens_seen": 3201091584 + }, + { + "epoch": 1.35, + "learning_rate": 2.447342026078235e-05, + "loss": 1.0835, + "theoretical_loss": 3.332809904680426, + "tokens_seen": 3201157120 + }, + { + "epoch": 1.35, + "learning_rate": 2.4457372116349046e-05, + "loss": 1.1375, + "theoretical_loss": 3.332804771765972, + "tokens_seen": 3201222656 + }, + { + "epoch": 1.35, + "learning_rate": 2.4441323971915747e-05, + "loss": 1.1879, + "theoretical_loss": 3.332799638986021, + "tokens_seen": 3201288192 + }, + { + "epoch": 1.35, + "learning_rate": 2.442527582748245e-05, + "loss": 1.1446, + "theoretical_loss": 3.332794506340567, + "tokens_seen": 3201353728 + }, + { + "epoch": 1.35, + "learning_rate": 2.4409227683049148e-05, + "loss": 1.222, + "theoretical_loss": 3.3327893738296037, + "tokens_seen": 3201419264 + }, + { + "epoch": 1.35, + "learning_rate": 2.439317953861585e-05, + "loss": 1.265, + "theoretical_loss": 3.3327842414531244, + "tokens_seen": 3201484800 + }, + { + "epoch": 1.35, + "learning_rate": 2.437713139418255e-05, + "loss": 1.183, + "theoretical_loss": 3.332779109211123, + "tokens_seen": 3201550336 + }, + { + "epoch": 1.35, + "learning_rate": 2.436108324974925e-05, + "loss": 1.2213, + "theoretical_loss": 3.332773977103594, + "tokens_seen": 3201615872 + }, + { + "epoch": 1.35, + "learning_rate": 2.434503510531595e-05, + "loss": 1.1978, + "theoretical_loss": 3.3327688451305297, + "tokens_seen": 3201681408 + }, + { + "epoch": 1.35, + "learning_rate": 2.4328986960882648e-05, + "loss": 1.1739, + "theoretical_loss": 3.332763713291925, + "tokens_seen": 3201746944 + }, + { + "epoch": 1.35, + "learning_rate": 2.431293881644935e-05, + "loss": 1.0744, + "theoretical_loss": 3.332758581587773, + "tokens_seen": 3201812480 + }, + { + "epoch": 1.35, + "learning_rate": 2.429689067201605e-05, + "loss": 1.1674, + "theoretical_loss": 3.332753450018068, + "tokens_seen": 3201878016 + }, + { + "epoch": 1.35, + "learning_rate": 2.428084252758275e-05, + "loss": 1.2262, + "theoretical_loss": 3.3327483185828033, + "tokens_seen": 3201943552 + }, + { + "epoch": 1.35, + "learning_rate": 2.426479438314945e-05, + "loss": 1.2661, + "theoretical_loss": 3.332743187281972, + "tokens_seen": 3202009088 + }, + { + "epoch": 1.35, + "learning_rate": 2.424874623871615e-05, + "loss": 1.1489, + "theoretical_loss": 3.332738056115569, + "tokens_seen": 3202074624 + }, + { + "epoch": 1.35, + "learning_rate": 2.4232698094282848e-05, + "loss": 1.2228, + "theoretical_loss": 3.3327329250835875, + "tokens_seen": 3202140160 + }, + { + "epoch": 1.35, + "learning_rate": 2.421664994984955e-05, + "loss": 1.1925, + "theoretical_loss": 3.3327277941860216, + "tokens_seen": 3202205696 + }, + { + "epoch": 1.35, + "learning_rate": 2.420060180541625e-05, + "loss": 1.1152, + "theoretical_loss": 3.3327226634228646, + "tokens_seen": 3202271232 + }, + { + "epoch": 1.35, + "learning_rate": 2.418455366098295e-05, + "loss": 1.1416, + "theoretical_loss": 3.33271753279411, + "tokens_seen": 3202336768 + }, + { + "epoch": 1.35, + "learning_rate": 2.416850551654965e-05, + "loss": 1.2858, + "theoretical_loss": 3.3327124022997516, + "tokens_seen": 3202402304 + }, + { + "epoch": 1.35, + "learning_rate": 2.4152457372116348e-05, + "loss": 1.2636, + "theoretical_loss": 3.332707271939784, + "tokens_seen": 3202467840 + }, + { + "epoch": 1.35, + "learning_rate": 2.4136409227683052e-05, + "loss": 1.1971, + "theoretical_loss": 3.3327021417142, + "tokens_seen": 3202533376 + }, + { + "epoch": 1.35, + "learning_rate": 2.4120361083249753e-05, + "loss": 1.2073, + "theoretical_loss": 3.3326970116229937, + "tokens_seen": 3202598912 + }, + { + "epoch": 1.35, + "learning_rate": 2.410431293881645e-05, + "loss": 1.1328, + "theoretical_loss": 3.332691881666159, + "tokens_seen": 3202664448 + }, + { + "epoch": 1.35, + "learning_rate": 2.408826479438315e-05, + "loss": 1.1751, + "theoretical_loss": 3.3326867518436893, + "tokens_seen": 3202729984 + }, + { + "epoch": 1.35, + "learning_rate": 2.407221664994985e-05, + "loss": 1.1949, + "theoretical_loss": 3.332681622155578, + "tokens_seen": 3202795520 + }, + { + "epoch": 1.35, + "learning_rate": 2.4056168505516552e-05, + "loss": 1.2994, + "theoretical_loss": 3.3326764926018195, + "tokens_seen": 3202861056 + }, + { + "epoch": 1.35, + "learning_rate": 2.4040120361083252e-05, + "loss": 1.1747, + "theoretical_loss": 3.332671363182407, + "tokens_seen": 3202926592 + }, + { + "epoch": 1.35, + "learning_rate": 2.402407221664995e-05, + "loss": 1.1644, + "theoretical_loss": 3.3326662338973354, + "tokens_seen": 3202992128 + }, + { + "epoch": 1.35, + "learning_rate": 2.400802407221665e-05, + "loss": 1.2357, + "theoretical_loss": 3.332661104746597, + "tokens_seen": 3203057664 + }, + { + "epoch": 1.35, + "learning_rate": 2.399197592778335e-05, + "loss": 1.1839, + "theoretical_loss": 3.3326559757301863, + "tokens_seen": 3203123200 + }, + { + "epoch": 1.35, + "learning_rate": 2.397592778335005e-05, + "loss": 1.1902, + "theoretical_loss": 3.3326508468480966, + "tokens_seen": 3203188736 + }, + { + "epoch": 1.35, + "learning_rate": 2.3959879638916752e-05, + "loss": 1.1378, + "theoretical_loss": 3.3326457181003217, + "tokens_seen": 3203254272 + }, + { + "epoch": 1.35, + "learning_rate": 2.3943831494483453e-05, + "loss": 1.1922, + "theoretical_loss": 3.3326405894868563, + "tokens_seen": 3203319808 + }, + { + "epoch": 1.35, + "learning_rate": 2.392778335005015e-05, + "loss": 1.1442, + "theoretical_loss": 3.3326354610076923, + "tokens_seen": 3203385344 + }, + { + "epoch": 1.35, + "learning_rate": 2.391173520561685e-05, + "loss": 1.176, + "theoretical_loss": 3.3326303326628253, + "tokens_seen": 3203450880 + }, + { + "epoch": 1.35, + "learning_rate": 2.389568706118355e-05, + "loss": 1.1304, + "theoretical_loss": 3.3326252044522477, + "tokens_seen": 3203516416 + }, + { + "epoch": 1.35, + "learning_rate": 2.3879638916750252e-05, + "loss": 1.1519, + "theoretical_loss": 3.332620076375954, + "tokens_seen": 3203581952 + }, + { + "epoch": 1.35, + "learning_rate": 2.3863590772316953e-05, + "loss": 1.1702, + "theoretical_loss": 3.3326149484339376, + "tokens_seen": 3203647488 + }, + { + "epoch": 1.35, + "learning_rate": 2.384754262788365e-05, + "loss": 1.1356, + "theoretical_loss": 3.332609820626192, + "tokens_seen": 3203713024 + }, + { + "epoch": 1.35, + "learning_rate": 2.3831494483450354e-05, + "loss": 1.2216, + "theoretical_loss": 3.3326046929527116, + "tokens_seen": 3203778560 + }, + { + "epoch": 1.35, + "learning_rate": 2.3815446339017055e-05, + "loss": 1.2024, + "theoretical_loss": 3.33259956541349, + "tokens_seen": 3203844096 + }, + { + "epoch": 1.35, + "learning_rate": 2.3799398194583752e-05, + "loss": 1.0974, + "theoretical_loss": 3.3325944380085204, + "tokens_seen": 3203909632 + }, + { + "epoch": 1.35, + "learning_rate": 2.3783350050150452e-05, + "loss": 1.2184, + "theoretical_loss": 3.332589310737797, + "tokens_seen": 3203975168 + }, + { + "epoch": 1.35, + "learning_rate": 2.3767301905717153e-05, + "loss": 1.2119, + "theoretical_loss": 3.3325841836013135, + "tokens_seen": 3204040704 + }, + { + "epoch": 1.35, + "learning_rate": 2.3751253761283854e-05, + "loss": 1.2015, + "theoretical_loss": 3.332579056599063, + "tokens_seen": 3204106240 + }, + { + "epoch": 1.35, + "learning_rate": 2.3735205616850554e-05, + "loss": 1.0562, + "theoretical_loss": 3.33257392973104, + "tokens_seen": 3204171776 + }, + { + "epoch": 1.35, + "learning_rate": 2.371915747241725e-05, + "loss": 1.1227, + "theoretical_loss": 3.3325688029972387, + "tokens_seen": 3204237312 + }, + { + "epoch": 1.35, + "learning_rate": 2.3703109327983952e-05, + "loss": 1.1721, + "theoretical_loss": 3.3325636763976516, + "tokens_seen": 3204302848 + }, + { + "epoch": 1.35, + "learning_rate": 2.3687061183550653e-05, + "loss": 1.1388, + "theoretical_loss": 3.3325585499322727, + "tokens_seen": 3204368384 + }, + { + "epoch": 1.35, + "learning_rate": 2.3671013039117353e-05, + "loss": 1.1916, + "theoretical_loss": 3.3325534236010967, + "tokens_seen": 3204433920 + }, + { + "epoch": 1.35, + "learning_rate": 2.3654964894684054e-05, + "loss": 1.1549, + "theoretical_loss": 3.3325482974041165, + "tokens_seen": 3204499456 + }, + { + "epoch": 1.35, + "learning_rate": 2.3638916750250755e-05, + "loss": 1.2014, + "theoretical_loss": 3.332543171341326, + "tokens_seen": 3204564992 + }, + { + "epoch": 1.35, + "learning_rate": 2.3622868605817452e-05, + "loss": 1.1286, + "theoretical_loss": 3.332538045412719, + "tokens_seen": 3204630528 + }, + { + "epoch": 1.35, + "objective/train/docs_used": 1181867, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.257744312286377, + "objective/train/theoretical_loss": 3.332532919618289, + "objective/train/tokens_used": 3225156064, + "theoretical_loss": 3.332532919618289, + "tokens_seen": 3204696064 + }, + { + "epoch": 1.35, + "learning_rate": 2.3606820461384156e-05, + "loss": 1.2577, + "theoretical_loss": 3.332532919618289, + "tokens_seen": 3204696064 + }, + { + "epoch": 1.35, + "learning_rate": 2.3590772316950853e-05, + "loss": 1.1175, + "theoretical_loss": 3.3325277939580302, + "tokens_seen": 3204761600 + }, + { + "epoch": 1.35, + "learning_rate": 2.3574724172517554e-05, + "loss": 1.2061, + "theoretical_loss": 3.332522668431936, + "tokens_seen": 3204827136 + }, + { + "epoch": 1.35, + "learning_rate": 2.3558676028084255e-05, + "loss": 1.1919, + "theoretical_loss": 3.3325175430400003, + "tokens_seen": 3204892672 + }, + { + "epoch": 1.35, + "learning_rate": 2.3542627883650952e-05, + "loss": 1.196, + "theoretical_loss": 3.3325124177822167, + "tokens_seen": 3204958208 + }, + { + "epoch": 1.35, + "learning_rate": 2.3526579739217656e-05, + "loss": 1.1119, + "theoretical_loss": 3.3325072926585793, + "tokens_seen": 3205023744 + }, + { + "epoch": 1.35, + "learning_rate": 2.3510531594784353e-05, + "loss": 1.155, + "theoretical_loss": 3.3325021676690816, + "tokens_seen": 3205089280 + }, + { + "epoch": 1.35, + "learning_rate": 2.3494483450351054e-05, + "loss": 1.1622, + "theoretical_loss": 3.332497042813717, + "tokens_seen": 3205154816 + }, + { + "epoch": 1.35, + "learning_rate": 2.3478435305917754e-05, + "loss": 1.1726, + "theoretical_loss": 3.33249191809248, + "tokens_seen": 3205220352 + }, + { + "epoch": 1.35, + "learning_rate": 2.346238716148445e-05, + "loss": 1.1671, + "theoretical_loss": 3.332486793505364, + "tokens_seen": 3205285888 + }, + { + "epoch": 1.35, + "learning_rate": 2.3446339017051156e-05, + "loss": 1.0992, + "theoretical_loss": 3.332481669052362, + "tokens_seen": 3205351424 + }, + { + "epoch": 1.35, + "learning_rate": 2.3430290872617856e-05, + "loss": 1.1458, + "theoretical_loss": 3.3324765447334688, + "tokens_seen": 3205416960 + }, + { + "epoch": 1.35, + "learning_rate": 2.3414242728184553e-05, + "loss": 1.2748, + "theoretical_loss": 3.332471420548678, + "tokens_seen": 3205482496 + }, + { + "epoch": 1.35, + "learning_rate": 2.3398194583751254e-05, + "loss": 1.2039, + "theoretical_loss": 3.3324662964979828, + "tokens_seen": 3205548032 + }, + { + "epoch": 1.35, + "learning_rate": 2.3382146439317955e-05, + "loss": 1.2066, + "theoretical_loss": 3.3324611725813775, + "tokens_seen": 3205613568 + }, + { + "epoch": 1.35, + "learning_rate": 2.3366098294884655e-05, + "loss": 1.1785, + "theoretical_loss": 3.3324560487988557, + "tokens_seen": 3205679104 + }, + { + "epoch": 1.35, + "learning_rate": 2.3350050150451356e-05, + "loss": 1.0962, + "theoretical_loss": 3.332450925150411, + "tokens_seen": 3205744640 + }, + { + "epoch": 1.35, + "learning_rate": 2.3334002006018053e-05, + "loss": 1.0594, + "theoretical_loss": 3.332445801636037, + "tokens_seen": 3205810176 + }, + { + "epoch": 1.35, + "learning_rate": 2.3317953861584754e-05, + "loss": 1.1017, + "theoretical_loss": 3.332440678255728, + "tokens_seen": 3205875712 + }, + { + "epoch": 1.35, + "learning_rate": 2.3301905717151458e-05, + "loss": 1.0954, + "theoretical_loss": 3.332435555009477, + "tokens_seen": 3205941248 + }, + { + "epoch": 1.35, + "learning_rate": 2.3285857572718155e-05, + "loss": 1.1196, + "theoretical_loss": 3.3324304318972784, + "tokens_seen": 3206006784 + }, + { + "epoch": 1.35, + "learning_rate": 2.3269809428284856e-05, + "loss": 1.126, + "theoretical_loss": 3.332425308919126, + "tokens_seen": 3206072320 + }, + { + "epoch": 1.35, + "learning_rate": 2.3253761283851556e-05, + "loss": 1.089, + "theoretical_loss": 3.332420186075013, + "tokens_seen": 3206137856 + }, + { + "epoch": 1.35, + "learning_rate": 2.3237713139418254e-05, + "loss": 1.115, + "theoretical_loss": 3.3324150633649334, + "tokens_seen": 3206203392 + }, + { + "epoch": 1.35, + "learning_rate": 2.3221664994984958e-05, + "loss": 1.2204, + "theoretical_loss": 3.332409940788881, + "tokens_seen": 3206268928 + }, + { + "epoch": 1.35, + "learning_rate": 2.3205616850551655e-05, + "loss": 1.1585, + "theoretical_loss": 3.33240481834685, + "tokens_seen": 3206334464 + }, + { + "epoch": 1.35, + "learning_rate": 2.3189568706118356e-05, + "loss": 1.2609, + "theoretical_loss": 3.3323996960388333, + "tokens_seen": 3206400000 + }, + { + "epoch": 1.35, + "learning_rate": 2.3173520561685056e-05, + "loss": 1.118, + "theoretical_loss": 3.332394573864825, + "tokens_seen": 3206465536 + }, + { + "epoch": 1.35, + "learning_rate": 2.3157472417251757e-05, + "loss": 1.1087, + "theoretical_loss": 3.3323894518248194, + "tokens_seen": 3206531072 + }, + { + "epoch": 1.35, + "learning_rate": 2.3141424272818457e-05, + "loss": 1.0549, + "theoretical_loss": 3.332384329918809, + "tokens_seen": 3206596608 + }, + { + "epoch": 1.35, + "learning_rate": 2.3125376128385158e-05, + "loss": 1.1895, + "theoretical_loss": 3.3323792081467887, + "tokens_seen": 3206662144 + }, + { + "epoch": 1.35, + "learning_rate": 2.3109327983951855e-05, + "loss": 1.3124, + "theoretical_loss": 3.332374086508752, + "tokens_seen": 3206727680 + }, + { + "epoch": 1.35, + "learning_rate": 2.3093279839518556e-05, + "loss": 1.1402, + "theoretical_loss": 3.3323689650046924, + "tokens_seen": 3206793216 + }, + { + "epoch": 1.35, + "learning_rate": 2.3077231695085257e-05, + "loss": 1.2018, + "theoretical_loss": 3.332363843634604, + "tokens_seen": 3206858752 + }, + { + "epoch": 1.35, + "learning_rate": 2.3061183550651957e-05, + "loss": 1.1715, + "theoretical_loss": 3.3323587223984803, + "tokens_seen": 3206924288 + }, + { + "epoch": 1.35, + "learning_rate": 2.3045135406218658e-05, + "loss": 1.1486, + "theoretical_loss": 3.332353601296315, + "tokens_seen": 3206989824 + }, + { + "epoch": 1.35, + "learning_rate": 2.3029087261785355e-05, + "loss": 1.1335, + "theoretical_loss": 3.332348480328102, + "tokens_seen": 3207055360 + }, + { + "epoch": 1.35, + "learning_rate": 2.3013039117352056e-05, + "loss": 1.1129, + "theoretical_loss": 3.332343359493835, + "tokens_seen": 3207120896 + }, + { + "epoch": 1.35, + "learning_rate": 2.299699097291876e-05, + "loss": 1.1926, + "theoretical_loss": 3.3323382387935077, + "tokens_seen": 3207186432 + }, + { + "epoch": 1.35, + "learning_rate": 2.2980942828485457e-05, + "loss": 1.1667, + "theoretical_loss": 3.332333118227114, + "tokens_seen": 3207251968 + }, + { + "epoch": 1.35, + "learning_rate": 2.2964894684052158e-05, + "loss": 1.1606, + "theoretical_loss": 3.3323279977946476, + "tokens_seen": 3207317504 + }, + { + "epoch": 1.35, + "learning_rate": 2.2948846539618858e-05, + "loss": 1.1571, + "theoretical_loss": 3.3323228774961025, + "tokens_seen": 3207383040 + }, + { + "epoch": 1.35, + "learning_rate": 2.293279839518556e-05, + "loss": 1.2508, + "theoretical_loss": 3.332317757331472, + "tokens_seen": 3207448576 + }, + { + "epoch": 1.35, + "learning_rate": 2.291675025075226e-05, + "loss": 1.2105, + "theoretical_loss": 3.33231263730075, + "tokens_seen": 3207514112 + }, + { + "epoch": 1.35, + "learning_rate": 2.2900702106318957e-05, + "loss": 1.1937, + "theoretical_loss": 3.3323075174039305, + "tokens_seen": 3207579648 + }, + { + "epoch": 1.35, + "learning_rate": 2.2884653961885657e-05, + "loss": 1.2905, + "theoretical_loss": 3.332302397641007, + "tokens_seen": 3207645184 + }, + { + "epoch": 1.35, + "learning_rate": 2.2868605817452358e-05, + "loss": 1.2168, + "theoretical_loss": 3.332297278011974, + "tokens_seen": 3207710720 + }, + { + "epoch": 1.35, + "learning_rate": 2.285255767301906e-05, + "loss": 1.1308, + "theoretical_loss": 3.332292158516824, + "tokens_seen": 3207776256 + }, + { + "epoch": 1.35, + "learning_rate": 2.283650952858576e-05, + "loss": 1.3374, + "theoretical_loss": 3.3322870391555517, + "tokens_seen": 3207841792 + }, + { + "epoch": 1.35, + "learning_rate": 2.282046138415246e-05, + "loss": 1.0569, + "theoretical_loss": 3.33228191992815, + "tokens_seen": 3207907328 + }, + { + "epoch": 1.35, + "learning_rate": 2.2804413239719157e-05, + "loss": 1.1437, + "theoretical_loss": 3.332276800834614, + "tokens_seen": 3207972864 + }, + { + "epoch": 1.35, + "learning_rate": 2.2788365095285858e-05, + "loss": 1.1817, + "theoretical_loss": 3.332271681874936, + "tokens_seen": 3208038400 + }, + { + "epoch": 1.35, + "learning_rate": 2.277231695085256e-05, + "loss": 1.2168, + "theoretical_loss": 3.332266563049111, + "tokens_seen": 3208103936 + }, + { + "epoch": 1.35, + "learning_rate": 2.275626880641926e-05, + "loss": 1.193, + "theoretical_loss": 3.332261444357132, + "tokens_seen": 3208169472 + }, + { + "epoch": 1.35, + "learning_rate": 2.274022066198596e-05, + "loss": 1.0759, + "theoretical_loss": 3.3322563257989932, + "tokens_seen": 3208235008 + }, + { + "epoch": 1.35, + "learning_rate": 2.2724172517552657e-05, + "loss": 1.1577, + "theoretical_loss": 3.332251207374688, + "tokens_seen": 3208300544 + }, + { + "epoch": 1.35, + "learning_rate": 2.270812437311936e-05, + "loss": 1.2169, + "theoretical_loss": 3.3322460890842103, + "tokens_seen": 3208366080 + }, + { + "epoch": 1.35, + "learning_rate": 2.269207622868606e-05, + "loss": 1.1767, + "theoretical_loss": 3.332240970927554, + "tokens_seen": 3208431616 + }, + { + "epoch": 1.35, + "learning_rate": 2.267602808425276e-05, + "loss": 1.2088, + "theoretical_loss": 3.3322358529047125, + "tokens_seen": 3208497152 + }, + { + "epoch": 1.35, + "learning_rate": 2.265997993981946e-05, + "loss": 1.1899, + "theoretical_loss": 3.33223073501568, + "tokens_seen": 3208562688 + }, + { + "epoch": 1.35, + "learning_rate": 2.264393179538616e-05, + "loss": 1.1015, + "theoretical_loss": 3.33222561726045, + "tokens_seen": 3208628224 + }, + { + "epoch": 1.35, + "learning_rate": 2.262788365095286e-05, + "loss": 1.1012, + "theoretical_loss": 3.332220499639017, + "tokens_seen": 3208693760 + }, + { + "epoch": 1.35, + "learning_rate": 2.261183550651956e-05, + "loss": 1.1879, + "theoretical_loss": 3.3322153821513734, + "tokens_seen": 3208759296 + }, + { + "epoch": 1.35, + "learning_rate": 2.259578736208626e-05, + "loss": 1.1218, + "theoretical_loss": 3.332210264797514, + "tokens_seen": 3208824832 + }, + { + "epoch": 1.35, + "learning_rate": 2.257973921765296e-05, + "loss": 1.1464, + "theoretical_loss": 3.332205147577432, + "tokens_seen": 3208890368 + }, + { + "epoch": 1.35, + "learning_rate": 2.256369107321966e-05, + "loss": 1.2133, + "theoretical_loss": 3.332200030491122, + "tokens_seen": 3208955904 + }, + { + "epoch": 1.35, + "learning_rate": 2.254764292878636e-05, + "loss": 1.2485, + "theoretical_loss": 3.3321949135385767, + "tokens_seen": 3209021440 + }, + { + "epoch": 1.35, + "learning_rate": 2.253159478435306e-05, + "loss": 1.1341, + "theoretical_loss": 3.3321897967197907, + "tokens_seen": 3209086976 + }, + { + "epoch": 1.35, + "learning_rate": 2.251554663991976e-05, + "loss": 1.1874, + "theoretical_loss": 3.332184680034757, + "tokens_seen": 3209152512 + }, + { + "epoch": 1.35, + "learning_rate": 2.249949849548646e-05, + "loss": 1.121, + "theoretical_loss": 3.3321795634834706, + "tokens_seen": 3209218048 + }, + { + "epoch": 1.35, + "learning_rate": 2.2483450351053163e-05, + "loss": 1.1108, + "theoretical_loss": 3.332174447065924, + "tokens_seen": 3209283584 + }, + { + "epoch": 1.35, + "learning_rate": 2.246740220661986e-05, + "loss": 1.2501, + "theoretical_loss": 3.3321693307821114, + "tokens_seen": 3209349120 + }, + { + "epoch": 1.35, + "learning_rate": 2.245135406218656e-05, + "loss": 1.2282, + "theoretical_loss": 3.332164214632027, + "tokens_seen": 3209414656 + }, + { + "epoch": 1.35, + "learning_rate": 2.243530591775326e-05, + "loss": 1.1925, + "theoretical_loss": 3.332159098615664, + "tokens_seen": 3209480192 + }, + { + "epoch": 1.35, + "learning_rate": 2.241925777331996e-05, + "loss": 1.3043, + "theoretical_loss": 3.3321539827330167, + "tokens_seen": 3209545728 + }, + { + "epoch": 1.35, + "learning_rate": 2.2403209628886663e-05, + "loss": 1.1987, + "theoretical_loss": 3.3321488669840784, + "tokens_seen": 3209611264 + }, + { + "epoch": 1.35, + "learning_rate": 2.238716148445336e-05, + "loss": 1.1775, + "theoretical_loss": 3.332143751368843, + "tokens_seen": 3209676800 + }, + { + "epoch": 1.35, + "learning_rate": 2.237111334002006e-05, + "loss": 1.2213, + "theoretical_loss": 3.332138635887304, + "tokens_seen": 3209742336 + }, + { + "epoch": 1.35, + "learning_rate": 2.235506519558676e-05, + "loss": 1.2236, + "theoretical_loss": 3.332133520539456, + "tokens_seen": 3209807872 + }, + { + "epoch": 1.35, + "learning_rate": 2.233901705115346e-05, + "loss": 1.2686, + "theoretical_loss": 3.3321284053252924, + "tokens_seen": 3209873408 + }, + { + "epoch": 1.35, + "learning_rate": 2.2322968906720163e-05, + "loss": 1.2417, + "theoretical_loss": 3.3321232902448066, + "tokens_seen": 3209938944 + }, + { + "epoch": 1.35, + "learning_rate": 2.2306920762286863e-05, + "loss": 1.1415, + "theoretical_loss": 3.3321181752979925, + "tokens_seen": 3210004480 + }, + { + "epoch": 1.35, + "learning_rate": 2.229087261785356e-05, + "loss": 1.0377, + "theoretical_loss": 3.3321130604848443, + "tokens_seen": 3210070016 + }, + { + "epoch": 1.35, + "learning_rate": 2.227482447342026e-05, + "loss": 1.1168, + "theoretical_loss": 3.3321079458053555, + "tokens_seen": 3210135552 + }, + { + "epoch": 1.35, + "learning_rate": 2.2258776328986962e-05, + "loss": 1.2335, + "theoretical_loss": 3.33210283125952, + "tokens_seen": 3210201088 + }, + { + "epoch": 1.35, + "learning_rate": 2.2242728184553662e-05, + "loss": 1.0938, + "theoretical_loss": 3.332097716847331, + "tokens_seen": 3210266624 + }, + { + "epoch": 1.35, + "learning_rate": 2.2226680040120363e-05, + "loss": 1.2263, + "theoretical_loss": 3.3320926025687827, + "tokens_seen": 3210332160 + }, + { + "epoch": 1.35, + "learning_rate": 2.221063189568706e-05, + "loss": 1.1525, + "theoretical_loss": 3.3320874884238694, + "tokens_seen": 3210397696 + }, + { + "epoch": 1.35, + "learning_rate": 2.219458375125376e-05, + "loss": 1.2147, + "theoretical_loss": 3.3320823744125843, + "tokens_seen": 3210463232 + }, + { + "epoch": 1.35, + "learning_rate": 2.2178535606820465e-05, + "loss": 1.2159, + "theoretical_loss": 3.332077260534921, + "tokens_seen": 3210528768 + }, + { + "epoch": 1.35, + "learning_rate": 2.2162487462387162e-05, + "loss": 1.1901, + "theoretical_loss": 3.332072146790874, + "tokens_seen": 3210594304 + }, + { + "epoch": 1.35, + "learning_rate": 2.2146439317953863e-05, + "loss": 1.1096, + "theoretical_loss": 3.332067033180436, + "tokens_seen": 3210659840 + }, + { + "epoch": 1.35, + "learning_rate": 2.2130391173520564e-05, + "loss": 1.2759, + "theoretical_loss": 3.3320619197036017, + "tokens_seen": 3210725376 + }, + { + "epoch": 1.35, + "learning_rate": 2.211434302908726e-05, + "loss": 1.1384, + "theoretical_loss": 3.332056806360365, + "tokens_seen": 3210790912 + }, + { + "epoch": 1.35, + "learning_rate": 2.2098294884653965e-05, + "loss": 1.1749, + "theoretical_loss": 3.3320516931507185, + "tokens_seen": 3210856448 + }, + { + "epoch": 1.35, + "learning_rate": 2.2082246740220662e-05, + "loss": 1.1281, + "theoretical_loss": 3.3320465800746573, + "tokens_seen": 3210921984 + }, + { + "epoch": 1.35, + "learning_rate": 2.2066198595787363e-05, + "loss": 1.1471, + "theoretical_loss": 3.3320414671321745, + "tokens_seen": 3210987520 + }, + { + "epoch": 1.35, + "learning_rate": 2.2050150451354063e-05, + "loss": 1.1694, + "theoretical_loss": 3.332036354323264, + "tokens_seen": 3211053056 + }, + { + "epoch": 1.35, + "learning_rate": 2.2034102306920764e-05, + "loss": 1.204, + "theoretical_loss": 3.3320312416479196, + "tokens_seen": 3211118592 + }, + { + "epoch": 1.35, + "learning_rate": 2.2018054162487465e-05, + "loss": 1.118, + "theoretical_loss": 3.3320261291061355, + "tokens_seen": 3211184128 + }, + { + "epoch": 1.35, + "objective/train/docs_used": 1184255, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1747627258300781, + "objective/train/theoretical_loss": 3.3320210166979045, + "objective/train/tokens_used": 3231709664, + "theoretical_loss": 3.3320210166979045, + "tokens_seen": 3211249664 + }, + { + "epoch": 1.35, + "learning_rate": 2.2002006018054165e-05, + "loss": 1.1748, + "theoretical_loss": 3.3320210166979045, + "tokens_seen": 3211249664 + }, + { + "epoch": 1.35, + "learning_rate": 2.1985957873620862e-05, + "loss": 1.2163, + "theoretical_loss": 3.3320159044232214, + "tokens_seen": 3211315200 + }, + { + "epoch": 1.35, + "learning_rate": 2.1969909729187563e-05, + "loss": 1.235, + "theoretical_loss": 3.3320107922820794, + "tokens_seen": 3211380736 + }, + { + "epoch": 1.35, + "learning_rate": 2.1953861584754264e-05, + "loss": 1.1506, + "theoretical_loss": 3.3320056802744724, + "tokens_seen": 3211446272 + }, + { + "epoch": 1.35, + "learning_rate": 2.1937813440320964e-05, + "loss": 1.1281, + "theoretical_loss": 3.332000568400394, + "tokens_seen": 3211511808 + }, + { + "epoch": 1.35, + "learning_rate": 2.1921765295887665e-05, + "loss": 1.268, + "theoretical_loss": 3.331995456659839, + "tokens_seen": 3211577344 + }, + { + "epoch": 1.35, + "learning_rate": 2.1905717151454362e-05, + "loss": 1.1566, + "theoretical_loss": 3.3319903450528, + "tokens_seen": 3211642880 + }, + { + "epoch": 1.35, + "learning_rate": 2.1889669007021063e-05, + "loss": 1.1081, + "theoretical_loss": 3.331985233579271, + "tokens_seen": 3211708416 + }, + { + "epoch": 1.35, + "learning_rate": 2.1873620862587767e-05, + "loss": 1.1637, + "theoretical_loss": 3.331980122239246, + "tokens_seen": 3211773952 + }, + { + "epoch": 1.35, + "learning_rate": 2.1857572718154464e-05, + "loss": 1.2169, + "theoretical_loss": 3.331975011032719, + "tokens_seen": 3211839488 + }, + { + "epoch": 1.35, + "learning_rate": 2.1841524573721165e-05, + "loss": 1.1577, + "theoretical_loss": 3.3319698999596836, + "tokens_seen": 3211905024 + }, + { + "epoch": 1.35, + "learning_rate": 2.1825476429287865e-05, + "loss": 1.1548, + "theoretical_loss": 3.3319647890201334, + "tokens_seen": 3211970560 + }, + { + "epoch": 1.35, + "learning_rate": 2.1809428284854563e-05, + "loss": 1.1558, + "theoretical_loss": 3.331959678214062, + "tokens_seen": 3212036096 + }, + { + "epoch": 1.35, + "learning_rate": 2.1793380140421267e-05, + "loss": 1.1497, + "theoretical_loss": 3.331954567541464, + "tokens_seen": 3212101632 + }, + { + "epoch": 1.35, + "learning_rate": 2.1777331995987964e-05, + "loss": 1.0929, + "theoretical_loss": 3.3319494570023327, + "tokens_seen": 3212167168 + }, + { + "epoch": 1.35, + "learning_rate": 2.1761283851554665e-05, + "loss": 1.2052, + "theoretical_loss": 3.331944346596662, + "tokens_seen": 3212232704 + }, + { + "epoch": 1.35, + "learning_rate": 2.1745235707121365e-05, + "loss": 1.2261, + "theoretical_loss": 3.3319392363244456, + "tokens_seen": 3212298240 + }, + { + "epoch": 1.35, + "learning_rate": 2.1729187562688066e-05, + "loss": 1.2489, + "theoretical_loss": 3.331934126185677, + "tokens_seen": 3212363776 + }, + { + "epoch": 1.35, + "learning_rate": 2.1713139418254766e-05, + "loss": 1.1066, + "theoretical_loss": 3.3319290161803505, + "tokens_seen": 3212429312 + }, + { + "epoch": 1.35, + "learning_rate": 2.1697091273821467e-05, + "loss": 1.2092, + "theoretical_loss": 3.3319239063084596, + "tokens_seen": 3212494848 + }, + { + "epoch": 1.35, + "learning_rate": 2.1681043129388164e-05, + "loss": 1.1895, + "theoretical_loss": 3.3319187965699983, + "tokens_seen": 3212560384 + }, + { + "epoch": 1.35, + "learning_rate": 2.1664994984954865e-05, + "loss": 1.2054, + "theoretical_loss": 3.33191368696496, + "tokens_seen": 3212625920 + }, + { + "epoch": 1.35, + "learning_rate": 2.1648946840521566e-05, + "loss": 1.2489, + "theoretical_loss": 3.331908577493339, + "tokens_seen": 3212691456 + }, + { + "epoch": 1.35, + "learning_rate": 2.1632898696088266e-05, + "loss": 1.1609, + "theoretical_loss": 3.331903468155129, + "tokens_seen": 3212756992 + }, + { + "epoch": 1.35, + "learning_rate": 2.1616850551654967e-05, + "loss": 1.1541, + "theoretical_loss": 3.3318983589503235, + "tokens_seen": 3212822528 + }, + { + "epoch": 1.35, + "learning_rate": 2.1600802407221664e-05, + "loss": 1.1821, + "theoretical_loss": 3.3318932498789167, + "tokens_seen": 3212888064 + }, + { + "epoch": 1.35, + "learning_rate": 2.1584754262788365e-05, + "loss": 1.1459, + "theoretical_loss": 3.331888140940902, + "tokens_seen": 3212953600 + }, + { + "epoch": 1.35, + "learning_rate": 2.1568706118355065e-05, + "loss": 1.2145, + "theoretical_loss": 3.3318830321362727, + "tokens_seen": 3213019136 + }, + { + "epoch": 1.35, + "learning_rate": 2.1552657973921766e-05, + "loss": 1.2205, + "theoretical_loss": 3.331877923465024, + "tokens_seen": 3213084672 + }, + { + "epoch": 1.35, + "learning_rate": 2.1536609829488467e-05, + "loss": 1.21, + "theoretical_loss": 3.3318728149271486, + "tokens_seen": 3213150208 + }, + { + "epoch": 1.35, + "learning_rate": 2.1520561685055164e-05, + "loss": 1.2153, + "theoretical_loss": 3.3318677065226407, + "tokens_seen": 3213215744 + }, + { + "epoch": 1.35, + "learning_rate": 2.1504513540621868e-05, + "loss": 1.2062, + "theoretical_loss": 3.3318625982514942, + "tokens_seen": 3213281280 + }, + { + "epoch": 1.35, + "learning_rate": 2.148846539618857e-05, + "loss": 1.1497, + "theoretical_loss": 3.331857490113703, + "tokens_seen": 3213346816 + }, + { + "epoch": 1.35, + "learning_rate": 2.1472417251755266e-05, + "loss": 1.1886, + "theoretical_loss": 3.3318523821092603, + "tokens_seen": 3213412352 + }, + { + "epoch": 1.35, + "learning_rate": 2.1456369107321966e-05, + "loss": 1.2109, + "theoretical_loss": 3.33184727423816, + "tokens_seen": 3213477888 + }, + { + "epoch": 1.35, + "learning_rate": 2.1440320962888667e-05, + "loss": 1.1985, + "theoretical_loss": 3.3318421665003966, + "tokens_seen": 3213543424 + }, + { + "epoch": 1.35, + "learning_rate": 2.1424272818455368e-05, + "loss": 1.1471, + "theoretical_loss": 3.331837058895963, + "tokens_seen": 3213608960 + }, + { + "epoch": 1.35, + "learning_rate": 2.140822467402207e-05, + "loss": 1.1136, + "theoretical_loss": 3.3318319514248538, + "tokens_seen": 3213674496 + }, + { + "epoch": 1.35, + "learning_rate": 2.1392176529588766e-05, + "loss": 1.133, + "theoretical_loss": 3.331826844087062, + "tokens_seen": 3213740032 + }, + { + "epoch": 1.35, + "learning_rate": 2.1376128385155466e-05, + "loss": 1.2289, + "theoretical_loss": 3.331821736882582, + "tokens_seen": 3213805568 + }, + { + "epoch": 1.35, + "learning_rate": 2.1360080240722167e-05, + "loss": 1.1566, + "theoretical_loss": 3.3318166298114082, + "tokens_seen": 3213871104 + }, + { + "epoch": 1.35, + "learning_rate": 2.1344032096288868e-05, + "loss": 1.142, + "theoretical_loss": 3.331811522873533, + "tokens_seen": 3213936640 + }, + { + "epoch": 1.35, + "learning_rate": 2.1327983951855568e-05, + "loss": 1.2131, + "theoretical_loss": 3.3318064160689507, + "tokens_seen": 3214002176 + }, + { + "epoch": 1.35, + "learning_rate": 2.131193580742227e-05, + "loss": 1.1857, + "theoretical_loss": 3.3318013093976555, + "tokens_seen": 3214067712 + }, + { + "epoch": 1.35, + "learning_rate": 2.1295887662988966e-05, + "loss": 1.1347, + "theoretical_loss": 3.331796202859641, + "tokens_seen": 3214133248 + }, + { + "epoch": 1.35, + "learning_rate": 2.127983951855567e-05, + "loss": 1.1963, + "theoretical_loss": 3.3317910964549005, + "tokens_seen": 3214198784 + }, + { + "epoch": 1.35, + "learning_rate": 2.1263791374122367e-05, + "loss": 1.132, + "theoretical_loss": 3.3317859901834286, + "tokens_seen": 3214264320 + }, + { + "epoch": 1.35, + "learning_rate": 2.1247743229689068e-05, + "loss": 1.0684, + "theoretical_loss": 3.3317808840452185, + "tokens_seen": 3214329856 + }, + { + "epoch": 1.35, + "learning_rate": 2.123169508525577e-05, + "loss": 1.2437, + "theoretical_loss": 3.331775778040265, + "tokens_seen": 3214395392 + }, + { + "epoch": 1.35, + "learning_rate": 2.1215646940822466e-05, + "loss": 1.193, + "theoretical_loss": 3.3317706721685605, + "tokens_seen": 3214460928 + }, + { + "epoch": 1.35, + "learning_rate": 2.119959879638917e-05, + "loss": 1.2326, + "theoretical_loss": 3.3317655664301, + "tokens_seen": 3214526464 + }, + { + "epoch": 1.35, + "learning_rate": 2.118355065195587e-05, + "loss": 1.2183, + "theoretical_loss": 3.331760460824876, + "tokens_seen": 3214592000 + }, + { + "epoch": 1.35, + "learning_rate": 2.1167502507522568e-05, + "loss": 1.1309, + "theoretical_loss": 3.331755355352884, + "tokens_seen": 3214657536 + }, + { + "epoch": 1.35, + "learning_rate": 2.115145436308927e-05, + "loss": 1.2216, + "theoretical_loss": 3.3317502500141165, + "tokens_seen": 3214723072 + }, + { + "epoch": 1.35, + "learning_rate": 2.113540621865597e-05, + "loss": 1.2216, + "theoretical_loss": 3.3317451448085675, + "tokens_seen": 3214788608 + }, + { + "epoch": 1.35, + "learning_rate": 2.111935807422267e-05, + "loss": 1.1735, + "theoretical_loss": 3.331740039736231, + "tokens_seen": 3214854144 + }, + { + "epoch": 1.35, + "learning_rate": 2.110330992978937e-05, + "loss": 1.1562, + "theoretical_loss": 3.3317349347971015, + "tokens_seen": 3214919680 + }, + { + "epoch": 1.35, + "learning_rate": 2.1087261785356068e-05, + "loss": 1.3353, + "theoretical_loss": 3.3317298299911715, + "tokens_seen": 3214985216 + }, + { + "epoch": 1.35, + "learning_rate": 2.1071213640922768e-05, + "loss": 1.1854, + "theoretical_loss": 3.331724725318436, + "tokens_seen": 3215050752 + }, + { + "epoch": 1.35, + "learning_rate": 2.1055165496489472e-05, + "loss": 1.2854, + "theoretical_loss": 3.3317196207788875, + "tokens_seen": 3215116288 + }, + { + "epoch": 1.35, + "learning_rate": 2.103911735205617e-05, + "loss": 1.2413, + "theoretical_loss": 3.331714516372521, + "tokens_seen": 3215181824 + }, + { + "epoch": 1.35, + "learning_rate": 2.102306920762287e-05, + "loss": 1.2256, + "theoretical_loss": 3.33170941209933, + "tokens_seen": 3215247360 + }, + { + "epoch": 1.36, + "learning_rate": 2.100702106318957e-05, + "loss": 1.2831, + "theoretical_loss": 3.331704307959308, + "tokens_seen": 3215312896 + }, + { + "epoch": 1.36, + "learning_rate": 2.0990972918756268e-05, + "loss": 1.1828, + "theoretical_loss": 3.331699203952449, + "tokens_seen": 3215378432 + }, + { + "epoch": 1.36, + "learning_rate": 2.0974924774322972e-05, + "loss": 1.2938, + "theoretical_loss": 3.331694100078747, + "tokens_seen": 3215443968 + }, + { + "epoch": 1.36, + "learning_rate": 2.095887662988967e-05, + "loss": 1.1175, + "theoretical_loss": 3.3316889963381953, + "tokens_seen": 3215509504 + }, + { + "epoch": 1.36, + "learning_rate": 2.094282848545637e-05, + "loss": 1.1945, + "theoretical_loss": 3.3316838927307884, + "tokens_seen": 3215575040 + }, + { + "epoch": 1.36, + "learning_rate": 2.092678034102307e-05, + "loss": 1.144, + "theoretical_loss": 3.3316787892565194, + "tokens_seen": 3215640576 + }, + { + "epoch": 1.36, + "learning_rate": 2.0910732196589768e-05, + "loss": 1.1315, + "theoretical_loss": 3.3316736859153826, + "tokens_seen": 3215706112 + }, + { + "epoch": 1.36, + "learning_rate": 2.0894684052156472e-05, + "loss": 1.2637, + "theoretical_loss": 3.3316685827073718, + "tokens_seen": 3215771648 + }, + { + "epoch": 1.36, + "learning_rate": 2.0878635907723172e-05, + "loss": 1.1266, + "theoretical_loss": 3.33166347963248, + "tokens_seen": 3215837184 + }, + { + "epoch": 1.36, + "learning_rate": 2.086258776328987e-05, + "loss": 1.2545, + "theoretical_loss": 3.3316583766907026, + "tokens_seen": 3215902720 + }, + { + "epoch": 1.36, + "learning_rate": 2.084653961885657e-05, + "loss": 1.1892, + "theoretical_loss": 3.3316532738820324, + "tokens_seen": 3215968256 + }, + { + "epoch": 1.36, + "learning_rate": 2.083049147442327e-05, + "loss": 1.3086, + "theoretical_loss": 3.331648171206463, + "tokens_seen": 3216033792 + }, + { + "epoch": 1.36, + "learning_rate": 2.081444332998997e-05, + "loss": 1.2843, + "theoretical_loss": 3.331643068663989, + "tokens_seen": 3216099328 + }, + { + "epoch": 1.36, + "learning_rate": 2.0798395185556672e-05, + "loss": 1.2034, + "theoretical_loss": 3.331637966254603, + "tokens_seen": 3216164864 + }, + { + "epoch": 1.36, + "learning_rate": 2.078234704112337e-05, + "loss": 1.2145, + "theoretical_loss": 3.3316328639783004, + "tokens_seen": 3216230400 + }, + { + "epoch": 1.36, + "learning_rate": 2.076629889669007e-05, + "loss": 1.165, + "theoretical_loss": 3.331627761835074, + "tokens_seen": 3216295936 + }, + { + "epoch": 1.36, + "learning_rate": 2.0750250752256774e-05, + "loss": 1.1388, + "theoretical_loss": 3.3316226598249177, + "tokens_seen": 3216361472 + }, + { + "epoch": 1.36, + "learning_rate": 2.073420260782347e-05, + "loss": 1.2136, + "theoretical_loss": 3.3316175579478253, + "tokens_seen": 3216427008 + }, + { + "epoch": 1.36, + "learning_rate": 2.0718154463390172e-05, + "loss": 1.2092, + "theoretical_loss": 3.331612456203791, + "tokens_seen": 3216492544 + }, + { + "epoch": 1.36, + "learning_rate": 2.0702106318956873e-05, + "loss": 1.1387, + "theoretical_loss": 3.331607354592808, + "tokens_seen": 3216558080 + }, + { + "epoch": 1.36, + "learning_rate": 2.068605817452357e-05, + "loss": 1.139, + "theoretical_loss": 3.331602253114871, + "tokens_seen": 3216623616 + }, + { + "epoch": 1.36, + "learning_rate": 2.0670010030090274e-05, + "loss": 1.185, + "theoretical_loss": 3.331597151769973, + "tokens_seen": 3216689152 + }, + { + "epoch": 1.36, + "learning_rate": 2.065396188565697e-05, + "loss": 1.1184, + "theoretical_loss": 3.3315920505581085, + "tokens_seen": 3216754688 + }, + { + "epoch": 1.36, + "learning_rate": 2.0637913741223672e-05, + "loss": 1.2616, + "theoretical_loss": 3.3315869494792705, + "tokens_seen": 3216820224 + }, + { + "epoch": 1.36, + "learning_rate": 2.0621865596790372e-05, + "loss": 1.2184, + "theoretical_loss": 3.331581848533453, + "tokens_seen": 3216885760 + }, + { + "epoch": 1.36, + "learning_rate": 2.0605817452357073e-05, + "loss": 1.2031, + "theoretical_loss": 3.331576747720651, + "tokens_seen": 3216951296 + }, + { + "epoch": 1.36, + "learning_rate": 2.0589769307923774e-05, + "loss": 1.1995, + "theoretical_loss": 3.3315716470408567, + "tokens_seen": 3217016832 + }, + { + "epoch": 1.36, + "learning_rate": 2.057372116349047e-05, + "loss": 1.167, + "theoretical_loss": 3.331566546494065, + "tokens_seen": 3217082368 + }, + { + "epoch": 1.36, + "learning_rate": 2.055767301905717e-05, + "loss": 1.2412, + "theoretical_loss": 3.3315614460802694, + "tokens_seen": 3217147904 + }, + { + "epoch": 1.36, + "learning_rate": 2.0541624874623872e-05, + "loss": 1.2592, + "theoretical_loss": 3.3315563457994637, + "tokens_seen": 3217213440 + }, + { + "epoch": 1.36, + "learning_rate": 2.0525576730190573e-05, + "loss": 1.138, + "theoretical_loss": 3.3315512456516414, + "tokens_seen": 3217278976 + }, + { + "epoch": 1.36, + "learning_rate": 2.0509528585757273e-05, + "loss": 1.1517, + "theoretical_loss": 3.3315461456367967, + "tokens_seen": 3217344512 + }, + { + "epoch": 1.36, + "learning_rate": 2.0493480441323974e-05, + "loss": 1.0993, + "theoretical_loss": 3.3315410457549235, + "tokens_seen": 3217410048 + }, + { + "epoch": 1.36, + "learning_rate": 2.047743229689067e-05, + "loss": 1.1645, + "theoretical_loss": 3.3315359460060154, + "tokens_seen": 3217475584 + }, + { + "epoch": 1.36, + "learning_rate": 2.0461384152457372e-05, + "loss": 1.217, + "theoretical_loss": 3.3315308463900664, + "tokens_seen": 3217541120 + }, + { + "epoch": 1.36, + "learning_rate": 2.0445336008024073e-05, + "loss": 1.3065, + "theoretical_loss": 3.3315257469070705, + "tokens_seen": 3217606656 + }, + { + "epoch": 1.36, + "learning_rate": 2.0429287863590773e-05, + "loss": 1.163, + "theoretical_loss": 3.3315206475570207, + "tokens_seen": 3217672192 + }, + { + "epoch": 1.36, + "learning_rate": 2.0413239719157474e-05, + "loss": 1.149, + "theoretical_loss": 3.3315155483399117, + "tokens_seen": 3217737728 + }, + { + "epoch": 1.36, + "objective/train/docs_used": 1186626, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.170747995376587, + "objective/train/theoretical_loss": 3.3315104492557372, + "objective/train/tokens_used": 3238263264, + "theoretical_loss": 3.3315104492557372, + "tokens_seen": 3217803264 + }, + { + "epoch": 1.36, + "learning_rate": 2.0413239719157474e-05, + "loss": 1.1707, + "theoretical_loss": 3.3315104492557372, + "tokens_seen": 3217803264 + }, + { + "epoch": 1.36, + "learning_rate": 2.039719157472417e-05, + "loss": 1.1291, + "theoretical_loss": 3.3315053503044902, + "tokens_seen": 3217868800 + }, + { + "epoch": 1.36, + "learning_rate": 2.0381143430290875e-05, + "loss": 1.2278, + "theoretical_loss": 3.331500251486166, + "tokens_seen": 3217934336 + }, + { + "epoch": 1.36, + "learning_rate": 2.0365095285857576e-05, + "loss": 1.142, + "theoretical_loss": 3.3314951528007573, + "tokens_seen": 3217999872 + }, + { + "epoch": 1.36, + "learning_rate": 2.0349047141424273e-05, + "loss": 1.1633, + "theoretical_loss": 3.331490054248258, + "tokens_seen": 3218065408 + }, + { + "epoch": 1.36, + "learning_rate": 2.0332998996990974e-05, + "loss": 1.1166, + "theoretical_loss": 3.331484955828662, + "tokens_seen": 3218130944 + }, + { + "epoch": 1.36, + "learning_rate": 2.0316950852557674e-05, + "loss": 1.251, + "theoretical_loss": 3.3314798575419635, + "tokens_seen": 3218196480 + }, + { + "epoch": 1.36, + "learning_rate": 2.0300902708124375e-05, + "loss": 1.228, + "theoretical_loss": 3.3314747593881564, + "tokens_seen": 3218262016 + }, + { + "epoch": 1.36, + "learning_rate": 2.0284854563691076e-05, + "loss": 1.3377, + "theoretical_loss": 3.331469661367234, + "tokens_seen": 3218327552 + }, + { + "epoch": 1.36, + "learning_rate": 2.0268806419257773e-05, + "loss": 1.1238, + "theoretical_loss": 3.3314645634791904, + "tokens_seen": 3218393088 + }, + { + "epoch": 1.36, + "learning_rate": 2.0252758274824473e-05, + "loss": 1.196, + "theoretical_loss": 3.331459465724019, + "tokens_seen": 3218458624 + }, + { + "epoch": 1.36, + "learning_rate": 2.0236710130391174e-05, + "loss": 1.306, + "theoretical_loss": 3.3314543681017144, + "tokens_seen": 3218524160 + }, + { + "epoch": 1.36, + "learning_rate": 2.0220661985957875e-05, + "loss": 1.2753, + "theoretical_loss": 3.3314492706122705, + "tokens_seen": 3218589696 + }, + { + "epoch": 1.36, + "learning_rate": 2.0204613841524575e-05, + "loss": 1.2598, + "theoretical_loss": 3.3314441732556803, + "tokens_seen": 3218655232 + }, + { + "epoch": 1.36, + "learning_rate": 2.0188565697091276e-05, + "loss": 1.2357, + "theoretical_loss": 3.3314390760319377, + "tokens_seen": 3218720768 + }, + { + "epoch": 1.36, + "learning_rate": 2.0172517552657973e-05, + "loss": 1.2605, + "theoretical_loss": 3.331433978941037, + "tokens_seen": 3218786304 + }, + { + "epoch": 1.36, + "learning_rate": 2.0156469408224677e-05, + "loss": 1.3225, + "theoretical_loss": 3.3314288819829727, + "tokens_seen": 3218851840 + }, + { + "epoch": 1.36, + "learning_rate": 2.0140421263791374e-05, + "loss": 1.1969, + "theoretical_loss": 3.331423785157737, + "tokens_seen": 3218917376 + }, + { + "epoch": 1.36, + "learning_rate": 2.0124373119358075e-05, + "loss": 1.2394, + "theoretical_loss": 3.331418688465325, + "tokens_seen": 3218982912 + }, + { + "epoch": 1.36, + "learning_rate": 2.0108324974924776e-05, + "loss": 1.3804, + "theoretical_loss": 3.3314135919057293, + "tokens_seen": 3219048448 + }, + { + "epoch": 1.36, + "learning_rate": 2.0092276830491473e-05, + "loss": 1.1797, + "theoretical_loss": 3.3314084954789456, + "tokens_seen": 3219113984 + }, + { + "epoch": 1.36, + "learning_rate": 2.0076228686058177e-05, + "loss": 1.2406, + "theoretical_loss": 3.331403399184966, + "tokens_seen": 3219179520 + }, + { + "epoch": 1.36, + "learning_rate": 2.0060180541624878e-05, + "loss": 1.1101, + "theoretical_loss": 3.3313983030237853, + "tokens_seen": 3219245056 + }, + { + "epoch": 1.36, + "learning_rate": 2.0044132397191575e-05, + "loss": 1.0769, + "theoretical_loss": 3.3313932069953966, + "tokens_seen": 3219310592 + }, + { + "epoch": 1.36, + "learning_rate": 2.0028084252758276e-05, + "loss": 1.3222, + "theoretical_loss": 3.3313881110997947, + "tokens_seen": 3219376128 + }, + { + "epoch": 1.36, + "learning_rate": 2.0012036108324976e-05, + "loss": 1.163, + "theoretical_loss": 3.331383015336973, + "tokens_seen": 3219441664 + }, + { + "epoch": 1.36, + "learning_rate": 1.9995987963891677e-05, + "loss": 1.1683, + "theoretical_loss": 3.3313779197069247, + "tokens_seen": 3219507200 + }, + { + "epoch": 1.36, + "learning_rate": 1.9979939819458377e-05, + "loss": 1.2679, + "theoretical_loss": 3.3313728242096445, + "tokens_seen": 3219572736 + }, + { + "epoch": 1.36, + "learning_rate": 1.9963891675025075e-05, + "loss": 1.1221, + "theoretical_loss": 3.3313677288451258, + "tokens_seen": 3219638272 + }, + { + "epoch": 1.36, + "learning_rate": 1.9947843530591775e-05, + "loss": 1.1923, + "theoretical_loss": 3.3313626336133626, + "tokens_seen": 3219703808 + }, + { + "epoch": 1.36, + "learning_rate": 1.993179538615848e-05, + "loss": 1.1112, + "theoretical_loss": 3.331357538514349, + "tokens_seen": 3219769344 + }, + { + "epoch": 1.36, + "learning_rate": 1.9915747241725177e-05, + "loss": 1.2423, + "theoretical_loss": 3.331352443548078, + "tokens_seen": 3219834880 + }, + { + "epoch": 1.36, + "learning_rate": 1.9899699097291877e-05, + "loss": 1.2849, + "theoretical_loss": 3.3313473487145444, + "tokens_seen": 3219900416 + }, + { + "epoch": 1.36, + "learning_rate": 1.9883650952858578e-05, + "loss": 1.1818, + "theoretical_loss": 3.331342254013742, + "tokens_seen": 3219965952 + }, + { + "epoch": 1.36, + "learning_rate": 1.9867602808425275e-05, + "loss": 1.2352, + "theoretical_loss": 3.3313371594456633, + "tokens_seen": 3220031488 + }, + { + "epoch": 1.36, + "learning_rate": 1.985155466399198e-05, + "loss": 0.9987, + "theoretical_loss": 3.3313320650103035, + "tokens_seen": 3220097024 + }, + { + "epoch": 1.36, + "learning_rate": 1.9835506519558676e-05, + "loss": 1.2315, + "theoretical_loss": 3.3313269707076563, + "tokens_seen": 3220162560 + }, + { + "epoch": 1.36, + "learning_rate": 1.9819458375125377e-05, + "loss": 1.2062, + "theoretical_loss": 3.3313218765377153, + "tokens_seen": 3220228096 + }, + { + "epoch": 1.36, + "learning_rate": 1.9803410230692078e-05, + "loss": 1.1663, + "theoretical_loss": 3.3313167825004744, + "tokens_seen": 3220293632 + }, + { + "epoch": 1.36, + "learning_rate": 1.9787362086258775e-05, + "loss": 1.1607, + "theoretical_loss": 3.331311688595927, + "tokens_seen": 3220359168 + }, + { + "epoch": 1.36, + "learning_rate": 1.977131394182548e-05, + "loss": 1.1745, + "theoretical_loss": 3.3313065948240674, + "tokens_seen": 3220424704 + }, + { + "epoch": 1.36, + "learning_rate": 1.975526579739218e-05, + "loss": 1.223, + "theoretical_loss": 3.3313015011848894, + "tokens_seen": 3220490240 + }, + { + "epoch": 1.36, + "learning_rate": 1.9739217652958877e-05, + "loss": 1.245, + "theoretical_loss": 3.331296407678387, + "tokens_seen": 3220555776 + }, + { + "epoch": 1.36, + "learning_rate": 1.9723169508525577e-05, + "loss": 1.1854, + "theoretical_loss": 3.3312913143045537, + "tokens_seen": 3220621312 + }, + { + "epoch": 1.36, + "learning_rate": 1.9707121364092278e-05, + "loss": 1.078, + "theoretical_loss": 3.3312862210633836, + "tokens_seen": 3220686848 + }, + { + "epoch": 1.36, + "learning_rate": 1.969107321965898e-05, + "loss": 1.103, + "theoretical_loss": 3.3312811279548704, + "tokens_seen": 3220752384 + }, + { + "epoch": 1.36, + "learning_rate": 1.967502507522568e-05, + "loss": 1.2377, + "theoretical_loss": 3.331276034979008, + "tokens_seen": 3220817920 + }, + { + "epoch": 1.36, + "learning_rate": 1.9658976930792377e-05, + "loss": 1.1844, + "theoretical_loss": 3.33127094213579, + "tokens_seen": 3220883456 + }, + { + "epoch": 1.36, + "learning_rate": 1.9642928786359077e-05, + "loss": 1.2969, + "theoretical_loss": 3.3312658494252108, + "tokens_seen": 3220948992 + }, + { + "epoch": 1.36, + "learning_rate": 1.9626880641925778e-05, + "loss": 1.2552, + "theoretical_loss": 3.3312607568472634, + "tokens_seen": 3221014528 + }, + { + "epoch": 1.36, + "learning_rate": 1.961083249749248e-05, + "loss": 1.2604, + "theoretical_loss": 3.331255664401943, + "tokens_seen": 3221080064 + }, + { + "epoch": 1.36, + "learning_rate": 1.959478435305918e-05, + "loss": 1.1937, + "theoretical_loss": 3.331250572089242, + "tokens_seen": 3221145600 + }, + { + "epoch": 1.36, + "learning_rate": 1.9578736208625876e-05, + "loss": 1.2485, + "theoretical_loss": 3.331245479909155, + "tokens_seen": 3221211136 + }, + { + "epoch": 1.36, + "learning_rate": 1.9562688064192577e-05, + "loss": 1.1822, + "theoretical_loss": 3.331240387861676, + "tokens_seen": 3221276672 + }, + { + "epoch": 1.36, + "learning_rate": 1.954663991975928e-05, + "loss": 1.2424, + "theoretical_loss": 3.331235295946798, + "tokens_seen": 3221342208 + }, + { + "epoch": 1.36, + "learning_rate": 1.9530591775325978e-05, + "loss": 1.1888, + "theoretical_loss": 3.331230204164516, + "tokens_seen": 3221407744 + }, + { + "epoch": 1.36, + "learning_rate": 1.951454363089268e-05, + "loss": 1.2057, + "theoretical_loss": 3.331225112514823, + "tokens_seen": 3221473280 + }, + { + "epoch": 1.36, + "learning_rate": 1.949849548645938e-05, + "loss": 1.2005, + "theoretical_loss": 3.331220020997713, + "tokens_seen": 3221538816 + }, + { + "epoch": 1.36, + "learning_rate": 1.9482447342026077e-05, + "loss": 1.184, + "theoretical_loss": 3.3312149296131803, + "tokens_seen": 3221604352 + }, + { + "epoch": 1.36, + "learning_rate": 1.946639919759278e-05, + "loss": 1.0805, + "theoretical_loss": 3.331209838361218, + "tokens_seen": 3221669888 + }, + { + "epoch": 1.36, + "learning_rate": 1.9450351053159478e-05, + "loss": 1.2692, + "theoretical_loss": 3.331204747241821, + "tokens_seen": 3221735424 + }, + { + "epoch": 1.36, + "learning_rate": 1.943430290872618e-05, + "loss": 1.209, + "theoretical_loss": 3.3311996562549817, + "tokens_seen": 3221800960 + }, + { + "epoch": 1.36, + "learning_rate": 1.941825476429288e-05, + "loss": 1.2669, + "theoretical_loss": 3.3311945654006956, + "tokens_seen": 3221866496 + }, + { + "epoch": 1.36, + "learning_rate": 1.940220661985958e-05, + "loss": 1.1587, + "theoretical_loss": 3.331189474678955, + "tokens_seen": 3221932032 + }, + { + "epoch": 1.36, + "learning_rate": 1.938615847542628e-05, + "loss": 1.1689, + "theoretical_loss": 3.331184384089755, + "tokens_seen": 3221997568 + }, + { + "epoch": 1.36, + "learning_rate": 1.937011033099298e-05, + "loss": 1.3453, + "theoretical_loss": 3.331179293633089, + "tokens_seen": 3222063104 + }, + { + "epoch": 1.36, + "learning_rate": 1.935406218655968e-05, + "loss": 1.1404, + "theoretical_loss": 3.3311742033089504, + "tokens_seen": 3222128640 + }, + { + "epoch": 1.36, + "learning_rate": 1.933801404212638e-05, + "loss": 1.1729, + "theoretical_loss": 3.3311691131173333, + "tokens_seen": 3222194176 + }, + { + "epoch": 1.36, + "learning_rate": 1.932196589769308e-05, + "loss": 1.1771, + "theoretical_loss": 3.331164023058232, + "tokens_seen": 3222259712 + }, + { + "epoch": 1.36, + "learning_rate": 1.930591775325978e-05, + "loss": 1.2292, + "theoretical_loss": 3.3311589331316402, + "tokens_seen": 3222325248 + }, + { + "epoch": 1.36, + "learning_rate": 1.928986960882648e-05, + "loss": 1.2533, + "theoretical_loss": 3.3311538433375514, + "tokens_seen": 3222390784 + }, + { + "epoch": 1.36, + "learning_rate": 1.9273821464393178e-05, + "loss": 1.1827, + "theoretical_loss": 3.3311487536759596, + "tokens_seen": 3222456320 + }, + { + "epoch": 1.36, + "learning_rate": 1.925777331995988e-05, + "loss": 1.1539, + "theoretical_loss": 3.3311436641468593, + "tokens_seen": 3222521856 + }, + { + "epoch": 1.36, + "learning_rate": 1.9241725175526583e-05, + "loss": 1.2183, + "theoretical_loss": 3.331138574750243, + "tokens_seen": 3222587392 + }, + { + "epoch": 1.36, + "learning_rate": 1.922567703109328e-05, + "loss": 1.2382, + "theoretical_loss": 3.331133485486106, + "tokens_seen": 3222652928 + }, + { + "epoch": 1.36, + "learning_rate": 1.920962888665998e-05, + "loss": 1.1931, + "theoretical_loss": 3.331128396354441, + "tokens_seen": 3222718464 + }, + { + "epoch": 1.36, + "learning_rate": 1.919358074222668e-05, + "loss": 1.1791, + "theoretical_loss": 3.3311233073552424, + "tokens_seen": 3222784000 + }, + { + "epoch": 1.36, + "learning_rate": 1.9177532597793382e-05, + "loss": 1.2354, + "theoretical_loss": 3.3311182184885038, + "tokens_seen": 3222849536 + }, + { + "epoch": 1.36, + "learning_rate": 1.9161484453360083e-05, + "loss": 1.2286, + "theoretical_loss": 3.33111312975422, + "tokens_seen": 3222915072 + }, + { + "epoch": 1.36, + "learning_rate": 1.914543630892678e-05, + "loss": 1.258, + "theoretical_loss": 3.3311080411523832, + "tokens_seen": 3222980608 + }, + { + "epoch": 1.36, + "learning_rate": 1.912938816449348e-05, + "loss": 1.1885, + "theoretical_loss": 3.331102952682989, + "tokens_seen": 3223046144 + }, + { + "epoch": 1.36, + "learning_rate": 1.911334002006018e-05, + "loss": 1.1966, + "theoretical_loss": 3.33109786434603, + "tokens_seen": 3223111680 + }, + { + "epoch": 1.36, + "learning_rate": 1.9097291875626882e-05, + "loss": 1.2407, + "theoretical_loss": 3.3310927761415003, + "tokens_seen": 3223177216 + }, + { + "epoch": 1.36, + "learning_rate": 1.9081243731193582e-05, + "loss": 1.0766, + "theoretical_loss": 3.3310876880693945, + "tokens_seen": 3223242752 + }, + { + "epoch": 1.36, + "learning_rate": 1.9065195586760283e-05, + "loss": 1.3575, + "theoretical_loss": 3.331082600129706, + "tokens_seen": 3223308288 + }, + { + "epoch": 1.36, + "learning_rate": 1.904914744232698e-05, + "loss": 1.0904, + "theoretical_loss": 3.331077512322428, + "tokens_seen": 3223373824 + }, + { + "epoch": 1.36, + "learning_rate": 1.903309929789368e-05, + "loss": 1.1583, + "theoretical_loss": 3.331072424647555, + "tokens_seen": 3223439360 + }, + { + "epoch": 1.36, + "learning_rate": 1.901705115346038e-05, + "loss": 1.1242, + "theoretical_loss": 3.3310673371050816, + "tokens_seen": 3223504896 + }, + { + "epoch": 1.36, + "learning_rate": 1.9001003009027082e-05, + "loss": 1.2569, + "theoretical_loss": 3.331062249695, + "tokens_seen": 3223570432 + }, + { + "epoch": 1.36, + "learning_rate": 1.8984954864593783e-05, + "loss": 1.1724, + "theoretical_loss": 3.3310571624173053, + "tokens_seen": 3223635968 + }, + { + "epoch": 1.36, + "learning_rate": 1.896890672016048e-05, + "loss": 1.2722, + "theoretical_loss": 3.331052075271991, + "tokens_seen": 3223701504 + }, + { + "epoch": 1.36, + "learning_rate": 1.8952858575727184e-05, + "loss": 1.1788, + "theoretical_loss": 3.331046988259051, + "tokens_seen": 3223767040 + }, + { + "epoch": 1.36, + "learning_rate": 1.8936810431293885e-05, + "loss": 1.1421, + "theoretical_loss": 3.3310419013784793, + "tokens_seen": 3223832576 + }, + { + "epoch": 1.36, + "learning_rate": 1.8920762286860582e-05, + "loss": 1.2427, + "theoretical_loss": 3.331036814630269, + "tokens_seen": 3223898112 + }, + { + "epoch": 1.36, + "learning_rate": 1.8904714142427283e-05, + "loss": 1.21, + "theoretical_loss": 3.3310317280144153, + "tokens_seen": 3223963648 + }, + { + "epoch": 1.36, + "learning_rate": 1.8888665997993983e-05, + "loss": 1.1368, + "theoretical_loss": 3.3310266415309107, + "tokens_seen": 3224029184 + }, + { + "epoch": 1.36, + "learning_rate": 1.8872617853560684e-05, + "loss": 1.1264, + "theoretical_loss": 3.33102155517975, + "tokens_seen": 3224094720 + }, + { + "epoch": 1.36, + "learning_rate": 1.8856569709127385e-05, + "loss": 1.2764, + "theoretical_loss": 3.331016468960927, + "tokens_seen": 3224160256 + }, + { + "epoch": 1.36, + "learning_rate": 1.8840521564694082e-05, + "loss": 1.1759, + "theoretical_loss": 3.331011382874435, + "tokens_seen": 3224225792 + }, + { + "epoch": 1.36, + "learning_rate": 1.8824473420260782e-05, + "loss": 1.2072, + "theoretical_loss": 3.3310062969202683, + "tokens_seen": 3224291328 + }, + { + "epoch": 1.36, + "objective/train/docs_used": 1188980, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2640712261199951, + "objective/train/theoretical_loss": 3.3310012110984206, + "objective/train/tokens_used": 3244816864, + "theoretical_loss": 3.3310012110984206, + "tokens_seen": 3224356864 + }, + { + "epoch": 1.36, + "learning_rate": 1.8808425275827483e-05, + "loss": 1.2641, + "theoretical_loss": 3.3310012110984206, + "tokens_seen": 3224356864 + }, + { + "epoch": 1.36, + "learning_rate": 1.8792377131394184e-05, + "loss": 1.1943, + "theoretical_loss": 3.330996125408886, + "tokens_seen": 3224422400 + }, + { + "epoch": 1.36, + "learning_rate": 1.8776328986960884e-05, + "loss": 1.1673, + "theoretical_loss": 3.330991039851658, + "tokens_seen": 3224487936 + }, + { + "epoch": 1.36, + "learning_rate": 1.8760280842527585e-05, + "loss": 1.1293, + "theoretical_loss": 3.3309859544267306, + "tokens_seen": 3224553472 + }, + { + "epoch": 1.36, + "learning_rate": 1.8744232698094282e-05, + "loss": 1.1985, + "theoretical_loss": 3.330980869134098, + "tokens_seen": 3224619008 + }, + { + "epoch": 1.36, + "learning_rate": 1.8728184553660986e-05, + "loss": 1.2825, + "theoretical_loss": 3.3309757839737535, + "tokens_seen": 3224684544 + }, + { + "epoch": 1.36, + "learning_rate": 1.8712136409227684e-05, + "loss": 1.1528, + "theoretical_loss": 3.3309706989456913, + "tokens_seen": 3224750080 + }, + { + "epoch": 1.36, + "learning_rate": 1.8696088264794384e-05, + "loss": 1.3087, + "theoretical_loss": 3.3309656140499055, + "tokens_seen": 3224815616 + }, + { + "epoch": 1.36, + "learning_rate": 1.8680040120361085e-05, + "loss": 1.1253, + "theoretical_loss": 3.3309605292863895, + "tokens_seen": 3224881152 + }, + { + "epoch": 1.36, + "learning_rate": 1.8663991975927782e-05, + "loss": 1.1829, + "theoretical_loss": 3.3309554446551375, + "tokens_seen": 3224946688 + }, + { + "epoch": 1.36, + "learning_rate": 1.8647943831494486e-05, + "loss": 1.2035, + "theoretical_loss": 3.330950360156143, + "tokens_seen": 3225012224 + }, + { + "epoch": 1.36, + "learning_rate": 1.8631895687061183e-05, + "loss": 1.2652, + "theoretical_loss": 3.330945275789401, + "tokens_seen": 3225077760 + }, + { + "epoch": 1.36, + "learning_rate": 1.8615847542627884e-05, + "loss": 1.1925, + "theoretical_loss": 3.3309401915549035, + "tokens_seen": 3225143296 + }, + { + "epoch": 1.36, + "learning_rate": 1.8599799398194585e-05, + "loss": 1.2433, + "theoretical_loss": 3.330935107452646, + "tokens_seen": 3225208832 + }, + { + "epoch": 1.36, + "learning_rate": 1.8583751253761282e-05, + "loss": 1.1765, + "theoretical_loss": 3.3309300234826216, + "tokens_seen": 3225274368 + }, + { + "epoch": 1.36, + "learning_rate": 1.8567703109327986e-05, + "loss": 1.2185, + "theoretical_loss": 3.330924939644824, + "tokens_seen": 3225339904 + }, + { + "epoch": 1.36, + "learning_rate": 1.8551654964894686e-05, + "loss": 1.3971, + "theoretical_loss": 3.330919855939248, + "tokens_seen": 3225405440 + }, + { + "epoch": 1.36, + "learning_rate": 1.8535606820461384e-05, + "loss": 1.127, + "theoretical_loss": 3.3309147723658863, + "tokens_seen": 3225470976 + }, + { + "epoch": 1.36, + "learning_rate": 1.8519558676028084e-05, + "loss": 1.2557, + "theoretical_loss": 3.330909688924734, + "tokens_seen": 3225536512 + }, + { + "epoch": 1.36, + "learning_rate": 1.8503510531594785e-05, + "loss": 1.3163, + "theoretical_loss": 3.3309046056157836, + "tokens_seen": 3225602048 + }, + { + "epoch": 1.36, + "learning_rate": 1.8487462387161486e-05, + "loss": 1.1194, + "theoretical_loss": 3.3308995224390303, + "tokens_seen": 3225667584 + }, + { + "epoch": 1.36, + "learning_rate": 1.8471414242728186e-05, + "loss": 1.1843, + "theoretical_loss": 3.330894439394467, + "tokens_seen": 3225733120 + }, + { + "epoch": 1.36, + "learning_rate": 1.8455366098294884e-05, + "loss": 1.3575, + "theoretical_loss": 3.3308893564820883, + "tokens_seen": 3225798656 + }, + { + "epoch": 1.36, + "learning_rate": 1.8439317953861584e-05, + "loss": 1.2606, + "theoretical_loss": 3.330884273701887, + "tokens_seen": 3225864192 + }, + { + "epoch": 1.36, + "learning_rate": 1.8423269809428288e-05, + "loss": 1.1087, + "theoretical_loss": 3.3308791910538584, + "tokens_seen": 3225929728 + }, + { + "epoch": 1.36, + "learning_rate": 1.8407221664994985e-05, + "loss": 1.1901, + "theoretical_loss": 3.3308741085379956, + "tokens_seen": 3225995264 + }, + { + "epoch": 1.36, + "learning_rate": 1.8391173520561686e-05, + "loss": 1.2059, + "theoretical_loss": 3.3308690261542924, + "tokens_seen": 3226060800 + }, + { + "epoch": 1.36, + "learning_rate": 1.8375125376128387e-05, + "loss": 1.1027, + "theoretical_loss": 3.330863943902743, + "tokens_seen": 3226126336 + }, + { + "epoch": 1.36, + "learning_rate": 1.8359077231695084e-05, + "loss": 1.3466, + "theoretical_loss": 3.3308588617833412, + "tokens_seen": 3226191872 + }, + { + "epoch": 1.36, + "learning_rate": 1.8343029087261788e-05, + "loss": 1.283, + "theoretical_loss": 3.3308537797960804, + "tokens_seen": 3226257408 + }, + { + "epoch": 1.36, + "learning_rate": 1.8326980942828485e-05, + "loss": 1.2081, + "theoretical_loss": 3.3308486979409553, + "tokens_seen": 3226322944 + }, + { + "epoch": 1.36, + "learning_rate": 1.8310932798395186e-05, + "loss": 1.1602, + "theoretical_loss": 3.330843616217959, + "tokens_seen": 3226388480 + }, + { + "epoch": 1.36, + "learning_rate": 1.8294884653961886e-05, + "loss": 1.1706, + "theoretical_loss": 3.330838534627086, + "tokens_seen": 3226454016 + }, + { + "epoch": 1.36, + "learning_rate": 1.8278836509528587e-05, + "loss": 1.1249, + "theoretical_loss": 3.33083345316833, + "tokens_seen": 3226519552 + }, + { + "epoch": 1.36, + "learning_rate": 1.8262788365095288e-05, + "loss": 1.2271, + "theoretical_loss": 3.330828371841685, + "tokens_seen": 3226585088 + }, + { + "epoch": 1.36, + "learning_rate": 1.824674022066199e-05, + "loss": 1.2483, + "theoretical_loss": 3.330823290647144, + "tokens_seen": 3226650624 + }, + { + "epoch": 1.36, + "learning_rate": 1.8230692076228686e-05, + "loss": 1.1369, + "theoretical_loss": 3.3308182095847023, + "tokens_seen": 3226716160 + }, + { + "epoch": 1.36, + "learning_rate": 1.8214643931795386e-05, + "loss": 1.1708, + "theoretical_loss": 3.330813128654353, + "tokens_seen": 3226781696 + }, + { + "epoch": 1.36, + "learning_rate": 1.8198595787362087e-05, + "loss": 1.2532, + "theoretical_loss": 3.3308080478560895, + "tokens_seen": 3226847232 + }, + { + "epoch": 1.36, + "learning_rate": 1.8182547642928788e-05, + "loss": 1.1045, + "theoretical_loss": 3.3308029671899066, + "tokens_seen": 3226912768 + }, + { + "epoch": 1.36, + "learning_rate": 1.8166499498495488e-05, + "loss": 1.1446, + "theoretical_loss": 3.3307978866557977, + "tokens_seen": 3226978304 + }, + { + "epoch": 1.36, + "learning_rate": 1.8150451354062185e-05, + "loss": 1.2306, + "theoretical_loss": 3.330792806253757, + "tokens_seen": 3227043840 + }, + { + "epoch": 1.36, + "learning_rate": 1.8134403209628886e-05, + "loss": 1.2397, + "theoretical_loss": 3.330787725983778, + "tokens_seen": 3227109376 + }, + { + "epoch": 1.36, + "learning_rate": 1.811835506519559e-05, + "loss": 1.3084, + "theoretical_loss": 3.3307826458458543, + "tokens_seen": 3227174912 + }, + { + "epoch": 1.36, + "learning_rate": 1.8102306920762287e-05, + "loss": 1.2293, + "theoretical_loss": 3.330777565839981, + "tokens_seen": 3227240448 + }, + { + "epoch": 1.36, + "learning_rate": 1.8086258776328988e-05, + "loss": 1.2019, + "theoretical_loss": 3.330772485966151, + "tokens_seen": 3227305984 + }, + { + "epoch": 1.36, + "learning_rate": 1.807021063189569e-05, + "loss": 1.1984, + "theoretical_loss": 3.3307674062243584, + "tokens_seen": 3227371520 + }, + { + "epoch": 1.36, + "learning_rate": 1.805416248746239e-05, + "loss": 1.2109, + "theoretical_loss": 3.3307623266145967, + "tokens_seen": 3227437056 + }, + { + "epoch": 1.36, + "learning_rate": 1.803811434302909e-05, + "loss": 1.2283, + "theoretical_loss": 3.330757247136861, + "tokens_seen": 3227502592 + }, + { + "epoch": 1.36, + "learning_rate": 1.8022066198595787e-05, + "loss": 1.2767, + "theoretical_loss": 3.330752167791144, + "tokens_seen": 3227568128 + }, + { + "epoch": 1.36, + "learning_rate": 1.8006018054162488e-05, + "loss": 1.2007, + "theoretical_loss": 3.3307470885774397, + "tokens_seen": 3227633664 + }, + { + "epoch": 1.36, + "learning_rate": 1.798996990972919e-05, + "loss": 1.2488, + "theoretical_loss": 3.3307420094957427, + "tokens_seen": 3227699200 + }, + { + "epoch": 1.36, + "learning_rate": 1.797392176529589e-05, + "loss": 1.1833, + "theoretical_loss": 3.330736930546046, + "tokens_seen": 3227764736 + }, + { + "epoch": 1.36, + "learning_rate": 1.795787362086259e-05, + "loss": 1.2987, + "theoretical_loss": 3.330731851728344, + "tokens_seen": 3227830272 + }, + { + "epoch": 1.36, + "learning_rate": 1.794182547642929e-05, + "loss": 1.1659, + "theoretical_loss": 3.330726773042631, + "tokens_seen": 3227895808 + }, + { + "epoch": 1.36, + "learning_rate": 1.7925777331995988e-05, + "loss": 1.2313, + "theoretical_loss": 3.3307216944888998, + "tokens_seen": 3227961344 + }, + { + "epoch": 1.36, + "learning_rate": 1.7909729187562688e-05, + "loss": 1.3589, + "theoretical_loss": 3.3307166160671455, + "tokens_seen": 3228026880 + }, + { + "epoch": 1.36, + "learning_rate": 1.789368104312939e-05, + "loss": 1.1375, + "theoretical_loss": 3.3307115377773613, + "tokens_seen": 3228092416 + }, + { + "epoch": 1.36, + "learning_rate": 1.787763289869609e-05, + "loss": 1.3774, + "theoretical_loss": 3.330706459619541, + "tokens_seen": 3228157952 + }, + { + "epoch": 1.36, + "learning_rate": 1.786158475426279e-05, + "loss": 1.1077, + "theoretical_loss": 3.330701381593679, + "tokens_seen": 3228223488 + }, + { + "epoch": 1.36, + "learning_rate": 1.7845536609829487e-05, + "loss": 1.4038, + "theoretical_loss": 3.330696303699768, + "tokens_seen": 3228289024 + }, + { + "epoch": 1.36, + "learning_rate": 1.782948846539619e-05, + "loss": 1.3449, + "theoretical_loss": 3.3306912259378034, + "tokens_seen": 3228354560 + }, + { + "epoch": 1.36, + "learning_rate": 1.7813440320962892e-05, + "loss": 1.1356, + "theoretical_loss": 3.3306861483077785, + "tokens_seen": 3228420096 + }, + { + "epoch": 1.36, + "learning_rate": 1.779739217652959e-05, + "loss": 1.1346, + "theoretical_loss": 3.3306810708096872, + "tokens_seen": 3228485632 + }, + { + "epoch": 1.36, + "learning_rate": 1.778134403209629e-05, + "loss": 1.332, + "theoretical_loss": 3.3306759934435233, + "tokens_seen": 3228551168 + }, + { + "epoch": 1.36, + "learning_rate": 1.776529588766299e-05, + "loss": 1.1203, + "theoretical_loss": 3.3306709162092805, + "tokens_seen": 3228616704 + }, + { + "epoch": 1.36, + "learning_rate": 1.774924774322969e-05, + "loss": 1.2429, + "theoretical_loss": 3.3306658391069535, + "tokens_seen": 3228682240 + }, + { + "epoch": 1.36, + "learning_rate": 1.7733199598796392e-05, + "loss": 1.1574, + "theoretical_loss": 3.330660762136535, + "tokens_seen": 3228747776 + }, + { + "epoch": 1.36, + "learning_rate": 1.771715145436309e-05, + "loss": 1.3608, + "theoretical_loss": 3.33065568529802, + "tokens_seen": 3228813312 + }, + { + "epoch": 1.36, + "learning_rate": 1.770110330992979e-05, + "loss": 1.199, + "theoretical_loss": 3.3306506085914016, + "tokens_seen": 3228878848 + }, + { + "epoch": 1.36, + "learning_rate": 1.768505516549649e-05, + "loss": 1.1815, + "theoretical_loss": 3.330645532016674, + "tokens_seen": 3228944384 + }, + { + "epoch": 1.36, + "learning_rate": 1.766900702106319e-05, + "loss": 1.1364, + "theoretical_loss": 3.3306404555738314, + "tokens_seen": 3229009920 + }, + { + "epoch": 1.36, + "learning_rate": 1.765295887662989e-05, + "loss": 1.264, + "theoretical_loss": 3.3306353792628673, + "tokens_seen": 3229075456 + }, + { + "epoch": 1.36, + "learning_rate": 1.763691073219659e-05, + "loss": 1.1467, + "theoretical_loss": 3.3306303030837756, + "tokens_seen": 3229140992 + }, + { + "epoch": 1.36, + "learning_rate": 1.762086258776329e-05, + "loss": 1.2112, + "theoretical_loss": 3.3306252270365504, + "tokens_seen": 3229206528 + }, + { + "epoch": 1.36, + "learning_rate": 1.7604814443329993e-05, + "loss": 1.2066, + "theoretical_loss": 3.3306201511211855, + "tokens_seen": 3229272064 + }, + { + "epoch": 1.36, + "learning_rate": 1.758876629889669e-05, + "loss": 1.2755, + "theoretical_loss": 3.3306150753376746, + "tokens_seen": 3229337600 + }, + { + "epoch": 1.36, + "learning_rate": 1.757271815446339e-05, + "loss": 1.1146, + "theoretical_loss": 3.330609999686012, + "tokens_seen": 3229403136 + }, + { + "epoch": 1.36, + "learning_rate": 1.7556670010030092e-05, + "loss": 1.285, + "theoretical_loss": 3.3306049241661917, + "tokens_seen": 3229468672 + }, + { + "epoch": 1.36, + "learning_rate": 1.754062186559679e-05, + "loss": 1.1659, + "theoretical_loss": 3.330599848778207, + "tokens_seen": 3229534208 + }, + { + "epoch": 1.36, + "learning_rate": 1.7524573721163493e-05, + "loss": 1.1619, + "theoretical_loss": 3.3305947735220522, + "tokens_seen": 3229599744 + }, + { + "epoch": 1.36, + "learning_rate": 1.750852557673019e-05, + "loss": 1.1619, + "theoretical_loss": 3.330589698397721, + "tokens_seen": 3229665280 + }, + { + "epoch": 1.36, + "learning_rate": 1.749247743229689e-05, + "loss": 1.2737, + "theoretical_loss": 3.3305846234052074, + "tokens_seen": 3229730816 + }, + { + "epoch": 1.36, + "learning_rate": 1.7476429287863592e-05, + "loss": 1.21, + "theoretical_loss": 3.3305795485445056, + "tokens_seen": 3229796352 + }, + { + "epoch": 1.36, + "learning_rate": 1.746038114343029e-05, + "loss": 1.085, + "theoretical_loss": 3.330574473815609, + "tokens_seen": 3229861888 + }, + { + "epoch": 1.36, + "learning_rate": 1.7444332998996993e-05, + "loss": 1.1199, + "theoretical_loss": 3.3305693992185113, + "tokens_seen": 3229927424 + }, + { + "epoch": 1.36, + "learning_rate": 1.7428284854563694e-05, + "loss": 1.1691, + "theoretical_loss": 3.3305643247532073, + "tokens_seen": 3229992960 + }, + { + "epoch": 1.36, + "learning_rate": 1.741223671013039e-05, + "loss": 1.2176, + "theoretical_loss": 3.3305592504196904, + "tokens_seen": 3230058496 + }, + { + "epoch": 1.36, + "learning_rate": 1.739618856569709e-05, + "loss": 1.1561, + "theoretical_loss": 3.3305541762179547, + "tokens_seen": 3230124032 + }, + { + "epoch": 1.36, + "learning_rate": 1.7380140421263792e-05, + "loss": 1.3148, + "theoretical_loss": 3.3305491021479936, + "tokens_seen": 3230189568 + }, + { + "epoch": 1.36, + "learning_rate": 1.7364092276830493e-05, + "loss": 1.1899, + "theoretical_loss": 3.3305440282098013, + "tokens_seen": 3230255104 + }, + { + "epoch": 1.36, + "learning_rate": 1.7348044132397193e-05, + "loss": 1.1899, + "theoretical_loss": 3.330538954403372, + "tokens_seen": 3230320640 + }, + { + "epoch": 1.36, + "learning_rate": 1.733199598796389e-05, + "loss": 1.1563, + "theoretical_loss": 3.3305338807286993, + "tokens_seen": 3230386176 + }, + { + "epoch": 1.36, + "learning_rate": 1.731594784353059e-05, + "loss": 1.2252, + "theoretical_loss": 3.330528807185777, + "tokens_seen": 3230451712 + }, + { + "epoch": 1.36, + "learning_rate": 1.7299899699097295e-05, + "loss": 1.1773, + "theoretical_loss": 3.330523733774599, + "tokens_seen": 3230517248 + }, + { + "epoch": 1.36, + "learning_rate": 1.7283851554663993e-05, + "loss": 1.1648, + "theoretical_loss": 3.3305186604951595, + "tokens_seen": 3230582784 + }, + { + "epoch": 1.36, + "learning_rate": 1.7267803410230693e-05, + "loss": 1.1177, + "theoretical_loss": 3.3305135873474523, + "tokens_seen": 3230648320 + }, + { + "epoch": 1.36, + "learning_rate": 1.7251755265797394e-05, + "loss": 1.0181, + "theoretical_loss": 3.3305085143314717, + "tokens_seen": 3230713856 + }, + { + "epoch": 1.36, + "learning_rate": 1.723570712136409e-05, + "loss": 1.1901, + "theoretical_loss": 3.3305034414472106, + "tokens_seen": 3230779392 + }, + { + "epoch": 1.36, + "learning_rate": 1.7219658976930795e-05, + "loss": 1.209, + "theoretical_loss": 3.3304983686946636, + "tokens_seen": 3230844928 + }, + { + "epoch": 1.36, + "objective/train/docs_used": 1191645, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1450773477554321, + "objective/train/theoretical_loss": 3.3304932960738247, + "objective/train/tokens_used": 3251370464, + "theoretical_loss": 3.3304932960738247, + "tokens_seen": 3230910464 + }, + { + "epoch": 1.36, + "learning_rate": 1.7203610832497492e-05, + "loss": 1.1451, + "theoretical_loss": 3.3304932960738247, + "tokens_seen": 3230910464 + }, + { + "epoch": 1.36, + "learning_rate": 1.7187562688064193e-05, + "loss": 1.1579, + "theoretical_loss": 3.3304882235846875, + "tokens_seen": 3230976000 + }, + { + "epoch": 1.36, + "learning_rate": 1.7171514543630894e-05, + "loss": 1.0722, + "theoretical_loss": 3.330483151227246, + "tokens_seen": 3231041536 + }, + { + "epoch": 1.36, + "learning_rate": 1.715546639919759e-05, + "loss": 1.1676, + "theoretical_loss": 3.3304780790014945, + "tokens_seen": 3231107072 + }, + { + "epoch": 1.36, + "learning_rate": 1.7139418254764295e-05, + "loss": 1.1638, + "theoretical_loss": 3.330473006907426, + "tokens_seen": 3231172608 + }, + { + "epoch": 1.36, + "learning_rate": 1.7123370110330996e-05, + "loss": 1.1233, + "theoretical_loss": 3.3304679349450352, + "tokens_seen": 3231238144 + }, + { + "epoch": 1.36, + "learning_rate": 1.7107321965897693e-05, + "loss": 1.1729, + "theoretical_loss": 3.330462863114316, + "tokens_seen": 3231303680 + }, + { + "epoch": 1.36, + "learning_rate": 1.7091273821464393e-05, + "loss": 1.3022, + "theoretical_loss": 3.3304577914152613, + "tokens_seen": 3231369216 + }, + { + "epoch": 1.36, + "learning_rate": 1.7075225677031094e-05, + "loss": 1.1985, + "theoretical_loss": 3.3304527198478664, + "tokens_seen": 3231434752 + }, + { + "epoch": 1.36, + "learning_rate": 1.7059177532597795e-05, + "loss": 1.1713, + "theoretical_loss": 3.3304476484121244, + "tokens_seen": 3231500288 + }, + { + "epoch": 1.36, + "learning_rate": 1.7043129388164495e-05, + "loss": 1.0944, + "theoretical_loss": 3.3304425771080295, + "tokens_seen": 3231565824 + }, + { + "epoch": 1.36, + "learning_rate": 1.7027081243731193e-05, + "loss": 1.1829, + "theoretical_loss": 3.3304375059355755, + "tokens_seen": 3231631360 + }, + { + "epoch": 1.36, + "learning_rate": 1.7011033099297893e-05, + "loss": 1.0547, + "theoretical_loss": 3.3304324348947563, + "tokens_seen": 3231696896 + }, + { + "epoch": 1.36, + "learning_rate": 1.6994984954864597e-05, + "loss": 1.2579, + "theoretical_loss": 3.330427363985566, + "tokens_seen": 3231762432 + }, + { + "epoch": 1.36, + "learning_rate": 1.6978936810431294e-05, + "loss": 1.1695, + "theoretical_loss": 3.330422293207998, + "tokens_seen": 3231827968 + }, + { + "epoch": 1.36, + "learning_rate": 1.6962888665997995e-05, + "loss": 1.1312, + "theoretical_loss": 3.3304172225620468, + "tokens_seen": 3231893504 + }, + { + "epoch": 1.36, + "learning_rate": 1.6946840521564696e-05, + "loss": 1.0552, + "theoretical_loss": 3.330412152047706, + "tokens_seen": 3231959040 + }, + { + "epoch": 1.36, + "learning_rate": 1.6930792377131393e-05, + "loss": 1.2636, + "theoretical_loss": 3.33040708166497, + "tokens_seen": 3232024576 + }, + { + "epoch": 1.36, + "learning_rate": 1.6914744232698097e-05, + "loss": 1.1361, + "theoretical_loss": 3.330402011413832, + "tokens_seen": 3232090112 + }, + { + "epoch": 1.36, + "learning_rate": 1.6898696088264794e-05, + "loss": 1.2267, + "theoretical_loss": 3.330396941294286, + "tokens_seen": 3232155648 + }, + { + "epoch": 1.36, + "learning_rate": 1.6882647943831495e-05, + "loss": 1.1011, + "theoretical_loss": 3.3303918713063263, + "tokens_seen": 3232221184 + }, + { + "epoch": 1.36, + "learning_rate": 1.6866599799398196e-05, + "loss": 1.2109, + "theoretical_loss": 3.330386801449947, + "tokens_seen": 3232286720 + }, + { + "epoch": 1.36, + "learning_rate": 1.6850551654964896e-05, + "loss": 1.163, + "theoretical_loss": 3.3303817317251414, + "tokens_seen": 3232352256 + }, + { + "epoch": 1.36, + "learning_rate": 1.6834503510531597e-05, + "loss": 1.1562, + "theoretical_loss": 3.3303766621319038, + "tokens_seen": 3232417792 + }, + { + "epoch": 1.36, + "learning_rate": 1.6818455366098297e-05, + "loss": 1.1994, + "theoretical_loss": 3.330371592670228, + "tokens_seen": 3232483328 + }, + { + "epoch": 1.36, + "learning_rate": 1.6802407221664995e-05, + "loss": 1.2429, + "theoretical_loss": 3.330366523340108, + "tokens_seen": 3232548864 + }, + { + "epoch": 1.36, + "learning_rate": 1.6786359077231695e-05, + "loss": 1.2045, + "theoretical_loss": 3.3303614541415376, + "tokens_seen": 3232614400 + }, + { + "epoch": 1.36, + "learning_rate": 1.6770310932798396e-05, + "loss": 1.1054, + "theoretical_loss": 3.330356385074511, + "tokens_seen": 3232679936 + }, + { + "epoch": 1.36, + "learning_rate": 1.6754262788365097e-05, + "loss": 1.1579, + "theoretical_loss": 3.3303513161390215, + "tokens_seen": 3232745472 + }, + { + "epoch": 1.36, + "learning_rate": 1.6738214643931797e-05, + "loss": 1.1755, + "theoretical_loss": 3.3303462473350636, + "tokens_seen": 3232811008 + }, + { + "epoch": 1.36, + "learning_rate": 1.6722166499498494e-05, + "loss": 1.1913, + "theoretical_loss": 3.330341178662631, + "tokens_seen": 3232876544 + }, + { + "epoch": 1.36, + "learning_rate": 1.6706118355065195e-05, + "loss": 1.1196, + "theoretical_loss": 3.330336110121718, + "tokens_seen": 3232942080 + }, + { + "epoch": 1.36, + "learning_rate": 1.6690070210631896e-05, + "loss": 1.1155, + "theoretical_loss": 3.3303310417123178, + "tokens_seen": 3233007616 + }, + { + "epoch": 1.36, + "learning_rate": 1.6674022066198596e-05, + "loss": 1.2298, + "theoretical_loss": 3.330325973434425, + "tokens_seen": 3233073152 + }, + { + "epoch": 1.36, + "learning_rate": 1.6657973921765297e-05, + "loss": 1.1544, + "theoretical_loss": 3.330320905288033, + "tokens_seen": 3233138688 + }, + { + "epoch": 1.36, + "learning_rate": 1.6641925777331994e-05, + "loss": 1.1815, + "theoretical_loss": 3.330315837273136, + "tokens_seen": 3233204224 + }, + { + "epoch": 1.36, + "learning_rate": 1.6625877632898698e-05, + "loss": 1.2367, + "theoretical_loss": 3.330310769389728, + "tokens_seen": 3233269760 + }, + { + "epoch": 1.36, + "learning_rate": 1.66098294884654e-05, + "loss": 1.2064, + "theoretical_loss": 3.330305701637802, + "tokens_seen": 3233335296 + }, + { + "epoch": 1.36, + "learning_rate": 1.6593781344032096e-05, + "loss": 1.2233, + "theoretical_loss": 3.3303006340173535, + "tokens_seen": 3233400832 + }, + { + "epoch": 1.36, + "learning_rate": 1.6577733199598797e-05, + "loss": 1.1686, + "theoretical_loss": 3.3302955665283758, + "tokens_seen": 3233466368 + }, + { + "epoch": 1.36, + "learning_rate": 1.6561685055165497e-05, + "loss": 1.2784, + "theoretical_loss": 3.3302904991708626, + "tokens_seen": 3233531904 + }, + { + "epoch": 1.36, + "learning_rate": 1.6545636910732198e-05, + "loss": 1.291, + "theoretical_loss": 3.330285431944808, + "tokens_seen": 3233597440 + }, + { + "epoch": 1.36, + "learning_rate": 1.65295887662989e-05, + "loss": 1.2196, + "theoretical_loss": 3.3302803648502053, + "tokens_seen": 3233662976 + }, + { + "epoch": 1.36, + "learning_rate": 1.6513540621865596e-05, + "loss": 1.1361, + "theoretical_loss": 3.330275297887049, + "tokens_seen": 3233728512 + }, + { + "epoch": 1.36, + "learning_rate": 1.6497492477432297e-05, + "loss": 1.2028, + "theoretical_loss": 3.330270231055333, + "tokens_seen": 3233794048 + }, + { + "epoch": 1.36, + "learning_rate": 1.6481444332998997e-05, + "loss": 1.2236, + "theoretical_loss": 3.3302651643550516, + "tokens_seen": 3233859584 + }, + { + "epoch": 1.36, + "learning_rate": 1.6465396188565698e-05, + "loss": 1.2602, + "theoretical_loss": 3.330260097786198, + "tokens_seen": 3233925120 + }, + { + "epoch": 1.36, + "learning_rate": 1.64493480441324e-05, + "loss": 1.184, + "theoretical_loss": 3.3302550313487664, + "tokens_seen": 3233990656 + }, + { + "epoch": 1.36, + "learning_rate": 1.64332998996991e-05, + "loss": 1.1947, + "theoretical_loss": 3.3302499650427513, + "tokens_seen": 3234056192 + }, + { + "epoch": 1.36, + "learning_rate": 1.6417251755265796e-05, + "loss": 1.1431, + "theoretical_loss": 3.3302448988681457, + "tokens_seen": 3234121728 + }, + { + "epoch": 1.36, + "learning_rate": 1.64012036108325e-05, + "loss": 1.1798, + "theoretical_loss": 3.330239832824944, + "tokens_seen": 3234187264 + }, + { + "epoch": 1.36, + "learning_rate": 1.6385155466399198e-05, + "loss": 1.3256, + "theoretical_loss": 3.33023476691314, + "tokens_seen": 3234252800 + }, + { + "epoch": 1.36, + "learning_rate": 1.6369107321965898e-05, + "loss": 1.1623, + "theoretical_loss": 3.330229701132728, + "tokens_seen": 3234318336 + }, + { + "epoch": 1.36, + "learning_rate": 1.63530591775326e-05, + "loss": 1.1423, + "theoretical_loss": 3.330224635483701, + "tokens_seen": 3234383872 + }, + { + "epoch": 1.36, + "learning_rate": 1.6337011033099296e-05, + "loss": 1.1542, + "theoretical_loss": 3.330219569966054, + "tokens_seen": 3234449408 + }, + { + "epoch": 1.36, + "learning_rate": 1.6320962888666e-05, + "loss": 1.1314, + "theoretical_loss": 3.3302145045797804, + "tokens_seen": 3234514944 + }, + { + "epoch": 1.36, + "learning_rate": 1.63049147442327e-05, + "loss": 1.2786, + "theoretical_loss": 3.3302094393248742, + "tokens_seen": 3234580480 + }, + { + "epoch": 1.36, + "learning_rate": 1.6288866599799398e-05, + "loss": 1.2544, + "theoretical_loss": 3.3302043742013296, + "tokens_seen": 3234646016 + }, + { + "epoch": 1.36, + "learning_rate": 1.62728184553661e-05, + "loss": 1.2097, + "theoretical_loss": 3.33019930920914, + "tokens_seen": 3234711552 + }, + { + "epoch": 1.36, + "learning_rate": 1.62567703109328e-05, + "loss": 1.1523, + "theoretical_loss": 3.3301942443483, + "tokens_seen": 3234777088 + }, + { + "epoch": 1.36, + "learning_rate": 1.62407221664995e-05, + "loss": 1.1858, + "theoretical_loss": 3.330189179618803, + "tokens_seen": 3234842624 + }, + { + "epoch": 1.36, + "learning_rate": 1.62246740220662e-05, + "loss": 1.2123, + "theoretical_loss": 3.330184115020643, + "tokens_seen": 3234908160 + }, + { + "epoch": 1.36, + "learning_rate": 1.6208625877632898e-05, + "loss": 1.2387, + "theoretical_loss": 3.330179050553814, + "tokens_seen": 3234973696 + }, + { + "epoch": 1.36, + "learning_rate": 1.61925777331996e-05, + "loss": 1.2235, + "theoretical_loss": 3.3301739862183095, + "tokens_seen": 3235039232 + }, + { + "epoch": 1.36, + "learning_rate": 1.6176529588766302e-05, + "loss": 1.2965, + "theoretical_loss": 3.330168922014124, + "tokens_seen": 3235104768 + }, + { + "epoch": 1.36, + "learning_rate": 1.6160481444333e-05, + "loss": 1.1427, + "theoretical_loss": 3.330163857941252, + "tokens_seen": 3235170304 + }, + { + "epoch": 1.36, + "learning_rate": 1.61444332998997e-05, + "loss": 1.2338, + "theoretical_loss": 3.3301587939996864, + "tokens_seen": 3235235840 + }, + { + "epoch": 1.36, + "learning_rate": 1.61283851554664e-05, + "loss": 1.336, + "theoretical_loss": 3.3301537301894215, + "tokens_seen": 3235301376 + }, + { + "epoch": 1.36, + "learning_rate": 1.6112337011033098e-05, + "loss": 1.1702, + "theoretical_loss": 3.330148666510451, + "tokens_seen": 3235366912 + }, + { + "epoch": 1.36, + "learning_rate": 1.6096288866599802e-05, + "loss": 1.1457, + "theoretical_loss": 3.330143602962769, + "tokens_seen": 3235432448 + }, + { + "epoch": 1.36, + "learning_rate": 1.60802407221665e-05, + "loss": 1.2707, + "theoretical_loss": 3.33013853954637, + "tokens_seen": 3235497984 + }, + { + "epoch": 1.36, + "learning_rate": 1.60641925777332e-05, + "loss": 1.1253, + "theoretical_loss": 3.3301334762612473, + "tokens_seen": 3235563520 + }, + { + "epoch": 1.36, + "learning_rate": 1.60481444332999e-05, + "loss": 1.192, + "theoretical_loss": 3.3301284131073947, + "tokens_seen": 3235629056 + }, + { + "epoch": 1.36, + "learning_rate": 1.6032096288866598e-05, + "loss": 1.2195, + "theoretical_loss": 3.330123350084807, + "tokens_seen": 3235694592 + }, + { + "epoch": 1.36, + "learning_rate": 1.6016048144433302e-05, + "loss": 1.2705, + "theoretical_loss": 3.3301182871934767, + "tokens_seen": 3235760128 + }, + { + "epoch": 1.36, + "learning_rate": 1.6000000000000003e-05, + "loss": 1.3154, + "theoretical_loss": 3.3301132244333993, + "tokens_seen": 3235825664 + }, + { + "epoch": 1.36, + "learning_rate": 1.59839518555667e-05, + "loss": 1.0701, + "theoretical_loss": 3.3301081618045676, + "tokens_seen": 3235891200 + }, + { + "epoch": 1.36, + "learning_rate": 1.59679037111334e-05, + "loss": 1.175, + "theoretical_loss": 3.3301030993069762, + "tokens_seen": 3235956736 + }, + { + "epoch": 1.36, + "learning_rate": 1.59518555667001e-05, + "loss": 1.1826, + "theoretical_loss": 3.3300980369406186, + "tokens_seen": 3236022272 + }, + { + "epoch": 1.36, + "learning_rate": 1.5935807422266802e-05, + "loss": 1.1595, + "theoretical_loss": 3.3300929747054893, + "tokens_seen": 3236087808 + }, + { + "epoch": 1.36, + "learning_rate": 1.5919759277833502e-05, + "loss": 1.2541, + "theoretical_loss": 3.3300879126015817, + "tokens_seen": 3236153344 + }, + { + "epoch": 1.36, + "learning_rate": 1.59037111334002e-05, + "loss": 1.1354, + "theoretical_loss": 3.3300828506288895, + "tokens_seen": 3236218880 + }, + { + "epoch": 1.36, + "learning_rate": 1.58876629889669e-05, + "loss": 1.2496, + "theoretical_loss": 3.3300777887874076, + "tokens_seen": 3236284416 + }, + { + "epoch": 1.36, + "learning_rate": 1.5871614844533604e-05, + "loss": 1.1404, + "theoretical_loss": 3.3300727270771295, + "tokens_seen": 3236349952 + }, + { + "epoch": 1.36, + "learning_rate": 1.58555667001003e-05, + "loss": 1.2552, + "theoretical_loss": 3.330067665498049, + "tokens_seen": 3236415488 + }, + { + "epoch": 1.36, + "learning_rate": 1.5839518555667002e-05, + "loss": 1.1026, + "theoretical_loss": 3.3300626040501595, + "tokens_seen": 3236481024 + }, + { + "epoch": 1.36, + "learning_rate": 1.5823470411233703e-05, + "loss": 1.3446, + "theoretical_loss": 3.330057542733456, + "tokens_seen": 3236546560 + }, + { + "epoch": 1.36, + "learning_rate": 1.58074222668004e-05, + "loss": 1.2223, + "theoretical_loss": 3.330052481547932, + "tokens_seen": 3236612096 + }, + { + "epoch": 1.36, + "learning_rate": 1.5791374122367104e-05, + "loss": 1.2203, + "theoretical_loss": 3.3300474204935817, + "tokens_seen": 3236677632 + }, + { + "epoch": 1.36, + "learning_rate": 1.57753259779338e-05, + "loss": 1.0962, + "theoretical_loss": 3.330042359570398, + "tokens_seen": 3236743168 + }, + { + "epoch": 1.36, + "learning_rate": 1.5759277833500502e-05, + "loss": 1.2078, + "theoretical_loss": 3.330037298778376, + "tokens_seen": 3236808704 + }, + { + "epoch": 1.36, + "learning_rate": 1.5743229689067203e-05, + "loss": 1.3386, + "theoretical_loss": 3.33003223811751, + "tokens_seen": 3236874240 + }, + { + "epoch": 1.36, + "learning_rate": 1.5727181544633903e-05, + "loss": 1.3839, + "theoretical_loss": 3.3300271775877923, + "tokens_seen": 3236939776 + }, + { + "epoch": 1.36, + "learning_rate": 1.5711133400200604e-05, + "loss": 1.1605, + "theoretical_loss": 3.330022117189218, + "tokens_seen": 3237005312 + }, + { + "epoch": 1.36, + "learning_rate": 1.56950852557673e-05, + "loss": 1.1575, + "theoretical_loss": 3.330017056921781, + "tokens_seen": 3237070848 + }, + { + "epoch": 1.36, + "learning_rate": 1.5679037111334002e-05, + "loss": 1.2237, + "theoretical_loss": 3.3300119967854744, + "tokens_seen": 3237136384 + }, + { + "epoch": 1.36, + "learning_rate": 1.5662988966900702e-05, + "loss": 1.1696, + "theoretical_loss": 3.3300069367802934, + "tokens_seen": 3237201920 + }, + { + "epoch": 1.36, + "learning_rate": 1.5646940822467403e-05, + "loss": 1.1033, + "theoretical_loss": 3.3300018769062314, + "tokens_seen": 3237267456 + }, + { + "epoch": 1.36, + "learning_rate": 1.5630892678034104e-05, + "loss": 1.1268, + "theoretical_loss": 3.3299968171632823, + "tokens_seen": 3237332992 + }, + { + "epoch": 1.36, + "learning_rate": 1.5614844533600804e-05, + "loss": 1.1522, + "theoretical_loss": 3.3299917575514395, + "tokens_seen": 3237398528 + }, + { + "epoch": 1.36, + "objective/train/docs_used": 1193651, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.3451507091522217, + "objective/train/theoretical_loss": 3.329986698070698, + "objective/train/tokens_used": 3257924064, + "theoretical_loss": 3.329986698070698, + "tokens_seen": 3237464064 + }, + { + "epoch": 1.36, + "learning_rate": 1.55987963891675e-05, + "loss": 1.3452, + "theoretical_loss": 3.329986698070698, + "tokens_seen": 3237464064 + }, + { + "epoch": 1.36, + "learning_rate": 1.5582748244734202e-05, + "loss": 1.2453, + "theoretical_loss": 3.3299816387210512, + "tokens_seen": 3237529600 + }, + { + "epoch": 1.36, + "learning_rate": 1.5566700100300903e-05, + "loss": 1.1823, + "theoretical_loss": 3.329976579502493, + "tokens_seen": 3237595136 + }, + { + "epoch": 1.36, + "learning_rate": 1.5550651955867603e-05, + "loss": 1.1532, + "theoretical_loss": 3.3299715204150173, + "tokens_seen": 3237660672 + }, + { + "epoch": 1.36, + "learning_rate": 1.5534603811434304e-05, + "loss": 1.2327, + "theoretical_loss": 3.329966461458618, + "tokens_seen": 3237726208 + }, + { + "epoch": 1.36, + "learning_rate": 1.5518555667001e-05, + "loss": 1.1658, + "theoretical_loss": 3.3299614026332898, + "tokens_seen": 3237791744 + }, + { + "epoch": 1.36, + "learning_rate": 1.5502507522567705e-05, + "loss": 1.1946, + "theoretical_loss": 3.329956343939026, + "tokens_seen": 3237857280 + }, + { + "epoch": 1.36, + "learning_rate": 1.5486459378134406e-05, + "loss": 1.1133, + "theoretical_loss": 3.32995128537582, + "tokens_seen": 3237922816 + }, + { + "epoch": 1.36, + "learning_rate": 1.5470411233701103e-05, + "loss": 1.34, + "theoretical_loss": 3.329946226943667, + "tokens_seen": 3237988352 + }, + { + "epoch": 1.36, + "learning_rate": 1.5454363089267804e-05, + "loss": 1.2377, + "theoretical_loss": 3.32994116864256, + "tokens_seen": 3238053888 + }, + { + "epoch": 1.36, + "learning_rate": 1.5438314944834505e-05, + "loss": 1.1054, + "theoretical_loss": 3.3299361104724943, + "tokens_seen": 3238119424 + }, + { + "epoch": 1.36, + "learning_rate": 1.5422266800401205e-05, + "loss": 1.1852, + "theoretical_loss": 3.3299310524334618, + "tokens_seen": 3238184960 + }, + { + "epoch": 1.36, + "learning_rate": 1.5406218655967906e-05, + "loss": 1.1313, + "theoretical_loss": 3.329925994525458, + "tokens_seen": 3238250496 + }, + { + "epoch": 1.36, + "learning_rate": 1.5390170511534603e-05, + "loss": 1.1151, + "theoretical_loss": 3.3299209367484757, + "tokens_seen": 3238316032 + }, + { + "epoch": 1.36, + "learning_rate": 1.5374122367101304e-05, + "loss": 1.1957, + "theoretical_loss": 3.32991587910251, + "tokens_seen": 3238381568 + }, + { + "epoch": 1.36, + "learning_rate": 1.5358074222668004e-05, + "loss": 1.1283, + "theoretical_loss": 3.3299108215875544, + "tokens_seen": 3238447104 + }, + { + "epoch": 1.36, + "learning_rate": 1.5342026078234705e-05, + "loss": 1.0961, + "theoretical_loss": 3.3299057642036027, + "tokens_seen": 3238512640 + }, + { + "epoch": 1.36, + "learning_rate": 1.5325977933801406e-05, + "loss": 1.2211, + "theoretical_loss": 3.329900706950649, + "tokens_seen": 3238578176 + }, + { + "epoch": 1.36, + "learning_rate": 1.5309929789368106e-05, + "loss": 1.0762, + "theoretical_loss": 3.3298956498286874, + "tokens_seen": 3238643712 + }, + { + "epoch": 1.36, + "learning_rate": 1.5293881644934803e-05, + "loss": 1.1899, + "theoretical_loss": 3.3298905928377116, + "tokens_seen": 3238709248 + }, + { + "epoch": 1.36, + "learning_rate": 1.5277833500501504e-05, + "loss": 1.1517, + "theoretical_loss": 3.3298855359777155, + "tokens_seen": 3238774784 + }, + { + "epoch": 1.36, + "learning_rate": 1.5261785356068205e-05, + "loss": 1.2475, + "theoretical_loss": 3.3298804792486933, + "tokens_seen": 3238840320 + }, + { + "epoch": 1.36, + "learning_rate": 1.5245737211634905e-05, + "loss": 1.2684, + "theoretical_loss": 3.3298754226506393, + "tokens_seen": 3238905856 + }, + { + "epoch": 1.36, + "learning_rate": 1.5229689067201604e-05, + "loss": 1.0695, + "theoretical_loss": 3.3298703661835463, + "tokens_seen": 3238971392 + }, + { + "epoch": 1.36, + "learning_rate": 1.5213640922768305e-05, + "loss": 1.2312, + "theoretical_loss": 3.3298653098474094, + "tokens_seen": 3239036928 + }, + { + "epoch": 1.36, + "learning_rate": 1.5197592778335007e-05, + "loss": 1.083, + "theoretical_loss": 3.329860253642222, + "tokens_seen": 3239102464 + }, + { + "epoch": 1.36, + "learning_rate": 1.5181544633901706e-05, + "loss": 1.1749, + "theoretical_loss": 3.329855197567978, + "tokens_seen": 3239168000 + }, + { + "epoch": 1.36, + "learning_rate": 1.5165496489468405e-05, + "loss": 1.2376, + "theoretical_loss": 3.329850141624672, + "tokens_seen": 3239233536 + }, + { + "epoch": 1.36, + "learning_rate": 1.5149448345035106e-05, + "loss": 1.2581, + "theoretical_loss": 3.3298450858122974, + "tokens_seen": 3239299072 + }, + { + "epoch": 1.36, + "learning_rate": 1.5133400200601805e-05, + "loss": 1.1637, + "theoretical_loss": 3.329840030130848, + "tokens_seen": 3239364608 + }, + { + "epoch": 1.36, + "learning_rate": 1.5117352056168507e-05, + "loss": 1.2468, + "theoretical_loss": 3.3298349745803186, + "tokens_seen": 3239430144 + }, + { + "epoch": 1.36, + "learning_rate": 1.5101303911735206e-05, + "loss": 1.2482, + "theoretical_loss": 3.3298299191607024, + "tokens_seen": 3239495680 + }, + { + "epoch": 1.36, + "learning_rate": 1.5085255767301907e-05, + "loss": 1.1853, + "theoretical_loss": 3.329824863871993, + "tokens_seen": 3239561216 + }, + { + "epoch": 1.36, + "learning_rate": 1.5069207622868606e-05, + "loss": 1.1547, + "theoretical_loss": 3.3298198087141855, + "tokens_seen": 3239626752 + }, + { + "epoch": 1.36, + "learning_rate": 1.5053159478435305e-05, + "loss": 1.1349, + "theoretical_loss": 3.3298147536872733, + "tokens_seen": 3239692288 + }, + { + "epoch": 1.36, + "learning_rate": 1.5037111334002007e-05, + "loss": 1.198, + "theoretical_loss": 3.32980969879125, + "tokens_seen": 3239757824 + }, + { + "epoch": 1.36, + "learning_rate": 1.5021063189568707e-05, + "loss": 1.1338, + "theoretical_loss": 3.32980464402611, + "tokens_seen": 3239823360 + }, + { + "epoch": 1.36, + "learning_rate": 1.5005015045135406e-05, + "loss": 1.2367, + "theoretical_loss": 3.3297995893918473, + "tokens_seen": 3239888896 + }, + { + "epoch": 1.36, + "learning_rate": 1.4988966900702105e-05, + "loss": 1.0916, + "theoretical_loss": 3.329794534888456, + "tokens_seen": 3239954432 + }, + { + "epoch": 1.36, + "learning_rate": 1.4972918756268808e-05, + "loss": 1.0562, + "theoretical_loss": 3.329789480515929, + "tokens_seen": 3240019968 + }, + { + "epoch": 1.36, + "learning_rate": 1.4956870611835508e-05, + "loss": 1.108, + "theoretical_loss": 3.329784426274262, + "tokens_seen": 3240085504 + }, + { + "epoch": 1.36, + "learning_rate": 1.4940822467402207e-05, + "loss": 1.1372, + "theoretical_loss": 3.329779372163448, + "tokens_seen": 3240151040 + }, + { + "epoch": 1.36, + "learning_rate": 1.4924774322968906e-05, + "loss": 1.1577, + "theoretical_loss": 3.3297743181834805, + "tokens_seen": 3240216576 + }, + { + "epoch": 1.36, + "learning_rate": 1.4908726178535607e-05, + "loss": 1.1485, + "theoretical_loss": 3.329769264334354, + "tokens_seen": 3240282112 + }, + { + "epoch": 1.36, + "learning_rate": 1.489267803410231e-05, + "loss": 1.1823, + "theoretical_loss": 3.3297642106160623, + "tokens_seen": 3240347648 + }, + { + "epoch": 1.36, + "learning_rate": 1.4876629889669008e-05, + "loss": 0.9768, + "theoretical_loss": 3.3297591570286, + "tokens_seen": 3240413184 + }, + { + "epoch": 1.36, + "learning_rate": 1.4860581745235707e-05, + "loss": 1.1936, + "theoretical_loss": 3.32975410357196, + "tokens_seen": 3240478720 + }, + { + "epoch": 1.36, + "learning_rate": 1.4844533600802408e-05, + "loss": 1.1927, + "theoretical_loss": 3.3297490502461375, + "tokens_seen": 3240544256 + }, + { + "epoch": 1.36, + "learning_rate": 1.4828485456369107e-05, + "loss": 1.2016, + "theoretical_loss": 3.3297439970511253, + "tokens_seen": 3240609792 + }, + { + "epoch": 1.36, + "learning_rate": 1.4812437311935809e-05, + "loss": 1.2045, + "theoretical_loss": 3.3297389439869183, + "tokens_seen": 3240675328 + }, + { + "epoch": 1.36, + "learning_rate": 1.4796389167502508e-05, + "loss": 1.1522, + "theoretical_loss": 3.3297338910535097, + "tokens_seen": 3240740864 + }, + { + "epoch": 1.36, + "learning_rate": 1.4780341023069209e-05, + "loss": 1.1222, + "theoretical_loss": 3.329728838250894, + "tokens_seen": 3240806400 + }, + { + "epoch": 1.36, + "learning_rate": 1.4764292878635907e-05, + "loss": 1.2536, + "theoretical_loss": 3.329723785579065, + "tokens_seen": 3240871936 + }, + { + "epoch": 1.36, + "learning_rate": 1.474824473420261e-05, + "loss": 1.1505, + "theoretical_loss": 3.3297187330380162, + "tokens_seen": 3240937472 + }, + { + "epoch": 1.36, + "learning_rate": 1.4732196589769309e-05, + "loss": 1.0504, + "theoretical_loss": 3.3297136806277425, + "tokens_seen": 3241003008 + }, + { + "epoch": 1.36, + "learning_rate": 1.471614844533601e-05, + "loss": 1.1223, + "theoretical_loss": 3.3297086283482376, + "tokens_seen": 3241068544 + }, + { + "epoch": 1.36, + "learning_rate": 1.4700100300902708e-05, + "loss": 1.1537, + "theoretical_loss": 3.3297035761994946, + "tokens_seen": 3241134080 + }, + { + "epoch": 1.36, + "learning_rate": 1.4684052156469407e-05, + "loss": 1.1942, + "theoretical_loss": 3.3296985241815085, + "tokens_seen": 3241199616 + }, + { + "epoch": 1.36, + "learning_rate": 1.466800401203611e-05, + "loss": 1.0416, + "theoretical_loss": 3.329693472294273, + "tokens_seen": 3241265152 + }, + { + "epoch": 1.36, + "learning_rate": 1.465195586760281e-05, + "loss": 1.0698, + "theoretical_loss": 3.3296884205377815, + "tokens_seen": 3241330688 + }, + { + "epoch": 1.36, + "learning_rate": 1.463590772316951e-05, + "loss": 1.1488, + "theoretical_loss": 3.329683368912029, + "tokens_seen": 3241396224 + }, + { + "epoch": 1.36, + "learning_rate": 1.4619859578736208e-05, + "loss": 1.1102, + "theoretical_loss": 3.3296783174170086, + "tokens_seen": 3241461760 + }, + { + "epoch": 1.36, + "learning_rate": 1.4603811434302909e-05, + "loss": 1.1396, + "theoretical_loss": 3.329673266052715, + "tokens_seen": 3241527296 + }, + { + "epoch": 1.36, + "learning_rate": 1.4587763289869611e-05, + "loss": 1.2715, + "theoretical_loss": 3.3296682148191414, + "tokens_seen": 3241592832 + }, + { + "epoch": 1.36, + "learning_rate": 1.457171514543631e-05, + "loss": 1.1586, + "theoretical_loss": 3.3296631637162823, + "tokens_seen": 3241658368 + }, + { + "epoch": 1.36, + "learning_rate": 1.4555667001003009e-05, + "loss": 1.1321, + "theoretical_loss": 3.3296581127441316, + "tokens_seen": 3241723904 + }, + { + "epoch": 1.36, + "learning_rate": 1.453961885656971e-05, + "loss": 1.1204, + "theoretical_loss": 3.329653061902683, + "tokens_seen": 3241789440 + }, + { + "epoch": 1.36, + "learning_rate": 1.4523570712136412e-05, + "loss": 1.1676, + "theoretical_loss": 3.3296480111919307, + "tokens_seen": 3241854976 + }, + { + "epoch": 1.36, + "learning_rate": 1.4507522567703111e-05, + "loss": 1.1805, + "theoretical_loss": 3.329642960611869, + "tokens_seen": 3241920512 + }, + { + "epoch": 1.36, + "learning_rate": 1.449147442326981e-05, + "loss": 1.1938, + "theoretical_loss": 3.329637910162491, + "tokens_seen": 3241986048 + }, + { + "epoch": 1.36, + "learning_rate": 1.447542627883651e-05, + "loss": 1.2354, + "theoretical_loss": 3.3296328598437914, + "tokens_seen": 3242051584 + }, + { + "epoch": 1.36, + "learning_rate": 1.445937813440321e-05, + "loss": 1.2135, + "theoretical_loss": 3.329627809655764, + "tokens_seen": 3242117120 + }, + { + "epoch": 1.36, + "learning_rate": 1.4443329989969912e-05, + "loss": 1.2165, + "theoretical_loss": 3.329622759598403, + "tokens_seen": 3242182656 + }, + { + "epoch": 1.36, + "learning_rate": 1.442728184553661e-05, + "loss": 1.1914, + "theoretical_loss": 3.329617709671702, + "tokens_seen": 3242248192 + }, + { + "epoch": 1.36, + "learning_rate": 1.4411233701103311e-05, + "loss": 1.188, + "theoretical_loss": 3.329612659875655, + "tokens_seen": 3242313728 + }, + { + "epoch": 1.36, + "learning_rate": 1.439518555667001e-05, + "loss": 1.2032, + "theoretical_loss": 3.3296076102102563, + "tokens_seen": 3242379264 + }, + { + "epoch": 1.36, + "learning_rate": 1.437913741223671e-05, + "loss": 1.3039, + "theoretical_loss": 3.3296025606754993, + "tokens_seen": 3242444800 + }, + { + "epoch": 1.36, + "learning_rate": 1.4363089267803411e-05, + "loss": 1.1107, + "theoretical_loss": 3.3295975112713787, + "tokens_seen": 3242510336 + }, + { + "epoch": 1.36, + "learning_rate": 1.4347041123370112e-05, + "loss": 1.1801, + "theoretical_loss": 3.329592461997888, + "tokens_seen": 3242575872 + }, + { + "epoch": 1.36, + "learning_rate": 1.4330992978936811e-05, + "loss": 1.1528, + "theoretical_loss": 3.3295874128550214, + "tokens_seen": 3242641408 + }, + { + "epoch": 1.36, + "learning_rate": 1.431494483450351e-05, + "loss": 1.1043, + "theoretical_loss": 3.3295823638427726, + "tokens_seen": 3242706944 + }, + { + "epoch": 1.36, + "learning_rate": 1.4298896690070212e-05, + "loss": 1.281, + "theoretical_loss": 3.329577314961136, + "tokens_seen": 3242772480 + }, + { + "epoch": 1.36, + "learning_rate": 1.4282848545636911e-05, + "loss": 1.2724, + "theoretical_loss": 3.3295722662101053, + "tokens_seen": 3242838016 + }, + { + "epoch": 1.36, + "learning_rate": 1.4266800401203612e-05, + "loss": 1.1193, + "theoretical_loss": 3.329567217589674, + "tokens_seen": 3242903552 + }, + { + "epoch": 1.36, + "learning_rate": 1.425075225677031e-05, + "loss": 1.2476, + "theoretical_loss": 3.3295621690998374, + "tokens_seen": 3242969088 + }, + { + "epoch": 1.36, + "learning_rate": 1.423470411233701e-05, + "loss": 1.062, + "theoretical_loss": 3.3295571207405885, + "tokens_seen": 3243034624 + }, + { + "epoch": 1.36, + "learning_rate": 1.4218655967903712e-05, + "loss": 1.1492, + "theoretical_loss": 3.3295520725119214, + "tokens_seen": 3243100160 + }, + { + "epoch": 1.36, + "learning_rate": 1.4202607823470413e-05, + "loss": 1.1645, + "theoretical_loss": 3.3295470244138303, + "tokens_seen": 3243165696 + }, + { + "epoch": 1.36, + "learning_rate": 1.4186559679037112e-05, + "loss": 1.243, + "theoretical_loss": 3.329541976446309, + "tokens_seen": 3243231232 + }, + { + "epoch": 1.36, + "learning_rate": 1.417051153460381e-05, + "loss": 1.1777, + "theoretical_loss": 3.3295369286093512, + "tokens_seen": 3243296768 + }, + { + "epoch": 1.36, + "learning_rate": 1.4154463390170511e-05, + "loss": 1.1949, + "theoretical_loss": 3.3295318809029517, + "tokens_seen": 3243362304 + }, + { + "epoch": 1.36, + "learning_rate": 1.4138415245737214e-05, + "loss": 1.1496, + "theoretical_loss": 3.329526833327104, + "tokens_seen": 3243427840 + }, + { + "epoch": 1.36, + "learning_rate": 1.4122367101303913e-05, + "loss": 1.1714, + "theoretical_loss": 3.329521785881802, + "tokens_seen": 3243493376 + }, + { + "epoch": 1.36, + "learning_rate": 1.4106318956870611e-05, + "loss": 1.2274, + "theoretical_loss": 3.32951673856704, + "tokens_seen": 3243558912 + }, + { + "epoch": 1.36, + "learning_rate": 1.4090270812437312e-05, + "loss": 1.1382, + "theoretical_loss": 3.329511691382811, + "tokens_seen": 3243624448 + }, + { + "epoch": 1.36, + "learning_rate": 1.4074222668004014e-05, + "loss": 1.152, + "theoretical_loss": 3.3295066443291104, + "tokens_seen": 3243689984 + }, + { + "epoch": 1.36, + "learning_rate": 1.4058174523570713e-05, + "loss": 1.071, + "theoretical_loss": 3.3295015974059314, + "tokens_seen": 3243755520 + }, + { + "epoch": 1.36, + "learning_rate": 1.4042126379137412e-05, + "loss": 1.1808, + "theoretical_loss": 3.329496550613268, + "tokens_seen": 3243821056 + }, + { + "epoch": 1.36, + "learning_rate": 1.4026078234704113e-05, + "loss": 1.2401, + "theoretical_loss": 3.3294915039511146, + "tokens_seen": 3243886592 + }, + { + "epoch": 1.36, + "learning_rate": 1.4010030090270812e-05, + "loss": 1.13, + "theoretical_loss": 3.3294864574194647, + "tokens_seen": 3243952128 + }, + { + "epoch": 1.36, + "objective/train/docs_used": 1196356, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.228031039237976, + "objective/train/theoretical_loss": 3.3294814110183126, + "objective/train/tokens_used": 3264477664, + "theoretical_loss": 3.3294814110183126, + "tokens_seen": 3244017664 + }, + { + "epoch": 1.36, + "learning_rate": 1.3993981945837514e-05, + "loss": 1.228, + "theoretical_loss": 3.3294814110183126, + "tokens_seen": 3244017664 + }, + { + "epoch": 1.36, + "learning_rate": 1.3977933801404213e-05, + "loss": 1.1982, + "theoretical_loss": 3.329476364747652, + "tokens_seen": 3244083200 + }, + { + "epoch": 1.36, + "learning_rate": 1.3961885656970914e-05, + "loss": 1.1802, + "theoretical_loss": 3.329471318607477, + "tokens_seen": 3244148736 + }, + { + "epoch": 1.36, + "learning_rate": 1.3945837512537613e-05, + "loss": 1.289, + "theoretical_loss": 3.3294662725977826, + "tokens_seen": 3244214272 + }, + { + "epoch": 1.36, + "learning_rate": 1.3929789368104312e-05, + "loss": 1.11, + "theoretical_loss": 3.329461226718561, + "tokens_seen": 3244279808 + }, + { + "epoch": 1.36, + "learning_rate": 1.3913741223671014e-05, + "loss": 1.1719, + "theoretical_loss": 3.329456180969807, + "tokens_seen": 3244345344 + }, + { + "epoch": 1.36, + "learning_rate": 1.3897693079237715e-05, + "loss": 1.1975, + "theoretical_loss": 3.3294511353515146, + "tokens_seen": 3244410880 + }, + { + "epoch": 1.36, + "learning_rate": 1.3881644934804414e-05, + "loss": 1.2569, + "theoretical_loss": 3.329446089863678, + "tokens_seen": 3244476416 + }, + { + "epoch": 1.36, + "learning_rate": 1.3865596790371113e-05, + "loss": 1.194, + "theoretical_loss": 3.329441044506291, + "tokens_seen": 3244541952 + }, + { + "epoch": 1.36, + "learning_rate": 1.3849548645937815e-05, + "loss": 1.158, + "theoretical_loss": 3.3294359992793483, + "tokens_seen": 3244607488 + }, + { + "epoch": 1.36, + "learning_rate": 1.3833500501504515e-05, + "loss": 1.1278, + "theoretical_loss": 3.3294309541828424, + "tokens_seen": 3244673024 + }, + { + "epoch": 1.36, + "learning_rate": 1.3817452357071214e-05, + "loss": 1.1709, + "theoretical_loss": 3.329425909216768, + "tokens_seen": 3244738560 + }, + { + "epoch": 1.36, + "learning_rate": 1.3801404212637913e-05, + "loss": 1.1289, + "theoretical_loss": 3.32942086438112, + "tokens_seen": 3244804096 + }, + { + "epoch": 1.36, + "learning_rate": 1.3785356068204614e-05, + "loss": 1.2242, + "theoretical_loss": 3.3294158196758907, + "tokens_seen": 3244869632 + }, + { + "epoch": 1.36, + "learning_rate": 1.3769307923771316e-05, + "loss": 1.1042, + "theoretical_loss": 3.3294107751010755, + "tokens_seen": 3244935168 + }, + { + "epoch": 1.36, + "learning_rate": 1.3753259779338015e-05, + "loss": 1.1321, + "theoretical_loss": 3.329405730656668, + "tokens_seen": 3245000704 + }, + { + "epoch": 1.36, + "learning_rate": 1.3737211634904714e-05, + "loss": 1.1316, + "theoretical_loss": 3.3294006863426615, + "tokens_seen": 3245066240 + }, + { + "epoch": 1.36, + "learning_rate": 1.3721163490471415e-05, + "loss": 1.135, + "theoretical_loss": 3.329395642159051, + "tokens_seen": 3245131776 + }, + { + "epoch": 1.36, + "learning_rate": 1.3705115346038114e-05, + "loss": 1.1228, + "theoretical_loss": 3.3293905981058303, + "tokens_seen": 3245197312 + }, + { + "epoch": 1.36, + "learning_rate": 1.3689067201604816e-05, + "loss": 1.1432, + "theoretical_loss": 3.3293855541829926, + "tokens_seen": 3245262848 + }, + { + "epoch": 1.36, + "learning_rate": 1.3673019057171515e-05, + "loss": 1.0919, + "theoretical_loss": 3.329380510390533, + "tokens_seen": 3245328384 + }, + { + "epoch": 1.36, + "learning_rate": 1.3656970912738216e-05, + "loss": 1.2036, + "theoretical_loss": 3.3293754667284445, + "tokens_seen": 3245393920 + }, + { + "epoch": 1.36, + "learning_rate": 1.3640922768304915e-05, + "loss": 1.1441, + "theoretical_loss": 3.3293704231967216, + "tokens_seen": 3245459456 + }, + { + "epoch": 1.36, + "learning_rate": 1.3624874623871617e-05, + "loss": 1.2748, + "theoretical_loss": 3.3293653797953584, + "tokens_seen": 3245524992 + }, + { + "epoch": 1.36, + "learning_rate": 1.3608826479438316e-05, + "loss": 1.0279, + "theoretical_loss": 3.3293603365243487, + "tokens_seen": 3245590528 + }, + { + "epoch": 1.36, + "learning_rate": 1.3592778335005017e-05, + "loss": 1.2023, + "theoretical_loss": 3.3293552933836863, + "tokens_seen": 3245656064 + }, + { + "epoch": 1.36, + "learning_rate": 1.3576730190571715e-05, + "loss": 1.1916, + "theoretical_loss": 3.329350250373366, + "tokens_seen": 3245721600 + }, + { + "epoch": 1.36, + "learning_rate": 1.3560682046138414e-05, + "loss": 1.1115, + "theoretical_loss": 3.3293452074933807, + "tokens_seen": 3245787136 + }, + { + "epoch": 1.36, + "learning_rate": 1.3544633901705117e-05, + "loss": 1.1927, + "theoretical_loss": 3.3293401647437255, + "tokens_seen": 3245852672 + }, + { + "epoch": 1.36, + "learning_rate": 1.3528585757271817e-05, + "loss": 1.1506, + "theoretical_loss": 3.329335122124393, + "tokens_seen": 3245918208 + }, + { + "epoch": 1.36, + "learning_rate": 1.3512537612838516e-05, + "loss": 1.1722, + "theoretical_loss": 3.329330079635379, + "tokens_seen": 3245983744 + }, + { + "epoch": 1.36, + "learning_rate": 1.3496489468405215e-05, + "loss": 1.027, + "theoretical_loss": 3.329325037276676, + "tokens_seen": 3246049280 + }, + { + "epoch": 1.36, + "learning_rate": 1.3480441323971916e-05, + "loss": 1.0692, + "theoretical_loss": 3.329319995048279, + "tokens_seen": 3246114816 + }, + { + "epoch": 1.36, + "learning_rate": 1.3464393179538618e-05, + "loss": 1.0926, + "theoretical_loss": 3.329314952950181, + "tokens_seen": 3246180352 + }, + { + "epoch": 1.36, + "learning_rate": 1.3448345035105317e-05, + "loss": 1.1817, + "theoretical_loss": 3.329309910982377, + "tokens_seen": 3246245888 + }, + { + "epoch": 1.36, + "learning_rate": 1.3432296890672016e-05, + "loss": 1.145, + "theoretical_loss": 3.32930486914486, + "tokens_seen": 3246311424 + }, + { + "epoch": 1.36, + "learning_rate": 1.3416248746238717e-05, + "loss": 1.1763, + "theoretical_loss": 3.329299827437625, + "tokens_seen": 3246376960 + }, + { + "epoch": 1.36, + "learning_rate": 1.3400200601805419e-05, + "loss": 1.1665, + "theoretical_loss": 3.3292947858606654, + "tokens_seen": 3246442496 + }, + { + "epoch": 1.36, + "learning_rate": 1.3384152457372118e-05, + "loss": 1.1397, + "theoretical_loss": 3.3292897444139755, + "tokens_seen": 3246508032 + }, + { + "epoch": 1.36, + "learning_rate": 1.3368104312938817e-05, + "loss": 1.1768, + "theoretical_loss": 3.3292847030975494, + "tokens_seen": 3246573568 + }, + { + "epoch": 1.36, + "learning_rate": 1.3352056168505518e-05, + "loss": 1.2717, + "theoretical_loss": 3.3292796619113805, + "tokens_seen": 3246639104 + }, + { + "epoch": 1.36, + "learning_rate": 1.3336008024072217e-05, + "loss": 1.1562, + "theoretical_loss": 3.329274620855463, + "tokens_seen": 3246704640 + }, + { + "epoch": 1.36, + "learning_rate": 1.3319959879638919e-05, + "loss": 1.1253, + "theoretical_loss": 3.3292695799297913, + "tokens_seen": 3246770176 + }, + { + "epoch": 1.36, + "learning_rate": 1.3303911735205618e-05, + "loss": 1.2547, + "theoretical_loss": 3.3292645391343596, + "tokens_seen": 3246835712 + }, + { + "epoch": 1.36, + "learning_rate": 1.3287863590772317e-05, + "loss": 1.1682, + "theoretical_loss": 3.329259498469161, + "tokens_seen": 3246901248 + }, + { + "epoch": 1.36, + "learning_rate": 1.3271815446339017e-05, + "loss": 1.2351, + "theoretical_loss": 3.3292544579341903, + "tokens_seen": 3246966784 + }, + { + "epoch": 1.36, + "learning_rate": 1.3255767301905716e-05, + "loss": 1.1573, + "theoretical_loss": 3.329249417529441, + "tokens_seen": 3247032320 + }, + { + "epoch": 1.36, + "learning_rate": 1.3239719157472419e-05, + "loss": 1.1094, + "theoretical_loss": 3.3292443772549074, + "tokens_seen": 3247097856 + }, + { + "epoch": 1.36, + "learning_rate": 1.3223671013039118e-05, + "loss": 1.1058, + "theoretical_loss": 3.3292393371105833, + "tokens_seen": 3247163392 + }, + { + "epoch": 1.36, + "learning_rate": 1.3207622868605818e-05, + "loss": 1.1088, + "theoretical_loss": 3.329234297096463, + "tokens_seen": 3247228928 + }, + { + "epoch": 1.36, + "learning_rate": 1.3191574724172517e-05, + "loss": 1.1586, + "theoretical_loss": 3.3292292572125404, + "tokens_seen": 3247294464 + }, + { + "epoch": 1.36, + "learning_rate": 1.317552657973922e-05, + "loss": 1.1614, + "theoretical_loss": 3.329224217458809, + "tokens_seen": 3247360000 + }, + { + "epoch": 1.36, + "learning_rate": 1.3159478435305918e-05, + "loss": 1.1894, + "theoretical_loss": 3.329219177835264, + "tokens_seen": 3247425536 + }, + { + "epoch": 1.36, + "learning_rate": 1.3143430290872619e-05, + "loss": 1.1663, + "theoretical_loss": 3.3292141383418983, + "tokens_seen": 3247491072 + }, + { + "epoch": 1.36, + "learning_rate": 1.3127382146439318e-05, + "loss": 1.0626, + "theoretical_loss": 3.329209098978706, + "tokens_seen": 3247556608 + }, + { + "epoch": 1.36, + "learning_rate": 1.3111334002006017e-05, + "loss": 1.1557, + "theoretical_loss": 3.329204059745682, + "tokens_seen": 3247622144 + }, + { + "epoch": 1.36, + "learning_rate": 1.309528585757272e-05, + "loss": 1.1684, + "theoretical_loss": 3.3291990206428195, + "tokens_seen": 3247687680 + }, + { + "epoch": 1.36, + "learning_rate": 1.307923771313942e-05, + "loss": 1.1455, + "theoretical_loss": 3.3291939816701124, + "tokens_seen": 3247753216 + }, + { + "epoch": 1.36, + "learning_rate": 1.3063189568706119e-05, + "loss": 1.1665, + "theoretical_loss": 3.3291889428275554, + "tokens_seen": 3247818752 + }, + { + "epoch": 1.36, + "learning_rate": 1.3047141424272818e-05, + "loss": 1.14, + "theoretical_loss": 3.3291839041151423, + "tokens_seen": 3247884288 + }, + { + "epoch": 1.36, + "learning_rate": 1.3031093279839518e-05, + "loss": 1.1422, + "theoretical_loss": 3.3291788655328665, + "tokens_seen": 3247949824 + }, + { + "epoch": 1.36, + "learning_rate": 1.301504513540622e-05, + "loss": 1.1396, + "theoretical_loss": 3.3291738270807225, + "tokens_seen": 3248015360 + }, + { + "epoch": 1.36, + "learning_rate": 1.299899699097292e-05, + "loss": 1.1525, + "theoretical_loss": 3.3291687887587047, + "tokens_seen": 3248080896 + }, + { + "epoch": 1.36, + "learning_rate": 1.2982948846539619e-05, + "loss": 1.1648, + "theoretical_loss": 3.3291637505668064, + "tokens_seen": 3248146432 + }, + { + "epoch": 1.36, + "learning_rate": 1.296690070210632e-05, + "loss": 1.1496, + "theoretical_loss": 3.3291587125050217, + "tokens_seen": 3248211968 + }, + { + "epoch": 1.36, + "learning_rate": 1.2950852557673018e-05, + "loss": 1.1594, + "theoretical_loss": 3.329153674573345, + "tokens_seen": 3248277504 + }, + { + "epoch": 1.37, + "learning_rate": 1.293480441323972e-05, + "loss": 1.0515, + "theoretical_loss": 3.3291486367717704, + "tokens_seen": 3248343040 + }, + { + "epoch": 1.37, + "learning_rate": 1.291875626880642e-05, + "loss": 1.1603, + "theoretical_loss": 3.3291435991002913, + "tokens_seen": 3248408576 + }, + { + "epoch": 1.37, + "learning_rate": 1.290270812437312e-05, + "loss": 1.1619, + "theoretical_loss": 3.329138561558902, + "tokens_seen": 3248474112 + }, + { + "epoch": 1.37, + "learning_rate": 1.2886659979939819e-05, + "loss": 1.2036, + "theoretical_loss": 3.329133524147597, + "tokens_seen": 3248539648 + }, + { + "epoch": 1.37, + "learning_rate": 1.2870611835506521e-05, + "loss": 1.1331, + "theoretical_loss": 3.3291284868663698, + "tokens_seen": 3248605184 + }, + { + "epoch": 1.37, + "learning_rate": 1.285456369107322e-05, + "loss": 1.1025, + "theoretical_loss": 3.3291234497152145, + "tokens_seen": 3248670720 + }, + { + "epoch": 1.37, + "learning_rate": 1.2838515546639921e-05, + "loss": 1.1227, + "theoretical_loss": 3.329118412694125, + "tokens_seen": 3248736256 + }, + { + "epoch": 1.37, + "learning_rate": 1.282246740220662e-05, + "loss": 1.1392, + "theoretical_loss": 3.3291133758030957, + "tokens_seen": 3248801792 + }, + { + "epoch": 1.37, + "learning_rate": 1.2806419257773319e-05, + "loss": 1.1194, + "theoretical_loss": 3.32910833904212, + "tokens_seen": 3248867328 + }, + { + "epoch": 1.37, + "learning_rate": 1.2790371113340021e-05, + "loss": 1.119, + "theoretical_loss": 3.3291033024111925, + "tokens_seen": 3248932864 + }, + { + "epoch": 1.37, + "learning_rate": 1.2774322968906722e-05, + "loss": 1.2915, + "theoretical_loss": 3.329098265910307, + "tokens_seen": 3248998400 + }, + { + "epoch": 1.37, + "learning_rate": 1.275827482447342e-05, + "loss": 1.2351, + "theoretical_loss": 3.3290932295394575, + "tokens_seen": 3249063936 + }, + { + "epoch": 1.37, + "learning_rate": 1.274222668004012e-05, + "loss": 1.141, + "theoretical_loss": 3.329088193298638, + "tokens_seen": 3249129472 + }, + { + "epoch": 1.37, + "learning_rate": 1.272617853560682e-05, + "loss": 1.2353, + "theoretical_loss": 3.329083157187843, + "tokens_seen": 3249195008 + }, + { + "epoch": 1.37, + "learning_rate": 1.2710130391173523e-05, + "loss": 1.1301, + "theoretical_loss": 3.3290781212070657, + "tokens_seen": 3249260544 + }, + { + "epoch": 1.37, + "learning_rate": 1.2694082246740222e-05, + "loss": 1.2417, + "theoretical_loss": 3.3290730853563004, + "tokens_seen": 3249326080 + }, + { + "epoch": 1.37, + "learning_rate": 1.267803410230692e-05, + "loss": 1.1332, + "theoretical_loss": 3.3290680496355414, + "tokens_seen": 3249391616 + }, + { + "epoch": 1.37, + "learning_rate": 1.2661985957873621e-05, + "loss": 1.1814, + "theoretical_loss": 3.3290630140447828, + "tokens_seen": 3249457152 + }, + { + "epoch": 1.37, + "learning_rate": 1.2645937813440323e-05, + "loss": 1.1182, + "theoretical_loss": 3.329057978584018, + "tokens_seen": 3249522688 + }, + { + "epoch": 1.37, + "learning_rate": 1.2629889669007022e-05, + "loss": 1.1235, + "theoretical_loss": 3.3290529432532416, + "tokens_seen": 3249588224 + }, + { + "epoch": 1.37, + "learning_rate": 1.2613841524573721e-05, + "loss": 1.0895, + "theoretical_loss": 3.3290479080524475, + "tokens_seen": 3249653760 + }, + { + "epoch": 1.37, + "learning_rate": 1.2597793380140422e-05, + "loss": 1.1653, + "theoretical_loss": 3.3290428729816295, + "tokens_seen": 3249719296 + }, + { + "epoch": 1.37, + "learning_rate": 1.2581745235707121e-05, + "loss": 1.1431, + "theoretical_loss": 3.3290378380407817, + "tokens_seen": 3249784832 + }, + { + "epoch": 1.37, + "learning_rate": 1.2565697091273823e-05, + "loss": 1.133, + "theoretical_loss": 3.3290328032298984, + "tokens_seen": 3249850368 + }, + { + "epoch": 1.37, + "learning_rate": 1.2549648946840522e-05, + "loss": 1.1739, + "theoretical_loss": 3.3290277685489733, + "tokens_seen": 3249915904 + }, + { + "epoch": 1.37, + "learning_rate": 1.2533600802407223e-05, + "loss": 1.1001, + "theoretical_loss": 3.3290227339980003, + "tokens_seen": 3249981440 + }, + { + "epoch": 1.37, + "learning_rate": 1.2517552657973922e-05, + "loss": 1.1769, + "theoretical_loss": 3.329017699576974, + "tokens_seen": 3250046976 + }, + { + "epoch": 1.37, + "learning_rate": 1.250150451354062e-05, + "loss": 1.2189, + "theoretical_loss": 3.3290126652858882, + "tokens_seen": 3250112512 + }, + { + "epoch": 1.37, + "learning_rate": 1.2485456369107321e-05, + "loss": 1.1964, + "theoretical_loss": 3.3290076311247367, + "tokens_seen": 3250178048 + }, + { + "epoch": 1.37, + "learning_rate": 1.2469408224674024e-05, + "loss": 1.0834, + "theoretical_loss": 3.3290025970935133, + "tokens_seen": 3250243584 + }, + { + "epoch": 1.37, + "learning_rate": 1.2453360080240723e-05, + "loss": 1.2504, + "theoretical_loss": 3.328997563192213, + "tokens_seen": 3250309120 + }, + { + "epoch": 1.37, + "learning_rate": 1.2437311935807423e-05, + "loss": 1.1554, + "theoretical_loss": 3.328992529420829, + "tokens_seen": 3250374656 + }, + { + "epoch": 1.37, + "learning_rate": 1.2421263791374122e-05, + "loss": 1.2592, + "theoretical_loss": 3.328987495779355, + "tokens_seen": 3250440192 + }, + { + "epoch": 1.37, + "learning_rate": 1.2405215646940825e-05, + "loss": 1.1121, + "theoretical_loss": 3.328982462267786, + "tokens_seen": 3250505728 + }, + { + "epoch": 1.37, + "objective/train/docs_used": 1198681, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2750874757766724, + "objective/train/theoretical_loss": 3.328977428886116, + "objective/train/tokens_used": 3271031264, + "theoretical_loss": 3.328977428886116, + "tokens_seen": 3250571264 + }, + { + "epoch": 1.37, + "learning_rate": 1.2389167502507523e-05, + "loss": 1.2751, + "theoretical_loss": 3.328977428886116, + "tokens_seen": 3250571264 + }, + { + "epoch": 1.37, + "learning_rate": 1.2373119358074222e-05, + "loss": 1.2773, + "theoretical_loss": 3.328972395634338, + "tokens_seen": 3250636800 + }, + { + "epoch": 1.37, + "learning_rate": 1.2357071213640923e-05, + "loss": 1.194, + "theoretical_loss": 3.3289673625124467, + "tokens_seen": 3250702336 + }, + { + "epoch": 1.37, + "learning_rate": 1.2341023069207624e-05, + "loss": 1.2202, + "theoretical_loss": 3.3289623295204365, + "tokens_seen": 3250767872 + }, + { + "epoch": 1.37, + "learning_rate": 1.2324974924774324e-05, + "loss": 1.2039, + "theoretical_loss": 3.3289572966583005, + "tokens_seen": 3250833408 + }, + { + "epoch": 1.37, + "learning_rate": 1.2308926780341023e-05, + "loss": 1.1668, + "theoretical_loss": 3.3289522639260336, + "tokens_seen": 3250898944 + }, + { + "epoch": 1.37, + "learning_rate": 1.2292878635907724e-05, + "loss": 1.1716, + "theoretical_loss": 3.328947231323629, + "tokens_seen": 3250964480 + }, + { + "epoch": 1.37, + "learning_rate": 1.2276830491474425e-05, + "loss": 1.2577, + "theoretical_loss": 3.328942198851082, + "tokens_seen": 3251030016 + }, + { + "epoch": 1.37, + "learning_rate": 1.2260782347041123e-05, + "loss": 1.2524, + "theoretical_loss": 3.328937166508385, + "tokens_seen": 3251095552 + }, + { + "epoch": 1.37, + "learning_rate": 1.2244734202607824e-05, + "loss": 1.2068, + "theoretical_loss": 3.3289321342955334, + "tokens_seen": 3251161088 + }, + { + "epoch": 1.37, + "learning_rate": 1.2228686058174523e-05, + "loss": 1.2189, + "theoretical_loss": 3.3289271022125204, + "tokens_seen": 3251226624 + }, + { + "epoch": 1.37, + "learning_rate": 1.2212637913741225e-05, + "loss": 1.0718, + "theoretical_loss": 3.3289220702593405, + "tokens_seen": 3251292160 + }, + { + "epoch": 1.37, + "learning_rate": 1.2196589769307924e-05, + "loss": 1.1433, + "theoretical_loss": 3.3289170384359874, + "tokens_seen": 3251357696 + }, + { + "epoch": 1.37, + "learning_rate": 1.2180541624874625e-05, + "loss": 1.0696, + "theoretical_loss": 3.3289120067424554, + "tokens_seen": 3251423232 + }, + { + "epoch": 1.37, + "learning_rate": 1.2164493480441324e-05, + "loss": 1.2199, + "theoretical_loss": 3.3289069751787386, + "tokens_seen": 3251488768 + }, + { + "epoch": 1.37, + "learning_rate": 1.2148445336008025e-05, + "loss": 1.2294, + "theoretical_loss": 3.3289019437448304, + "tokens_seen": 3251554304 + }, + { + "epoch": 1.37, + "learning_rate": 1.2132397191574725e-05, + "loss": 1.2049, + "theoretical_loss": 3.328896912440726, + "tokens_seen": 3251619840 + }, + { + "epoch": 1.37, + "learning_rate": 1.2116349047141424e-05, + "loss": 1.2256, + "theoretical_loss": 3.328891881266418, + "tokens_seen": 3251685376 + }, + { + "epoch": 1.37, + "learning_rate": 1.2100300902708125e-05, + "loss": 1.2686, + "theoretical_loss": 3.3288868502219016, + "tokens_seen": 3251750912 + }, + { + "epoch": 1.37, + "learning_rate": 1.2084252758274825e-05, + "loss": 1.1789, + "theoretical_loss": 3.32888181930717, + "tokens_seen": 3251816448 + }, + { + "epoch": 1.37, + "learning_rate": 1.2068204613841526e-05, + "loss": 1.2159, + "theoretical_loss": 3.3288767885222184, + "tokens_seen": 3251881984 + }, + { + "epoch": 1.37, + "learning_rate": 1.2052156469408225e-05, + "loss": 1.2126, + "theoretical_loss": 3.3288717578670393, + "tokens_seen": 3251947520 + }, + { + "epoch": 1.37, + "learning_rate": 1.2036108324974926e-05, + "loss": 1.1273, + "theoretical_loss": 3.328866727341628, + "tokens_seen": 3252013056 + }, + { + "epoch": 1.37, + "learning_rate": 1.2020060180541626e-05, + "loss": 1.2269, + "theoretical_loss": 3.3288616969459777, + "tokens_seen": 3252078592 + }, + { + "epoch": 1.37, + "learning_rate": 1.2004012036108325e-05, + "loss": 1.1381, + "theoretical_loss": 3.328856666680083, + "tokens_seen": 3252144128 + }, + { + "epoch": 1.37, + "learning_rate": 1.1987963891675026e-05, + "loss": 1.0499, + "theoretical_loss": 3.328851636543938, + "tokens_seen": 3252209664 + }, + { + "epoch": 1.37, + "learning_rate": 1.1971915747241726e-05, + "loss": 1.3118, + "theoretical_loss": 3.328846606537536, + "tokens_seen": 3252275200 + }, + { + "epoch": 1.37, + "learning_rate": 1.1955867602808425e-05, + "loss": 1.2908, + "theoretical_loss": 3.3288415766608717, + "tokens_seen": 3252340736 + }, + { + "epoch": 1.37, + "learning_rate": 1.1939819458375126e-05, + "loss": 1.0849, + "theoretical_loss": 3.328836546913939, + "tokens_seen": 3252406272 + }, + { + "epoch": 1.37, + "learning_rate": 1.1923771313941825e-05, + "loss": 1.2025, + "theoretical_loss": 3.3288315172967318, + "tokens_seen": 3252471808 + }, + { + "epoch": 1.37, + "learning_rate": 1.1907723169508527e-05, + "loss": 1.1394, + "theoretical_loss": 3.328826487809244, + "tokens_seen": 3252537344 + }, + { + "epoch": 1.37, + "learning_rate": 1.1891675025075226e-05, + "loss": 1.2841, + "theoretical_loss": 3.32882145845147, + "tokens_seen": 3252602880 + }, + { + "epoch": 1.37, + "learning_rate": 1.1875626880641927e-05, + "loss": 1.1341, + "theoretical_loss": 3.328816429223404, + "tokens_seen": 3252668416 + }, + { + "epoch": 1.37, + "learning_rate": 1.1859578736208626e-05, + "loss": 1.138, + "theoretical_loss": 3.3288114001250397, + "tokens_seen": 3252733952 + }, + { + "epoch": 1.37, + "learning_rate": 1.1843530591775326e-05, + "loss": 1.1533, + "theoretical_loss": 3.3288063711563707, + "tokens_seen": 3252799488 + }, + { + "epoch": 1.37, + "learning_rate": 1.1827482447342027e-05, + "loss": 1.2317, + "theoretical_loss": 3.328801342317392, + "tokens_seen": 3252865024 + }, + { + "epoch": 1.37, + "learning_rate": 1.1811434302908726e-05, + "loss": 1.0933, + "theoretical_loss": 3.3287963136080974, + "tokens_seen": 3252930560 + }, + { + "epoch": 1.37, + "learning_rate": 1.1795386158475427e-05, + "loss": 1.198, + "theoretical_loss": 3.32879128502848, + "tokens_seen": 3252996096 + }, + { + "epoch": 1.37, + "learning_rate": 1.1779338014042127e-05, + "loss": 1.2424, + "theoretical_loss": 3.3287862565785353, + "tokens_seen": 3253061632 + }, + { + "epoch": 1.37, + "learning_rate": 1.1763289869608828e-05, + "loss": 1.1579, + "theoretical_loss": 3.328781228258256, + "tokens_seen": 3253127168 + }, + { + "epoch": 1.37, + "learning_rate": 1.1747241725175527e-05, + "loss": 1.2037, + "theoretical_loss": 3.328776200067637, + "tokens_seen": 3253192704 + }, + { + "epoch": 1.37, + "learning_rate": 1.1731193580742226e-05, + "loss": 1.1769, + "theoretical_loss": 3.328771172006672, + "tokens_seen": 3253258240 + }, + { + "epoch": 1.37, + "learning_rate": 1.1715145436308928e-05, + "loss": 1.2009, + "theoretical_loss": 3.3287661440753555, + "tokens_seen": 3253323776 + }, + { + "epoch": 1.37, + "learning_rate": 1.1699097291875627e-05, + "loss": 1.1464, + "theoretical_loss": 3.328761116273681, + "tokens_seen": 3253389312 + }, + { + "epoch": 1.37, + "learning_rate": 1.1683049147442328e-05, + "loss": 1.1996, + "theoretical_loss": 3.3287560886016427, + "tokens_seen": 3253454848 + }, + { + "epoch": 1.37, + "learning_rate": 1.1667001003009027e-05, + "loss": 1.1616, + "theoretical_loss": 3.3287510610592346, + "tokens_seen": 3253520384 + }, + { + "epoch": 1.37, + "learning_rate": 1.1650952858575729e-05, + "loss": 1.1365, + "theoretical_loss": 3.3287460336464507, + "tokens_seen": 3253585920 + }, + { + "epoch": 1.37, + "learning_rate": 1.1634904714142428e-05, + "loss": 1.2688, + "theoretical_loss": 3.3287410063632854, + "tokens_seen": 3253651456 + }, + { + "epoch": 1.37, + "learning_rate": 1.1618856569709127e-05, + "loss": 1.1864, + "theoretical_loss": 3.3287359792097324, + "tokens_seen": 3253716992 + }, + { + "epoch": 1.37, + "learning_rate": 1.1602808425275827e-05, + "loss": 1.1845, + "theoretical_loss": 3.328730952185786, + "tokens_seen": 3253782528 + }, + { + "epoch": 1.37, + "learning_rate": 1.1586760280842528e-05, + "loss": 1.2535, + "theoretical_loss": 3.3287259252914403, + "tokens_seen": 3253848064 + }, + { + "epoch": 1.37, + "learning_rate": 1.1570712136409229e-05, + "loss": 1.0461, + "theoretical_loss": 3.3287208985266887, + "tokens_seen": 3253913600 + }, + { + "epoch": 1.37, + "learning_rate": 1.1554663991975928e-05, + "loss": 1.0798, + "theoretical_loss": 3.328715871891526, + "tokens_seen": 3253979136 + }, + { + "epoch": 1.37, + "learning_rate": 1.1538615847542628e-05, + "loss": 1.2007, + "theoretical_loss": 3.328710845385946, + "tokens_seen": 3254044672 + }, + { + "epoch": 1.37, + "learning_rate": 1.1522567703109329e-05, + "loss": 1.1656, + "theoretical_loss": 3.328705819009943, + "tokens_seen": 3254110208 + }, + { + "epoch": 1.37, + "learning_rate": 1.1506519558676028e-05, + "loss": 1.1689, + "theoretical_loss": 3.32870079276351, + "tokens_seen": 3254175744 + }, + { + "epoch": 1.37, + "learning_rate": 1.1490471414242729e-05, + "loss": 1.1028, + "theoretical_loss": 3.328695766646642, + "tokens_seen": 3254241280 + }, + { + "epoch": 1.37, + "learning_rate": 1.1474423269809429e-05, + "loss": 1.1335, + "theoretical_loss": 3.328690740659333, + "tokens_seen": 3254306816 + }, + { + "epoch": 1.37, + "learning_rate": 1.145837512537613e-05, + "loss": 1.0753, + "theoretical_loss": 3.328685714801577, + "tokens_seen": 3254372352 + }, + { + "epoch": 1.37, + "learning_rate": 1.1442326980942829e-05, + "loss": 1.2076, + "theoretical_loss": 3.328680689073368, + "tokens_seen": 3254437888 + }, + { + "epoch": 1.37, + "learning_rate": 1.142627883650953e-05, + "loss": 1.159, + "theoretical_loss": 3.3286756634747, + "tokens_seen": 3254503424 + }, + { + "epoch": 1.37, + "learning_rate": 1.141023069207623e-05, + "loss": 1.3029, + "theoretical_loss": 3.3286706380055673, + "tokens_seen": 3254568960 + }, + { + "epoch": 1.37, + "learning_rate": 1.1394182547642929e-05, + "loss": 1.182, + "theoretical_loss": 3.3286656126659633, + "tokens_seen": 3254634496 + }, + { + "epoch": 1.37, + "learning_rate": 1.137813440320963e-05, + "loss": 1.2654, + "theoretical_loss": 3.3286605874558823, + "tokens_seen": 3254700032 + }, + { + "epoch": 1.37, + "learning_rate": 1.1362086258776329e-05, + "loss": 1.1399, + "theoretical_loss": 3.328655562375319, + "tokens_seen": 3254765568 + }, + { + "epoch": 1.37, + "learning_rate": 1.134603811434303e-05, + "loss": 1.2064, + "theoretical_loss": 3.328650537424267, + "tokens_seen": 3254831104 + }, + { + "epoch": 1.37, + "learning_rate": 1.132998996990973e-05, + "loss": 1.1759, + "theoretical_loss": 3.32864551260272, + "tokens_seen": 3254896640 + }, + { + "epoch": 1.37, + "learning_rate": 1.131394182547643e-05, + "loss": 1.1999, + "theoretical_loss": 3.328640487910673, + "tokens_seen": 3254962176 + }, + { + "epoch": 1.37, + "learning_rate": 1.129789368104313e-05, + "loss": 1.2024, + "theoretical_loss": 3.3286354633481188, + "tokens_seen": 3255027712 + }, + { + "epoch": 1.37, + "learning_rate": 1.128184553660983e-05, + "loss": 1.1218, + "theoretical_loss": 3.3286304389150523, + "tokens_seen": 3255093248 + }, + { + "epoch": 1.37, + "learning_rate": 1.126579739217653e-05, + "loss": 1.1387, + "theoretical_loss": 3.3286254146114675, + "tokens_seen": 3255158784 + }, + { + "epoch": 1.37, + "learning_rate": 1.124974924774323e-05, + "loss": 1.0959, + "theoretical_loss": 3.3286203904373584, + "tokens_seen": 3255224320 + }, + { + "epoch": 1.37, + "learning_rate": 1.123370110330993e-05, + "loss": 1.1366, + "theoretical_loss": 3.328615366392719, + "tokens_seen": 3255289856 + }, + { + "epoch": 1.37, + "learning_rate": 1.121765295887663e-05, + "loss": 1.201, + "theoretical_loss": 3.328610342477543, + "tokens_seen": 3255355392 + }, + { + "epoch": 1.37, + "learning_rate": 1.1201604814443331e-05, + "loss": 1.0882, + "theoretical_loss": 3.328605318691825, + "tokens_seen": 3255420928 + }, + { + "epoch": 1.37, + "learning_rate": 1.118555667001003e-05, + "loss": 1.2532, + "theoretical_loss": 3.3286002950355584, + "tokens_seen": 3255486464 + }, + { + "epoch": 1.37, + "learning_rate": 1.116950852557673e-05, + "loss": 1.1199, + "theoretical_loss": 3.328595271508738, + "tokens_seen": 3255552000 + }, + { + "epoch": 1.37, + "learning_rate": 1.1153460381143432e-05, + "loss": 1.281, + "theoretical_loss": 3.3285902481113583, + "tokens_seen": 3255617536 + }, + { + "epoch": 1.37, + "learning_rate": 1.113741223671013e-05, + "loss": 1.1486, + "theoretical_loss": 3.328585224843412, + "tokens_seen": 3255683072 + }, + { + "epoch": 1.37, + "learning_rate": 1.1121364092276831e-05, + "loss": 1.1586, + "theoretical_loss": 3.3285802017048933, + "tokens_seen": 3255748608 + }, + { + "epoch": 1.37, + "learning_rate": 1.110531594784353e-05, + "loss": 1.1774, + "theoretical_loss": 3.3285751786957976, + "tokens_seen": 3255814144 + }, + { + "epoch": 1.37, + "learning_rate": 1.1089267803410233e-05, + "loss": 1.2103, + "theoretical_loss": 3.3285701558161174, + "tokens_seen": 3255879680 + }, + { + "epoch": 1.37, + "learning_rate": 1.1073219658976931e-05, + "loss": 1.2157, + "theoretical_loss": 3.328565133065848, + "tokens_seen": 3255945216 + }, + { + "epoch": 1.37, + "learning_rate": 1.105717151454363e-05, + "loss": 1.0901, + "theoretical_loss": 3.3285601104449825, + "tokens_seen": 3256010752 + }, + { + "epoch": 1.37, + "learning_rate": 1.1041123370110331e-05, + "loss": 1.2367, + "theoretical_loss": 3.3285550879535157, + "tokens_seen": 3256076288 + }, + { + "epoch": 1.37, + "learning_rate": 1.1025075225677032e-05, + "loss": 1.2088, + "theoretical_loss": 3.3285500655914415, + "tokens_seen": 3256141824 + }, + { + "epoch": 1.37, + "learning_rate": 1.1009027081243732e-05, + "loss": 1.2358, + "theoretical_loss": 3.3285450433587536, + "tokens_seen": 3256207360 + }, + { + "epoch": 1.37, + "learning_rate": 1.0992978936810431e-05, + "loss": 1.1645, + "theoretical_loss": 3.328540021255446, + "tokens_seen": 3256272896 + }, + { + "epoch": 1.37, + "learning_rate": 1.0976930792377132e-05, + "loss": 1.1511, + "theoretical_loss": 3.3285349992815134, + "tokens_seen": 3256338432 + }, + { + "epoch": 1.37, + "learning_rate": 1.0960882647943833e-05, + "loss": 1.1214, + "theoretical_loss": 3.3285299774369497, + "tokens_seen": 3256403968 + }, + { + "epoch": 1.37, + "learning_rate": 1.0944834503510531e-05, + "loss": 1.1878, + "theoretical_loss": 3.3285249557217482, + "tokens_seen": 3256469504 + }, + { + "epoch": 1.37, + "learning_rate": 1.0928786359077232e-05, + "loss": 1.1821, + "theoretical_loss": 3.3285199341359037, + "tokens_seen": 3256535040 + }, + { + "epoch": 1.37, + "learning_rate": 1.0912738214643933e-05, + "loss": 1.17, + "theoretical_loss": 3.3285149126794105, + "tokens_seen": 3256600576 + }, + { + "epoch": 1.37, + "learning_rate": 1.0896690070210633e-05, + "loss": 1.2028, + "theoretical_loss": 3.3285098913522617, + "tokens_seen": 3256666112 + }, + { + "epoch": 1.37, + "learning_rate": 1.0880641925777332e-05, + "loss": 1.0527, + "theoretical_loss": 3.328504870154452, + "tokens_seen": 3256731648 + }, + { + "epoch": 1.37, + "learning_rate": 1.0864593781344033e-05, + "loss": 1.1068, + "theoretical_loss": 3.328499849085976, + "tokens_seen": 3256797184 + }, + { + "epoch": 1.37, + "learning_rate": 1.0848545636910734e-05, + "loss": 1.1847, + "theoretical_loss": 3.3284948281468267, + "tokens_seen": 3256862720 + }, + { + "epoch": 1.37, + "learning_rate": 1.0832497492477433e-05, + "loss": 1.1336, + "theoretical_loss": 3.328489807336999, + "tokens_seen": 3256928256 + }, + { + "epoch": 1.37, + "learning_rate": 1.0816449348044133e-05, + "loss": 1.1668, + "theoretical_loss": 3.328484786656486, + "tokens_seen": 3256993792 + }, + { + "epoch": 1.37, + "learning_rate": 1.0800401203610832e-05, + "loss": 1.1205, + "theoretical_loss": 3.3284797661052825, + "tokens_seen": 3257059328 + }, + { + "epoch": 1.37, + "objective/train/docs_used": 1201041, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1913856267929077, + "objective/train/theoretical_loss": 3.3284747456833825, + "objective/train/tokens_used": 3277584864, + "theoretical_loss": 3.3284747456833825, + "tokens_seen": 3257124864 + }, + { + "epoch": 1.37, + "learning_rate": 1.0784353059177533e-05, + "loss": 1.1914, + "theoretical_loss": 3.3284747456833825, + "tokens_seen": 3257124864 + }, + { + "epoch": 1.37, + "learning_rate": 1.0768304914744233e-05, + "loss": 1.2102, + "theoretical_loss": 3.3284697253907805, + "tokens_seen": 3257190400 + }, + { + "epoch": 1.37, + "learning_rate": 1.0752256770310934e-05, + "loss": 0.9996, + "theoretical_loss": 3.328464705227469, + "tokens_seen": 3257255936 + }, + { + "epoch": 1.37, + "learning_rate": 1.0736208625877633e-05, + "loss": 1.1517, + "theoretical_loss": 3.328459685193444, + "tokens_seen": 3257321472 + }, + { + "epoch": 1.37, + "learning_rate": 1.0720160481444334e-05, + "loss": 1.2085, + "theoretical_loss": 3.3284546652886986, + "tokens_seen": 3257387008 + }, + { + "epoch": 1.37, + "learning_rate": 1.0704112337011034e-05, + "loss": 1.203, + "theoretical_loss": 3.328449645513227, + "tokens_seen": 3257452544 + }, + { + "epoch": 1.37, + "learning_rate": 1.0688064192577733e-05, + "loss": 1.1859, + "theoretical_loss": 3.328444625867023, + "tokens_seen": 3257518080 + }, + { + "epoch": 1.37, + "learning_rate": 1.0672016048144434e-05, + "loss": 1.1781, + "theoretical_loss": 3.3284396063500807, + "tokens_seen": 3257583616 + }, + { + "epoch": 1.37, + "learning_rate": 1.0655967903711134e-05, + "loss": 1.2782, + "theoretical_loss": 3.328434586962395, + "tokens_seen": 3257649152 + }, + { + "epoch": 1.37, + "learning_rate": 1.0639919759277835e-05, + "loss": 1.174, + "theoretical_loss": 3.328429567703959, + "tokens_seen": 3257714688 + }, + { + "epoch": 1.37, + "learning_rate": 1.0623871614844534e-05, + "loss": 1.2088, + "theoretical_loss": 3.328424548574767, + "tokens_seen": 3257780224 + }, + { + "epoch": 1.37, + "learning_rate": 1.0607823470411233e-05, + "loss": 1.18, + "theoretical_loss": 3.328419529574813, + "tokens_seen": 3257845760 + }, + { + "epoch": 1.37, + "learning_rate": 1.0591775325977935e-05, + "loss": 1.2102, + "theoretical_loss": 3.328414510704092, + "tokens_seen": 3257911296 + }, + { + "epoch": 1.37, + "learning_rate": 1.0575727181544634e-05, + "loss": 1.2103, + "theoretical_loss": 3.3284094919625966, + "tokens_seen": 3257976832 + }, + { + "epoch": 1.37, + "learning_rate": 1.0559679037111335e-05, + "loss": 1.2094, + "theoretical_loss": 3.3284044733503224, + "tokens_seen": 3258042368 + }, + { + "epoch": 1.37, + "learning_rate": 1.0543630892678034e-05, + "loss": 1.2903, + "theoretical_loss": 3.328399454867262, + "tokens_seen": 3258107904 + }, + { + "epoch": 1.37, + "learning_rate": 1.0527582748244736e-05, + "loss": 1.1317, + "theoretical_loss": 3.3283944365134106, + "tokens_seen": 3258173440 + }, + { + "epoch": 1.37, + "learning_rate": 1.0511534603811435e-05, + "loss": 1.1869, + "theoretical_loss": 3.3283894182887614, + "tokens_seen": 3258238976 + }, + { + "epoch": 1.37, + "learning_rate": 1.0495486459378134e-05, + "loss": 1.2212, + "theoretical_loss": 3.3283844001933094, + "tokens_seen": 3258304512 + }, + { + "epoch": 1.37, + "learning_rate": 1.0479438314944835e-05, + "loss": 1.2779, + "theoretical_loss": 3.328379382227048, + "tokens_seen": 3258370048 + }, + { + "epoch": 1.37, + "learning_rate": 1.0463390170511535e-05, + "loss": 1.1901, + "theoretical_loss": 3.3283743643899717, + "tokens_seen": 3258435584 + }, + { + "epoch": 1.37, + "learning_rate": 1.0447342026078236e-05, + "loss": 1.1404, + "theoretical_loss": 3.328369346682074, + "tokens_seen": 3258501120 + }, + { + "epoch": 1.37, + "learning_rate": 1.0431293881644935e-05, + "loss": 1.2375, + "theoretical_loss": 3.3283643291033496, + "tokens_seen": 3258566656 + }, + { + "epoch": 1.37, + "learning_rate": 1.0415245737211635e-05, + "loss": 1.1122, + "theoretical_loss": 3.328359311653792, + "tokens_seen": 3258632192 + }, + { + "epoch": 1.37, + "learning_rate": 1.0399197592778336e-05, + "loss": 1.1927, + "theoretical_loss": 3.3283542943333955, + "tokens_seen": 3258697728 + }, + { + "epoch": 1.37, + "learning_rate": 1.0383149448345035e-05, + "loss": 1.1371, + "theoretical_loss": 3.3283492771421543, + "tokens_seen": 3258763264 + }, + { + "epoch": 1.37, + "learning_rate": 1.0367101303911736e-05, + "loss": 1.1877, + "theoretical_loss": 3.328344260080063, + "tokens_seen": 3258828800 + }, + { + "epoch": 1.37, + "learning_rate": 1.0351053159478436e-05, + "loss": 1.203, + "theoretical_loss": 3.3283392431471146, + "tokens_seen": 3258894336 + }, + { + "epoch": 1.37, + "learning_rate": 1.0335005015045137e-05, + "loss": 1.1405, + "theoretical_loss": 3.3283342263433036, + "tokens_seen": 3258959872 + }, + { + "epoch": 1.37, + "learning_rate": 1.0318956870611836e-05, + "loss": 1.1567, + "theoretical_loss": 3.3283292096686243, + "tokens_seen": 3259025408 + }, + { + "epoch": 1.37, + "learning_rate": 1.0302908726178537e-05, + "loss": 1.1873, + "theoretical_loss": 3.3283241931230707, + "tokens_seen": 3259090944 + }, + { + "epoch": 1.37, + "learning_rate": 1.0286860581745235e-05, + "loss": 1.1291, + "theoretical_loss": 3.328319176706637, + "tokens_seen": 3259156480 + }, + { + "epoch": 1.37, + "learning_rate": 1.0270812437311936e-05, + "loss": 1.1021, + "theoretical_loss": 3.3283141604193167, + "tokens_seen": 3259222016 + }, + { + "epoch": 1.37, + "learning_rate": 1.0254764292878637e-05, + "loss": 1.1496, + "theoretical_loss": 3.3283091442611044, + "tokens_seen": 3259287552 + }, + { + "epoch": 1.37, + "learning_rate": 1.0238716148445336e-05, + "loss": 1.1206, + "theoretical_loss": 3.328304128231994, + "tokens_seen": 3259353088 + }, + { + "epoch": 1.37, + "learning_rate": 1.0222668004012036e-05, + "loss": 1.1892, + "theoretical_loss": 3.32829911233198, + "tokens_seen": 3259418624 + }, + { + "epoch": 1.37, + "learning_rate": 1.0206619859578737e-05, + "loss": 1.1813, + "theoretical_loss": 3.328294096561056, + "tokens_seen": 3259484160 + }, + { + "epoch": 1.37, + "learning_rate": 1.0190571715145438e-05, + "loss": 1.2674, + "theoretical_loss": 3.328289080919216, + "tokens_seen": 3259549696 + }, + { + "epoch": 1.37, + "learning_rate": 1.0174523570712136e-05, + "loss": 1.0729, + "theoretical_loss": 3.3282840654064545, + "tokens_seen": 3259615232 + }, + { + "epoch": 1.37, + "learning_rate": 1.0158475426278837e-05, + "loss": 1.2607, + "theoretical_loss": 3.3282790500227657, + "tokens_seen": 3259680768 + }, + { + "epoch": 1.37, + "learning_rate": 1.0142427281845538e-05, + "loss": 1.3104, + "theoretical_loss": 3.3282740347681425, + "tokens_seen": 3259746304 + }, + { + "epoch": 1.37, + "learning_rate": 1.0126379137412237e-05, + "loss": 1.1256, + "theoretical_loss": 3.3282690196425806, + "tokens_seen": 3259811840 + }, + { + "epoch": 1.37, + "learning_rate": 1.0110330992978937e-05, + "loss": 1.1172, + "theoretical_loss": 3.3282640046460727, + "tokens_seen": 3259877376 + }, + { + "epoch": 1.37, + "learning_rate": 1.0094282848545638e-05, + "loss": 1.1911, + "theoretical_loss": 3.328258989778614, + "tokens_seen": 3259942912 + }, + { + "epoch": 1.37, + "learning_rate": 1.0078234704112339e-05, + "loss": 1.2022, + "theoretical_loss": 3.328253975040198, + "tokens_seen": 3260008448 + }, + { + "epoch": 1.37, + "learning_rate": 1.0062186559679038e-05, + "loss": 1.1839, + "theoretical_loss": 3.328248960430819, + "tokens_seen": 3260073984 + }, + { + "epoch": 1.37, + "learning_rate": 1.0046138415245736e-05, + "loss": 1.2001, + "theoretical_loss": 3.328243945950471, + "tokens_seen": 3260139520 + }, + { + "epoch": 1.37, + "learning_rate": 1.0030090270812439e-05, + "loss": 1.1755, + "theoretical_loss": 3.328238931599148, + "tokens_seen": 3260205056 + }, + { + "epoch": 1.37, + "learning_rate": 1.0014042126379138e-05, + "loss": 1.1585, + "theoretical_loss": 3.328233917376844, + "tokens_seen": 3260270592 + }, + { + "epoch": 1.37, + "learning_rate": 9.997993981945838e-06, + "loss": 1.1612, + "theoretical_loss": 3.328228903283553, + "tokens_seen": 3260336128 + }, + { + "epoch": 1.37, + "learning_rate": 9.981945837512537e-06, + "loss": 1.1402, + "theoretical_loss": 3.3282238893192697, + "tokens_seen": 3260401664 + }, + { + "epoch": 1.37, + "learning_rate": 9.96589769307924e-06, + "loss": 1.2365, + "theoretical_loss": 3.3282188754839876, + "tokens_seen": 3260467200 + }, + { + "epoch": 1.37, + "learning_rate": 9.949849548645939e-06, + "loss": 1.1953, + "theoretical_loss": 3.328213861777701, + "tokens_seen": 3260532736 + }, + { + "epoch": 1.37, + "learning_rate": 9.933801404212638e-06, + "loss": 1.1184, + "theoretical_loss": 3.328208848200404, + "tokens_seen": 3260598272 + }, + { + "epoch": 1.37, + "learning_rate": 9.917753259779338e-06, + "loss": 1.1979, + "theoretical_loss": 3.328203834752091, + "tokens_seen": 3260663808 + }, + { + "epoch": 1.37, + "learning_rate": 9.901705115346039e-06, + "loss": 1.187, + "theoretical_loss": 3.328198821432755, + "tokens_seen": 3260729344 + }, + { + "epoch": 1.37, + "learning_rate": 9.88565697091274e-06, + "loss": 1.2479, + "theoretical_loss": 3.328193808242392, + "tokens_seen": 3260794880 + }, + { + "epoch": 1.37, + "learning_rate": 9.869608826479438e-06, + "loss": 1.1951, + "theoretical_loss": 3.328188795180994, + "tokens_seen": 3260860416 + }, + { + "epoch": 1.37, + "learning_rate": 9.853560682046139e-06, + "loss": 1.0928, + "theoretical_loss": 3.3281837822485567, + "tokens_seen": 3260925952 + }, + { + "epoch": 1.37, + "learning_rate": 9.83751253761284e-06, + "loss": 1.1934, + "theoretical_loss": 3.328178769445073, + "tokens_seen": 3260991488 + }, + { + "epoch": 1.37, + "learning_rate": 9.821464393179539e-06, + "loss": 1.1511, + "theoretical_loss": 3.3281737567705374, + "tokens_seen": 3261057024 + }, + { + "epoch": 1.37, + "learning_rate": 9.80541624874624e-06, + "loss": 1.169, + "theoretical_loss": 3.3281687442249446, + "tokens_seen": 3261122560 + }, + { + "epoch": 1.37, + "learning_rate": 9.789368104312938e-06, + "loss": 1.1744, + "theoretical_loss": 3.3281637318082877, + "tokens_seen": 3261188096 + }, + { + "epoch": 1.37, + "learning_rate": 9.77331995987964e-06, + "loss": 1.2657, + "theoretical_loss": 3.3281587195205615, + "tokens_seen": 3261253632 + }, + { + "epoch": 1.37, + "learning_rate": 9.75727181544634e-06, + "loss": 1.1938, + "theoretical_loss": 3.32815370736176, + "tokens_seen": 3261319168 + }, + { + "epoch": 1.37, + "learning_rate": 9.741223671013038e-06, + "loss": 1.1332, + "theoretical_loss": 3.328148695331877, + "tokens_seen": 3261384704 + }, + { + "epoch": 1.37, + "learning_rate": 9.725175526579739e-06, + "loss": 1.1346, + "theoretical_loss": 3.3281436834309073, + "tokens_seen": 3261450240 + }, + { + "epoch": 1.37, + "learning_rate": 9.70912738214644e-06, + "loss": 1.0891, + "theoretical_loss": 3.3281386716588437, + "tokens_seen": 3261515776 + }, + { + "epoch": 1.37, + "learning_rate": 9.69307923771314e-06, + "loss": 1.2162, + "theoretical_loss": 3.3281336600156814, + "tokens_seen": 3261581312 + }, + { + "epoch": 1.37, + "learning_rate": 9.67703109327984e-06, + "loss": 1.1293, + "theoretical_loss": 3.3281286485014143, + "tokens_seen": 3261646848 + }, + { + "epoch": 1.37, + "learning_rate": 9.66098294884654e-06, + "loss": 1.2013, + "theoretical_loss": 3.328123637116036, + "tokens_seen": 3261712384 + }, + { + "epoch": 1.37, + "learning_rate": 9.64493480441324e-06, + "loss": 1.2022, + "theoretical_loss": 3.328118625859541, + "tokens_seen": 3261777920 + }, + { + "epoch": 1.37, + "learning_rate": 9.62888665997994e-06, + "loss": 1.1426, + "theoretical_loss": 3.3281136147319237, + "tokens_seen": 3261843456 + }, + { + "epoch": 1.37, + "learning_rate": 9.61283851554664e-06, + "loss": 1.0907, + "theoretical_loss": 3.3281086037331775, + "tokens_seen": 3261908992 + }, + { + "epoch": 1.37, + "learning_rate": 9.59679037111334e-06, + "loss": 1.0714, + "theoretical_loss": 3.328103592863297, + "tokens_seen": 3261974528 + }, + { + "epoch": 1.37, + "learning_rate": 9.580742226680041e-06, + "loss": 1.2019, + "theoretical_loss": 3.3280985821222764, + "tokens_seen": 3262040064 + }, + { + "epoch": 1.37, + "learning_rate": 9.56469408224674e-06, + "loss": 1.2063, + "theoretical_loss": 3.328093571510109, + "tokens_seen": 3262105600 + }, + { + "epoch": 1.37, + "learning_rate": 9.548645937813441e-06, + "loss": 1.1996, + "theoretical_loss": 3.3280885610267896, + "tokens_seen": 3262171136 + }, + { + "epoch": 1.37, + "learning_rate": 9.532597793380142e-06, + "loss": 1.1659, + "theoretical_loss": 3.328083550672312, + "tokens_seen": 3262236672 + }, + { + "epoch": 1.37, + "learning_rate": 9.51654964894684e-06, + "loss": 1.2274, + "theoretical_loss": 3.3280785404466706, + "tokens_seen": 3262302208 + }, + { + "epoch": 1.37, + "learning_rate": 9.500501504513541e-06, + "loss": 1.0971, + "theoretical_loss": 3.3280735303498594, + "tokens_seen": 3262367744 + }, + { + "epoch": 1.37, + "learning_rate": 9.48445336008024e-06, + "loss": 1.0937, + "theoretical_loss": 3.3280685203818723, + "tokens_seen": 3262433280 + }, + { + "epoch": 1.37, + "learning_rate": 9.468405215646942e-06, + "loss": 1.1554, + "theoretical_loss": 3.3280635105427034, + "tokens_seen": 3262498816 + }, + { + "epoch": 1.37, + "learning_rate": 9.452357071213641e-06, + "loss": 1.1061, + "theoretical_loss": 3.328058500832347, + "tokens_seen": 3262564352 + }, + { + "epoch": 1.37, + "learning_rate": 9.436308926780342e-06, + "loss": 1.1329, + "theoretical_loss": 3.328053491250797, + "tokens_seen": 3262629888 + }, + { + "epoch": 1.37, + "learning_rate": 9.420260782347041e-06, + "loss": 1.0973, + "theoretical_loss": 3.328048481798048, + "tokens_seen": 3262695424 + }, + { + "epoch": 1.37, + "learning_rate": 9.404212637913742e-06, + "loss": 1.1355, + "theoretical_loss": 3.3280434724740937, + "tokens_seen": 3262760960 + }, + { + "epoch": 1.37, + "learning_rate": 9.388164493480442e-06, + "loss": 1.1663, + "theoretical_loss": 3.328038463278928, + "tokens_seen": 3262826496 + }, + { + "epoch": 1.37, + "learning_rate": 9.372116349047141e-06, + "loss": 1.0601, + "theoretical_loss": 3.3280334542125454, + "tokens_seen": 3262892032 + }, + { + "epoch": 1.37, + "learning_rate": 9.356068204613842e-06, + "loss": 1.1592, + "theoretical_loss": 3.3280284452749394, + "tokens_seen": 3262957568 + }, + { + "epoch": 1.37, + "learning_rate": 9.340020060180542e-06, + "loss": 1.1167, + "theoretical_loss": 3.328023436466105, + "tokens_seen": 3263023104 + }, + { + "epoch": 1.37, + "learning_rate": 9.323971915747243e-06, + "loss": 1.1107, + "theoretical_loss": 3.3280184277860356, + "tokens_seen": 3263088640 + }, + { + "epoch": 1.37, + "learning_rate": 9.307923771313942e-06, + "loss": 1.0373, + "theoretical_loss": 3.3280134192347255, + "tokens_seen": 3263154176 + }, + { + "epoch": 1.37, + "learning_rate": 9.291875626880641e-06, + "loss": 1.2032, + "theoretical_loss": 3.328008410812169, + "tokens_seen": 3263219712 + }, + { + "epoch": 1.37, + "learning_rate": 9.275827482447343e-06, + "loss": 1.2003, + "theoretical_loss": 3.32800340251836, + "tokens_seen": 3263285248 + }, + { + "epoch": 1.37, + "learning_rate": 9.259779338014042e-06, + "loss": 1.2083, + "theoretical_loss": 3.3279983943532927, + "tokens_seen": 3263350784 + }, + { + "epoch": 1.37, + "learning_rate": 9.243731193580743e-06, + "loss": 1.0724, + "theoretical_loss": 3.327993386316961, + "tokens_seen": 3263416320 + }, + { + "epoch": 1.37, + "learning_rate": 9.227683049147442e-06, + "loss": 1.1594, + "theoretical_loss": 3.3279883784093593, + "tokens_seen": 3263481856 + }, + { + "epoch": 1.37, + "learning_rate": 9.211634904714144e-06, + "loss": 1.2393, + "theoretical_loss": 3.3279833706304816, + "tokens_seen": 3263547392 + }, + { + "epoch": 1.37, + "learning_rate": 9.195586760280843e-06, + "loss": 1.1318, + "theoretical_loss": 3.3279783629803217, + "tokens_seen": 3263612928 + }, + { + "epoch": 1.37, + "objective/train/docs_used": 1203655, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.1544163227081299, + "objective/train/theoretical_loss": 3.3279733554588744, + "objective/train/tokens_used": 3284138464, + "theoretical_loss": 3.3279733554588744, + "tokens_seen": 3263678464 + }, + { + "epoch": 1.37, + "learning_rate": 9.179538615847542e-06, + "loss": 1.1544, + "theoretical_loss": 3.3279733554588744, + "tokens_seen": 3263678464 + }, + { + "epoch": 1.37, + "learning_rate": 9.163490471414243e-06, + "loss": 1.1499, + "theoretical_loss": 3.327968348066133, + "tokens_seen": 3263744000 + }, + { + "epoch": 1.37, + "learning_rate": 9.147442326980943e-06, + "loss": 1.2014, + "theoretical_loss": 3.327963340802092, + "tokens_seen": 3263809536 + }, + { + "epoch": 1.37, + "learning_rate": 9.131394182547644e-06, + "loss": 1.092, + "theoretical_loss": 3.327958333666746, + "tokens_seen": 3263875072 + }, + { + "epoch": 1.37, + "learning_rate": 9.115346038114343e-06, + "loss": 1.1797, + "theoretical_loss": 3.3279533266600883, + "tokens_seen": 3263940608 + }, + { + "epoch": 1.37, + "learning_rate": 9.099297893681043e-06, + "loss": 1.2863, + "theoretical_loss": 3.3279483197821134, + "tokens_seen": 3264006144 + }, + { + "epoch": 1.37, + "learning_rate": 9.083249749247744e-06, + "loss": 1.083, + "theoretical_loss": 3.327943313032815, + "tokens_seen": 3264071680 + }, + { + "epoch": 1.37, + "learning_rate": 9.067201604814443e-06, + "loss": 1.1749, + "theoretical_loss": 3.3279383064121877, + "tokens_seen": 3264137216 + }, + { + "epoch": 1.37, + "learning_rate": 9.051153460381144e-06, + "loss": 1.2243, + "theoretical_loss": 3.3279332999202254, + "tokens_seen": 3264202752 + }, + { + "epoch": 1.37, + "learning_rate": 9.035105315947844e-06, + "loss": 1.1423, + "theoretical_loss": 3.3279282935569228, + "tokens_seen": 3264268288 + }, + { + "epoch": 1.37, + "learning_rate": 9.019057171514545e-06, + "loss": 1.0666, + "theoretical_loss": 3.327923287322273, + "tokens_seen": 3264333824 + }, + { + "epoch": 1.37, + "learning_rate": 9.003009027081244e-06, + "loss": 1.1299, + "theoretical_loss": 3.3279182812162706, + "tokens_seen": 3264399360 + }, + { + "epoch": 1.37, + "learning_rate": 8.986960882647944e-06, + "loss": 1.2001, + "theoretical_loss": 3.3279132752389096, + "tokens_seen": 3264464896 + }, + { + "epoch": 1.37, + "learning_rate": 8.970912738214645e-06, + "loss": 1.2205, + "theoretical_loss": 3.3279082693901847, + "tokens_seen": 3264530432 + }, + { + "epoch": 1.37, + "learning_rate": 8.954864593781344e-06, + "loss": 1.1786, + "theoretical_loss": 3.327903263670089, + "tokens_seen": 3264595968 + }, + { + "epoch": 1.37, + "learning_rate": 8.938816449348045e-06, + "loss": 1.0929, + "theoretical_loss": 3.3278982580786174, + "tokens_seen": 3264661504 + }, + { + "epoch": 1.37, + "learning_rate": 8.922768304914744e-06, + "loss": 1.1725, + "theoretical_loss": 3.3278932526157634, + "tokens_seen": 3264727040 + }, + { + "epoch": 1.37, + "learning_rate": 8.906720160481446e-06, + "loss": 1.1622, + "theoretical_loss": 3.327888247281522, + "tokens_seen": 3264792576 + }, + { + "epoch": 1.37, + "learning_rate": 8.890672016048145e-06, + "loss": 1.1762, + "theoretical_loss": 3.3278832420758864, + "tokens_seen": 3264858112 + }, + { + "epoch": 1.37, + "learning_rate": 8.874623871614846e-06, + "loss": 1.1676, + "theoretical_loss": 3.327878236998851, + "tokens_seen": 3264923648 + }, + { + "epoch": 1.37, + "learning_rate": 8.858575727181544e-06, + "loss": 1.1481, + "theoretical_loss": 3.3278732320504103, + "tokens_seen": 3264989184 + }, + { + "epoch": 1.37, + "learning_rate": 8.842527582748245e-06, + "loss": 1.1463, + "theoretical_loss": 3.327868227230558, + "tokens_seen": 3265054720 + }, + { + "epoch": 1.37, + "learning_rate": 8.826479438314946e-06, + "loss": 1.1468, + "theoretical_loss": 3.327863222539288, + "tokens_seen": 3265120256 + }, + { + "epoch": 1.37, + "learning_rate": 8.810431293881645e-06, + "loss": 1.1884, + "theoretical_loss": 3.3278582179765954, + "tokens_seen": 3265185792 + }, + { + "epoch": 1.37, + "learning_rate": 8.794383149448345e-06, + "loss": 1.1759, + "theoretical_loss": 3.327853213542473, + "tokens_seen": 3265251328 + }, + { + "epoch": 1.37, + "learning_rate": 8.778335005015046e-06, + "loss": 1.1454, + "theoretical_loss": 3.327848209236916, + "tokens_seen": 3265316864 + }, + { + "epoch": 1.37, + "learning_rate": 8.762286860581747e-06, + "loss": 1.2319, + "theoretical_loss": 3.327843205059918, + "tokens_seen": 3265382400 + }, + { + "epoch": 1.37, + "learning_rate": 8.746238716148446e-06, + "loss": 1.1397, + "theoretical_loss": 3.327838201011473, + "tokens_seen": 3265447936 + }, + { + "epoch": 1.37, + "learning_rate": 8.730190571715144e-06, + "loss": 1.1768, + "theoretical_loss": 3.327833197091576, + "tokens_seen": 3265513472 + }, + { + "epoch": 1.37, + "learning_rate": 8.714142427281847e-06, + "loss": 1.0903, + "theoretical_loss": 3.3278281933002196, + "tokens_seen": 3265579008 + }, + { + "epoch": 1.37, + "learning_rate": 8.698094282848546e-06, + "loss": 1.2029, + "theoretical_loss": 3.3278231896373995, + "tokens_seen": 3265644544 + }, + { + "epoch": 1.37, + "learning_rate": 8.682046138415246e-06, + "loss": 1.1568, + "theoretical_loss": 3.3278181861031086, + "tokens_seen": 3265710080 + }, + { + "epoch": 1.37, + "learning_rate": 8.665997993981945e-06, + "loss": 1.1207, + "theoretical_loss": 3.3278131826973416, + "tokens_seen": 3265775616 + }, + { + "epoch": 1.37, + "learning_rate": 8.649949849548648e-06, + "loss": 1.0957, + "theoretical_loss": 3.3278081794200927, + "tokens_seen": 3265841152 + }, + { + "epoch": 1.37, + "learning_rate": 8.633901705115347e-06, + "loss": 1.164, + "theoretical_loss": 3.3278031762713556, + "tokens_seen": 3265906688 + }, + { + "epoch": 1.37, + "learning_rate": 8.617853560682046e-06, + "loss": 1.2382, + "theoretical_loss": 3.327798173251125, + "tokens_seen": 3265972224 + }, + { + "epoch": 1.37, + "learning_rate": 8.601805416248746e-06, + "loss": 1.1316, + "theoretical_loss": 3.3277931703593944, + "tokens_seen": 3266037760 + }, + { + "epoch": 1.37, + "learning_rate": 8.585757271815447e-06, + "loss": 1.1762, + "theoretical_loss": 3.3277881675961583, + "tokens_seen": 3266103296 + }, + { + "epoch": 1.37, + "learning_rate": 8.569709127382147e-06, + "loss": 1.1487, + "theoretical_loss": 3.327783164961411, + "tokens_seen": 3266168832 + }, + { + "epoch": 1.37, + "learning_rate": 8.553660982948846e-06, + "loss": 1.2529, + "theoretical_loss": 3.3277781624551457, + "tokens_seen": 3266234368 + }, + { + "epoch": 1.37, + "learning_rate": 8.537612838515547e-06, + "loss": 1.1883, + "theoretical_loss": 3.3277731600773577, + "tokens_seen": 3266299904 + }, + { + "epoch": 1.37, + "learning_rate": 8.521564694082248e-06, + "loss": 1.2322, + "theoretical_loss": 3.3277681578280403, + "tokens_seen": 3266365440 + }, + { + "epoch": 1.37, + "learning_rate": 8.505516549648947e-06, + "loss": 1.1224, + "theoretical_loss": 3.327763155707188, + "tokens_seen": 3266430976 + }, + { + "epoch": 1.37, + "learning_rate": 8.489468405215647e-06, + "loss": 1.1009, + "theoretical_loss": 3.327758153714795, + "tokens_seen": 3266496512 + }, + { + "epoch": 1.37, + "learning_rate": 8.473420260782348e-06, + "loss": 1.0385, + "theoretical_loss": 3.327753151850855, + "tokens_seen": 3266562048 + }, + { + "epoch": 1.37, + "learning_rate": 8.457372116349048e-06, + "loss": 1.2473, + "theoretical_loss": 3.3277481501153625, + "tokens_seen": 3266627584 + }, + { + "epoch": 1.37, + "learning_rate": 8.441323971915747e-06, + "loss": 1.3113, + "theoretical_loss": 3.3277431485083118, + "tokens_seen": 3266693120 + }, + { + "epoch": 1.37, + "learning_rate": 8.425275827482448e-06, + "loss": 1.1688, + "theoretical_loss": 3.3277381470296965, + "tokens_seen": 3266758656 + }, + { + "epoch": 1.37, + "learning_rate": 8.409227683049149e-06, + "loss": 1.2015, + "theoretical_loss": 3.327733145679511, + "tokens_seen": 3266824192 + }, + { + "epoch": 1.37, + "learning_rate": 8.393179538615848e-06, + "loss": 1.2641, + "theoretical_loss": 3.3277281444577493, + "tokens_seen": 3266889728 + }, + { + "epoch": 1.37, + "learning_rate": 8.377131394182548e-06, + "loss": 1.171, + "theoretical_loss": 3.3277231433644054, + "tokens_seen": 3266955264 + }, + { + "epoch": 1.37, + "learning_rate": 8.361083249749247e-06, + "loss": 1.1748, + "theoretical_loss": 3.3277181423994744, + "tokens_seen": 3267020800 + }, + { + "epoch": 1.37, + "learning_rate": 8.345035105315948e-06, + "loss": 1.1079, + "theoretical_loss": 3.327713141562949, + "tokens_seen": 3267086336 + }, + { + "epoch": 1.37, + "learning_rate": 8.328986960882648e-06, + "loss": 1.1348, + "theoretical_loss": 3.327708140854824, + "tokens_seen": 3267151872 + }, + { + "epoch": 1.37, + "learning_rate": 8.312938816449349e-06, + "loss": 1.2588, + "theoretical_loss": 3.327703140275094, + "tokens_seen": 3267217408 + }, + { + "epoch": 1.37, + "learning_rate": 8.296890672016048e-06, + "loss": 1.1935, + "theoretical_loss": 3.3276981398237524, + "tokens_seen": 3267282944 + }, + { + "epoch": 1.37, + "learning_rate": 8.280842527582749e-06, + "loss": 1.1609, + "theoretical_loss": 3.327693139500793, + "tokens_seen": 3267348480 + }, + { + "epoch": 1.37, + "learning_rate": 8.26479438314945e-06, + "loss": 1.1973, + "theoretical_loss": 3.327688139306211, + "tokens_seen": 3267414016 + }, + { + "epoch": 1.37, + "learning_rate": 8.248746238716148e-06, + "loss": 1.2153, + "theoretical_loss": 3.3276831392400004, + "tokens_seen": 3267479552 + }, + { + "epoch": 1.37, + "learning_rate": 8.232698094282849e-06, + "loss": 1.0886, + "theoretical_loss": 3.327678139302155, + "tokens_seen": 3267545088 + }, + { + "epoch": 1.37, + "learning_rate": 8.21664994984955e-06, + "loss": 1.0797, + "theoretical_loss": 3.3276731394926684, + "tokens_seen": 3267610624 + }, + { + "epoch": 1.37, + "learning_rate": 8.20060180541625e-06, + "loss": 1.225, + "theoretical_loss": 3.3276681398115353, + "tokens_seen": 3267676160 + }, + { + "epoch": 1.37, + "learning_rate": 8.184553660982949e-06, + "loss": 1.187, + "theoretical_loss": 3.3276631402587498, + "tokens_seen": 3267741696 + }, + { + "epoch": 1.37, + "learning_rate": 8.168505516549648e-06, + "loss": 1.1598, + "theoretical_loss": 3.3276581408343064, + "tokens_seen": 3267807232 + }, + { + "epoch": 1.37, + "learning_rate": 8.15245737211635e-06, + "loss": 1.2223, + "theoretical_loss": 3.3276531415381982, + "tokens_seen": 3267872768 + }, + { + "epoch": 1.37, + "learning_rate": 8.13640922768305e-06, + "loss": 1.1744, + "theoretical_loss": 3.3276481423704203, + "tokens_seen": 3267938304 + }, + { + "epoch": 1.37, + "learning_rate": 8.12036108324975e-06, + "loss": 1.0884, + "theoretical_loss": 3.3276431433309663, + "tokens_seen": 3268003840 + }, + { + "epoch": 1.37, + "learning_rate": 8.104312938816449e-06, + "loss": 1.1389, + "theoretical_loss": 3.327638144419831, + "tokens_seen": 3268069376 + }, + { + "epoch": 1.37, + "learning_rate": 8.088264794383151e-06, + "loss": 1.0921, + "theoretical_loss": 3.3276331456370074, + "tokens_seen": 3268134912 + }, + { + "epoch": 1.37, + "learning_rate": 8.07221664994985e-06, + "loss": 1.1447, + "theoretical_loss": 3.327628146982491, + "tokens_seen": 3268200448 + }, + { + "epoch": 1.37, + "learning_rate": 8.056168505516549e-06, + "loss": 1.1359, + "theoretical_loss": 3.327623148456275, + "tokens_seen": 3268265984 + }, + { + "epoch": 1.37, + "learning_rate": 8.04012036108325e-06, + "loss": 1.2177, + "theoretical_loss": 3.327618150058353, + "tokens_seen": 3268331520 + }, + { + "epoch": 1.37, + "learning_rate": 8.02407221664995e-06, + "loss": 1.2747, + "theoretical_loss": 3.3276131517887206, + "tokens_seen": 3268397056 + }, + { + "epoch": 1.37, + "learning_rate": 8.008024072216651e-06, + "loss": 1.2646, + "theoretical_loss": 3.327608153647371, + "tokens_seen": 3268462592 + }, + { + "epoch": 1.37, + "learning_rate": 7.99197592778335e-06, + "loss": 1.0977, + "theoretical_loss": 3.3276031556342986, + "tokens_seen": 3268528128 + }, + { + "epoch": 1.37, + "learning_rate": 7.97592778335005e-06, + "loss": 1.1, + "theoretical_loss": 3.327598157749498, + "tokens_seen": 3268593664 + }, + { + "epoch": 1.37, + "learning_rate": 7.959879638916751e-06, + "loss": 1.1151, + "theoretical_loss": 3.3275931599929622, + "tokens_seen": 3268659200 + }, + { + "epoch": 1.37, + "learning_rate": 7.94383149448345e-06, + "loss": 1.1421, + "theoretical_loss": 3.327588162364686, + "tokens_seen": 3268724736 + }, + { + "epoch": 1.37, + "learning_rate": 7.92778335005015e-06, + "loss": 1.226, + "theoretical_loss": 3.327583164864664, + "tokens_seen": 3268790272 + }, + { + "epoch": 1.37, + "learning_rate": 7.911735205616851e-06, + "loss": 1.1667, + "theoretical_loss": 3.3275781674928893, + "tokens_seen": 3268855808 + }, + { + "epoch": 1.37, + "learning_rate": 7.895687061183552e-06, + "loss": 1.1765, + "theoretical_loss": 3.327573170249357, + "tokens_seen": 3268921344 + }, + { + "epoch": 1.37, + "learning_rate": 7.879638916750251e-06, + "loss": 1.2265, + "theoretical_loss": 3.3275681731340603, + "tokens_seen": 3268986880 + }, + { + "epoch": 1.37, + "learning_rate": 7.863590772316952e-06, + "loss": 1.2362, + "theoretical_loss": 3.3275631761469944, + "tokens_seen": 3269052416 + }, + { + "epoch": 1.37, + "learning_rate": 7.84754262788365e-06, + "loss": 1.1851, + "theoretical_loss": 3.3275581792881526, + "tokens_seen": 3269117952 + }, + { + "epoch": 1.37, + "learning_rate": 7.831494483450351e-06, + "loss": 1.1568, + "theoretical_loss": 3.327553182557529, + "tokens_seen": 3269183488 + }, + { + "epoch": 1.37, + "learning_rate": 7.815446339017052e-06, + "loss": 1.1262, + "theoretical_loss": 3.327548185955119, + "tokens_seen": 3269249024 + }, + { + "epoch": 1.37, + "learning_rate": 7.79939819458375e-06, + "loss": 1.0589, + "theoretical_loss": 3.327543189480915, + "tokens_seen": 3269314560 + }, + { + "epoch": 1.37, + "learning_rate": 7.783350050150451e-06, + "loss": 1.2165, + "theoretical_loss": 3.327538193134912, + "tokens_seen": 3269380096 + }, + { + "epoch": 1.37, + "learning_rate": 7.767301905717152e-06, + "loss": 1.0948, + "theoretical_loss": 3.3275331969171047, + "tokens_seen": 3269445632 + }, + { + "epoch": 1.37, + "learning_rate": 7.751253761283853e-06, + "loss": 1.1444, + "theoretical_loss": 3.3275282008274862, + "tokens_seen": 3269511168 + }, + { + "epoch": 1.37, + "learning_rate": 7.735205616850552e-06, + "loss": 1.1899, + "theoretical_loss": 3.3275232048660515, + "tokens_seen": 3269576704 + }, + { + "epoch": 1.37, + "learning_rate": 7.719157472417252e-06, + "loss": 1.1597, + "theoretical_loss": 3.3275182090327937, + "tokens_seen": 3269642240 + }, + { + "epoch": 1.37, + "learning_rate": 7.703109327983953e-06, + "loss": 1.1876, + "theoretical_loss": 3.327513213327708, + "tokens_seen": 3269707776 + }, + { + "epoch": 1.37, + "learning_rate": 7.687061183550652e-06, + "loss": 1.1661, + "theoretical_loss": 3.327508217750788, + "tokens_seen": 3269773312 + }, + { + "epoch": 1.37, + "learning_rate": 7.671013039117352e-06, + "loss": 1.2048, + "theoretical_loss": 3.327503222302028, + "tokens_seen": 3269838848 + }, + { + "epoch": 1.37, + "learning_rate": 7.654964894684053e-06, + "loss": 1.1889, + "theoretical_loss": 3.327498226981422, + "tokens_seen": 3269904384 + }, + { + "epoch": 1.37, + "learning_rate": 7.638916750250752e-06, + "loss": 1.1475, + "theoretical_loss": 3.327493231788964, + "tokens_seen": 3269969920 + }, + { + "epoch": 1.37, + "learning_rate": 7.622868605817453e-06, + "loss": 1.1617, + "theoretical_loss": 3.327488236724649, + "tokens_seen": 3270035456 + }, + { + "epoch": 1.37, + "learning_rate": 7.6068204613841525e-06, + "loss": 1.3185, + "theoretical_loss": 3.32748324178847, + "tokens_seen": 3270100992 + }, + { + "epoch": 1.37, + "learning_rate": 7.590772316950853e-06, + "loss": 1.1332, + "theoretical_loss": 3.327478246980422, + "tokens_seen": 3270166528 + }, + { + "epoch": 1.37, + "objective/train/docs_used": 1206013, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.0468988418579102, + "objective/train/theoretical_loss": 3.3274732523004986, + "objective/train/tokens_used": 3290692064, + "theoretical_loss": 3.3274732523004986, + "tokens_seen": 3270232064 + }, + { + "epoch": 1.37, + "learning_rate": 7.574724172517553e-06, + "loss": 1.0469, + "theoretical_loss": 3.3274732523004986, + "tokens_seen": 3270232064 + }, + { + "epoch": 1.37, + "learning_rate": 7.5586760280842535e-06, + "loss": 1.092, + "theoretical_loss": 3.327468257748694, + "tokens_seen": 3270297600 + }, + { + "epoch": 1.37, + "learning_rate": 7.542627883650953e-06, + "loss": 1.2025, + "theoretical_loss": 3.327463263325003, + "tokens_seen": 3270363136 + }, + { + "epoch": 1.37, + "learning_rate": 7.526579739217652e-06, + "loss": 1.189, + "theoretical_loss": 3.327458269029419, + "tokens_seen": 3270428672 + }, + { + "epoch": 1.37, + "learning_rate": 7.510531594784354e-06, + "loss": 1.2616, + "theoretical_loss": 3.327453274861936, + "tokens_seen": 3270494208 + }, + { + "epoch": 1.37, + "learning_rate": 7.494483450351053e-06, + "loss": 1.1981, + "theoretical_loss": 3.327448280822549, + "tokens_seen": 3270559744 + }, + { + "epoch": 1.37, + "learning_rate": 7.478435305917754e-06, + "loss": 1.1366, + "theoretical_loss": 3.327443286911252, + "tokens_seen": 3270625280 + }, + { + "epoch": 1.37, + "learning_rate": 7.462387161484453e-06, + "loss": 1.1478, + "theoretical_loss": 3.327438293128038, + "tokens_seen": 3270690816 + }, + { + "epoch": 1.37, + "learning_rate": 7.446339017051155e-06, + "loss": 1.1765, + "theoretical_loss": 3.327433299472903, + "tokens_seen": 3270756352 + }, + { + "epoch": 1.37, + "learning_rate": 7.4302908726178535e-06, + "loss": 1.2154, + "theoretical_loss": 3.3274283059458396, + "tokens_seen": 3270821888 + }, + { + "epoch": 1.37, + "learning_rate": 7.414242728184553e-06, + "loss": 1.2019, + "theoretical_loss": 3.3274233125468426, + "tokens_seen": 3270887424 + }, + { + "epoch": 1.37, + "learning_rate": 7.398194583751254e-06, + "loss": 1.1045, + "theoretical_loss": 3.327418319275906, + "tokens_seen": 3270952960 + }, + { + "epoch": 1.37, + "learning_rate": 7.382146439317954e-06, + "loss": 1.1779, + "theoretical_loss": 3.3274133261330237, + "tokens_seen": 3271018496 + }, + { + "epoch": 1.37, + "learning_rate": 7.366098294884654e-06, + "loss": 1.272, + "theoretical_loss": 3.3274083331181905, + "tokens_seen": 3271084032 + }, + { + "epoch": 1.37, + "learning_rate": 7.350050150451354e-06, + "loss": 1.3217, + "theoretical_loss": 3.3274033402314003, + "tokens_seen": 3271149568 + }, + { + "epoch": 1.37, + "learning_rate": 7.334002006018055e-06, + "loss": 1.1922, + "theoretical_loss": 3.3273983474726467, + "tokens_seen": 3271215104 + }, + { + "epoch": 1.37, + "learning_rate": 7.317953861584755e-06, + "loss": 1.2222, + "theoretical_loss": 3.3273933548419246, + "tokens_seen": 3271280640 + }, + { + "epoch": 1.37, + "learning_rate": 7.301905717151454e-06, + "loss": 1.1887, + "theoretical_loss": 3.327388362339228, + "tokens_seen": 3271346176 + }, + { + "epoch": 1.37, + "learning_rate": 7.285857572718155e-06, + "loss": 1.1515, + "theoretical_loss": 3.3273833699645508, + "tokens_seen": 3271411712 + }, + { + "epoch": 1.37, + "learning_rate": 7.269809428284855e-06, + "loss": 1.2077, + "theoretical_loss": 3.327378377717887, + "tokens_seen": 3271477248 + }, + { + "epoch": 1.37, + "learning_rate": 7.2537612838515554e-06, + "loss": 1.1463, + "theoretical_loss": 3.3273733855992313, + "tokens_seen": 3271542784 + }, + { + "epoch": 1.37, + "learning_rate": 7.237713139418255e-06, + "loss": 1.1842, + "theoretical_loss": 3.3273683936085776, + "tokens_seen": 3271608320 + }, + { + "epoch": 1.37, + "learning_rate": 7.221664994984956e-06, + "loss": 1.2434, + "theoretical_loss": 3.3273634017459197, + "tokens_seen": 3271673856 + }, + { + "epoch": 1.37, + "learning_rate": 7.205616850551656e-06, + "loss": 1.181, + "theoretical_loss": 3.3273584100112523, + "tokens_seen": 3271739392 + }, + { + "epoch": 1.37, + "learning_rate": 7.189568706118355e-06, + "loss": 1.1478, + "theoretical_loss": 3.327353418404569, + "tokens_seen": 3271804928 + }, + { + "epoch": 1.37, + "learning_rate": 7.173520561685056e-06, + "loss": 1.1457, + "theoretical_loss": 3.3273484269258646, + "tokens_seen": 3271870464 + }, + { + "epoch": 1.37, + "learning_rate": 7.157472417251755e-06, + "loss": 1.0824, + "theoretical_loss": 3.3273434355751332, + "tokens_seen": 3271936000 + }, + { + "epoch": 1.37, + "learning_rate": 7.141424272818456e-06, + "loss": 1.2545, + "theoretical_loss": 3.3273384443523684, + "tokens_seen": 3272001536 + }, + { + "epoch": 1.37, + "learning_rate": 7.125376128385155e-06, + "loss": 1.2592, + "theoretical_loss": 3.3273334532575642, + "tokens_seen": 3272067072 + }, + { + "epoch": 1.37, + "learning_rate": 7.109327983951856e-06, + "loss": 1.0739, + "theoretical_loss": 3.327328462290716, + "tokens_seen": 3272132608 + }, + { + "epoch": 1.37, + "learning_rate": 7.093279839518556e-06, + "loss": 1.1792, + "theoretical_loss": 3.3273234714518165, + "tokens_seen": 3272198144 + }, + { + "epoch": 1.37, + "learning_rate": 7.077231695085256e-06, + "loss": 1.1749, + "theoretical_loss": 3.327318480740861, + "tokens_seen": 3272263680 + }, + { + "epoch": 1.37, + "learning_rate": 7.061183550651956e-06, + "loss": 1.2196, + "theoretical_loss": 3.327313490157843, + "tokens_seen": 3272329216 + }, + { + "epoch": 1.37, + "learning_rate": 7.045135406218656e-06, + "loss": 1.1751, + "theoretical_loss": 3.327308499702757, + "tokens_seen": 3272394752 + }, + { + "epoch": 1.37, + "learning_rate": 7.029087261785357e-06, + "loss": 1.154, + "theoretical_loss": 3.3273035093755965, + "tokens_seen": 3272460288 + }, + { + "epoch": 1.37, + "learning_rate": 7.0130391173520565e-06, + "loss": 1.0934, + "theoretical_loss": 3.3272985191763564, + "tokens_seen": 3272525824 + }, + { + "epoch": 1.37, + "learning_rate": 6.996990972918757e-06, + "loss": 1.1766, + "theoretical_loss": 3.327293529105031, + "tokens_seen": 3272591360 + }, + { + "epoch": 1.37, + "learning_rate": 6.980942828485457e-06, + "loss": 1.221, + "theoretical_loss": 3.3272885391616134, + "tokens_seen": 3272656896 + }, + { + "epoch": 1.37, + "learning_rate": 6.964894684052156e-06, + "loss": 1.2078, + "theoretical_loss": 3.327283549346099, + "tokens_seen": 3272722432 + }, + { + "epoch": 1.37, + "learning_rate": 6.948846539618857e-06, + "loss": 1.1196, + "theoretical_loss": 3.327278559658481, + "tokens_seen": 3272787968 + }, + { + "epoch": 1.37, + "learning_rate": 6.932798395185556e-06, + "loss": 1.2184, + "theoretical_loss": 3.327273570098754, + "tokens_seen": 3272853504 + }, + { + "epoch": 1.37, + "learning_rate": 6.916750250752258e-06, + "loss": 1.1326, + "theoretical_loss": 3.3272685806669124, + "tokens_seen": 3272919040 + }, + { + "epoch": 1.37, + "learning_rate": 6.900702106318957e-06, + "loss": 1.0874, + "theoretical_loss": 3.32726359136295, + "tokens_seen": 3272984576 + }, + { + "epoch": 1.37, + "learning_rate": 6.884653961885658e-06, + "loss": 1.1088, + "theoretical_loss": 3.327258602186861, + "tokens_seen": 3273050112 + }, + { + "epoch": 1.37, + "learning_rate": 6.868605817452357e-06, + "loss": 1.0814, + "theoretical_loss": 3.327253613138639, + "tokens_seen": 3273115648 + }, + { + "epoch": 1.37, + "learning_rate": 6.852557673019057e-06, + "loss": 1.1808, + "theoretical_loss": 3.3272486242182797, + "tokens_seen": 3273181184 + }, + { + "epoch": 1.37, + "learning_rate": 6.8365095285857575e-06, + "loss": 1.1424, + "theoretical_loss": 3.3272436354257757, + "tokens_seen": 3273246720 + }, + { + "epoch": 1.37, + "learning_rate": 6.820461384152457e-06, + "loss": 1.2076, + "theoretical_loss": 3.3272386467611224, + "tokens_seen": 3273312256 + }, + { + "epoch": 1.37, + "learning_rate": 6.804413239719158e-06, + "loss": 1.1102, + "theoretical_loss": 3.327233658224313, + "tokens_seen": 3273377792 + }, + { + "epoch": 1.37, + "learning_rate": 6.788365095285858e-06, + "loss": 1.2779, + "theoretical_loss": 3.3272286698153417, + "tokens_seen": 3273443328 + }, + { + "epoch": 1.37, + "learning_rate": 6.772316950852558e-06, + "loss": 1.1633, + "theoretical_loss": 3.3272236815342033, + "tokens_seen": 3273508864 + }, + { + "epoch": 1.37, + "learning_rate": 6.756268806419258e-06, + "loss": 1.2145, + "theoretical_loss": 3.3272186933808916, + "tokens_seen": 3273574400 + }, + { + "epoch": 1.37, + "learning_rate": 6.740220661985958e-06, + "loss": 1.1911, + "theoretical_loss": 3.3272137053554007, + "tokens_seen": 3273639936 + }, + { + "epoch": 1.37, + "learning_rate": 6.724172517552659e-06, + "loss": 1.1986, + "theoretical_loss": 3.327208717457725, + "tokens_seen": 3273705472 + }, + { + "epoch": 1.37, + "learning_rate": 6.708124373119358e-06, + "loss": 1.1179, + "theoretical_loss": 3.3272037296878585, + "tokens_seen": 3273771008 + }, + { + "epoch": 1.37, + "learning_rate": 6.692076228686059e-06, + "loss": 1.0658, + "theoretical_loss": 3.3271987420457956, + "tokens_seen": 3273836544 + }, + { + "epoch": 1.37, + "learning_rate": 6.676028084252759e-06, + "loss": 1.2704, + "theoretical_loss": 3.32719375453153, + "tokens_seen": 3273902080 + }, + { + "epoch": 1.37, + "learning_rate": 6.6599799398194594e-06, + "loss": 1.1118, + "theoretical_loss": 3.3271887671450564, + "tokens_seen": 3273967616 + }, + { + "epoch": 1.37, + "learning_rate": 6.643931795386158e-06, + "loss": 1.162, + "theoretical_loss": 3.3271837798863686, + "tokens_seen": 3274033152 + }, + { + "epoch": 1.37, + "learning_rate": 6.627883650952858e-06, + "loss": 1.2302, + "theoretical_loss": 3.327178792755461, + "tokens_seen": 3274098688 + }, + { + "epoch": 1.37, + "learning_rate": 6.611835506519559e-06, + "loss": 1.1716, + "theoretical_loss": 3.3271738057523272, + "tokens_seen": 3274164224 + }, + { + "epoch": 1.37, + "learning_rate": 6.5957873620862586e-06, + "loss": 1.2065, + "theoretical_loss": 3.327168818876962, + "tokens_seen": 3274229760 + }, + { + "epoch": 1.37, + "learning_rate": 6.579739217652959e-06, + "loss": 1.2233, + "theoretical_loss": 3.3271638321293597, + "tokens_seen": 3274295296 + }, + { + "epoch": 1.37, + "learning_rate": 6.563691073219659e-06, + "loss": 1.1678, + "theoretical_loss": 3.3271588455095142, + "tokens_seen": 3274360832 + }, + { + "epoch": 1.37, + "learning_rate": 6.54764292878636e-06, + "loss": 1.0739, + "theoretical_loss": 3.3271538590174194, + "tokens_seen": 3274426368 + }, + { + "epoch": 1.37, + "learning_rate": 6.531594784353059e-06, + "loss": 1.2397, + "theoretical_loss": 3.32714887265307, + "tokens_seen": 3274491904 + }, + { + "epoch": 1.37, + "learning_rate": 6.515546639919759e-06, + "loss": 1.0684, + "theoretical_loss": 3.3271438864164593, + "tokens_seen": 3274557440 + }, + { + "epoch": 1.37, + "learning_rate": 6.49949849548646e-06, + "loss": 1.2928, + "theoretical_loss": 3.3271389003075824, + "tokens_seen": 3274622976 + }, + { + "epoch": 1.37, + "learning_rate": 6.48345035105316e-06, + "loss": 1.0315, + "theoretical_loss": 3.327133914326433, + "tokens_seen": 3274688512 + }, + { + "epoch": 1.37, + "learning_rate": 6.46740220661986e-06, + "loss": 1.2151, + "theoretical_loss": 3.3271289284730052, + "tokens_seen": 3274754048 + }, + { + "epoch": 1.37, + "learning_rate": 6.45135406218656e-06, + "loss": 1.1406, + "theoretical_loss": 3.327123942747294, + "tokens_seen": 3274819584 + }, + { + "epoch": 1.37, + "learning_rate": 6.435305917753261e-06, + "loss": 1.147, + "theoretical_loss": 3.3271189571492923, + "tokens_seen": 3274885120 + }, + { + "epoch": 1.37, + "learning_rate": 6.4192577733199605e-06, + "loss": 1.22, + "theoretical_loss": 3.327113971678995, + "tokens_seen": 3274950656 + }, + { + "epoch": 1.37, + "learning_rate": 6.403209628886659e-06, + "loss": 1.1877, + "theoretical_loss": 3.3271089863363965, + "tokens_seen": 3275016192 + }, + { + "epoch": 1.37, + "learning_rate": 6.387161484453361e-06, + "loss": 1.2834, + "theoretical_loss": 3.3271040011214903, + "tokens_seen": 3275081728 + }, + { + "epoch": 1.37, + "learning_rate": 6.37111334002006e-06, + "loss": 1.1933, + "theoretical_loss": 3.3270990160342713, + "tokens_seen": 3275147264 + }, + { + "epoch": 1.37, + "learning_rate": 6.355065195586761e-06, + "loss": 1.2159, + "theoretical_loss": 3.327094031074733, + "tokens_seen": 3275212800 + }, + { + "epoch": 1.37, + "learning_rate": 6.33901705115346e-06, + "loss": 1.1012, + "theoretical_loss": 3.32708904624287, + "tokens_seen": 3275278336 + }, + { + "epoch": 1.37, + "learning_rate": 6.322968906720162e-06, + "loss": 1.198, + "theoretical_loss": 3.327084061538676, + "tokens_seen": 3275343872 + }, + { + "epoch": 1.37, + "learning_rate": 6.306920762286861e-06, + "loss": 1.1355, + "theoretical_loss": 3.327079076962146, + "tokens_seen": 3275409408 + }, + { + "epoch": 1.37, + "learning_rate": 6.2908726178535605e-06, + "loss": 1.3235, + "theoretical_loss": 3.327074092513273, + "tokens_seen": 3275474944 + }, + { + "epoch": 1.37, + "learning_rate": 6.274824473420261e-06, + "loss": 1.1515, + "theoretical_loss": 3.3270691081920525, + "tokens_seen": 3275540480 + }, + { + "epoch": 1.37, + "learning_rate": 6.258776328986961e-06, + "loss": 1.1648, + "theoretical_loss": 3.327064123998478, + "tokens_seen": 3275606016 + }, + { + "epoch": 1.37, + "learning_rate": 6.242728184553661e-06, + "loss": 1.1985, + "theoretical_loss": 3.327059139932544, + "tokens_seen": 3275671552 + }, + { + "epoch": 1.37, + "learning_rate": 6.226680040120361e-06, + "loss": 1.1239, + "theoretical_loss": 3.3270541559942437, + "tokens_seen": 3275737088 + }, + { + "epoch": 1.37, + "learning_rate": 6.210631895687061e-06, + "loss": 1.165, + "theoretical_loss": 3.3270491721835724, + "tokens_seen": 3275802624 + }, + { + "epoch": 1.37, + "learning_rate": 6.194583751253762e-06, + "loss": 1.159, + "theoretical_loss": 3.327044188500524, + "tokens_seen": 3275868160 + }, + { + "epoch": 1.37, + "learning_rate": 6.1785356068204615e-06, + "loss": 1.0631, + "theoretical_loss": 3.327039204945092, + "tokens_seen": 3275933696 + }, + { + "epoch": 1.37, + "learning_rate": 6.162487462387162e-06, + "loss": 1.1982, + "theoretical_loss": 3.327034221517272, + "tokens_seen": 3275999232 + }, + { + "epoch": 1.37, + "learning_rate": 6.146439317953862e-06, + "loss": 1.1678, + "theoretical_loss": 3.3270292382170563, + "tokens_seen": 3276064768 + }, + { + "epoch": 1.37, + "learning_rate": 6.130391173520562e-06, + "loss": 1.1798, + "theoretical_loss": 3.327024255044441, + "tokens_seen": 3276130304 + }, + { + "epoch": 1.37, + "learning_rate": 6.1143430290872615e-06, + "loss": 1.0797, + "theoretical_loss": 3.3270192719994185, + "tokens_seen": 3276195840 + }, + { + "epoch": 1.37, + "learning_rate": 6.098294884653962e-06, + "loss": 1.1857, + "theoretical_loss": 3.3270142890819843, + "tokens_seen": 3276261376 + }, + { + "epoch": 1.37, + "learning_rate": 6.082246740220662e-06, + "loss": 1.1567, + "theoretical_loss": 3.3270093062921324, + "tokens_seen": 3276326912 + }, + { + "epoch": 1.37, + "learning_rate": 6.0661985957873626e-06, + "loss": 1.1018, + "theoretical_loss": 3.3270043236298563, + "tokens_seen": 3276392448 + }, + { + "epoch": 1.37, + "learning_rate": 6.050150451354062e-06, + "loss": 1.1305, + "theoretical_loss": 3.3269993410951506, + "tokens_seen": 3276457984 + }, + { + "epoch": 1.37, + "learning_rate": 6.034102306920763e-06, + "loss": 1.2875, + "theoretical_loss": 3.3269943586880095, + "tokens_seen": 3276523520 + }, + { + "epoch": 1.37, + "learning_rate": 6.018054162487463e-06, + "loss": 1.1931, + "theoretical_loss": 3.3269893764084273, + "tokens_seen": 3276589056 + }, + { + "epoch": 1.37, + "learning_rate": 6.0020060180541626e-06, + "loss": 1.1922, + "theoretical_loss": 3.326984394256398, + "tokens_seen": 3276654592 + }, + { + "epoch": 1.37, + "learning_rate": 5.985957873620863e-06, + "loss": 1.289, + "theoretical_loss": 3.326979412231916, + "tokens_seen": 3276720128 + }, + { + "debugging/Compilability": 1.0, + "debugging/distinct-1-grams": 0.7605763182765615, + "debugging/entropy-1-grams": 5.658532076395022, + "debugging/length": 457.57142857142856, + "debugging/num_segments": 21, + "debugging/score": 0.005049310483209717, + "debugging/score_std": 0.0032977059034618567, + "epoch": 1.37, + "objective/train/docs_used": 1208393, + "objective/train/instantaneous_batch_size": 64, + "objective/train/instantaneous_microbatch_size": 65536, + "objective/train/original_loss": 1.2265136241912842, + "objective/train/theoretical_loss": 3.3269744303349746, + "objective/train/tokens_used": 3297245664, + "theoretical_loss": 3.3269744303349746, + "tokens_seen": 3276785664 + }, + { + "epoch": 1.37, + "learning_rate": 5.969909729187563e-06, + "loss": 1.2265, + "theoretical_loss": 3.3269744303349746, + "tokens_seen": 3276785664 + } + ], + "max_steps": 50354, + "num_train_epochs": 9223372036854775807, + "total_flos": 1.672261688623104e+18, + "trial_name": null, + "trial_params": null +}